%\section{Computational Methods and Software}
\label{sec:software}

The previous chapter described a set of models
for the system of interest.
%These models are too
%complicated to solve exactly, and must instead be
%instantiated with software to produce a numerical
%result.
This chapter details the numerical formulation
and solution of these models. It begins with a
discussion of the numerical discretization of the equations
of interest. The mesh discretization is then described.
Next, the scientific software in which these numerical
models are used is discussed. Finally, the tool
chain and supercomputer systems are briefly introduced.

\section{Discretization Scheme}
\label{sec:discretization}
The finite element method (FEM) is used to numerically solve the
Navier-Stokes equations. The starting point for the FEM is
to cast the equations in Section~\ref{sub_sec:ns_en} into a weak
form. Use of the weak form reduces the continuity requirements on the
basis functions, thereby allowing the use of functions that are
easy to construct and implement, such as piece-wise polynomials.
Manipulating these partial differential equations into a
variational formulation is accomplished by multiplying the equations by
appropriate test functions and integrating over the domain,
$\Omega$. The resulting weak problem is: find $({\bf u},p,T) \in
H^1(\Omega)^3 \times L_2(\Omega) \times H^1(\Omega)$ such that

%
% http://www.numerik.uni-hd.de/Oberwolfach-Seminar/CFD-Course.pdf
%
\begin{align}
  (\frac{\partial{\bf u}}{\partial t},{\bf v}) + ({\bf u} \cdot \nabla{\bf
 u},{\bf v}) + (\nu \,\nabla{\bf u}, \nabla{\bf v})
  -(p,\nabla \cdot{\bf u}) &= ({\bf g} \, T'/T_0,v),
 \label{eqn:ns_weak} \\
 (\nabla \cdot{\bf u},q) &= 0,
 \label{eqn:cont_weak} \\
 (\frac{\partial T}{\partial t}, w) + ({\bf u} \cdot \nabla T,
 w) + (k \, \nabla T, \nabla  w) &= 0,\label{eqn:en_weak}
\end{align}

%\todo{boundary terms}

$\forall ({\bf v},q,w) \in H^1(\Omega)^3 \times L_2(\Omega) \times
H^1(\Omega)$, where $(\cdot,\cdot)$ denotes the $L_2$ inner product
$({\bf u},{\bf v}) = \int_\Omega {\bf u} \cdot {\bf v} \, dx$ and
$H^1(\Omega)$ is the Sobolov space with one square integrable derivative
on the domain
$\Omega$\cite{oden2012introduction}. As noted previously, boldface letters
denote vector quantities (such as ${\bf u} = \left \{ u,v,w \right \}$).
Some of the simulations presented here were conducted under
steady conditions, for which the $\frac{\partial}{\partial t}$ terms
vanish. An FEM scheme is obtained by posing the weak form in
terms of finite-dimensional subspaces of the function spaces specified above
defined using piecewise-polynomial basis functions. This discretization
has the form, $ {\bf v_h} \in {\bf v}$, where ${\bf v_h}$ is
%an approximation of ${\bf v}$
formed through a linear combination of a finite number (N) of basis
functions $\phi_i$,
\begin{equation}
 {\bf v_h} = \sum_{i=1}^N \alpha_i \phi_i,
\end{equation}
where $\alpha_i$ are coefficients\cite{becker1981introduction}.
%
%are constants and the N basis functions, $\left \{
%\psi_1, \psi_2, \ldots, \psi_N \right \}$, define an N-dimensional
%subspace of $H^1$\cite{becker1981introduction}.
%
All of the simulations discussed in this work were
accomplished using piecewise linear basis functions for both the
velocity and pressure. Typically, the use of equal order elements for
velocity and pressure is ruled out in the standard Galerkin FEM
formulation because of the Babuska-Brezzi condition\cite{bb-cond}.
This is overcome here by
%However, the weak form equations shown above are stable with equal-order
%elements for velocity and pressure due to
the introduction of a pressure stabilization
term\cite{becker1981introduction}. The resulting system is still
susceptible to convective instabilities, and so streamline upwind/Petrov-Galerkin
(SUPG) stabilization terms are used, as first described by
Hughes\cite{Hughes198685,supg} and extended to natural convection as in
Becker and Braack\cite{Becker2002428}. These stabilization terms add a
residual dependent artificial dissipation that approaches zero as the
solution converges. This scheme is called consistent because the
underlying order of convergence of the numerical method is not
affected\cite{hughes2000finite}.

The stabilization described above is accomplished by introducing an additional term,
$\langle L{\bf c},S {\bf \phi} \rangle_\tau$, to the weak form defined in Equations
\ref{eqn:ns_weak}-\ref{eqn:en_weak}. Here $L$ is the operator for the PDEs
in \ref{sub_sec:ns_en}, and S is a stabilization operator
which is chosen to be the negative adjoint of the differential operator
terms of $L$, and ${\bf c}$ and ${\bf \phi}$ are state and test
function vectors, i.e. $ {\bf c}= ({\bf u},p,T)$, and ${\bf \phi} = (
{\bf v},w,q )$. The angle brackets $\langle \cdot,\cdot \rangle$ signify
integration of the element interiors for each of the K elements, that is:
\begin{equation}
 \langle {\bf u},{\bf v} \rangle_\tau = \sum_K \tau_K({\bf u},{\bf
  v})_K.\label{eq:tau}
\end{equation}
This results in three stabilization parameters, $\tau_P, \tau_v, \tau_T$,
which are selected as proposed by Becker and Braack\footnote{\normalsize
Derivations of the weak form and stabilization terms are provided
in Appendix~\ref{app:stab}}.

After spatial discretization, the system of ODEs are discretized in time
using the backward Euler method\cite{moin2010fundamentals}. The time
interval $(0,T)$ is sliced into $N_t$ steps of uniform temporal length,
$\Delta t$, where $n = 0,\dots,N_t$.
This has the form,
\begin{equation}
 {\bf y}_{n+1} = {\bf y_n } + \Delta t \, f({\bf y_{n+1}},t_{n+1}).
\end{equation}
Where ${\bf y_{n+1}}$ denotes the solution vector at the time step $n+1$, for
instance. As $f$ is non-linear, a Newton-Raphson method is used to solve
the resulting implicit nonlinear problem.

%While an iterative method is
%significantly more computationally expensive per timestep than a similar
%explicit method, the method was selected due to its unconditional
%stability and ease of statistical sampling for a uniform timestep.

%
% gave not completely described numerical methods
% for instance, have not indicated the stabilization schemes
% do not need complete equations, but should permit someone to access
% the literature and construct precisely the numerical formulations used
%

\section{Mesh Discretization}

%
% what about mesh...
%
The domains described in Section~\ref{sec:bc} are consistently
discretized. This means that the domain extents, $\{L_x,L_y,L_z\}$, are
scaled by system diameter (D) but the same number of grid points are
used for every simulation. Thus, while the ratio of the domain length to
system diameter remains fixed, the grid spacing $\{\Delta x,\Delta
y,\Delta z\}$ increases proportionally with domain length.

The eddy diffusivities ($\nu_{t},K_{t}$) in Equations~\ref{eqn:visc_tot}
and \ref{eqn:therm_tot} are proportionally scaled with grid spacing to
ensure that the cell Reynolds number,
\begin{equation}
 \text{Re}_\text{cell} = \frac{\text{max}(\Delta x,\Delta y) \, u}{\nu_t},
\end{equation}
is maintained for every simulation, to ensure stability.
In this way, larger domains have higher eddy viscosities which serve to
dissipate effects of scales that are smaller than the grid spacing.

The mesh has a uniform spacing in the lateral directions, except for a
single refinement in the region of the vanes. Typically, the grid is
roughly one hundred points in the streamwise and spanwise directions
before the refinement. The refinement halves the spacing (doubles the
number of points) in all three coordinate directions, \{x,y,z\}, in this
region. The refinement is introduced from the ground to 1.5 times the
height of the vanes and cone.

%
% makes no sense
%
% describe a mapping what C's are used
%
% describe a grid refinement verification
%
%

The vertical mesh spacing is non-uniform to resolve the boundary
layer. This is accomplished by redistributing a mesh which is uniformly
spaced over the interval $\{0,L_z\}$ in height, $\hat z$, to a
non-uniform mesh, $z$, over the same interval.
%This non-uniform ``boundary-layer'' mesh is designed to resolve the small
%scale variations near the wall.
% what about chi?
A mapping $z = \chi(\hat z)$ is defined by the following conditions.
The minimum and maximum extents are preserved so that $ \chi(0) = 0 $
and $ \chi(L_z) = L_z $.
The map spacing between points is initially a specified minimum value,
\begin{equation}
  h_{\text{min}} = \chi'(0) \, \hat h,
\end{equation}
where $\hat h = L_z/\text{nz}$. The map is strictly increasing until the
point $\hat z_b$, where the spacing reaches a constant,
$h_{\text{max}} = \chi'(\hat z_b) \, \hat h$.
From this point onward, the mesh has a constant uniform spacing,
$  h_{\text{max}} = \chi'(L_z) \, \hat h$.
%
Finally, the mapping is assumed to hold an exponential functional form,
$  \chi(\hat z) = A e^{\beta \hat z}$.

%
% mesh mapping here
%
Then, by specifying $h_{\text{min}}$, $h_{\text{max}}$ and
$z_b$ (chosen thickness of the boundary layer mesh),
these conditions are sufficient to define a mapping,
\begin{equation}
 z = \chi(\hat z) =
 \begin{cases} C_1 (\hat z - L_z) + L_z & \text{if } \hat z \geq \hat z_b, \\
   C_2 \text{ exp}(C_3 \, \hat z - 1)      & \text{if } \hat z < \hat z_b. \\
 \end{cases}
\end{equation}
%
Here, $C_1-C_3$ are coefficients, and are defined as, %\todo{show coefficients}.
\begin{align}
  C_1 &= C_2 \, C_3 e^{C_3 \, \hat z_b}, \\
  C_2 &= \frac{\hat z_b}{h_r -1}, \\
  C_3 &= \frac{\text{ln }(h_r) }{\hat z_b},
\end{align}
where $h_r$ is the ratio between the largest and smallest spacing,
\begin{equation}
  h_r = \frac{h_{\text{max}}}{h_{\text{min}}},
\end{equation}
and,
\begin{equation}
  \hat z_b = \frac{L_z}{1 + \frac{L_z - z_b}{C_2 \, h_r \text{ ln }(h_r)}}.
\end{equation}
%% \begin{equation}
%%  z = \begin{cases} C_1 (z-L_z)+L_z,& \text{if } z \geq z_\delta\\
%%       C_2 \text{ exp}(C_3 z - 1),                 & \text{otherwise}
%%      \end{cases}
%% \end{equation}

%The mesh has an exponentially varying character,
%$h_{\text{min}}$, $h_{\text{max}}$ and $z_b$
%are chosen to ensure ten or more
%points in the boundary layer, isotropic spacing in cells outside of
%it, and smooth blending between these two regimes.

Grid refinement verification was performed, where the spacing was
tested against a mesh with twice the grid points in the vertical
direction for a representative test case. The results were not sensitive
to the choice of spacing. A horizontal slice though a
typical domain is shown in Figure~\ref{fig:meshing}. The single
refinement in the region of the vanes is visible, along with the finer
meshed boundary layer region near the ground.

  \begin{figure}[!htb]
    \begin{center}
     \includegraphics[width = 10 cm]{figs/meshing}
     \caption{Horizontal slice through the domain, to show a
     representative meshing. The single refinement region around the
     vanes is visible, along with the finer boundary layer mesh near the
     ground.}
     \label{fig:meshing}
    \end{center}
  \end{figure}

% (01:40:40 PM) Roy Stogner: Not just looks boundary layer ish.
% (01:40:43 PM) Roy Stogner: We had a sweet derivation for this.
% (01:41:20 PM) Roy Stogner: Geometric progression of the nodes, solved the inverse problem for the redistribution, the whole works.
%

%% hmin = 0.001
%% hmax = 0.4
%% zb = 2.0
%% hrat = ${/ ${mesh-options/hmax} ${mesh-options/hmin}}
%% zmax = ${mesh-options/domain_x3_max}
%% loghrat = ${= log ${mesh-options/hrat}}
%% c2 = ${/ ${mesh-options/zb} ${- ${mesh-options/hrat} 1}}
%% zetab = ${/ ${mesh-options/zmax} ${+ 1 ${/ ${- ${mesh-options/zmax} ${mesh-options/zb}} ${* ${mesh-options/c2} ${mesh-options/hrat} ${mesh-options/loghrat}}}}}
%% c3 = ${/ ${mesh-options/loghrat} ${mesh-options/zetab}}
%% mesh_nx3 = ${= ceil ${/ ${* ${mesh-options/zmax} ${mesh-options/c2} ${mesh-options/c3}} ${mesh-options/hmin}}}
%% c1 = ${* ${mesh-options/c2} ${mesh-options/c3} ${= exp ${* ${mesh-options/c3} ${mesh-options/zetab}}}}
%% redistribute = '{x}{y}{if(z>${mesh-options/zetab},${mesh-options/c1}*(z-${mesh-options/zmax})+${mesh-options/zmax},${mesh-options/c2}*(exp(${mesh-options/c3}*z)-1))}'

%After operation, solutions are evaluated to ensure that
%the qualitative character of the solution does not change.

\section{Software}

The numerical formulations described in Section~\ref{sec:discretization}
is implemented with the GRINS library\cite{GRINSpaper} by Bauman
and Stogner using the libMesh\cite{libMeshPaper} FEM
infrastructure. Designed to support multiphysics FEM applications,
GRINS is a flexible library that effectively addresses a wide range of
science and engineering problems.

%the reusability
%and extensibility of mathematical modeling kernels, supporting
%interfaces to existing solver and discretization libraries to enable
%modern solution strategies, while, at the same time, retaining

%GRINS provides a platform that enables powerful numerical algorithms
%such as adjoint-based AMR, adaptive modeling, sensitivity analysis,
%and, eventually, enabling uncertainty quantification. While few of these
%capabilities are in use for the present work, they could be useful in
%future investigations.

GRINS stands for, ``General Reacting Incompressible Navier-Stokes'',
which roughly encapsulates the physical regimes it was originally
designed to simulate. GRINS is open-source, and available on
\hyperref[www.github.com/grinsfem/grins]{GitHub}. It is released
under LGPL2.1.

%The remainder of this Section is devoted to
%discussing the underlying libraries used and the description of the
%GRINS framework.

GRINS uses the fparser~\cite{fparser}
library to support both parsing and compilation of mathematical
functions into high performance kernels. This capability allows for
easy specification of boundary conditions, initial conditions, or
constitutive equations from an input file. Some of these inputs are
detailed in Appendix~\ref{sec:archiving}.

GRINS/libMesh are built on the PETSc\cite{petsc} solver package, which
provides the numerical linear algebra packages used for constructing and
using sparse matrices, finding the solution of linear systems,
and for preconditioning.

While a variety of solver options have been tested in PETSc, all the
results shown in this document use GMRES with block Jacobi for
preconditioning\cite{Saad:2003} the linear solve. This uses the
inverse of the diagonal block for that processor for preconditioning of
the entire linear system. In addition, a preconditioner is used for the
solution of the diagonal block. This is approximated with incomplete LU
factorization\cite{chan1997approximate}. Here, the ``incomplete'' refers
to the level of fill, with greater levels of fill approaching
the ``complete'' LU factorization.

In principle, alternative software libraries/frameworks such as
FEniCS\cite{AlnaesBlechta2015a} or OpenFOAM\cite{jasak2007openfoam}
would be capable of simulating this problem. While these and
other libraries have various strengths and weaknesses, the pre-eminent
concern is the parallel performance at the intended processor count, due
to the rapid design iterations necessary for this research
campaign. Given these concerns, the GRINS library is a satisfactory tool.

%% (11:41:54 AM) nick: ``-ksp_view -ksp_type gmres -pc_type bjacobi
%% -sub_pc_type ilu -sub_pc_factor_levels 0''
%% (11:42:00 AM) Paul Bauman: OK
%% (11:42:17 AM) Paul Bauman: -pc_type is the preconditioner for the entire linear system
%% (11:42:25 AM) Paul Bauman: You're doing bjacobi = block Jacobi
%% (11:42:28 AM) nick: right
%% (11:42:36 AM) nick: and does anyone have a good reference I can learn
%% this better? i feel as if I cant look this up, for some reason
%% (11:42:39 AM) Paul Bauman: That is just using the inverse of the
%% diagonal block for that processor
%% (11:42:46 AM) Paul Bauman: Now
%% (11:42:55 AM) Paul Bauman: that is a linear solve
%% (11:43:17 AM) Paul Bauman: So you can use all the linear solver
%% technology to solve or approximately that block
%% (11:43:24 AM) Paul Bauman: Hence, -sub_pc_type
%% (11:43:32 AM) hil left the room.
%% (11:43:46 AM) Paul Bauman: That's the preconditioner it's going to
%% use to precondition the linear system for the solution of the
%% diagonal block
%% (11:43:53 AM) Paul Bauman: You're telling it to use incomplete lu
%% (11:44:26 AM) Paul Bauman: Now the -sub_pc_factor_levels options
%% applies to ilu
%% (11:44:28 AM) hil entered the room.
%% (11:44:44 AM) Paul Bauman: The incomplete part of imcomplete LU is
%% about the level of fill you use
%% (11:45:05 AM) Paul Bauman: The more levels of fill you have, the more
%% ``complete'' the incomplete LU will be
%% (11:45:08 AM) Paul Bauman: Does that make sense?
%% (11:45:23 AM) nick: no, that is where you lost me
%% (11:45:39 AM) nick: i dont think i know this level of fill
%% (11:46:17 AM) Paul Bauman: Check out Youssef Saad's book if more
%% curious about the subject
%% (11:46:37 AM) nick: cool thanks
%% (11:46:38 AM) Paul Bauman: Suffice it to say, you heopfully shouldn't
%% ever need to go past 3 or 4 levels of fill
%% (11:47:03 AM) Paul Bauman: Also, if you've got superlu installed with
%% the PETSc, consider using -sub_pc_factor_mat_solver_package superlu
%% (11:47:15 AM) Paul Bauman: That's a *much* faster/better
%% implementation than PETSc's

At the time of this writing, GRINS has 94 regression tests, which
provides a reasonable degree of confidence in verification testing of
the library. Several of these tests directly test the capabilities in
GRINS used in this study. In particular, several of the tests were
contributed to GRINS by the author over the course of this
work during the addition of several of the models detailed in
Chapter~\ref{sec:mathmodel}.

\section{Tool Chain and Simulation Custodianship}

Simulations are performed on the Texas Advanced Computing Center's
(TACC) supercomputers Lonestar Four, Lonestar Five, and Stampede. Run
durations for transient cases are typically twelve hours to perform
several hundred timesteps.  The steady runs are considerably shorter,
and require less than ten minute runtimes. Typically  the wall clock
times of the steady-state runs are two or three minutes to solution.
%As a result, the queue time is significantly longer than the actual
%production runtime.
These runs use 264-528 processing cores, or 22-44 nodes on Lonestar4
(with 12 cores per node), and a similar number for Stampede. The runs have
several million degrees of freedom (DoF), and the local number of DoF
per core is maintained at $O(10^4)$. This was selected due to memory
constraints, after a strong scaling analysis of the performance of the
code on these resources, and after consulting with the software developers.
At the time of this writing libMesh has been scaled to tens of thousands of
cores and has been run on over 100,000 cores on the BG/Q machine Mira at
Argonne National Lab\cite{gaston2013massive}, and the scaling results
here are consistent with the performance expectations for this library.

%In November, 2015, runs were also staged during early access into full
%production period on TACC's new system, Lonestar5. This machine is
%closer in terms of software stack to Stampede,
%using the Simple Linux Utility for Resource Management
%(Slurm) scheduling system.

Runs are submitted via job scripts to the SLURM\cite{yoo2003slurm}
scheduling system. After a run terminates, several additional scripts
are automatically invoked.
These archive the run (outside of the volatile /scratch
production directories) and simultaneously, label the concluded run with
unique metadata that defines the system environment; the jobs input
files and run definitions; and information detailing the
hypothesis or physics the job was intended to investigate. Finally, once
a week a script performs \textbf{rsync} on the entire archived database to
%ensure more than single redundancy for the runs.
maintain redundant storage of the simulation
data. Appendix~\ref{sec:archiving} details the archived simulations and
storage patterns.

In other words, the workflow is designed to permit rapid queuing of a
series of runs (in parallel) to investigate a variety of conditions or
scenario parameters. This capability is necessary for the optimization
campaign detailed in Section~\ref{sec:results}, where running many
concurrent investigations are required to sample the configuration
space.

%\section{Testing and Verification}
%grid convergence?

%The validation of the  runs is detailed in the next Chapter.


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{physics}
\usepackage{amsmath}
\usepackage{showlabels}
\usepackage{amssymb}
\title{Statistical Computing for Scientists and Engineers\\[1em] Homework 3}
\author{Jiale Shi}
\date{September/21/2018}

\usepackage{natbib}
\usepackage{graphicx}
\usepackage{array}
\begin{document}
\maketitle

\newpage
\section{Problem 1}
The problem of interest consists of a scalar output $y$ (the logarithmic of the number of caterpillars colonies in a $500 m^2$ area) and $k=10$ explanatory variables $x_{i} (i=1,...,10)$ which correlate to features such as altitude, landscape slope, vegetation count, etc. The exact meaning does not matter in this context. Using the caterpillar.mat data, plot the semi-log-y plots of the caterpillar colony (last column of the data file) versus each feature reproducing Figure 3.1 in Bayesian Core.

Answer: we run the code that provided to us. We get the semi-log-$y$ plots are shown below.

\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{Figure3_1.png}
\caption{semi-$\log$-$y$ plots}
%\label{fig:universe}
\end{figure}

\newpage
\section{Problem 2}
From the book, 3.1.1 linear model

The ordinary normal linear regression model is such that
\begin{equation}
    y|\beta,\sigma^{2},X \sim \mathcal{N}_{n}(X\beta,\sigma^{2}I_{n})
\end{equation}

the likelihood of the ordinary normal linear model is
\begin{equation}
    l(\beta,\sigma^{2}|y,X) = (2\pi\sigma^{2})^{n/2} \exp\left[-\frac{1}{2\sigma^2}(y-X\beta)^{T}(y-X\beta)\right]
\end{equation}

The maximum likelihood estimator of $\beta$ is then the solution of the (least squares) minimization problem.

\begin{equation}
\min_{\beta}(y-X\beta)^{T}(y-X\beta) = \min_{\beta}\sum_{i=1}^{n}(y_{i}-\beta_{0}-\beta_{1}x_{i1}-...-\beta{k}x_{ik})^2
\end{equation}
namely,
\begin{equation}
\hat{\beta} = (X^{T} X)^{-1}X^{T}y
\end{equation}
which is also the orthogonal projection of y on the linear subspace spanned by the columns of $X$.

Similarly, an unbiased estimator of $\sigma^{2}$ is
\begin{equation}
  \hat{\sigma}^2 = \frac{1}{n-k-1}(y-X\hat{\beta})^{T} (y-X\hat{\beta}) = \frac{s^{2}}{n-k-1}
\end{equation}
and $\hat{\sigma}^2(X^{T}X)^{-1}$ approximates the covariance matrix of $\hat{\beta}$. We can then define the standard t-statistic as

\begin{equation}
    T_{i} = \frac{\hat{\beta}_{i}-\beta_{i}}{\sqrt{\hat{\sigma}^2\omega_{(i,i)}}} ~\mathcal{J}_{n-k-1} =
\end{equation}
where $\omega_{(i,i)}$ denotes the $(i,i)$th element of the matrix $(X^{T}X)^{-1}$. For this problem $\beta_{i}=0$

the p value
\begin{equation}
    p_{i}= P_{H_{0}}(|T_{i}|-|t_{i}|) < \alpha
\end{equation}

Note that this statistic $T_{i}$ can also be used to build on the $\beta_{i}$s as a (marginal) frequentist confidence interval, of the form.

\begin{equation}
    {\beta_{i};|\beta_{i}-\hat{\beta}_{i}|<=\sqrt{\omega_{ii}}F^{-1}_{n-k-1}(1-\alpha/2)}
\end{equation}

Then we run the code that is offered to us. We reproduce Figure 3.2
\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{Figure3_2.png}
\caption{Dataset caterpillar: output providing the maximum likelihood estimates of the regression coefficients and their standard significance analysis.}
%\label{fig:universe}
\end{figure}

\section{Problem 3}
We mainly focus on the section 3.2.1 Conjugate Priors.

We use the conjugate prior:
\begin{equation}
\begin{aligned}
    &\beta|\sigma^{2}, X \sim \mathcal{N}_{k+1}(\Tilde{\beta},\sigma^{2}M^{-1})  \\
    &\sigma^{2} | X \sim \mathcal{IG}(a,b)
\end{aligned}
\end{equation}
%Use the hyper-parameter values of $\Tilde{\beta}$,k=10,a=2.1andb=2.0forcomputingthetablevalues.
The prior is conjugate, leading us to the following normal-inverse-gamma posterior distribution.

\begin{equation}
\begin{aligned}
    &\beta|\sigma^{2},y, X \sim \mathcal{N}_{k+1}((M+X^{T}X)^{-1}X^{T}X\Tilde{\beta},\sigma^{2}[M^{-1}+(X^{T}X)^{-1}]^{-1})  \\
    &\sigma^{2} | y,X \sim \mathcal{IG}(\frac{n}{2},b+\frac{s^2}{2}+\frac{\hat{\beta}^{T}[M^{-1}+(X^{T}X)^{-1}]^{-1}\hat{\beta}}{2})
\end{aligned}
\end{equation}
In this setting, the Bayes estimators of $\beta$ and $\sigma^{2}$ associated with squared error losses, namely the posterior means of $\beta$ and $\sigma^{2}$, can be computed in closed form. In fact, a simple computation shows that they are given by
\begin{equation}
\begin{aligned}
    \mathbb{E}^{\pi}[\beta|y,X] &= \mathbb{E}^{\pi}[\mathbb{E}^{\pi}(\beta|\sigma^2,y,X)|y,X] \\
    & = (M+X^{T}X)^{-1}{(X^{T}X)\hat{\beta}+M\hat{\beta}}
\end{aligned}
\end{equation}

and for $n\geq 2$

\begin{equation}
\begin{aligned}
    \mathbb{E}^{\pi}[\beta|y,X] &= \frac{2b+s^{2}+(\tilde{\beta}-\hat{\beta})^{T}{M^{-1}+(X^{T}X)^{-1}}^{-1}(\tilde{\beta}-\hat{\beta})}{n+2a-2}
\end{aligned}
\end{equation}

Integrating Eq.10 in $\sigma^{2}$ leads to a multivariate $t$ marginal posterior distribution on $\beta$ since.

\begin{equation}
\begin{aligned}
\pi(\beta|y,X) & \propto [ (\beta-{M+X^{T}X}^{-1}[{X^{T}X}\hat{\beta}+M\tilde{\beta}])^{T} (M+X^{T}X)  \\
&\times (\beta-{M+X^{T}X}^{-1}[{X^{T}X}\hat{\beta}+M\tilde{\beta}]) +2b+s^2  \\
&+(\tilde{\beta}-\hat{\beta})^{T} (M^{-1}+(X^{T}X)^{-1})^{-1} (\tilde{\beta}-\hat{\beta})
]^{-(n/2+k/2+a)}
\end{aligned}
\end{equation}

We thus have that, marginally and a posteriori,
\begin{equation}
    \beta|y,X \sim \mathcal{J}_{k+1}(n+2a,\hat{\mu},\hat{\Sigma})
\end{equation}

with
\begin{equation}
\begin{aligned}
    & \hat{\mu} = (M+X^{T}X)^{-1}((X^{T}X)\hat{\beta}+M\tilde{\beta}) \\
    & \hat{\Sigma} = \frac{2b+s^2 +(\tilde{\beta}-\hat{\beta})^{T} (M^{-1}+(X^{T}X)^{-1})^{-1} (\tilde{\beta}-\hat{\beta})}{n+2a}(M+X^{T}X)^{-1}
\end{aligned}
\end{equation}

\begin{equation}
\begin{aligned}
    \mathbb{V}^{\pi} &= \frac{n+2a}{n+2a-4} \hat{\Sigma} \\
    &=  \frac{2b+s^2 +(\tilde{\beta}-\hat{\beta})^{T} (M^{-1}+(X^{T}X)^{-1})^{-1} (\tilde{\beta}-\hat{\beta})}{n+2a-4}(M+X^{T}X)^{-1}
\end{aligned}
\end{equation}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{Table3_1.png}
\caption{Dataset caterpillar: Influence of the prior scale c on the Bayes estimates of $\sigma^2$ and $\beta_{0}$.}
%\label{fig:universe}
\end{figure}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{Table3_2.png}
\caption{Dataset caterpillar: Bayes estimates of $\beta$ for $c=100$.}
%\label{fig:universe}
\end{figure}

\newpage
\section{Problem 4}
The posterior distribution can be  derived as
\begin{equation}
\begin{aligned}
    \pi(\beta,\sigma^2|y,X) & \propto f(y|\beta,\sigma^2,X)\pi(\beta,\sigma^2|X) \\
    &\propto (\sigma^{2})^{-n/2-1} \exp [-\frac{1}{2\sigma^2}(y-X\hat{\beta})^{T}(y-X\hat{\beta})-\frac{1}{2\sigma^2}(\beta-\hat{\beta})^{T}(X^{T}X)(\beta-\hat{\beta})]\\
    &\cdot (\sigma^2)^{-k/2} \exp[-\frac{1}{2c\sigma^2}(\beta^T X^T X\beta)]
\end{aligned}
\end{equation}
Conduct marginalization and derive the conditional and marginal posteriors on $\beta$ and $\sigma^{2}$:

\begin{equation}
\begin{aligned}
    & \beta | \sigma^{2},y, X \sim \mathcal{N}_{k+1}(\frac{c}{c+1}\hat{\beta}, \frac{\sigma^2 c}{c+1}(X^{T} X)^{-1}) \\
     & \sigma^2 | y, X \sim \mathcal{IG}(\frac{n}{2}, \frac{s^2}{2}+\frac{1}{2(c+1)}\hat{\beta}^T X^T X \hat{\beta}) \\
\end{aligned}
\end{equation}

By integrating out $\sigma^2$ on the conditional posterior $\beta$, we can show that
\begin{equation}
\begin{aligned}
    \beta | y,X \sim \mathbb{T}_{k+1} (n,\frac{c}{c+1}\hat{\beta},\frac{c(s^2+\frac{\hat{\beta}^{T} X^{T} X \hat{\beta}}{c+1})}{n(c+1)}(X^T X)^{-1})
\end{aligned}
\end{equation}

\begin{equation}
\begin{aligned}
    & \mathcal{E}[\beta|y,X] = \frac{c}{c+1} \hat{\beta} \\
    & \mathcal{V}[\beta|y,X] = \frac{c(s^2+\frac{\hat{\beta}^{T} X^{T} X \hat{\beta}}{c+1})}{(n-2)(c+1)}(X^T X)^{-1}
\end{aligned}
\end{equation}

And the expression of Bayes factor has been given.

\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{Table3_3.png}
\caption{Dataset caterpillar: Posterior mean and variance of $\beta$ for $c=100$ using Zellar's G-prior.}
%\label{fig:universe}
\end{figure}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{Table3_4.png}
\caption{Dataset caterpillar: Same legend as Table 3.3 for c=1000}
%\label{fig:universe}
\end{figure}

The dependence of the Bayes factor on the pair ($c,c_{0}$) cannot be bypassed in the sense that the Bayes factor varies between $0$ and $\infty$ when $c_{0}/c$ goes from $0$ to $\infty$.

Although these Bayes factors should not be used simultaneously, an informal conclusion is that the most important variables besides the intercept seem to be $X_{1},X_{2},X_{4}$ and $X_{5}$.

\newpage
\section{Problem 5}
The marginal posterior distribution from Problem 4:

\begin{equation}
\begin{aligned}
    \beta | y,X \sim \mathbb{T}_{k+1} (n,\frac{c}{c+1}\hat{\beta},\frac{c(s^2+\frac{\hat{\beta}^{T} X^{T} X \hat{\beta}}{c+1})}{n(c+1)}(X^T X)^{-1})
\end{aligned}
\end{equation}

As the standard student-t is one symmetrical distribution, we can analytically calculate its $90\%$ HPD region as the two points whose respective CDF is $5\%$ and $95\%$. Denote them as $\delta_{5}$ and $\delta_{95}$. The corresponding HPD of $\beta$ would than be

\begin{equation}
\begin{aligned}
    & \beta_{5} = \frac{c}{c+1}\hat{\beta} + \delta_{5} \cdot \frac{c(s^2+\frac{\hat{\beta}^{T} X^{T} X \hat{\beta}}{c+1})}{n(c+1)}(X^T X)^{-1}) \\
    & \beta_{95} = \frac{c}{c+1}\hat{\beta} + \delta_{95} \cdot \frac{c(s^2+\frac{\hat{\beta}^{T} X^{T} X \hat{\beta}}{c+1})}{n(c+1)}(X^T X)^{-1})
\end{aligned}
\end{equation}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{Table3_5.png}
\caption{Dataset caterpillar:$95\%$ HPD intervals for the components of $\beta$ for $c=100$}
%\label{fig:universe}
\end{figure}

\newpage
\section{Problem 6}
We now consider c an unknown hyper-parameter, that is we use the same G-prior distribution, and we now introduce a diffuse prior distribution on c.

\begin{equation}
\begin{aligned}
    \pi(c)=c^{-1} \mathcal{I}_{\mathcal{N*}}(c)
\end{aligned}
\end{equation}

the corresponding marginal posterior on the parameters of interest is then
\begin{equation}
\begin{aligned}
    \pi(\beta,\sigma^2|y,X) \sum_{c=1}^{\infty} \pi(\beta,\sigma^{2}|y,X,c)f(y|X,c)c^{-1}
\end{aligned}
\end{equation}

\begin{equation}
\begin{aligned}
    f(y|X) &\propto_{c} f(y|X,c)\pi(c) \\
    & \sum_{c}c^{-1}(c+1)^{-\frac{k+1}{2}} [y^T y-\frac{c}{c+1}y^T X(X^T X)^{-1} X^{T} y]^{-n/2}
\end{aligned}
\end{equation}

For the posterior mean and variance of $\beta$, we integrate out $c$ by summation:
\begin{equation}
\begin{aligned}
    \mathcal{E}[\beta|y,X] = \sum_{c}\mathcal{E}[\beta|y,X,c]\pi(c|y,X)
\end{aligned}
\end{equation}

because $\pi(c|y,X) \propto f(y|X,c) \pi(c)$

\begin{equation}
\begin{aligned}
    \mathcal{E}[\beta|y,X] = \sum_{c} \frac{c}{c+1} \hat{\beta} \pi(c|y,X) = \hat{\beta} \sum_{c} [\frac{c}{c+1}\frac{f(y|X,c)\pi(c)}{\sum_{c}f(y|X,c)\pi(c)}]
\end{aligned}
\end{equation}

\begin{equation}
\begin{aligned}
    \mathcal{V}[\beta|y,X] = \mathcal{E}[\mathcal{V}(\beta|y,X,c)|y,X] + \mathcal{V}(\mathcal{E}[\beta,X,c]|y,X)
\end{aligned}
\end{equation}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{Table3_6.png}
\caption{Table of Problem 6}
%\label{fig:universe}
\end{figure}

An important point with this approach is that the marginal distribution of the dataset is available in closed form. It is therefore possible to produce a Bayes regression output, just as in the first-level prior. This is one additional reason why this non-informative prior is introduced.

\newpage
\section{Problem 7}

The algorithm is stochastic search for the most likely model.
When k is large, it becomes computationally intractable to compute the posterior probabilites of the $2^{k}$ models. Need of a tailored algorithm that samples from $p(\gamma|y, X)$ and selects the most likely models.
This can be done by Gibbs sampling, given the availability of the full conditional posterior probabilities of the $\gamma_{i}$'s.
If the $\pi$ with the that model parameter is much larger than that without that model parameter, that model parameter should be turned on.
If the $\pi$ with the that model parameter is much smaller than that without that model parameter, that model parameter should be turned off.

\begin{figure}[ht]
\centering
\includegraphics[scale=0.6]{Table3_7.png}
\caption{Table of Problem 7(a)}
%\label{fig:universe}
\end{figure}

\begin{figure}[ht]
\centering
\includegraphics[scale=0.6]{Table3_8.png}
\caption{Table of Problem 7(b)}
%\label{fig:universe}
\end{figure}

\begin{figure}[ht]
\centering
\includegraphics[scale=0.6]{Table3_9.png}
\caption{Table of Problem 7(c)}
%\label{fig:universe}
\end{figure}

\begin{figure}[ht]
\centering
\includegraphics[scale=0.6]{Table3_10.png}
\caption{Table of Problem 7(d)}
%\label{fig:universe}
\end{figure}

\begin{figure}[ht]
\centering
\includegraphics[scale=0.6]{Table3_11.png}
\caption{Table of Problem 7(e)}
%\label{fig:universe}
\end{figure}

%\bibliographystyle{plain}
%\bibliography{references}
\end{document}


\subsubsection{Parabolic Cylinders}
\noindent
A paraboloid cylinder look like a parabola that has been extruded into 3D space. One of its forms is
\begin{equation*}
	z = cy^2
\end{equation*}
where $c \in \R$.

[INSERT IMAGE]


\section{Overview}\label{sec:overviews}

\def\sm{\texttt{sm}}
\def\mc{\texttt{mconcat}}

\subsection{Monoids and Parallelism}\label{subsec:monoids}

As noted in section~\ref{sec:intro}, associativity ultimately
allows monoidal operations to be run in parallel. However, for our
purposes, we would like a way of breaking an input into pieces and
running our computation, string matching, in parallel on all the
pieces. For this, we will focus on the standard Haskell function \mc,
\begin{code}
  mconcat :: Monoid m => [m] -> m
  mconcat [] = mempty
  mconcat (x:xs) = x <> mconcat xs
\end{code}
where @mappend@ and @mempty@ are the monoid operation and identity for
monoid \texttt{m} (usually written \texttt{mappend} and
\texttt{mempty})~\footnote{\mc\, is usually defined as \texttt{foldr mappend mempty}}.

\mc is a monoid morphism from lists of values of
a monoid to that monoid, i.e.
%
$$
\begin{array}{rcl}
\mc\;[] & \quad \equiv \quad & \mempty \\
\mc\;([x_0,\;\ldots x_m]\;\texttt{++}\;[y_0,\ldots y_n])
& \quad \equiv \quad &
\mc\;[x_0, \ldots x_m]\;\mappend\;\mc\;[y_0,\ldots y_n]
\end{array}
$$
%
where \texttt{++} is list append.

$$
\begin{array}{l}
\mc\;[\sm\;x_0,\;\ldots \sm\;x_i, \sm\;x_{i+1},\ldots \sm\;x_j, \sm\;x_{j+1},\ldots \sm\;x_n] \\
\;\;\equiv\;\; \quad
\mc\;([\sm\;x_0,\;\ldots \sm\;x_i]\;\texttt{++}\;[\sm\;x_{i+1},\ldots \sm\;x_j]\;\texttt{++}\;[\sm\;x_{j+1},\ldots \sm\;x_n]) \\
\;\;\equiv\;\; \quad
%\mc\;[\mc\;[\sm\;x_0,\;\ldots \sm\;x_i]\;,\;\mc\;[\sm\;x_{i+1},\ldots \sm\;x_n]] \\
%\;\;\equiv\;\; \quad
\mc\;[\sm\;x_0, \ldots \sm\;x_i]\;\mappend\;\mc\;[\sm\;x_{i+1},\ldots \sm\;x_j]\;\mappend\;\mc\;[\sm\;x_{j+1},\ldots \sm\;x_n]
\end{array}
$$

\subsection{String Matching}\label{subsec:string-matching}

String matching is determining all the indices in a source string
where a given target string begins; for example, for source string
\texttt{abababa} and target \texttt{aba} the results of string
matching would be \texttt{[0, 2, 4]}. We will use the following code
as a specification of string matching:
\begin{code}
makeIndices inp trg = go 0 where
  trgLen = length trg
  go i | i >= length inp = []
       | take trgLen (drop i inp) == trg = i : rest
       | otherwise = rest
  where rest = go (i + 1)

*Main> makeIndices "abababa" "aba" == [0, 2, 4]
\end{code}
Though this implementation is not efficient, it is simple and provides
a good starting point from which to create a verifiable function (see
sec~\ref{sec:monoid}).

\subsection{Monoidal String Matching}\label{subsec:monoidal-string-matching}


\chapter{Introduction to Recurrent Neural Networks}

	\section{What is a Recurrent Neural Network?}

	\begin{itemize}
		\item Recurrent neural networks or RNNs (Rumelhart et al.,1986a) are a family of
		neural networks for processing sequential data.
		\item  Much as a convolutional network is a neural network that is specialized for processing a grid of values X such as an image, a recurrent neural network is a neural network that is specialized for
		processing a sequence of values $X = \{x^{(1)}, . . . , x^{(\tau)}\}$.
		\item Most recurrent networks can also process sequences of \textbf{variable} length. For each sample $X$, the $\tau$ doesn't have to bethe same.
	\end{itemize}

	\subsection{Motivation of RNN}
		\textbf{Parameter sharing!} \\
		Reasons why we need parameter sharing:
		\begin{itemize}
			\item  Parameter sharing makes it possible to extend and apply the model to examples of different forms
			(or different lengths) and generalize across them.
			\item  If we had separate parameters for each value of the time index, we could not generalize to sequence lengths not seen during training, nor share statistical characteristic across different sequence lengths
			and across different positions in time.
		\end{itemize}

		\textbf{How to use the parameter sharing?}
		\begin{itemize}
			\item In the Recurrent networks, each member of the
			output is a function of the previous members of the output.
			\item Each member of the output is produced using \textbf{the same update rule} applied to the previous outputs. This recurrent formulation results in the sharing of parameters through a very deep computational graph.
		\end{itemize}

	\subsection{Vanilla RNN}
	\begin{itemize}
		\item
		\textbf{Training Data of the model} \\
		Here we use the sentence generation as the model, which aims to predict the next character in a sentence given all the characters before. \\
		 In this model, each $\mathbf{x}_i^{(t)}$ is a character in the sentence(represented as a $0-1$ vector), and the target is the next character which is also represented as a $0-1$ vector.
		\item
		\textbf{The structure of the network} \\
		It's an example of a recurrent network that maps an
		input sequence to an output sequence of the same length. \\
		The input of the network is $\{\mathbf{x}_i^{(1)},\mathbf{x}_i^{(2)},\dots,\mathbf{x}_i^{(\tau)}\}_{i=1}^N$, where N is the number of the samples, $\tau$ is the length of each input, and each $\mathbf{x}_i^{(t)}$ is a vector in the vector space $\mathbb{R}^n$. \\
		 The output is $\{\mathbf{y}_i^{(1)},\mathbf{y}_i^{(2)},\dots,\mathbf{y}_i^{(\tau)}\}_{i=1}^N$, the meaning of N and $\tau$ is the same, while each $\mathbf{y}_i^{(t)}$ is a vector in the vector space $\mathbb{R}^m$. \\
		\begin{figure}[H]
			\centering
			\includegraphics[width=0.9\textwidth]{VaRNN}
			\caption{The structure of Vanilla RNN}
		\end{figure}

		\item
		\textbf{The update equations for the forward propagation} is
		\begin{flalign*}
		\mathbf{a}^{(t)} &= \mathbf{b} + \mathbf{Wh}^{(t-1)} + \mathbf{Ux}^{(t)} \\
		\mathbf{h}^{(t)} &= tanh(\mathbf{a}^{(t)}) \\
		\mathbf{O}^{(t)} &= \mathbf{c} + \mathbf{Vh}^{(t)} \\
		\hat{\mathbf{y}}^{(t)} &= softmax(\mathbf{O}^{(t)})
		\end{flalign*}
		where the parameters are bias vectors $\mathbf{b}$ and $\mathbf{c}$ along with the weight matrices
		$\mathbf{U}$, $\mathbf{V}$ and $\mathbf{W}$, respectively for input-to-hidden, hidden-to-output and hidden-to- hidden connections.

		\item
		\textbf{The computation of the loss} \\
		For a given sequence of $\mathbf{x}$ values paired with a sequence of $\mathbf{y}$ values, the total loss would then be just the sum of the losses over all the time steps.\\
		For example, if $L^{(t)}$ is the negative log-likelihood of $y^{t}$ given $\{\mathbf{x}^{(1)},\ldots,\mathbf{x}^{(t)} \}$, the total loss is
		\begin{flalign*}
		L({\mathbf{x}^{(1)},\ldots,\mathbf{x}^{(\tau)}},{\mathbf{\hat{y}}^{(1)},\ldots,\mathbf{\hat{y}}^{(\tau)}}) \\
		= \sum_{t=1}^{\tau} L^{(t)}  \\
		= - \sum_{t=1}^{\tau} log \ p_{model}(\mathbf{\hat{y}}^{(t)} |{\mathbf{x}^{(1)},\ldots,\mathbf{x}^{(t)}})
		\end{flalign*}
		where $p_{model}(\mathbf{\hat{y}}^{(t)} |{\mathbf{x}^{(1)},\ldots,\mathbf{x}^{(t)}}) $ is given by the following rule:\\
		Denote the $\mathbb{R}^m$ output vector $\hat{\mathbf{y}}^{t}$ as $(\hat{y}_1^{(t)},\hat{y}_2^{(t)},\dots,\hat{y}_m^{(t)})$,and the target vector (a $0-1$ vector) is $(y_1^{(t)},y_2^{(t)},\dots,y_m^{(t)})$ while $y_i^{(t)} = 1$ and all the other elements of the vector is 0. Then
		\begin{equation}
		p_{model}(\mathbf{\hat{y}}^{(t)} |{\mathbf{x}^{(1)},\ldots,\mathbf{x}^{(t)}}) = \hat{y}_i^{(t)}
		\end{equation}
	\end{itemize}

	\subsection{Some important design patterns for RNN}
		\begin{itemize}
			\item Recurrent networks that produce an output at each time step and have recurrent connections only from the output at one time step to the hidden units at the next time step.
			\begin{figure}[H]
				\centering
				\includegraphics[width=0.9\textwidth]{O2HRNN}
				\caption{The structure of RNN(Output to Hidden)}
			\end{figure}
			\textbf{The update equations for the forward propagation} is
			\begin{flalign*}
			\mathbf{a}^{(t)} &= \mathbf{b} + \mathbf{WO}^{(t-1)} + \mathbf{Ux}^{(t)} \\
			\mathbf{h}^{(t)} &= tanh(\mathbf{a}^{(t)}) \\
			\mathbf{O}^{(t)} &= \mathbf{c} + \mathbf{Vh}^{(t)} \\
			\hat{\mathbf{y}}^{(t)} &= softmax(\mathbf{O}^{(t)})
			\end{flalign*}
			where the parameters are  bias vectors $\mathbf{b}$ and $\mathbf{c}$ along with the weight matrices
			$\mathbf{U}$, $\mathbf{V}$ and $\mathbf{W}$, respectively for input-to-hidden, hidden-to-output and output-to- hidden connections.
			\item Recurrent networks with recurrent connections between hidden units, that read an entire sequence and then produce a single output.
			\begin{figure}[H]
				\centering
				\includegraphics[width=0.9\textwidth]{OFRNN}
				\caption{The structure of RNN(Only One Output)}
			\end{figure}
			\textbf{The update equations for the forward propagation is}
			\begin{flalign*}
			\mathbf{a}^{(t)} &= \mathbf{b} + \mathbf{Wh}^{(t-1)} + \mathbf{Ux}^{(t)} \qquad 1 \leq t \leq \tau\\
			\mathbf{h}^{(t)} &= tanh(\mathbf{a}^{(t)}) \qquad 1 \leq t \leq \tau\\
			\mathbf{O}^{(\tau)} &= \mathbf{c} + \mathbf{Vh}^{(\tau)} \\
			\hat{\mathbf{y}}^{(\tau)} &= softmax(\mathbf{O}^{(\tau)})
			\end{flalign*}
			where the parameters are bias vectors $\mathbf{b}$ and $\mathbf{c}$ along with the weight matrices
			$\mathbf{U}$, $\mathbf{V}$ and $\mathbf{W}$, respectively for input-to-hidden, hidden-to-output and hidden-to- hidden connections
	\end{itemize}

%	\subsection{Teaching Force}
%		Note that in the second pattern,it lacks hidden-to-hidden
%		recurrence.
%		\begin{itemize}
%			\item It is less powerful.
%			\item However,for any loss function based on comparing the prediction at time t to the training target at time t, all the time steps are decoupled.
%			\item Training can thus be parallelized, with the gradient for each step t computed in isolation.
%		\end{itemize}
%		\begin{figure}[H]
%			\centering
%			\includegraphics[width=0.9\textwidth]{TF}
%			\caption{The representation of Teaching Force}
%		\end{figure}
%	The parallelization is the concept of \textbf{Teaching Force}.
%		\begin{itemize}
%			\item Teacher forcing is a procedure that emerges from the maximum likelihood criterion, in which during training the
%			model receives \textbf{the ground truth output} $y^{(t)}$ as input at time t + 1.
%			\item Take a sequence with two time steps as an example. The conditional maximum likelihood criterion is
%			\begin{flalign*}
%			log p(\mathbf{\hat{y}}^{(1)},\mathbf{\hat{y}}^{(2)}|\mathbf{x}^{(1)},\mathbf{x}^{(2)}) \\
%			= log p(\mathbf{\hat{y}}^{(2)} | \mathbf{\hat{y}}^{(1)},\mathbf{x}^{(1)},\mathbf{x}^{(2)}) + log p(\mathbf{\hat{y}}^{(1)} | \mathbf{x}^{(1)},\mathbf{x}^{(2)})
%			\end{flalign*}
%			\item In this example, we see that at time t = 2, the model is trained to maximize the \textbf{conditional probability} of $\mathbf{y}^{(2)}$ given both the $\mathbf{x}$ sequence so far and the previous $\mathbf{y}$ value from the training.
%		\end{itemize}

	\section{Development of RNN}

	\subsection{Encoder-Decoder}
		\begin{figure}[H]
			\centering
			\includegraphics[width=0.9\textwidth]{Encoder-Decoder}
			\caption{The structure of Encoder-Decoder}
		\end{figure}

		\begin{itemize}
			\item It's an RNN that can be trained to map an input sequence to an output sequence which is not necessarily of the same length.
			\item First, we have an \textbf{encoder} or \textbf{input} RNN that processes the input sequence. The encoder emits the context C, usually as a simple function of its final hidden state.
			\item The context C might be a vector or sequence of
			vectors that summarize the input sequence $\mathbf{X} = (\mathbf{x}^{(1)},\ldots,\mathbf{x}^{(n_x)})$.
			\item Then we have a \textbf{decoder} or \textbf{output} RNN that is conditioned on that fixed-length vector $\mathbf{X}$ to generate the output sequence $(\mathbf{y}^{(1)},\ldots,\mathbf{y}^{(n_y)})$.
			\item The two RNNs are trained jointly to maximize the average of \\
			$log P(\mathbf{y}^{(1)},\ldots,\mathbf{y}^{(n_y)}|\mathbf{x}^{(1)},\ldots,\mathbf{x}^{(n_x)})$ over all the pairs of x and y
			sequences in the training set.
		\end{itemize}
	\subsection{Deep RNN}
		The computation in most RNNs can be decomposed into three blocks of parameters and associated transformations:

		\begin{itemize}
			\item[1.] from the input to the hidden state
			\item[2.] from the previous hidden state to the next hidden state
			\item[3.] from the hidden state to the output
		\end{itemize}
		Experimental evidence (Graves et al., 2013; Pascanu et al., 2014a) strongly suggests that we need \textbf{enough depth} in order to perform the required mappings in each of these operations.
		\begin{figure}[H]
			\centering
			\includegraphics[width=0.9\textwidth]{DeepRNN}
			\caption{Some kind of Structure for Deep RNN}
		\end{figure}
	A recurrent neural network can be made deep in many ways (Pascanu
	et al., 2014a).
	\begin{itemize}
		\item (a) The hidden recurrent state can be broken down into groups organized hierarchically.
		\item (b) Deeper computation can be introduced in the input-to- hidden, hidden-to-hidden and hidden-to-output parts. This may lengthen the shortest path linking different time steps.
		\item (c) The path-lengthening effect can be mitigated by introducing skip connections.
	\end{itemize}

%	\subsection{Recursive Neural Network}
%		\begin{itemize}
%			\item Recursive neural networks represent yet another generalization of recurrent networks, with a different kind of computational graph, which is structured as a \textbf{deep tree}, rather than the chain-like structure of RNNs.
%			\item One clear advantage of recursive nets over recurrent nets is that for a sequence of the same length $\tau$, the depth (measured as the number of compositions of nonlinear operations) can be drastically reduced from $\tau$ to $O(log \tau)$.
%			\item However,how to \textbf{best structure the tree} is an open question.
%		\end{itemize}
%		\begin{figure}[H]
%			\centering
%			\includegraphics[width=0.9\textwidth]{ReNN}
%			\caption{The structure of Recursive Neural Network}
%		\end{figure}

\section{The Challenge of long term dependency}
\subsection{Computation of the gradient}
Computing the gradient through a recurrent neural network is straightforward. One simply applies the generalized back-propagation algorithm to the network. \\
What we need to take care of is that the back propagation should be computed through \textbf{the time step}. We will use the example of computing the gradients of Vanilla RNN above to show how the BP algorithm behaves. \\
Denote the final loss as $L$,we can see
\begin{equation}
\frac{\partial L}{\partial L^{(t)}} = 1  \ ,1\le t \le \tau
\end{equation}
Assume the output function is softmax function and the loss is negative log-likelihood function, The gradient $\nabla_{o^{(t)}}L$ on the outputs at time step t, for all i,t, is as follows:
\begin{equation}
(\nabla_{o^{(t)}}L)_i = \frac{\partial L}{\partial L^{(t)}}\frac{\partial L^{(t)}}{\partial o_i^{(t)}} = \hat{y_i}^{(t)} - 1_{i,y^{(t)}}
\end{equation}
At the final time step $\tau$, we have
\begin{equation}
\nabla_{h^{(\tau)}}L = V^{T}\nabla_{o^{(\tau)}}L
\end{equation}
with
\begin{eqnarray}
\nabla_{h^{(t)}}L & = & (\frac{\partial h^{(t+1)}}{\partial h^{(t)}})^{T}\nabla_{h^{(t+1)}}L + (\frac{\partial o^{(t)}}{\partial h^{(t)}})^{T}\nabla_{o^{(t)}}L \\
 & = & W^T\nabla_{h^{(t+1)}}L diag(1 - (h^{(t+1)})^2) + V^T\nabla_{o^{(t)}}L
\end{eqnarray}
from $t = \tau -1$ down to $t = 1$. \\
With these, the gradient on the remaining parameters is given by:
\begin{eqnarray}
\nabla_{c}L & = & \sum_{t=1}^{\tau} \nabla_{o^{(t)}}L \\
\nabla_{b}L & = & \sum_{t=1}^{\tau} diag(1 - (h^{(t)})^2)\nabla_{h^{(t)}}L \\
\nabla_{V}L & = & \sum_{t=1}^{\tau}(\nabla_{o^{(t)}}L)h^{(t)^T} \\
\nabla_{W}L & = & \sum_{t=1}^{\tau} diag(1 - (h^{(t)})^2)(\nabla_{h^{(t)}}L)h^{(t-1)^T} \\
\nabla_{U}L & = & \sum_{t=1}^{\tau} diag(1 - (h^{(t)})^2)(\nabla_{h^{(t)}}L)x^{(t)^T}
\end{eqnarray}
\subsection{Challenge of long term dependency}
From the computation above, we can see in the recurrent network, it involves the composition of the same function multiple times, once per time step. Recall the equation:
\begin{equation}
\nabla_{h^{(t)}}L = W^T\nabla_{h^{(t+1)}}L diag(1 - (h^{(t+1)})^2) + V^T\nabla_{o^{(t)}}L
\end{equation}
From the knowledge of linear algebra, we can see the recurrent composition causes eigenvalues with magnitude less than one to decay to zero and eigenvalues with magnitude greater than one to explode.  \\
Let's first focus on the impact of vanishing gradient.Specifically, whenever the model is able to represent long term dependencies, the gradient of a long term interaction has \textbf{exponentially smaller magnitude} than
the gradient of a short term interaction. It does not mean that it is impossible to learn, but that it might take a very long time to learn long-term dependencies, because the signal about these dependencies will tend to be hidden by the smallest fluctuations arising from short-term dependencies. \\
We will discuss various approaches that have been proposed to reduce the difficulty of learning long- term dependencies, but the problem of learning long-term dependencies remains one of the main challenges in deep learning.

\subsection{Long Short Term Memory}
In the LSTM model, we use a forget unit $f_i^{(t)}$,that sets this weight to a value between 0 and 1 via a sigmoid unit:
\begin{equation}
f_i^{(t)} = \sigma(b_i^f + \sum_jU_{i,j}^fx_j^{(t)} + \sum_jW_{i,j}^fh_j^{(t-1)})
\end{equation}
where $x^{(t)}$ is the current input vector and $h^{(t)}$ is the current hidden layer vector, containing the outputs of all the LSTM cells, and $b^f$,$U^f$,$W^f$,are respectively biases, input weights and recurrent weights for the forget gates. \\
 The LSTM cell internal state is thus updated as the following, with a conditional self-loop weight $f_i^{(t)}$
\begin{equation}
s_i^{(t)} = f_i^{(t)} \ast s_i^{(t-1)} + g_i^{(t)} \ast \sigma(b_i + \sum_jU_{i,j}x_j^{(t)} + \sum_jW_{i,j}h_j^{(t-1)})
\end{equation}
where b, U and W respectively denote the biases, input weights and recurrent weights into the LSTM cell and $\ast$ means the multiplication by elements($.\ast$ in MATLAB). \\
The external input gate unit $g_i^{(t)}$ is computed similarly to the forget gate (with a sigmoid unit to obtain a gating value between 0 and 1, but with its own parameters:

\begin{equation}
g_i^{(t)} = \sigma(b_i^g + \sum_jU_{i,j}^gx_j^{(t)} + \sum_jW_{i,j}^gh_j^{(t-1)})
\end{equation}The output $h_i^{(t)}$ of the LSTM cell can also be shut off, via the output gate $o_i^{(t)}$, which also uses a sigmoid unit for gating:

\begin{equation}
h_i^{(t)} = tanh(s_i^{(t)}) \ast o_i^{(t)}
\end{equation}
\begin{equation}
o_i^{(t)} = \sigma(b_i^o + \sum_jU_{i,j}^ox_j^{(t)} + \sum_jW_{i,j}^oh_j^{(t-1)})
\end{equation}
LSTM networks have been shown to learn long-term dependencies more easily than the simple recurrent architectures.
\begin{figure}[H]
	\centering
	\includegraphics[width=0.9\textwidth]{LSTM}
	\caption{Block diagram of the cell of LSTM recurrent network. Cells are connected recurrently to each other, replacing the usual hidden units of ordinary recurrent networks. An input feature is computed with a regular artificial neuron unit. Its value can be accumulated into the state if the sigmoidal input gate allows it. The state unit has a linear self-loop whose weight is controlled by the forget gate. The output of the cell can be shut off by the output gate. All the gating units have a sigmoid nonlinearity, while the input unit can have any squashing nonlinearity. The state unit can also be used as an extra input to the gating units. The black square indicates a delay of a single time step.}
\end{figure}
From the equations above, we'll give a brief explanation(not strict!) here why the LSTM works for the long-term dependency. We will just focus on the gradient vanishing. \\
First, we have
\begin{equation}
s_i^{(t)} = f_i^{(t)} \ast s_i^{(t-1)} + g_i^{(t)} \ast \sigma(b_i + \sum_jU_{i,j}x_j^{(t)} + \sum_jW_{i,j}h_j^{(t-1)})
\end{equation}
hence we get
\begin{equation}
\frac{\partial s_i^{(t+k)}}{\partial s_i^{(t)}} \approx \prod_{j=t+1}^{t+k} diag(f_i^{(j)}) + ...
\end{equation}
where the $...$ things are some products which are vanishing to zero as k increases(just like we talked above). If in each time step j($t \le j \le t+k$), $f_i^{(j)} \approx \textbf{1}$(It forgets nothing!),we can see that $\frac{\partial s_i^{(t+k)}}{\partial s_i^{(t)}}$ is almost an identity matrix in the long term, and that means we can connect the information between long time steps together. We can go to the paper <LONG SHORT-TERM MEMORY>(Schmidhuber $\&$ Hochreiter,1997) for the detailed discussion.
\subsection{Other Gated RNNs}
Let's take a look at the recent work on gated RNNs, whose units are also known as gated recurrent units or GRUs (Cho et al., 2014b;Chung et al., 2014, 2015a; Jozefowicz et al., 2015; Chrupala et al., 2015). \\
The main difference with the LSTM is that a single gating unit simultaneously controls the forgetting factor and the decision to update the state unit. The update equations are the following:

\begin{equation}
h_i^{(t)} = u_i^{(t)} \ast h_i^{(t-1)} + (1 - u_i^{t}) \ast \sigma(b_i + \sum_jU_{i,j}x_j^{(t-1)} + \sum_jW_{i,j}r_j^{(t)}h_j^{(t-1)})
\end{equation}
where $u$ is 'update' gate and $r$ is 'reset' gate. The values are determined by:
\begin{equation}
u_i^{(t)} = \sigma(b_i^u + \sum_jU_{i,j}^ux_j^{(t)} + \sum_jW_{i,j}^uh_j^{(t-1)})
\end{equation}
and
\begin{equation}
r_i^{(t)} = \sigma(b_i^r + \sum_jU_{i,j}^rx_j^{(t)} + \sum_jW_{i,j}^rh_j^{(t-1)})
\end{equation}
\section{Optimization methods for RNN}
\subsection{Clipping gradients}
As discussed before, strongly nonlinear functions such as those computed by a recurrent net over many time steps tend to have derivatives that can be either very large or very small in magnitude. \\
The difficulty that arises is that when the parameter gradient is very large, a gradient descent parameter update could throw the parameters very far, into a region where the objective function is larger, undoing much of the work before.The gradient tells us the direction that corresponds to the steepest descent within an infinitesimal region surrounding the current parameters. \\
A simple type of solution has been in use by practitioners for many years:
clipping the gradient. One step is to clip the norm $||g||$ of the gradient g (Pascanu et al., 2013) just before the parameter update:
\begin{eqnarray}
\text{if} \ & ||g||  > v \\
& g\leftarrow \frac{g v}{||g||}
\end{eqnarray}
Because the gradient of all the parameters (including different groups of parameters, such as weights and biases) is renormalized jointly with a single scaling factor, the method has the advantage that it guarantees that each step is \textbf{still in the gradient direction}. Although the parameter updating has the same direction as the true gradient, with gradient norm clipping, the parameter update vector norm is now bounded. This bounded gradient avoids performing a detrimental step when the gradient explodes.
\begin{figure}[H]
	\centering
	\includegraphics[width=0.9\textwidth]{Clipgrad}
	\caption{Example of the effect of gradient clipping in a recurrent network with two parameters w and b. Gradient clipping can make gradient descent perform more reasonably in the vicinity of extremely steep cliffs. These steep cliffs commonly occur in recurrent networks near where a recurrent network behaves approximately linearly. The cliff is exponentially steep in the number of time steps because the weight matrix is multiplied by itself once for each time step. (Left)Gradient descent without gradient clipping overshoots the bottom of this small ravine, then receives a very large gradient from the cliff face. The large gradient catastrophically propels the parameters outside the axes of the plot. (Right)Gradient descent with gradient clipping has a more moderate reaction to the cliff. While it does ascend the cliff face, the step size is restricted so that it cannot be propelled away from steep region near the solution. Figure adapted with permission from Pascanu et al. (2013).
}
\end{figure}
\subsection{Regularizing to Encourage Information Flow}
Gradient clipping helps to deal with exploding gradients, but it does not help with vanishing gradients.To address vanishing gradients and better capture long-term dependencies, an idea is to regularize or constrain the parameters so as to encourage 'information flow'. \\
In particular, we would like the gradient vector $\nabla_{h^{(t)}}L$ being back-propagated to maintain its magnitude, even if the loss function only penalizes the output at the end of the sequence. Formally, we want
\begin{equation}
(\nabla_{h^{(t)}}L) \frac{\partial h^{(t)}}{\partial h^{(t-1)}}
\end{equation}
to be as large as
\begin{equation}
\nabla_{h^{(t)}}L
\end{equation}
With this objective, Pascanu et al. (2013) propose the following regularizer:
\begin{equation}
\Omega = \sum_{t}(\frac{|(\nabla_{h^{(t)}}L) \frac{\partial h^{(t)}}{\partial h^{(t-1)}}|}{|\nabla_{h^{(t)}}L|} - 1)^2
\end{equation}
The experiments with this regularizer suggest that, if combined with the norm clipping, the regularizer can considerably increase the span of the dependencies that an RNN can learn.


\section{Dynamics (formally)}
\subsection{}

\begin{frame}
\frametitleTC{System -- recap and systematisation}
\framesubtitleTC{A mathematical representation (model) of something that evolves}
\myPause
 \begin{columns}
  \column[T]{0.45\textwidth}
   \begin{itemize}[<+-| alert@+>]
   \item[] Simple representation
   \item[] \vspace{2mm}\begin{center}
            \input{./Unit-02/img/DS-SimpleRepresentation.tex}
           \end{center}
           \begin{itemize}[<+-| alert@+>]
           \item[] $u$ -- input(s)
           \item[] $y$ -- output(s)
           \end{itemize}
   \end{itemize}
  \column[T]{0.55\textwidth}
   \begin{itemize}[<+-| alert@+>]
   \item[] Model ingredients
           \begin{itemize}[<+-| alert@+>]
           \item what the system evolves upon:
                 \begin{itemize}[<+-| alert@+>]
                 \item the \TC{continuous time} $t$;
                 \item an integer index $k$ counting some events,
                       frequently called\\the \TC{discrete time}.
                 \end{itemize}
           \item the \TC{evolution law}:
                 \begin{itemize}[<+-| alert@+>]
                 \item $u[t_0,t]\;\rightarrow\;y[t_0,t]$;
                 \item $u[k_0,k]\;\rightarrow\;y[k_0,k]$.
                 \end{itemize}
           \end{itemize}
   \end{itemize}
 \end{columns} \myPause
 \vfill
 \begin{center}
  \vfill \textbf{Is anything missing? \myPause Sometimes, yes.}
 \end{center}
\end{frame}

\begin{frame}
\frametitleTC{\underline{Dynamic} system -- recap and systematisation}
\framesubtitleTC{The most general definition}
\myPause
\begin{center}
 {\Large
 If the knowledge of $u[t_0,t]$ --- or $u[k_0,k]$ \\ \myPause
 allows to determine $y[t_0,t]$ --- or $y[k_0,k]$ \\ \myPause
 the system is said to be \TC{non dynamic},\\ \myPause
 \vspace{1.5mm}\TC{dynamic} otherwise. \myPause
 }
\end{center}
\end{frame}

\begin{frame}
\frametitleTC{Dynamic system -- recap and systematisation}
\framesubtitleTC{Input, output, and \emph{state}}
\myPause
\begin{itemize}[<+-| alert@+>]
\item Non dynamic system:\\
      $y[t_0,t]$ or $y[k_0,k]$ depends only on $u[t_0,t]$ or $u[k_0,k]$.
\item Dynamic system:\\
      $y[t_0,t]$ or $y[k_0,k]$ depends on $u[t_0,t]$ or $u[k_0,k]$,\\
      and on the initial values $x[t_0]$ or $x[k_0]$ of some quantities.
\item These are called the \TC{state variables}, and form the \TC{state (vector)}.
\item The number of state variables is called the \TC{order} of the system.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Dynamic system -- recap and systematisation}
\framesubtitleTC{How can we express this in mathematical terms?}
\myPause
In several ways. We see the only two relevant for us.\myPause
\begin{itemize}[<+-| alert@+>]
\item Continuous-Time (CT) system:\\
      \begin{displaymath}
       \left\{
        \begin{array}{rl}
         \frac{dx(t)}{dt} &= f \big( x(t),u(t),t  \big) \\
         y(t)             &= g \big( x(t),u(t),t  \big)
        \end{array}
       \right.
      \end{displaymath}
\item Discrete-Time (DT) system:\\
      \begin{displaymath}
       \left\{
        \begin{array}{rl}
         x(k) &= f \big( x(k-1),u(k-1),k  \big) \\
         y(k) &= g \big( x(k),u(k),k  \big)
        \end{array}
       \right.
      \end{displaymath}

\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Dynamic system -- recap and systematisation}
\framesubtitleTC{Some more definitions (for both the CT and the DT case)}
\myPause
\begin{itemize}[<+-| alert@+>]
\item \TC{Linear (L)} system:\\
      $f(\cdot,\cdot,\cdot)$ and $g(\cdot,\cdot,\cdot)$ linear in $x$ and $u$.
\item \TC{Time-Invariant (TI)} system:\\
      $f(\cdot,\cdot,\cdot)$ and $g(\cdot,\cdot,\cdot)$ not depending on $t$ or $k$.
\item \TC{Proper} (sometimes, \emph{strictly} proper) system:\\
      $g(\cdot,\cdot,\cdot)$ not depending on $x$,\\
      i.e., the input acts on the output only through the state.
\item \TC{SISO} (Single-Input, Single-Output) system:\\
      $u$ and $y$ -- not necessarily $x$ -- scalars.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Motion}
\framesubtitleTC{General definition}
\myPause
\centerline{Note: from now on we mostly speak DT, for CT just replace $k$ with $t$.} \myPause
\vspace{3mm}\begin{itemize}[<+-| alert@+>]
\item Initial state + input at a certain start time $\Rightarrow$ motion, i.e.,
      \begin{displaymath}
       \left.
        \begin{array}{l}
         x(k_0) \\ u(k),\,k \geq k_0
        \end{array}
       \right\}
       \quad \Rightarrow \quad
        x(k),y(k),\,k \geq k_0.
      \end{displaymath}
\item We call $x(k)$ and $y(k)$, respectively,
      \begin{itemize}[<+-| alert@+>]
      \item the \TC{state motion}
      \item and the \TC{output motion}
      \end{itemize}
\item [] produced by the initial state $x(k_0)$ and the input $u(k)$, starting\\
      at time $k_0$.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Motion}
\framesubtitleTC{for the Time-Invariant (TI) case, to which we restrict the scope from now on}
\myPause
\begin{itemize}[<+-| alert@+>]
\item Initial state + input $\Rightarrow$ motion independently of the start time, i.e.,
      \begin{displaymath}
       \left.
        \begin{array}{l}
         x(0) \\ u(k),\,k \geq 0
        \end{array}
       \right\}
       \quad \Rightarrow \quad
        x(k),y(k),\,k \geq 0.
      \end{displaymath}
\item Alternatively, we can say that with TI systems one can set the origin\\
      of the time axis wherever one wants (for convenience, at zero).
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Equilibrium}
\framesubtitleTC{Definition (in the TI case)}
\myPause
\begin{center}
 {\Large
 If there exist some state vectors $\overline{x}$\\ \myPause
 such that $x(0)=\overline{x}$ and $u(k)=\overline{u}$, $k \geq 0$,\\ \myPause
 produce the constant state motion $x(k)=\overline{x}$, $k \geq 0$,\\ \myPause
 then those vectors are called \TC{equilibrium states},\\ \myPause
 or \TC{equilibria} for short,\\ \myPause
 \vspace{1.5mm}corresponding to the constant input $\overline{u}$.
 }
\end{center}
\end{frame}

\begin{frame}
\frametitleTC{Equilibrium}
\framesubtitleTC{Finding equilibrium states and outputs}
\myPause
\begin{itemize}[<+-| alert@+>]
\item A state vector $\overline{x}$ is an equilibrium for a given $\overline{u}$ if the consequent motion is
      $x(k)=x(k-1)=\overline{x}$ $\forall k$.
\item Thus to find equilibria one solves
      \begin{displaymath}
       \overline{x} = f(\overline{x},\overline{u}).
      \end{displaymath}
\item If some equilibrium state exists, and $g(\overline{x},\overline{u})$ does not lose significance,\\
      the result is the corresponding \TC{equilibrium output} $\overline{y}$.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Stability}
\framesubtitleTC{Preliminaries}
\myPause
\begin{itemize}[<+-| alert@+>]
\item The existence of an equilibrium implies NOTHING on what happens if the system does not start exactly
      at the equilibrium.
\item Discussing this is a matter of \TC{stability}.
\item One can talk about stability of equilibria, motions, and sometimes systems.
\item We define stability for an equilibrium, do not talk about motions,\\
      and move to systems later on.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Stability}
\framesubtitleTC{Stable equilibrium}
\myPause
\begin{itemize}[<+-| alert@+>]
\item Let $\overline{x}$ be an equilibrium for the constant input $\overline{u}$.
\item Denote by $x_{\Delta}(k)$ the \TC{perturbed motion} produced by
      \begin{itemize}[<+-| alert@+>]
      \item the input $\overline{u}$,
      \item and the \TC{perturbed initial state} $x(0)=\overline{x}+\Delta\overline{x}$.
      \end{itemize}
\item Not that in general, $x_{\Delta}(k)$ will not be constant.
\item Denote by $||x||$ the norm (think here of the Euclidean norm)\\
      of vector $x$.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Stability}
\framesubtitleTC{Stable equilibrium}
\myPause
\begin{itemize}[<+-| alert@+>]
\item An equilibrium $\overline{x}$ corresponding to the constant input $\overline{u}$ is said to be \TC{stable} if
      \begin{displaymath}
       \forall \varepsilon>0 \;
       \exists \delta_{\varepsilon}>0 \; : \;
       ||\Delta\overline{x}||<\delta_{\varepsilon}
       \Rightarrow ||x_{\Delta}(k)-\overline{x}||<\varepsilon \; \forall k \geq 0.
      \end{displaymath}
\item Interpretation: stable equilibrium means that
      \begin{itemize}[<+-| alert@+>]
      \item no matter how close one wants the \TC{entire} perturbed motion to remain to the equilibrium,
      \item a maximum distance of the initial state from the equilibrium can be\\
            found, that fulfils the desire.
      \end{itemize}
      \item Graphically,
            \begin{center}
             \includegraphics[width=0.60\columnwidth]{./Unit-02/img/StableEq-general.pdf}
            \end{center}
\item If the above does not hold true, the equilibrium is \TC{unstable}.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Stability}
\framesubtitleTC{Asymptotically stable equilibrium}
\myPause
\begin{itemize}[<+-| alert@+>]
\item An equilibrium $\overline{x}$ corresponding to $\overline{u}$ is said to be \TC{asymptotically stable} if
      \begin{itemize}[<+-| alert@+>]
      \item it is stable,
      \item and \underline{in addition}
            \begin{displaymath}
             \lim_{k\rightarrow\infty} ||x_{\Delta}(k)-\overline{x}||=0.
            \end{displaymath}
      \end{itemize}
      \item Graphically,
            \begin{center}
             \includegraphics[width=0.60\columnwidth]{./Unit-02/img/AsympStableEq-general.pdf}
            \end{center}
\item Clearly asymptotic stability implies stability, but not \emph{vice versa}.
\end{itemize}
\end{frame}


\subsection{Adaboost}

\subsubsection{Introduction}

Boosting is a way to create multiple learners for use in an ensemble predictor.

The goal is to create many predictors, which may not be themselves very accuracte, but have a high degree of independence.

\subsubsection{AdaBoost}

AdaBoost is a popular algorithm for boosting.

It works by:

\begin{itemize}
\item Creating a set of weak learners using different restrictions on features in the training data.
\item Choosing the weak learner that most reduces the error of the combined learners, and give it a weighting which most reduces the error of the combined learners.
\item Creating a new weighting for the dataset, where ones poorly predicted (by the combination of learners) are given high weights.
\item Repeating the process a fixed number of times.
\end{itemize}


\Opensolutionfile{solutions}[ex]
\section{Additional exercises}{}{}

\begin{enumialphparenastyle}

These problems require the techniques of this chapter, and are in no
particular order. Some problems may be done in more than one way.

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int (t+4)^3\,dt$
\begin{sol}
 $\ds{(t+4)^4\over4}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int t(t^2-9)^{3/2}\,dt$
\begin{sol}
 $\ds{(t^2-9)^{5/2}\over5}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int (e^{t^2}+16)te^{t^2}\,dt$
\begin{sol}
 $\ds{(e^{t^2}+16)^2\over 4}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% use addition and subtraction formulas sin(A+B)=sinAcosB+sinBcosA and sin(A-B)=sinAcosB-sinBcosA to solve for sinAcosB and substitute into integral
 $\ds\int \sin t\cos 2t\,dt$
\begin{sol}
 $\ds\cos t-{2\over3}\cos^3 t+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int \tan t\sec^2t\,dt$
\begin{sol}
 $\ds{\tan^2 t\over 2}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int {2t+1\over t^2+t+3}\,dt$
\begin{sol}
 $\ds\ln|t^2+t+3|+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% partial fractions
 $\ds\int {1\over t(t^2-4)}\,dt$
\begin{sol}
 $\ds {1\over8} \ln|1-4/t^2|+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% trig sub x=5sin(theta)
 $\ds\int {1\over (25-t^2)^{3/2}}\,dt$
\begin{sol}
 $\ds{1\over25}\tan(\arcsin(t/5))+C={t\over25\sqrt{25-t^2}}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int {\cos 3t\over\sqrt{\sin3t}}\,dt$
\begin{sol}
 $\ds{2\over3}\sqrt{\sin 3t}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% integration by parts
 $\ds\int t\sec^2 t\,dt$
\begin{sol}
 $\ds t\tan t+\ln|\cos t|+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int {e^t\over \sqrt{e^t+1}}\,dt$
\begin{sol}
 $\ds 2\sqrt{e^t+1}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% powers of trigonometric functions [use trig identity cos^2(x)=1/2(1+cos(2x))]
 $\ds\int \cos^4 t\,dt$
\begin{sol}
 $\ds{3t\over 8}+{\sin 2t\over4}+ {\sin 4t\over 32}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% partial fractions
 $\ds\int {1\over t^2+3t}\,dt$
\begin{sol}
 $\ds{\ln |t|\over 3} - {\ln |t+3|\over 3}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% trig sub t=tan(theta), then u-substitution
 $\ds\int {1\over t^2\sqrt{1+t^2}}\,dt$
\begin{sol}
 $\ds{-1\over \sin\arctan t}+C=-\sqrt{1+t^2}/t+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int {\sec^2t\over (1+\tan t)^3}\,dt$
\begin{sol}
 $\ds{-1\over 2(1+\tan t)^2}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int t^3\sqrt{t^2+1}\,dt$
\begin{sol}
 $\ds{(t^2+1)^{5/2}\over 5}-{(t^2+1)^{3/2}\over 3}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% integration by parts
 $\ds\int e^t\sin t\,dt$
\begin{sol}
 $\ds{e^t\sin t-e^t\cos t\over 2}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int (t^{3/2}+47)^3\sqrt{t}\,dt$
\begin{sol}
 $\ds{(t^{3/2}+47)^4\over6}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int {t^3\over (2-t^2)^{5/2}}\,dt$
\begin{sol}
 $\ds{2\over 3(2-t^2)^{3/2}}-{1\over(2-t^2)^{1/2}}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% trig sub x=(3/2)tan(theta), then u-substitution
 $\ds\int {1\over t(9+4t^2)}\,dt$
\begin{sol}
 $\ds{\ln|\sin(\arctan(2t/3))|\over9}+C =
(\ln(4t^2)-\ln(9+4t^2))/18 + C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int {\arctan 2t\over 1+4t^2}\,dt$
\begin{sol}
 $\ds{(\arctan(2t))^2\over4}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% partial fractions
 $\ds\int {t\over t^2+2t-3}\,dt$
\begin{sol}
 $\ds{3\ln|t+3|\over 4}+{\ln|t-1|\over4}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% powers of trigonometric functions with u=cos(t)
 $\ds\int \sin^3 t\cos^4 t\,dt$
\begin{sol}
 $\ds{\cos^7 t\over 7}-{\cos^5 t\over 5}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% factoring then simple integration
 $\ds\int {1\over t^2-6t+9}\,dt$
\begin{sol}
 $\ds{-1\over t-3}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% u-substitution
 $\ds\int {1\over t(\ln t)^2}\,dt$
\begin{sol}
 $\ds{-1\over \ln t}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% integration by parts
 $\ds\int t(\ln t)^2\,dt$
\begin{sol}
 $\ds{t^2(\ln t)^2\over 2}-{t^2\ln t\over 2}+{t^2\over4}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% integration by parts
 $\ds\int t^3e^{t}\,dt$
\begin{sol}
 $\ds(t^3-3t^2+6t-6)e^t+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}	% split into two fractions: (2x+1)/(2(x^2+x-1)) + 1/(2(x^2+x-1)). Integrate first by u-sub, and second by trig sub (x+1/2)=(sqrt(5)/2)sec(theta)
 $\ds\int {t+1\over t^2+t-1}\,dt$
\begin{sol}
 $\ds{5+\sqrt5\over10}
\ln(2t+1-\sqrt5)+{5-\sqrt5\over10}\ln(2t+1+\sqrt5)+C$
\end{sol}
\end{ex}
%%%%%%%%%%

\end{enumialphparenastyle}


\begin{frame}{Finding the autocorrelation}
To find the parameters $a_1, \ldots, a_N$, we need to compute the autocorrelation $r[0], \ldots, r[N-1]$ from $M+1$ samples of the signal $s[n]$.

Two methods:
\begin{enumerate}
	\item \textbf{Autocorrelation method}
	This method assumes that the signal $s[n]$ is \underline{zero outside the interval} $[0, M+1]$. Therefore, for $s[n]$ deterministic we have
	\begin{equation*}
	r_{ss}[|m|] = \sum_{n =0}^{M-|m|}s[n]s[n+|m|], |m| \leq N-1
	\end{equation*}
	This implies $r_{ss}[m] = 0, |m| > M$.
	\item \textbf{Covariance method}
	This method assumes that the signal $s[n]$ is \underline{unknown outside the interval} $[0, M+1]$. Therefore, for $s[n]$ deterministic we have
	\begin{equation*}
	r_{ss}[m] = \sum_{n = N}^{M}s[n]s[n+|m|], m = 0, 1, \ldots, N-1
	\end{equation*}
\end{enumerate}
\end{frame}

\begin{frame}{Convolution as a matrix-vector product}
We can write a convolution as a matrix-vector product
\begin{align*}
y[n] &= \sum_{m = 0}^N h_nx[n-m] \\
y &= Xh \\
\begin{bmatrix}
y[0] \\
y[1] \\
y[2] \\
\vdots \\
y[L-1] \\
\end{bmatrix}
&= \begin{bmatrix}
x[0] & 0 & \ldots & 0 \\
x[1] & x[0] & \ddots & \vdots \\
\vdots & x[1] & \ddots &  0\\
x[L-1] & \vdots &\ddots &  x[0]\\
0 & x[L-1] & \ddots & x[1] \\
\vdots & \vdots & \ddots &  \vdots \\
0  & 0 & \ldots & x[L-1]
\end{bmatrix}_{L\times N}\cdot\begin{bmatrix}
h_0 \\
h_1 \\
h_2 \\
\vdots \\
h_{N-1} \\
\end{bmatrix}
\end{align*}

The matrix $X$ is called \textbf{Toeplitz matrix}.

\vspace{0.25cm}
In Matlab: \texttt{>> X = toeplitz(c, r)}, where the vector \texttt{c} is the first column of $X$, and the vector \texttt{r} is the first row of $X$.

\end{frame}

\begin{frame}{All-pole model for deterministic signals}
Back to our problem when $s[n]$ is deterministic:
\begin{equation}
\sum_{n = 0}^{L-1} |v[n] - \sum_{m = 0}^N a_ns[n-m]|^2 \tag{if $s[n]$ is deterministic}
\end{equation}

Now we can write this in matrix notation
\begin{equation*}
||v - Sh||^2_2
\end{equation*}

This is a \textbf{least-squares problem}, and its solutions is
\begin{equation}
h = S^{\dagger}v \tag{least-squares solution}
\end{equation}
where $S^{\dagger} = (S^TS)^{-1}S^T$ is the \textbf{Moore-Penrose pseudo-inverse}.

\vspace{0.25cm}
When $s[n]$ is deterministic, $v[n]$ is assumed to be an impulse. Hence, $v = [1, 0, \ldots, 0]^T$.
\end{frame}

\begin{frame}{All-pole model for random signals}
When $s[n]$ is random, we want to minimize
\begin{equation*}
\E\Big(|v[n] - \sum_{m = 0}^N a_ns[n-m]|^2\Big)
\end{equation*}

\begin{equation*}
\E(s[n]v[n]) = \sum_{m = 0}^N a_n\E(s[n]s[n-m]), \quad n = 0, \ldots, L-1
\end{equation*}

\end{frame}

%
\begin{frame}{Revisiting the Bartlett or triangular window}
\begin{columns}
	\begin{column}{0.5\textwidth}
		\textbf{Time}
		\begin{equation*}
		w_B[n] = \begin{cases}
		1 - |n|/M, & |n| < M \\
		0, & \text{otherwise}
		\end{cases}
		\end{equation*}
	\end{column}

	\begin{column}{0.5\textwidth}
		\textbf{Frequency}
		\begin{equation*}
		W_B[n] = \frac{1}{M}\bigg(\frac{\sin(\omega M/2)}{\sin(\omega/2)}\bigg)^2
		\end{equation*}
	\end{column}
\end{columns}

\hspace*{-0.75cm}\resizebox{0.95\paperwidth}{!}{\input{figs/barlett_time_freq.tex}}
\end{frame}

%
\begin{frame}{Design specifications}
Specifications of \textbf{frequency-selective} filters
\begin{center}
	\resizebox{0.7\linewidth}{!}{\input{figs/design_specs.tex}}
\end{center}

\textbf{Terminology}
\begin{itemize}
	\item $\delta_1$ passband ripple
	\item $\delta_2$ stopband ripple
	\item $\omega_p$ passband edge frequency
	\item $\omega_p$ stopband edge frequency
\end{itemize}
\end{frame}

\begin{frame}{Poles, zeros, and the frequency response}

\textbf{Adding one more zero:} $H(z) = (1 - r_1e^{j\theta}z^{-1})(1 - r_2e^{j\psi}z^{-1})$

\begin{columns}[t]
	\begin{column}{0.7\textwidth}
		\vspace{-0.5cm}
		\begin{center}
			\resizebox{\linewidth}{!}{\input{figs/pole_zero_freq_resp_2zeros.tex}}
		\end{center}
	\end{column}

	\begin{column}{0.3\textwidth}
		\only<1|handout:1>{
			\textbf{Magnitude}
			\flushleft
			\begin{align*}
			H(z) &= \frac{z - r_1e^{j\theta}}{z}\cdot\frac{z - r_2e^{j\psi}}{z} \\
			|H(e^{j\omega})| &= \bigg|\frac{{\color{red2} v_1} - {\color{black} v_2}}{{\color{red2} v_1}}\bigg|\bigg|\frac{{\color{red2} v_1} - {\color{black} v_4}}{{\color{red2} v_1}}\bigg|\\
			&= \frac{||{\color{blue2} v_3}||}{||{\color{red2} v_1}||}\frac{||{\color{green2} v_5}||}{||{\color{red2} v_1}||} \\
			&= ||{\color{blue2} v_3}||||{\color{green2} v_5}||
			\end{align*}
		}

		\only<2|handout:2>{
			\textbf{Phase}
			\flushleft
			\begin{align*}
			H(z) &= \frac{z - r_1e^{j\theta}}{z}\cdot\frac{z - r_2e^{j\psi}}{z} \\
			\angle H(e^{j\omega})  &= \angle\frac{{\color{red2} v_1} - {\color{black} v_2}}{{\color{red2} v_1}} + \angle\frac{{\color{red2} v_1} - {\color{black} v_4}}{{\color{red2} v_1}}\\
			&= \angle {\color{blue2} v_3} + \angle {\color{green2} v_5} - 2\omega
			\end{align*}
		}
	\end{column}
\end{columns}

\only<1|handout:1>{\textbf{Conclusion:} As $e^{j\omega}$ approaches a zero, $|H(e^{j\omega})| \to 0$}
\only<2|handout:2>{\textbf{Conclusion:} A zero causes a negative to positive phase change of the phase response (\textbf{phase advance})}
\end{frame}

%
\begin{frame}{The unilateral $z$-transform}

The bilateral $z$-transform we've seen so far does not account for initial conditions.

\begin{block}{The unilateral $z$-transform}
	\begin{equation*} \tag{Direct transform}
	X^+(z) = \sum_{\tikz[baseline]{
			\node[fill=blue!20,anchor=base] (t1) {$n = 0$};
	}}^{\infty} x[n]z^{-n}
	\end{equation*}
\end{block}

\begin{itemize}
	\item For causal signals, $X(z) = X^+(z)$. \\
	\item For any signal $y[n] \Longleftrightarrow Y(z)$, we have $y[n]u[n] \Longleftrightarrow Y^+(z)$. \\
	\item As the unilateral $z$-transform only takes into account the causal part of the signal, the ROC will always the exterior of a circle (ROC = $\{|z|>|r|\}$). Hence, there's no ambiguity.
\end{itemize}

\end{frame}

\subsection{Sampling and reconstruction}
%
\begin{frame}{Families of $z$-transforms}
\begin{block}{Finite-length sequences}
	When $A(z) = 1$, we're left with $z$-transforms of the form
	\begin{align*}
	X(z) = b_0 + b_1z^{-1}+\ldots+b_Mz^{-M}
	\end{align*}
	\begin{itemize}
		\item These correspond to finite-length sequences or systems with \textbf{finite impulse response (FIR)}
		\item In this case, we have $M$ zeros, which are the roots of $X(z)$, and we have $M$ poles at $z = 0$. As a result, the ROC is the entire $z$-plane with the exception of $z = 0$, and possible poles at infinity i.e., $X(p = \infty) = \infty$.
		\item Since finite-length sequences are always absolute summable, the ROC of their $z$-trasnform is the entire $z$ plan, with the exception of $z = 0$ or $z = \infty$ (the poles).
	\end{itemize}
\end{block}

\end{frame}

%
\begin{frame}{Obtaining frequency response from difference equation}

	\begin{block}{Difference equation}
		\begin{equation*}
		\sum_{k=0}^N a_k y[n-k] = \sum_{k=0}^Mb_k x[n-k]
		\end{equation*}
	\end{block}

	Using the linearity and time shift properties of the DTFT it follows that

	\begin{align*}
	\sum_{k=0}^N a_kY(e^{j\omega})e^{-j\omega k} = \sum_{k=0}^M b_kX(e^{j\omega})e^{-j\omega k} \\
	H(e^{j\omega}) = \frac{Y(e^{j\omega})}{X(e^{j\omega})} = \frac{\sum_{k=0}^M b_ke^{-j\omega k}}{\sum_{k=0}^N a_ke^{-j\omega k}}
	\end{align*}
\end{frame}

%%%%%%% Discrete Fourier Transform