\documentclass{article} % For LaTeX2e
\usepackage[a4paper,margin=1.5in]{geometry}
\usepackage[mathlines]{lineno}
\usepackage{hyperref}
\usepackage{url}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{bm}
\usepackage{hyperref}
\usepackage{natbib}
\usepackage{latexsym,amsbsy,amssymb,color,xspace, booktabs}

\include{macros}
\include{local_macros}
\def\linenumberfont{\normalfont\small\sffamily}

\title{Multiple Output Gaussian Processes Regression}

\newcommand{\fix}{\marginpar{FIX}}
\newcommand{\new}{\marginpar{NEW}}

%\nipsfinalcopy % Uncomment for camera-ready version

\begin{document}
\maketitle
\linenumbers

\section{Model Description}
We describe a multiple-output regression model.
Suppose that we have inputs $\X = \{\x_n\}_{n=1}^N$ and outputs $\y = \{\y_n\}_{n=1}^N$, with $\x_n \in \calR^D$ and $\y_n \in \calR^P$, i.e. the output is $P$-dimensional.
For ease of exposition, we assume that there are no missing values in any output dimension.
The most dominant approach in GP-based multiple output regression is to compose the different outputs as some \textit{linear} mixing (combination) of some basic processes (random functions).
Our model is also underpinned by this composition approach.

\noindent Consider the semiparametric latent factor models (SLFM) \cite{teh-et-al-aistats-05}.
It assumes $Q$ basic independent processes, $g_j(\x) \sim \GP(0, k(\x,\x';\vectheta_j))$, where $\vectheta_j$ is the hyperparameters of the covariance function of $g_j(\x)$.
Each (latent) output is a weighted combination of these basic processes, i.e.
\begin{align}
f_i(\x) = \sum_{j=1}^Q w_{ij} g_j(\x), \quad i = 1 \hdots P,
\end{align}
In \cite{seeger2005semiparametric}, an extra degree of freedom is given by allowing each output $i$ to have its own process $h_i(\x) \sim \GP(0, k(\x,\x'; \vectheta^h_i)$ leading to
\begin{align}
f_i(\x) = \sum_{j=1}^Q w_{ij} g_j(\x) + h_i(\x),
\end{align}
We emphasize that, although $k$ denotes covariance functions in general, each independent process can have its own covariance function.

\noindent
\textbf{Augmented sparse GPs}
Toward scalable modeling, we replace standard GPs with sparse GPs augmented with \textit{different} set of inducing inputs.
This adds much flexibility to the model as each of the shared process $g_j(\x)$ can model a different pattern in the data with its own covariance function and inducing inputs. The roles of $g_j(\x)$ and $h_i(\x)$ can be quite different, so it is necessary that each has its own inducing inputs.
Furthermore, $g_j(\x)$ can be seen as a \textit{global} function operating on the entire input space (of all output dimensions), while each $h_i(\x)$ operates only on the inputs of the $i$-th output, which can be a subspace of the input.

%However, to really address the issue of scalability, this model is built upon sparse GPs from the ground up.

\subsection{Prior and Likelihood}
In this section we specify the prior and likelihood of this model.
\newcommand{\Zj}{\Z_j}
\newcommand{\Zhi}{\Z^h_i}
Let the set of inducing inputs for $g_j(\x)$ and $h_i(\x)$ be $\Z_j$ and $\Z^h_i$, and the corresponding inducing points $\u_j$ and $\v_i$, respectively.
We denote the collective variables: $\g = \{\g_j\}$, $\h = \{\h_i \}$, $\u = \{\u_j\}$, $\v = \{\v_i\}$, $\Z = \{\Zj\}$, and $\Z^h = \{\Zhi \}$ where $\g_j = \{g_j(\x_n)\}$, $\h_i = \{h_i(\x_n)\}$.
The subscripts are $i = 1 \hdots P$, $j = 1 \hdots Q$, and $n = 1 \hdots N$.

\noindent
\textbf{Prior}
The prior of the augmented model is given by:
\begin{align}
p(\g | \u) &= \prod_{j=1}^Q p(\g_j | \u_j) = \prod_{j=1}^Q \Normal(\g_j; \BigMu_j, \tilde{\K}_j )\\
p(\u) &= \prod_{j=1}^Q p(\u_j) = \prod_{j=1}^Q \Normal(\u_j; \vec{0}, k(\Zj, \Zj)) \\
p(\h | \v) &= \prod_{i=1}^P p(\h_i | \v_i) = \prod_{i=1}^P \Normal(\h_i; \BigMu^h_i, \tilde{\K}^h_i)\\
p(\v) &= \prod_{i=1}^P p(\v_i) = \prod_{i=1}^P \Normal(\v_i; \vec{0}, k(\Zhi, \Zhi)),
\end{align}
where $\BigMu_j = k(\X,\Zj)k(\Zj,\Zj)^{-1}\u$ and $\tilde{\K}_j = k_j(\X,\X) - k(\X,\Zj)k(\Zj,\Zj)^{-1}k(\Zj,\X)$ and $\BigMu^h_i, \tilde{\K}^h_i$ are similarly defined.
To avoid notational clutter, we omit the subscripts $j,h,i$ from the kernels $k_j(\cdot,\cdot)$ and $k^h_i(\cdot,\cdot)$ when it is clear from the parameters inside the parentheses which covariance function is in action.

\noindent
\textbf{Likelihood}
The likelihood as usual follows the standard iid Gaussian likelihood,
\begin{align}
p(\y | \g, \h ) = \prod_{i=1}^P p( \y_i ; \g, \h_i) = \prod_{i=1}^P \Normal( \y_i ; \sum_{j=1}^Q w_{ij} \g_j + \h_i, \beta_i^{-1} \I).
\end{align}

\subsection{Variational Inference in Sparse GPs Revisited}
In this section we review inference in sparse GPs (as presented in Titsias).
The posterior in an augmented model is $p(\f, \u | \y) = p(\f | \u, \y) p(\u | \y)$.
The key property of such augmented GPs  is the notation of \textit{sufficient statistics}: given the inducing points $\u$, the latent values $\f$ are independent with any other set of latent values (e.g. the test set).
In the optimal setting when $\u$ is the sufficient statistics of $\f$, it should hold that $p(\f | \u, \y) = p(\f | \u)$ as $\y$ is only the noisy version of $\f$.
This leads to choosing a variational approximation of the posterior which factorizes as $q(\f, \u | \y) = p(\f | \u) q(\u | \y)$.
Since the conditional $p(\f | \u)$ is known, variational inference becomes learning an optimal posterior $q(\u | \y)$ only. \\

\noindent Also as a consequence of sufficient statistics, the approximate prediction for test targets $\vfstar$ at test inputs $\X_*$ is
\begin{align}
\nonumber
p(\vfstar | \y, \X_*) &= \int p(\vfstar | \f, \u, \X_*) q(\f, \u | \y) \der \f \der \u \\
\nonumber
&= \int p(\vfstar | \u) q(\u| \y) p(\f | \u) \der \f \der \u \\ \nonumber
&= \int p(\vfstar | \u) q(\u| \y) \der \u \\
\label{eq:sorprediction}
&= \Normal(\vfstar; \bs{\mu_*},\vec{s_*})
\end{align}
where,
\begin{align}
\nonumber
\bs{\mu_*} &= \K_{*z} \K_{zz}^{-1}\m \\
\nonumber
\S_* &= \K_{**} - \K_{*z} \left(\K_{zz}^{-1} - \K_{zz}^{-1} \S \K_{zz}^{-1} \right) \K_{*z}^T.
\end{align}
 Here $\K_{*z}$ is the covariance matrix between test and inducing inputs, $\K_{zz}$ is the covariance matrix of the inducing inputs.

\subsection{Variational Inference}
\newcommand{\ug}{\u_g}
\newcommand{\uh}{\u^h}
\newcommand{\mgj}{\m_j}
\newcommand{\mhi}{\m^h_i}
\newcommand{\Sgj}{\S_j}
\newcommand{\Shi}{\S^h_i}
Our goal of inference is to find the posterior $p(\g, \h, \u, \v | \y)$.
Following the previous discussion, we assume a variational distribution which factorizes as:
\begin{align}
\nonumber
q(\g, \h, \u, \v | \y)
\nonumber
 &= p(\g|\u) p(\h|\v) q(\u,\v)  \\
 &= p(\g|\u) p(\h|\v) \prod_{j=1}^Q q(\u_j) \prod_{i=1}^P  q(\v_i)
\end{align}
Since the conditionals $p(\g | \u)$ and $p(\h | \v)$ are given, we need only to find the optimum $q(\u_j)$ and $q(\v_i)$.
Let $q(\u_j) = \Normal(\u_j; \mgj, \Sgj)$ and $q(\v_i) = \Normal(\v_i; \mhi, \Shi)$. \\

\noindent
To find the optimum $q(\u, \v)$ we optimize the evidence lower bound (ELBO) of the log marginal,
\begin{align}
\nonumber
\log p(\y) \ge& \int q(\u, \v) \log \frac{p(\y | \u, \v) p(\u, \v)}{q(\u, \v)} \der \u \der \v \\
\nonumber
=& \int q(\u, \v) \log p(\y | \u, \v)  \der \u \der \v
+ \int q(\u, \v) \log \frac{p(\u, \v)}{q(\u, \v)} \der \u \der \v \\
\label{eq:elbo}
=& \int q(\u, \v) \log p(\y | \u, \v)  \der \u \der \v
- \left(\sum_{j=1}^Q \KL[q(\u_j) || p(\u_j)] + \sum_{i=1}^P \KL[q(\u_i) || p(\u_i)] \right),
\end{align}
where the last equality occurs because both of $q(\u, \v)$ and $p(\u, \v)$ fully factorize.
Since $q(\u_j), q(\v_i), p(\u_j), p(\v_i)$ are all multivariate Gaussian distributions, the KL divergences are analytically tractable and require $\calO(M^3)$ computation, where $M$ is the largest number of inducing inputs (recall that $g_j(\x)$ and $h_i(\x)$ use separate set of inducing inputs). \\

\noindent To compute the ELBO we first focus on the term:
\newcommand{\llangle}{\left\langle}
\newcommand{\rrangle}{\right\rangle}
\begin{align}
\nonumber
\log p(\y | \u, \v)
 &= \log \Eb{p(\y | \g, \h)}_{p(\g,\h | \u, \v)} \\
 \nonumber
&\ge \Eb{\log p(\y | \g, \h)}_{p(\g,\h | \u, \v)} \quad &\text{(Jensen's inequality)} \\
&= \sum_{i=1}^P \sum_{n=1}^N \Eb{\log p(y_{in} | \g_n, h_{in}) }_{p(\g_n | \u) p(\h_{in} | \v_i)}, \quad
 &\text{(factorized likelihood)}
\end{align}
where $\g_n = \{g_{jn} = (\g_j)_n\}_{j=1}^Q$.
Each individual term $l_{in} \define \Eb{\log p(y_{in} | \g_n, h_{in}) }_{p(\g_n | \u) p(\h_in | \v_i)}$ can be computed using the identity in eq. \ref{eq:identity} given in the appendix to give:
\begin{align}
\nonumber
l_{in} &= \int \log p(y_{in} | \g_n, h_{in}) \prod_{j=1}^Q p(g_{jn} | \u_j) p(h_{in} | \u_i) \der \g_n \der h_{in} \\
&= \log \Normal(y_{in}; \sum_{j=1}^Q w_{ij} \mu_{jn}+ \mu^h_{in}, \beta_i^{-1})
- \frac{1}{2} \beta_i \sum_{j=1}^Q w_{ij}^2 \tilde{k}_{nn}
- \frac{1}{2} \beta_i \tilde{k}^h_{inn},
\end{align}
where $\tilde{k}_{nn} = (\tilde{\K})_{nn}, \tilde{k}^h_{inn} = (\tilde{\K}^h_i)_{nn}, \mu_{jn} = (\Mu_j)_n, \text{ and } \mu^h_{in} = (\Mu^h_i)_n$.

% previous detailed derivation
%\begin{align}
%\nonumber
%l_{in} &= \int \log p(y_{in} | g_n, h_{in}) p(\g | \ug) p(\h_i | \u_i) \der \g \der \h_i \\
%\nonumber
%&= \int \log \Normal(y_{in} ; w_i g_n + h_{in}, \beta_i^{-1})
%\Normal(g_n ; \mu_{gn}, \tilde{k}_{gnn})
%\Normal(h_{in} ; \mu_{in}, (\tilde{k}_{inn}) \der g_n \der h_{in} \\
%\nonumber
%&= -\frac{1}{2} \log 2 \pi \beta_i^{-1} - \frac{1}{2} \int (y_{in} - w_i g_n - h_{in}) \beta_i (y_{in} - w_i g_n - h_{in})
%\Normal(g_n ; \mu_{gn}, \tilde{k}_{gnn})
%\Normal(h_{in} ; \mu_{in}, \tilde{k}_{inn}) \der g_n \der h_{in} \\
%\nonumber
%&= -\frac{1}{2} \log 2 \pi \beta_i^{-1} - \frac{1}{2} \int \left[(y_{in} - h_{in} - w_i \mu_{gn}) \beta_i (y_{in} - h_{in} - w_i \mu_{gn}) + w_i^2 \beta_i \tilde{k}_{gnn} \right]
%\Normal(h_{in} ; \mu_{in}, \tilde{k}_{inn}) \der h_{in} \\
%\nonumber
%&= -\frac{1}{2} \log 2 \pi \beta_i^{-1} - \frac{1}{2} w_i^2 \beta_i \tilde{k}_{gnn}
%- \frac{1}{2} \beta_i \tilde{k}_{inn} - \frac{1}{2} (y_{in} - w_i \mu_{gn} - \mu_{in}) \beta_i (y_{in} - w_i \mu_{gn} - \mu_{in}) \\
%&= \log \Normal(y_{in}; w_i \mu_{gn} + \mu_{in}, \beta_i^{-1})  - \frac{1}{2} \beta_i (w_i^2 \tilde{k}_{gnn} + \tilde{k}_{inn}),
%\end{align}

\noindent Substituting $l_{in}$ into equation \ref{eq:elbo} and carrying out the integral using the identity in eq. \ref{eq:identity} we get:
\begin{align}
\nonumber
\log p(\y)
\ge& \sum_{i=1}^P \sum_{n=1}^N
\bigg( \log \Normal(y_{in}; \tilde{\mu}_{in}, \beta_i^{-1})
          - \frac{1}{2} \beta_i \sum_{j=1}^Q w_{ij}^2        \tilde{k}_{nn} - \frac{1}{2} \beta_i \tilde{k}^h_{inn}
 \\ \nonumber
         &\quad \quad \quad \quad - \frac{1}{2} \beta_i \trace  \sum_{j=1}^Q w_{ij}^2 \S_j \mat{\Lambda}_{jn} - \beta_i \frac{1}{2} \trace \S^h_i \mat{\Lambda}_{in}
\bigg) \\
&- \left(\sum_{j=1}^Q \KL[q(\u_j) || p(\u_j)] + \sum_{i=1}^P \KL[q(\v_i) || p(\v_i)] \right) \define \calL,
\end{align}
where
%TODO \Lambda_gn depends on the output i so need a better notation
\newcommand{\Zg}{\Z_g}
\newcommand{\Zi}{\Z_i}
\begin{align}
\tilde{\mu}_{in}
&= \sum_{j=1}^Q w_{ij} k(\x_n, \Zj)k(\Zj,\Zj)^{-1}\m_j + k(\x_n, \Zhi)k(\Zhi,\Zhi)^{-1}\mhi \\
\mat{\Lambda}_{jn}
&= k(\Zj,\Zj)^{-1} k(\Zj, \x_n) k(\x_n, \Zj) k(\Zj,\Zj)^{-1} \\
\mat{\Lambda}_{in}
&= k(\Zhi,\Zhi)^{-1} k(\Zhi, \x_n) k(\x_n, \Zhi) k(\Zhi,\Zhi)^{-1}.
\end{align}

\noindent Notice that this ELBO clearly generalizes the standard GP regression. In particular, setting $P = Q = 1$, $w_i = 1$ and $h_i(\x) = 0$ we recover the bound in Hensman et al \cite{hensmangaussian}.
Due to the decomposition of this bound, we can use stochastic gradient descent to learn the variational parameters.

\subsubsection{Variational Parameters Derivatives}
\newcommand{\oi}{\vec{o}_i}
% some notation
Before diving into the details, we first define the indexing operator of a matrix: $\B(\vec{r},\vec{c})$ extracts the submatrix in rows $\vec{r}$ and columns $\vec{c}$ of $\B$.
To index all columns we use $\B(\vec{r},:)$ and similarly for all rows $\B(:,\vec{c})$.
Readers familiar with this operator will recognize that this is the MATLAB indexing operator.

The optimal posteriors are found by setting the gradients of the lowerbound $\calL$ wrt to the parameters of $q(\u_j)$ and $q(\v_i)$.
Recall that in the model, different outputs can be observed at different inputs (i.e. the case of missing values).
Let $\oi$ be the indice of the observed inputs of the output dimension $i$.
We denote its set of observed inputs and targets as: $\X_i = \X(\oi,:)$ and $\y_i = y_{i}(\oi)$.

\noindent As a function of the parameters of $q(\u_j)$, the lowerbound $\calL$ is:
\newcommand{\Ahi}{\A^h_i}
\begin{align}
\nonumber
\calL^g_j \define&
 \sum_{i=1}^P \log \Normal(\y_i; \sum_{j=1}^Q w_{ij} \A_j(\oi,:) \m_j + \Ahi \mhi, \beta_i^{-1} \I)  \\
 \nonumber
 &- \frac{1}{2} \sum_{i=1}^P \bigg(\beta_i \trace w_{ij}^2 \tilde{\K}_j(\oi,\oi)
 + \beta_i \trace w_{ij}^2 \S_j \A_j(\oi,:)^T \A_j(\oi,:) \bigg)
 \\
 &- \frac{1}{2} \log |k(\Zj,\Zj) \S_j^{-1}| -\frac{1}{2} \trace k(\Zj,\Zj)^{-1} (\m_j \m_j^T + \S_j) ,
\end{align}
where $\A_j = k(\X,\Zj)k(\Zj,\Zj)^{-1}$, which gives $\A_j(\oi,:) = k(\X_i,\Zj) k(\Zj,\Zj)^{-1}$, and
$\Ahi = k(\X_i,\Zhi)k(\Zhi,\Zhi)^{-1}$. \\

%------------------------------------------
% derivatives of q(u_j)
\newcommand{\Lgj}{\calL^g_j}
\newcommand{\ynoj}{\y_i^{\backslash j}}
\noindent The derivatives of $\Lgj$ wrt $\m_j$ and $\S_j$ are given by:
\begin{align}
\deriv{\Lgj}{\m_j}
& = \sum_{i=1}^P \beta_i w_{ij} \A_j(\oi,:)^T \ynoj - \bigg[k_g(\Zg,\Zg)^{-1} + \sum_{i=1}^P \beta_i w_{ij}^2 \A_j(\oi,:)^T \A_j(\oi,:) \bigg] \m_j \\
\deriv{\Lgj}{\S_j}
&= \frac{1}{2} \S_j^{-1} - \frac{1}{2} \bigg[ k(\Zj,\Zj)^{-1} + \sum_{i=1}^P \beta_i w_{ij}^2 \A_j(\oi,:)^T \A_j(\oi,:) \bigg],
\end{align}
where $\y_i^{\backslash j} = \y_i - \Ahi \mhi - \sum_{j' \neq j} w_{ij'} \A_{j'}(\oi,:) \m_{j'}$.

%-------------------------------------------
%  derivatives of q(v_i)
\newcommand{\Lhi}{\calL^h_i}
\noindent As a function of the parameters of $q(\v_i)$, the lower bound $\calL$ is:
\begin{align}
\nonumber
\Lhi \define&
 \log \Normal(\y_i; \sum_{j=1}^Q w_{ij} \A_j(\oi,:) \m_j + \Ahi \mhi, \beta_i^{-1} \I)
 - \frac{1}{2} \beta_i \trace \tilde{\K}^h_i(\oi,\oi)
 - \frac{1}{2} \beta_i \trace \Shi (\Ahi)^T \Ahi
 \\
  &- \frac{1}{2} \log |k(\Zhi,\Zhi) (\Shi)^{-1}| -\frac{1}{2} \trace k(\Zhi,\Zhi)^{-1} (\mhi (\mhi)^T + \Shi) ,
\end{align}

\noindent The derivatives of $\Lhi$ wrt $\mhi$ and $\Shi$ are given by:
\newcommand{\ynoh}{\y_i^{\backslash h}}
\begin{align}
\deriv{\Lhi}{\mhi}
& = \beta_i \A_i^T \ynoh - \left[k(\Zhi,\Zhi)^{-1} +  \beta_i \A_i^T \A_i \right] \m_i \\
\deriv{\Lhi}{\Shi}
&= \frac{1}{2} \S_i^{-1} - \frac{1}{2} \left[ k(\Zhi,\Zhi)^{-1} + \beta_i \A_i^T \A_i \right] ,
\end{align}
where $\ynoh = \y_i - \sum_{j=1}^Q w_{ij} \A_j(\oi,:) \m_j$.

% comment on computation
\noindent It can be seen that the derivatives of the parameters of $q(\v_i)$ only involve the observations of the output dimension $i$.
The derivatives of the parameters of $q(\u_j)$ involve the observations across all output dimensions but decompose as a sum of contributions from individual outputs.
Therefore, computation of the derivatives (and hence the update equations) can be distributed or parallelized easily.
This attractive property allows the model to scale to a very large number of inputs and outputs.

% Comment on other hyperparameters

%\subsubsection{Update Equations}
% and comment on the intuition of the update equations: e.g. y(x) - g(x) for h(x)
\subsection{Prediction}
The predictive distribution of the $i$-th output for a test input $\x_*$ is
\begin{align}
p(\fstar | \y, \x_*) = \int \Normal(\fstar; \sum_{j=1}^Q w_j g_{j*} + h_{i*}, 0) p(\g_* | \y, \x_*) p(h_{i*} | \y, \x_*) \der \g_* \der h_{i*},
\end{align}
where $p(\g_* | \y, \x_*) = \prod_{j=1}^Q \Normal(g_{j*}; \mu_{j*}, s_{j*})$ and $p(h_{i*} | \y, \x_*) = p(h_{i*}; \mu^h_{i*}, s^h_{i*})$ are the predictive distributions of the sparse GPs as given in eq. \ref{eq:sorprediction}.
Therefore we have:
\begin{align}
p(\fstar | \y, \x_*) = \Normal(\fstar; \sum_{j=1}^Q w_{ij} \mu_{j*} + \mu_{i*}, w_{ij}^2 s_{j*} + s_{i*}).
\end{align}

%\begin{linenomath}
%\begin{align}
%\sum_{i,h} \lambda_i \lambda_h cov [f(\x, \x')]
%&= \sum_{i,h}   \sum_{j,j'} \lambda_i g(\x_i - \vs_j) k(\vs_j, \vs_{j'}) \lambda_h g(\x_h - \vs_j') \\
%\end{align}
%\end{linenomath}

\section{Toy Experiments}
The first toy experiment uses two identical outputs which are noisy version of the same version plus some noise: $y_1(x) = sin(x) + \epsilon$ and $y_2(x) = sin(x) + \epsilon$, $\epsilon \sim \Normal(0,0.01)$.
In this case the shared function $g(x)$ is $sin(x)$ and the independent processes $h_1(x)$ and $h_2(x)$ are just white noises.
Each output has missing values in one region of the input space.
The predictive distributions of the multiple-gp model compared to that of independent gps are shown in \ref{fig4}.

\noindent The second toy experiment uses similar setting as the first one, except that now $y_1(x) = sin(x) + \epsilon$ and $y_2(x) = -sin(x) + \epsilon$.
In this case the shared function is still $g(x) = sin(x)$, but the weights should be opposite i.e. $w_1 = 1$ and $w_2 = 1$.
The learning procedure was indeed able to recovered this relation and learned that $w_1 = 1.2$ and $w_2 = -1.3$.
The predictive distributions are shown in \ref{fig5}.

\begin{figure*}
\centering
\begin{tabular}{cc}
\includegraphics[scale=0.5]{figures/ssvi-y1.eps} &
\includegraphics[scale=0.5]{figures/ssvi-svi1.eps} \\
\includegraphics[scale=0.5]{figures/ssvi-y2.eps} &
\includegraphics[scale=0.5]{figures/ssvi-svi2.eps} \\
\multicolumn{2}{c}{\includegraphics[scale=0.5]{figures/ssvi-y1byg.eps} }
\end{tabular}
\label{fig4}
\caption{Predictive distributions of the multipe-output gps (left column) and independent gps (right column) for the first toy example. The predictive distribution by $g(x)$ for $y_1(x)$ is shown in the bottom figure.}
\end{figure*}

\begin{figure*}
\centering
\begin{tabular}{cc}
\includegraphics[scale=0.5]{figures/ssvi2-y1.eps} &
\includegraphics[scale=0.5]{figures/ssvi2-svi1.eps} \\
\includegraphics[scale=0.5]{figures/ssvi2-y2.eps} &
\includegraphics[scale=0.5]{figures/ssvi2-svi2.eps} \\
\multicolumn{2}{c}{\includegraphics[scale=0.5]{figures/ssvi-y1byg.eps} }
\end{tabular}
\label{fig5}
\caption{Predictive distributions of the multipe-output gps (left column) and independent gps (right column) for the second toy experiment. The predictive distribution by $g(x)$ for $y_1(x)$ is shown in the bottom figure.}
\end{figure*}

\section{Appendix 1: Derivatives}
Here we derive the gradients of the lower bound wrt the  hyperparameters.
As a template, consider the lower bound in Hensman et al (as a function of the hyperparameters):
\begin{align}
\nonumber
\calL
=& \log \Normal(\y; \K_{NM} \K_{MM}^{-1} \m, \beta^{-1}\I)
 - \frac{1}{2} \beta \trace \tilde{\K}
 - \frac{1}{2} \beta \trace (\S\K_{MM}^{-1} \K_{MN} \K_{NM} \K_{MM}^{-1}) \\  \nonumber
&- \frac{1}{2} \left( \log |\K_{MM}| + \trace(\K_{MM}^{-1}(\m \m^T + \S)) \right) \\ \nonumber
=& \underbrace{\log \Normal(\y; \A \m, \beta^{-1}\I)}_{\calL_1}
 - \underbrace{\frac{1}{2} \beta \trace (\K_{NN} - \A\K_{MN})}_{\calL_2}
 - \underbrace{\frac{1}{2} \beta \trace (\S\A^T\A)}_{\calL_3} \\
&- \underbrace{\frac{1}{2} \left( \log |\K_{MM}| + \trace(\K_{MM}^{-1}(\m \m^T + \S)) \right)}_{\calL_4},
\end{align}
where $\A = \K_{NM} \K_{MM}^{-1}$.
Notice that we have re-written the sum of individual terms in matrix form which will make the derivation and also computation easier.

\subsection{Derivative of the Noise Hyperparameter}
The derivative of the noise hyperparameter $\beta$ is easily computed as:
\begin{align}
\deriv{\calL}{\beta} = \frac{N}{2\beta} - \frac{1}{2} (\y - \A\m)^T (\y - \A\m) - \frac{\calL_2}{\beta} - \frac{\calL_3}{\beta}.
\end{align}
\subsection{Derivatives of the Covariance Hyperparameters}
To simplify the math, we utilize the matrix $\A$ defined above.
Firstly, the derivative of $\A$ wrt a covariance hyperparameter $t$ is given by:
\begin{align}
\deriv{\A}{t} = \left(\deriv{\K_{NM}}{t} - \A \deriv{\K_{MM}}{t}\right)\K_{MM}^{-1}.
\end{align}
The derivatives of $\calL_1, \calL_2, \calL_3 \text{ and } \calL_4$ are thus given by:
\begin{align}
\deriv{\calL_1}{t} &= \beta (\y - \A\m)^T \deriv{\A}{t} \m \\
\deriv{\calL_2}{t} &= \frac{1}{2}\beta \trace \left(\deriv{\K_{NN}}{t} - \A \deriv{\K_{MN}}{t} - \deriv{\A}{t} \K_{MN}\right) \\
\deriv{\calL_3}{t} &= \beta \trace \left(\A \S \deriv{\A^T}{t} \right) \\
\deriv{\calL_4}{t} &= \frac{1}{2}  \trace \left(\K_{MM}^{-1} \deriv{ \K_{MM}}{t}\right) - \frac{1}{2} \trace \left(\K_{MM}^{-1} \deriv{\K_{MM}}{t} \K_{MM}^{-1} (\m \m^T + \S) \right)
\end{align}
The derivatives are then computed by taking the derivatives of the covariance matrices $\K_{NN} (\text{the diagonal only}), \K_{NM} \text{ and }  \K_{MM}$, hence the covariance function, wrt the hyperparameters.

\subsection{Derivatives of the Inducing Inputs}
To compute the derivatives of $\calL$ wrt the inducing inputs, first notice that $\Z = \{\z_m\}_{m=1}^M$ are also parameters of the covariance matrices $\K_{NM}$ and $\K_{MM}$.
Hence the derivative wrt a single dimension of an inducing input, i.e. $z_{mj}$, is the same as that of $\deriv{ \calL}{t}$.

%Since $MD$ parameters are needed for the inducing inputs, it appears that the derivatives of all inducing inputs would require $\calO(MD \times M^3)$ in computation.
%However, this complexity can actually be reduced to $\calO(DM^3)$ using the following lemma. \\
%
%\noindent \textbf{Lemma} Let $A, B$ be two matrices of size $N \times M$ and $M \times N$, respectively. Furthermore, $B$ has the property that only one of its rows or columns is non-zero. The complexity of $\trace(AB)$ is only $\calO(N)$. \\
%
%\noindent \textbf{Proof} Let $m <= M$ be the non-zero row of $B$. We have
%\begin{align}
%\trace (AB) = \sum_{i=1}^N \sum_{j=1}^M A_{ij} B_{ji} = \sum_{i=1}^N A_{im} B_{mi},
%\end{align}
%which clearly takes $\calO(N)$. It is easy to see that the lemma also holds when $B$ is symmetric and only one of its row and the corresponding column is non-zero. \\
%
%\noindent To exploit the property in the Lemma, we re-write $\frac{d \calL_1}{dt}, \frac{d \calL_2}{dt}, \frac{d \calL_3}{dt}, \frac{d \calL_4}{dt}$ by expanding $\frac{d\A}{dt}$ (here $t = z_{mj}$):

\noindent We re-write $\deriv{\calL_1}{t}, \deriv{\calL_2}{t}, \deriv{ \calL_3}{t}, \deriv{\calL_4}{t}$ by expanding $\deriv{\A}{t}$ (here $t = z_{mj}$):
\begin{align}
\nonumber
% dL1
\deriv{\calL_1}{t}
 &= \beta \trace (\y - \A\m)^T \left(\deriv{\K_{NM}}{t} -  \A \deriv{ \K_{MM}}{t}\right)\K_{MM}^{-1} \m \\
&= \beta \trace \K_{MM}^{-1} \m (\y - \A\m)^T \deriv{\K_{NM}}{t}
-\beta \trace \K_{MM}^{-1} \m (\y - \A\m)^T \A \deriv{\K_{MM}}{t} \\
%dL2
\deriv{\calL_2}{t}
&= - \beta \trace \A^T \deriv{\K_{NM}}{t}
 + \frac{1}{2} \beta \trace \A^T \A \deriv{\K_{MM}}{t}  \\
% dL3
\deriv{\calL_3}{t}
&= \beta \trace \K_{MM}^{-1} \S \A^T \deriv{\K_{NM}}{t}
 - \beta \trace \K_{MM}^{-1} \S \A^T \A \deriv{\K_{MM}}{t}\\
% dL4
\deriv{\calL_4}{t}
 &= \frac{1}{2}  \trace \K_{MM}^{-1} \deriv{\K_{MM}}{t}
  - \frac{1}{2} \trace \K_{MM}^{-1} (\m \m^T + \S) \K_{MM}^{-1} \deriv{\K_{MM}}{t}
\end{align}
From the above equations we get,
\begin{align}
\deriv{\calL}{t} = \trace \D_1 \deriv{\K_{NM}}{t} + \trace \D_2 \deriv{ \K_{MM}}{t},
\end{align}
where
\begin{align}
\D_1 =& \beta \K_{MM}^{-1} \m (\y - \A\m)^T
 + \beta \A^T
 - \beta \K_{MM}^{-1} \S \A^T \\ \nonumber
\D_2 =& -\beta \trace \K_{MM}^{-1} \m (\y - \A\m)^T \A
 - \frac{1}{2} \beta \A^T \A
  + \beta \K_{MM}^{-1} \S \A^T \A	 \\
  &-\frac{1}{2} \K_{MM}^{-1} + \frac{1}{2} \K_{MM}^{-1} (\m \m^T + \S) \K_{MM}^{-1}
\end{align}
Notice that $\D_1$ and $\D_2$ can be pre-computed with a cost of $\calO(M^3)$ (or $\calO(N_bM^2)$ if the minibatch size $N_b > M$).
The computational cost of taking derivatives of $MD$ inducing parameters is thus $\calO(M^3 + MDM) = \calO(M^3)$ as the cost of the two $\trace$ operators is $\calO(M)$ due to the fact that only $\calO(M)$ elements of $\deriv{\K_{MM}}{t}$ or $\deriv{\K_{NM}}{t}$ are non-zero.
This fact can be further exploited to perform vectorized operation, for e.g. in Matlab, such that the iteration over all inducing inputs can be avoided.

\section{Appendix 2: Gaussian Identity}
Let $\y$, $\g = \{\g_j\}_{j=1}^q$, and $\h$ be random variables with multivariate Gaussian distributions:
$p(\y | \g, \h) = \Normal(\y; \sum_{j=1}^Q \W_j \g_j + \W \h, \beta^{-1} \I)$, $p(\g_j) = \Normal(\g_j; \m_j, \S_j)$, and $p(\h) = \Normal(\h; \m, \S)$.
The following identity is important in deriving the evidence lower bound:
\begin{align}
\nonumber
\int \log p(\y | \g, \h) \prod_{j=1}^q p(\g_j) p(\h) \der \g \der \h
=& \log \Normal(\y; \sum_{j=1}^q \W_j \m_j + \W \m, \beta^{-1} \I) - \frac{1}{2} \beta \trace \W^T \W \S \\
\label{eq:identity}
&- \frac{1}{2} \beta \trace \sum_{j=1}^q \W_j^T \W_j \S_j.
\end{align}
The identity can be proved by using this fact:
\begin{align}
\nonumber
\int (\W\x - \Mu)^T \BigSigma^{-1} &(\W \x - \Mu) \Normal(\x; \m, \S) \der \x \\
\nonumber
&= (\Mu - \W\m)^T \BigSigma^{-1} (\Mu - \W\m) + \trace \W^T \BigSigma^{-1} \W \S.
\end{align}

\section{Appendix: Analysis on Learning of the Inducing Inputs}
In this section we present some analysis on learning of the inducing inputs under the stochastic variational inference procedure on some toy problems.
We use three real-valued truth functions of scalar inputs: the first one was used in Snelson et al, the second one is a function $f(x) = \sin(x) + \cos(x)$, and the third one is a sample function generated from a GP with squared exponential with ARD covariance function with a lengthscale of 2 and signal variance of 1.

\noindent \paragraph{Experimental Settings} We use gpsvi to optimize for \textit{all} of the parameters in the model, i.e. the variational parameters of the posterior, the covariance hyperparameters, the noise hyperparameter, and the inducing inputs.
For the hyperparameters, we use a momentum of $0.9$ and a fixed learn rate of $1e-5$.
For the variational parameters, we use a learn rate of $0.01$ (no momentum was used).
We vary the learn rate of the inducing inputs from $1e{-2}$ to $1e{-6}$.
Using momentum for the inducing inputs does not seem to have much effects.
The maximum number of iterations is 500, the batch size 5, and the number of inducing values 10.

% effect of learn rate on inducing inputs
\noindent \paragraph{Effects of the Learn Rates }
In Figure \ref{fig1}, \ref{fig2}, and \ref{fig3}, we show the lower bounds, predictive distributions, and the learned inducing inputs using stochastic variational inference with varying learn rates for the inducing inputs.
We also show the results with FITC for comparison.
It can be seen from the figures that, as typical in stochastic optimization, the behavior of GPSVI is very sensitive to the learn rate.
In particular, when the rate is slow e.g. $1e{-5}$, almost no progress was made in learning the inducing inputs.
However, when the rate increases to e.g. $1e{-3}$, the inducing inputs are spread out more evenly in the input space compared to the initial locations.
Although not shown the in figures, when the learn rate is too large, e.g. $1e{-02}$, the inducing inputs may spread to regions far outside of the training intervals.
An example of this phenomenon can be observed in the top left figure in Figure \ref{fig1}.
The learn rates also affect the evidence lower bound which seems to wiggle more compared to not learning the inducing inputs.
The predictive distributions are sensible particularly because the toy functions exhibit sparsity and are easy to learn.  \\

\noindent The results of this qualitative analysis do not seem to suggest that learning of the inducing inputs does not work in GPSVI.
Its effectiveness perhaps rests upon empirical performance on real datasets.

\noindent \paragraph{The Noise Parameter} It seems harder to set the initial value and the learning rate of the noise parameter compared to standard GP.
While a large range of values of noise can be used in standard gp or fitc (e.g. 0.5), a very small value of noise $1e{-02}$, which is the true noise, is required for gpsvi to work well.
It appears that an adaptive learning rate may be required for the noise parameter.

\begin{figure*}
\centering
\begin{tabular}{ccc}
\includegraphics[scale=0.3]{figures/func1-svi-lrate1e-03.eps} &
\includegraphics[scale=0.3]{figures/func1-svi-lrate1e-04.eps} &
\includegraphics[scale=0.3]{figures/func1-svi-lrate1e-05.eps} \\
\includegraphics[scale=0.3]{figures/func1-svi-lrate1e-03-bound.eps} &
\includegraphics[scale=0.3]{figures/func1-svi-lrate1e-04-bound.eps} &
\includegraphics[scale=0.3]{figures/func1-svi-lrate1e-05-bound.eps} \\
(a) learn rate = $1e{-3}$ & (b) learn rate = $1e{-4}$ & (c) learn rate = $1e{-5}$ \\
\multicolumn{3}{c}{\includegraphics[scale=0.4]{figures/func1-fitc.eps}}
\end{tabular}
\caption{Predictive distributions and learned inducing inputs by GPSVI and FITC for the first function. Top row: GPSVI with different learning rates (the rates decreases from left to right). Bottom row: FITC. Magenta dots : training points. Solid blue line: predictive mean. Grey-shaded area and solid black lines: two standard deviations of the predictive distributions. Black (+) crosses: initial locations of inducing inputs. Magenta (+) crosses: learned locations of inducing inputs.  Middle row: the evidence lower bound of gpsvi vs. iteration.}
\label{fig1}
\end{figure*}

\begin{figure*}
\centering
\begin{tabular}{ccc}
\includegraphics[scale=0.3]{figures/func2-svi-lrate1e-03.eps} &
\includegraphics[scale=0.3]{figures/func2-svi-lrate1e-04.eps} &
\includegraphics[scale=0.3]{figures/func2-svi-lrate1e-05.eps} \\
\includegraphics[scale=0.3]{figures/func2-svi-lrate1e-03-bound.eps} &
\includegraphics[scale=0.3]{figures/func2-svi-lrate1e-04-bound.eps} &
\includegraphics[scale=0.3]{figures/func2-svi-lrate1e-05-bound.eps} \\
(a) learn rate = $1e{-3}$ & (b) learn rate = $1e{-4}$ & (c) learn rate = $1e{-5}$ \\
\multicolumn{3}{c}{\includegraphics[scale=0.4]{figures/func2-fitc.eps}}
\end{tabular}
\caption{Predictive distributions and learned inducing inputs by GPSVI and FITC for the second function. The legends are same as in Figure 1.}
\label{fig2}
\end{figure*}

\begin{figure*}
\centering
\begin{tabular}{ccc}
\includegraphics[scale=0.3]{figures/func3-svi-lrate1e-03.eps} &
\includegraphics[scale=0.3]{figures/func3-svi-lrate1e-04.eps} &
\includegraphics[scale=0.3]{figures/func3-svi-lrate1e-05.eps} \\
\includegraphics[scale=0.3]{figures/func3-svi-lrate1e-03-bound.eps} &
\includegraphics[scale=0.3]{figures/func3-svi-lrate1e-04-bound.eps} &
\includegraphics[scale=0.3]{figures/func3-svi-lrate1e-05-bound.eps} \\
(a) learn rate = $1e{-3}$ & (b) learn rate = $1e{-4}$ & (c) learn rate = $1e{-5}$ \\
\multicolumn{3}{c}{\includegraphics[scale=0.4]{figures/func3-fitc.eps}}
\end{tabular}
\caption{Predictive distributions and learned inducing inputs by GPSVI and FITC for the third function. The legends are same as in Figure 1.}
\label{fig3}
\end{figure*}

\bibliographystyle{abbrv}
\bibliography{references}

\end{document}


% !TeX spellcheck = en_GB
\section{Integrated Vapour Transport}
\label{sec:weather:atm_riv}
% An atmospheric river (AR) is a filament structure of intense moisture transport from the tropics to higher latitudes. Heavy precipitation can be associated with it, because the air is warm and moist. This can often be observed at mountain ranges at west coasts such as in Norway \citep{azad_extreme_2017}. Due to orographic lifting the moisture will be released and follow high amounts of precipitation.
% \\
% An atmospheric river is characterised if the integrated vapour transport shows values higher than \SI{250}{\IVT} and a continuous region larger than \SI{2000}{\km} \citep{rutz_climatological_2014}.
% \\
\Cref{fig:AR24_pres} shows coloured contours of the integrated vapour transport (IVT) in \SI{}{\IVT}, where warmer colours indicate higher IVT.
Stream vectors in \Cref{fig:AR24_pres} indicate the direction and intensity of the IVT flow.
% The integrated vapour transport (IVT) was calculated from the ECMWF data as followed:
% \begin{align}
% IVT = \frac{1}{g} \int\limits_{p_{sfc}}^{\SI{100}{\hPa}} q \mathbf{V} dp \qquad [\SI{}{\IVT}]
% \label{eq:IVT}
% \end{align}
% where $g$ is the standard gravity, $q$ the specific humidity, and $\mathbf{V}$ the total wind vector at each pressure level $p$. The numerical, trapezoidal integration is performed by using data from the surface pressure $p_{sfc}$ to \SI{850}{\hPa} in \SI{50}{\hPa} intervals and from \SIrange{700}{100}{\hPa} in \SI{100}{\hPa} intervals.
\\
Analysing integrated vapour transport maps is important, since extreme precipitation events in Norway are often influenced by moist, warm air advection from the tropics \citep{azad_extreme_2017}. \Cref{fig:AtmRiv_00} and \ref{fig:AtmRiv_01} shows integrated vapour transport from the tropics to the midlatitudes, but it also presents that the occurrence of the atmospheric river was not the main factor which led to intense precipitation during the 2016 Christmas storm.
Since it showed not to be intense it will not be further discussed.
%%% Atmospheric river maps %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[!b]
	\centering
	%%%%%% 29/12
	\begin{subfigure}[b]{0.49\textwidth}
		\includegraphics[trim={4.2cm 3.9cm 4.3cm 5.1cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161219_12}
		\caption{}\label{fig:AR19}
	\end{subfigure}
	%%%%%% 20/12
	\begin{subfigure}[b]{0.49\textwidth}
		\includegraphics[trim={4.2cm 3.9cm 4.3cm 5.1cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161220_12}
		\caption{}\label{fig:AR20}
	\end{subfigure}
	%%%%%% 21/12
	\begin{subfigure}[b]{0.49\textwidth}
		\includegraphics[trim={4.2cm 3.9cm 4.3cm 5.1cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161221_12}
		\caption{}\label{fig:AR21}
	\end{subfigure}
	%%%%%% 22/12
	\begin{subfigure}[b]{0.49\textwidth}
		\includegraphics[trim={4.2cm 3.9cm 4.3cm 5.1cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161222_12}
		\caption{}\label{fig:AR22}
		%\label{fig:sfc2100}
	\end{subfigure}
	%%%%%% label
	\begin{subfigure}[b]{\textwidth}
		\includegraphics[trim={4.2cm 0cm 4.3cm 36.8cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161225_12}
	\end{subfigure}
	\caption{Atmospheric river analysis map, data from ECMWF. During \num{19} to \SI{22}{\dec} at \SI{12}{\UTC}, chronologically (\protect\subref{fig:AR19} to \protect\subref{fig:AR22}). IVT, shaded according to the colour bar [\SI{}{\IVT}]. Vectors, indicating the direction and magnitude of the IVT. }\label{fig:AtmRiv_00}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\noindent
%% Atmospheric river maps %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[t!]%\ContinuedFloat
	%%%%%% 23/12
	\begin{subfigure}[b]{0.49\textwidth}
		\includegraphics[trim={4.2cm 3.9cm 4.3cm 5.1cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161223_12}
		\caption{}\label{fig:AR23}
	\end{subfigure}
	%%%%%% 24/12
	\begin{subfigure}[b]{0.49\textwidth}
		\includegraphics[trim={4.2cm 3.9cm 4.3cm 5.1cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161224_12}
		\caption{}\label{fig:AR24}
	\end{subfigure}
	%%%%%% 25/12
	\begin{subfigure}[b]{0.49\textwidth}
		\includegraphics[trim={4.2cm 3.9cm 4.3cm 5.1cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161225_12}
		\caption{}\label{fig:AR25}
	\end{subfigure}
	%	\centering
	%%%%%% 26/12
	\begin{subfigure}[b]{0.49\textwidth}
		\includegraphics[trim={4.2cm 3.9cm 4.3cm 5.1cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161226_12}
		\caption{}\label{fig:AR26}
	\end{subfigure}
	%%%%%% label
	\begin{subfigure}[b]{\textwidth}
		\includegraphics[trim={4.2cm 0cm 4.3cm 36.8cm},clip,
		width=\textwidth]{./fig_Atm_Riv/20161225_12}
	\end{subfigure}
	\caption{\textit{(As \Cref{fig:AtmRiv_00}.)} During \num{23} to \SI{26}{\dec} at \SI{12}{\UTC}, chronologically (\protect\subref{fig:AR23} to \protect\subref{fig:AR26})} \label{fig:AtmRiv_01}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\section{Deep Learning approach}
\label{sec32}

Deep Learning (DL) is a subfield of Machine Learning.
DL emerged as a solution to the feature engineering extraction issue which requires high expertise and skills to extract damage-sensitive features for specific SHM applications.
DL is a representation learning that automatically distinguishes  the proper data representations required for models like classification and detection.

It can be said that DL techniques developed rapidly in recent years due to the huge development that occurred in the computational powers (e.g. central processing units (CPU), graphical processing units (GPU), etc.), in addition to the availability of big data, and the development of new learning algorithms~\cite{Yuan2020}.
Consequently, DL-based SHM methods have been utilized to overcome issues related to ML-based SHM.

Originally, DL was inspired by the human brain method of learning.
In which it has a huge number of neurons that are densely connected to form a hierarchical structure that is capable of receiving data at the visual cortex which can identify distinct shapes of edges of an object.
Then, these learned patterns  are shifted down to the brain area which is capable to detect more complex patterns.

DL is a hierarchical learning~\cite{Ongsulee2018}, in which
data representations is acquired from the raw input data using non-linear function~\cite{Lecun2015}.
At shallow levels, the acquired representation data has simple learnable extracted features, those extracted features keep shifting into more complex learnable features as moving into deeper levels.

\subsection{Multilayer Perceptrons}

The simplest DL networks are called multilayer perceptron (MLP) that are constructed from a group of multiple layers of perceptrons (artificial neurons), hence, the term "deep" came from the multiple layers.
A perceptron has several inputs and outputs that are weighted connections with a nonlinear activation function.
By performing this operation a non-linearity is injected to the network, which is important for the learning process.
Therefore, if the non-linearity was not considered, no matter how many layers there are  in the network, it would act like a single-layer neuron.
Simply, because by just linearly adding these layers it will produce another linear output, consequently, the neuron can not update its weights therefore, no learning happens. Artificial neuron structure is presented in Fig.~\ref{fig:artificial Neuron}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure} [!ht]
	\begin{center}
		\centering
		\includegraphics[scale=1]{Figures/Chapter_1/artificial_neuron.png}
	\end{center}
	\caption{Structure of Perceptron}
	\label{fig:artificial Neuron}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
There are several non-linear functions used in artificial neural networks such as the Rectified Linear unit (Relu) which is used commonly, shown in Equation~\ref{Eq:relu}. Other non-linear functions are the Sigmoid logistic function as shown in Equation~\ref{sigmoid} and hyperbolic tangent function tanh as shown in Equation~\ref{tanh} ~\cite{Lecun2015}, where \(z\) is the summation of adjustable weights \(\{w_0,w_1,...,w_n \}\) multiplied by input variables (from previous layer) \(\{x_0,x_1,_...,x_n\}\) and a bias \(b\) as shown in Equation~\ref{z}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{equation}
	Relu(z) =
	\begin{cases}
		0,  \text{  if}\ z<0\\
		z,  \text{  otherwise}
	\end{cases}
	\label{Eq:relu}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{equation}
	\sigma(z) = \frac{1}{1+e^{-z}}
	\label{sigmoid}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{equation}
	\tanh(z)=  \frac{e^z-e^{-z}}{e^z+e^{-z}}
	\label{tanh}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{equation}
	z= \sum_{i=0}^{n}  w_i\times x_i +b
	\label{z}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Supervised learning is the traditional approach for learning in which a neural network builds its knowledge from the given labelled dataset, where the ground truth output is known previously~\cite{Lecun2015}.

\subsubsection{Optimization and Deep Learning}
MLP models learn to find the desired output by updating the network parameters such as weights and biases.
Accordingly, a model does a comparison between the calculated output (predicted) and the ground truth output (target).
For this purpose, an objective function (cost function) is applied to estimate the loss (error) between the predicted output and the target.
Accordingly, this process needs to be optimized to minimize the estimated value of the loss.
A well-know optimization algorithm utilised in DL is the gradient descent (GD)~\cite{Lecun2015}.
Fig.~\ref{fig:GD} illustrates the concept of GD in one dimension, in which weights \(\{w_0,w_1,...,w_n\}\) are initially assigned randomly.
GD aims to reduce the cost function \(J(w)\) at each step to reach the \(J_{min}(w)\) (global minimum) by calculating the gradient which represent the slope of the cost function.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[!ht]
	\begin{center}
		\centering
		\includegraphics[scale=1]{Figures/Chapter_3/Gradient_decent.png}
	\end{center}
	\caption{The process of Gradient Descent}
	\label{fig:GD}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Accordingly, the weights are modified as shown in Eqn.~\ref{weight_updates} where the partial derivative \(\frac{\partial J(w)}{\partial w_i}\) is the gradient, and \(\alpha \) is the learning rate which is the amount that the weights are updated during the learning process~\cite{Russell2010}.
Therefore, learning rate monitors the rate at which the neural network learns.
Hence, small learning rates require more training time because of the small changes made to the weights when updated, whereas large learning rates result in accelerated changes, consequently, require less training time.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{equation}
	w_{i+1}= w_{i} -\alpha \frac{\partial J(w)}{\partial w_i}
	\label{weight_updates}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The backpropagation algorithm is the most widely used learning algorithm for neural networks, in which it back propagates the calculated gradients across all the perceptrons.
Accordingly, all weights and biases are updated, which leads to minimizing the loss value.
\subsubsection{Optimization Challenges in Deep Learning}
The optimization algorithm in DL aims to reach the global minimum value of the cost function \(J(w)\).
However, some challenges during the training process may occur.
The most tricky challenges are the local minima, saddle points, vanishing gradients, and exploding gradients.

The local minima occurs during training when the optimization algorithm ends up with a value of the cost function \(J(w)\) that is the smaller than values of \(J(w)\) at any other points in the local neighbourhood of \(w\).
Figure~\ref{fig:local_minima} shows that during the optimization process it is possible to end up in a local minima.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[!ht]
	\begin{center}
		\centering
		\includegraphics[scale=1]{Figures/Chapter_3/local_minima.png}
	\end{center}
	\caption{Local minima}
	\label{fig:local_minima}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Moreover, a saddle point is any location where all gradients of a cost function vanish but which is neither a global nor a local minima as shown if Fig.\ref{fig:saddle_point}.

The vanishing gradients is the most encountered problem during optimization process as the gradients become too small causing the learning process stuck for a long time  before it makes any progress or to stop at all~\cite{Brownlee2017a}.
Whereas, the exploding gradients problem occurs when the gradients become so large that leads to numerical overflow and results in \say{NaN} values~\cite{Brownlee2017a}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[!ht]
	\begin{center}
		\centering
		\includegraphics[scale=1]{Figures/Chapter_3/saddle_point.png}
	\end{center}
	\caption{Saddle point}
	\label{fig:saddle_point}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Convolutional Neural Network}
Convolutional Neural Networks (CNNs) are a special type of artificial neural network (ANN) that were initially developed in 1980s by ~\textcite{Fukushima1980} who was inspired by the discoveries of Hubel and Wiesel regarding the cat's visual cortex.
CNNs are one of the most utilised architectures in DL for image processing as they can recognise complex patterns of images by performing convolution operations.

In mathematics, a convolution is an operation performed between any two functions, as for example \(f, g:\mathbb{R}^{d} \to \mathbb{R}\) to produce at third function \((f\ast g)\) depicted in Eqn.~\ref{eqn:convolution}.
In which, we measure the overlap between \(f\) and \(g\), as one function is flipped and shifted by \(x\).
\begin{equation}
	(f\ast g)(x) = \int_{}^{} f(z).g(x-z)dz
	\label{eqn:convolution}
\end{equation}
In the case of discrete objects defined on the set \(\mathbb{Z}\) of integers, the integral operation turns into a summation,  as depicted in Eqn.~\ref{eqn:discrete_conv}
\begin{equation}
	(f\ast g)(x) = \sum_{a}^{} f(a)g(i-a)
	\label{eqn:discrete_conv}
\end{equation}
For inputs with two dimensions, we have a corresponding sum with indices \((a,b)\) for \(f\) and \((i-a, j-b)\) for \(y\) respectively as depicted in Eqn.~\ref{eqn:2d_conv} and that describes a cross correlation operation.
\begin{equation}
	(f\ast g)(i,j) = \sum_{a}^{}\sum_{b}^{}f(a,b)g(i-a,j-b)
	\label{eqn:2d_conv}
\end{equation}
%%%%%%%%%%%%%%%%%%%% from here
Convolution operation for image processing is essentially a cross-correlation operation also known as a sliding dot product or sliding inner-product.
%%%%%%%%%%%%%%%%%%%%
CNNs was designed to process data as tensors with different dimensions.
For a 1D data tensor, it can represent various data forms, such as signals and sequences, in addition to languages.
For a 2D data tensor, it can represent an image in grey scale (one channel),
moreover, by combining three 2D tensors a coloured 3D image is produced due to different intensities of the pixels in the (RGB) channels.
A 4D tensors represents volumetric data, such as a sequence of 3D images  or a video.

%Commonly, a CNN consists of three main parts: convolutional layers, downsampling layers, and dense layers.
A convolutional layer, has a number \( n\) of convolution kernels (filters), in  which,  each kernel has a set of weights, of a size \((w_k,h_k,d_k)\).
The kernel slides over an input image of a size \((w,h,d)\) performing a convolution operation (dot product), where \(w\) and \(h\) represent the image width and height, respectively, while \(d\) represents the depth (number of channels).
The output of the convolution operation are feature maps, moreover, each feature map is locally connected to the previous layer.
Figure~\ref{fig:convolution_3d} illustrates the convolution operation for a 3D input and the calculated output (feature map) with a new shape of \((h_{n}\times w_{n} \times d_{n})\).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure} [!ht]
	\begin{center}
		\centering
		\includegraphics[width=0.75\textwidth]{Figures/Chapter_3/convolution_operation_3D.png}
	\end{center}
	\caption{Convolution operation with a sliding kernel.}
	\label{fig:convolution_3d}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Typically, the feature map size is diminished due to the convolution operation, however, the feature map can keep the same size of the input by applying some padding over the input.
Equations~\ref{new_hight} and~\ref{new_width} illustrates the calculations of new height and width of the output, where \(h_{n}\) and \(w_{n}\) are the new height and width dimensions of the feature map respectively after applying the convolution.
The padding \(p\) which is added to the input image of a feature map to guarantee that both the input and the output have the same dimensions, \(h_{k}\) and \(w_{k}\) are the height and the width of the convolutional kernel, respectively.
The stride \(s\) defines how much the convolutional kernel slides each step during convolution.
The number channels at the output feature map \((d_{n})\) equals to the applied number of convolutional kernels \((n)\).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{equation}
	h_{n} = \frac{h+2\times p-h_{k}}{s}+1
	\label{new_hight}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{equation}
	w_{n} = \frac{w+2\times p-w_{k}}{s}+1
	\label{new_width}
\end{equation}
Typically,  while training a CNN model, the kernel weights are initialised randomly.
Accordingly, during the backpropagation process, all learnable parameters (kernels weights and biases) are updated.
Consequently, kernels learn to detect different of types of edges (vertical, horizontal, and diagonal edges), color intensities, etc.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Commonly, a convolutional operation is followed by a non-linear activation function such as (relu, sigmoid, tanh), followed by a downsampling operation (pooling).
The idea behind pooling operation is to aggregate related features into one by reducing the spatial dimensions of the feature maps(e.g., width, height, and depth)~\cite{Lecun2015}, which reduces the computation complexity.
Figure~\ref{fig:downsampling} presents common downsampling operations, which are Max and average pooling, further, the pool size is \(2 \times 2\) with strides of \(2\).
The Maxpool picks the maximum value in the local pool filter in a feature map, whereas the average pool picks the average value in the local pool filter in a feature map.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure} [!ht]
	\begin{center}
		\centering
		\includegraphics[scale=1]{Figures/Chapter_3/downsampling.png}
	\end{center}
	\caption{Types of downsampling operations}
	\label{fig:downsampling}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
A convolution operation, followed by a non-linear activation function, and pooling is referred to as a convolutional block.
Moreover, a convolutional block can be stacked and repeated several times.
Finally, to pass the output from the convolutional block to the dense layer, a flattened layer is utilised to produce a 1D tensor.
Figure~\ref{fig:CNN} presents the default architecture of a CNN.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure} [!ht]
	\begin{center}
		\centering
		\includegraphics[width=1\textwidth]{Figures/Chapter_3/cnn.png}
	\end{center}
	\caption{Convolutional Neural Network architecture}
	\label{fig:CNN}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

CNNs became popular after the competition of the Large Scale Visual Recognition Challenge 2012 (ILSVRC2012).
When \textcite{Krizhevsky2012} introduced AlexNet~\cite{Krizhevsky2012}, which is a deep CNN applied on a large dataset of \(1,000,000\) images and \(1,000\) different classes.
AlexNet results were magnificent.
The success has stimulated the progress of the development in GPUs technology and the use of the non-linear activation function Relu~\cite{Lecun2015}.
In next years, several spectacular CNNs architectures were presented (e.g VGG-16, ResNet, Inception-v4 and others).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Recurrent neural networks}
\label{sec222}
Recurrent neural network (RNN) is a class of ANN that was introduced to work with time-series data (sequential data).
RNN technique can remember its data input, because of its internal memory which makes it a powerful and promising technique in the field of DL.
Since there are temporal problems such as natural language processing, language translation, image captioning and so on, they require to be handled sequentially.
In the traditional deep neural networks (feed-forward) the information only moves in one direction from input layer through hidden layers to the output layers.
However, this is not the case for the RNN technique, which implies the current output of an RNN depends on prior input sequence.
Accordingly, future events are also utilised for predicting the output of a given sequence.
Figure~\ref{fig:rnn_vs_FFNN} depicts the difference between RNN and feed-forward deep neural networks.
As shown in the Fig.~\ref{fig:rrn}, for the RNN, the output of a certain layer is looped back to its input which helps in making the prediction.
However, in feed-forward networks shown in Fig.~\ref{fig:FFNN} the inputs and outputs are independent, as there is only one direction for the data to move.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[!ht]
	\centering
	\begin{subfigure}{0.49\textwidth}
		\centering
		\includegraphics[scale=1]{Figures/Chapter_3/recurrent_NN.png}
		\caption{}
		\label{fig:rrn}
	\end{subfigure}
	\hfill
	\begin{subfigure}{0.49\textwidth}
		\centering
		\includegraphics[scale=1]{Figures/Chapter_3/feedforward_NN.png}
		\caption{}
		\label{fig:FFNN}
	\end{subfigure}
	\caption{(a) RNN v.s. (b) Feed-forward neural network}
	\label{fig:rnn_vs_FFNN}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure~\ref{unrolled_rnn} presents a visualisation of an unrolled RNN, where \(x_{t}\) corresponds to the sequential timestamped input at time \(t\), \(h_{t}\) corresponds to internal state  and \(Y_{t}\) corresponds to the predicted timestamped output at time \(t\).
An unrolled RNN can be seen as a cascaded sequence of feed-forward networks.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}
	\begin{center}
	\includegraphics[scale=1]{Figures/Chapter_3/unrolled_rnn.png}
	\end{center}
	\captionof{figure}{Unrolled RNN.}
	\label{unrolled_rnn}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In feed-forward neural networks, as mentioned earlier, the learnable parameters (adjustable weights) are available only for the forward path of data propagation that are updated through back-propagation algorithm.
In RNNs, there are two paths of data propagation (forward and backward), hence, there are learnable weights for both directions.
In RNN, weights are updated using back-propagation through time (BBTT)~\cite{Werbos1990}.
BBTT depends on the number of timestamps, so it is computationally expensive when there are a high number of timestamps as BBTT performs a back-propagation algorithm on unrolled RNN.
Consequently, when implementing RNNs, issues may arise during updating the learnable weights using BBTT which are vanishing and exploding gradients.
To overcome such issues, ~\textcite{Hochreiter1997} introduced a long short-term memory (LSTM), which is a memory extension for a regular RNN to address the problem of long-term dependencies.
Further, LSTMs handle inputs or outputs of any length that makes LSTMs powerful for solving very complex sequential problems.
LSTM is composed of four units: an input gate, a cell state, a forget gate, and a output gate as presented in Fig.~\ref{fig:lstm}.
These gates help regulating the flow of information that is added to or removed from the cell state.
The hidden states in LSTM hold the short-term memory, while the cells state holds the long-term memory.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[h!]
	\begin{center}
		\includegraphics[scale=1]{Figures/Chapter_3/lstm.png}
	\end{center}
	\captionof{figure}{LSTM architecture.}
	\label{fig:lstm}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The forget gate is utilised to determine which information to consider and which to neglect.
The current input \(x_t\) and the previous hidden state \(h_{t-1}\) are passed through a sigmoid function which will produce values between \(0\) and \(1\).
Then the outputs of the sigmoid are multiplied with the previous cell state \(c_{t-1}\) to discard outputs equal to zero.
Equation~\ref{eqn:forget_gate} depicts the calculation at the forget gate.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{equation}
	\centering
	f_t = \sigma(W_f.[h_{t-1}, X_{t}]+ b_f)
	\label{eqn:forget_gate}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
where \(W\) represents the learnable weights, and \(b\) represents the bias term.

The input gate \(i_{t}\) takes the current input \(X_t\) with the previous hidden state \(h_{t-1}\) then apply the sigmoid function to get values in a range between 0 (not important) and 1 (important), then the
same current input \(X_t\), and the hidden state \(h_{t-1}\) are passed through a \(\tanh\) function at \(\tilde{C}_{t}\) that will regulate the network by transferring the values into a range between \(-1\) and \(1\).
Then, the outputs from the sigmoid and \(\tanh\) functions are multiplied point-by-point to eliminate \(0\) values.
Equation~(\ref{eq:eq2}) depicts the calculation at the input gate:
\begin{equation}
	\begin{aligned}
		i_{t} &=\sigma\left(W_{i} \cdot\left[h_{t-1}, X_{t}\right]+b_{i}\right)
		\\
		\tilde{C}_{t} &=\tanh \left(W_{s} \cdot\left[h_{t-1}, X_{t}\right]+b_{c}\right)
	\end{aligned} \label{eq:eq2}
\end{equation}
At this point, the network has sufficient information obtained from the input and forget gates.
Hence, the current cell state \(C_t\) can be calculated by multiplying the previous cell state \(C_{t-1}\) with the output of the forget gate, then the result is added to the calculated input values as depicted in Eqn.~(\ref{eq:eq3}).
\begin{equation}
	C_{t}=f_{t} * C_{t-1}+i_{t} * \tilde{C}_{t}
	\label{eq:eq3}
\end{equation}
The output gate \(o_{t}\) computes the next hidden state \(h_{t}\) which
holds information related to the current inputs.
Accordingly, the current input \(X_{t}\) and the previous hidden state \(h_{t-1}\) are passed through a third sigmoid function to produce values between \(0\) and \(1\).
The current cell state \(C_{t}\) is passed through a \(\tanh\) function and multiplied point-by-point with \(o_{t}\) to produce the new hidden state \(h_{t}\) which is transferred to the next timestamp.
Equation~(\ref{eq:eq4}) illustrates the calculations at the output gate:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{equation}
	\begin{aligned}
		o_{t} &=\sigma\left(W_{o}\left[h_{t-1}, X_{t}\right]+b_{o}\right) \\
		h_{t} &=o_{t} * \tanh \left(C_{t}\right)
	\end{aligned}
	\label{eq:eq4}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Recently, LSTMs have been widely used for large-scale learning of language translation models, speech recognition systems, chatbots, forecasting stock markets, text data analysis, and many more~\cite{graves2014towards, cho2014properties}.
However, LSTMs are inefficient regarding capturing spatial information by themselves when the time series inputs are consecutive images.
Accordingly, ConvLSTM layer which is a combination of CNN and LSTM unit was introduced by Shi et al.~\cite{xingjian2015convolutional} to solve such a problem.
For ConvLSTM, the convolution operations are applied both at the input-to-state transition and at the state-to-state transitions.
ConvLSTM shown in Fig.~\ref{fig:ConvLSTM} is a variation of the LSTM cell as it performs a convolution operation within the LSTM cell.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[h!]
	\begin{center}
		\includegraphics[scale=1]{Figures/Chapter_3/convlstm_image.png}
	\end{center}
	\captionof{figure}{ConvLSTM architecture.}
	\label{fig:ConvLSTM}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
ConvLSTM is a combination of a convolution operation and an LSTM cell.
Thus, ConvLSTM can capture the time-correlated and spatial features in a series of consecutive images.
Equation~(\ref{eq:eq5}) depicts the ConvLSTM operations as the inputs \(X_1, \dots, X_t\), hidden states \(h_1, \dots, h_t\), cell states \(C_1, \dots, C_t\) and input, forget and output gates are represented as \(i_t, f_t\), and \(o_t\), respectively:
\begin{equation}
	\begin{aligned}
		i_{t} &=\sigma\left(W_{x i} * X_{t}+W_{h i} * h_{t-1}+W_{c i} \odot C_{t-1}+b_{i}\right)
		\\
		f_{t} &=\sigma\left(W_{x f} * X_{t}+W_{h f} * h_{t-1}+W_{c f} \odot C_{t-1}+b_{f}\right) \\
		C_{t} &=f_{t} \odot C_{t-1}+i_{t} \odot \tanh \left(W_{x c} * X_{t}+W_{h c} * h_{t-1}+b_{c}\right)
		\\
		o_{t} &=\sigma\left(W_{x o} * X_{t}+W_{h o} * h_{t-1}+W_{c o} \odot C_{t}+b_{o}\right) \\
		h_{t} &=o_{t} \odot \tanh \left(C_{t}\right)
	\end{aligned}
	\label{eq:eq5}
\end{equation}
where \(*\) indicates the convolution operation, and \(\odot\) represents the
Hadamard product.
Recently, ConvLSTM has become very popular and is increasingly being used in
more and more image processing applications.


\section{Accelerated Sampling Methods}
\label{section:accel}

\subsection{Accelerated Molecular Dynamics}
\label{section:accelmd}
Accelerated molecular dynamics (aMD)~\cite{HAME2004mc} is an enhanced-sampling method that
improves the conformational space sampling by
reducing energy barriers separating different states of a system.
The method modifies the potential
energy landscape by raising energy wells that are below
a certain threshold level, while leaving those above this level unaffected.
As a result, barriers separating adjacent energy basins are reduced, allowing the system to sample
conformational space that cannot be easily accessed in a classical MD simulation.

Please include the following two references in your work using the NAMD implementation of aMD:
\begin{itemize}
  \item {Accelerated Molecular Dynamics: A Promising and Efficient Simulation Method for Biomolecules, D.\,Hamelberg, J.\,Mongan, and J.\,A. McCammon. {\it J. Chem. Phys.}, 120:11919-11929, 2004.}
  \item{Implementation of Accelerated Molecular Dynamics in NAMD, Y.\,Wang, C.\,Harrison, K.\,Schulten, and J.\,A. McCammon, {\it Comp.~Sci.~Discov.}, 4:015002, 2011.}
\end{itemize}

\subsubsection{Theoretical background}
In the original form of aMD~\cite{HAME2004mc}, when the system's potential energy falls
below a threshold energy, $E$, a boost potential is added,
such that the modified potential, $V^*({\bf r})$, is related to the original
potential, $V({\bf r})$, via
\begin{equation}
V^*({\bf r})= V({\bf r}) + \Delta V({\bf r}),
\end{equation}
where $\Delta V({\bf r})$ is the boost potential,
\begin{equation}
\Delta V({\bf r})= \left \{
\begin{array}{l l}
0   & \quad \quad V({\bf r})\geq E \\
\frac{(E-V({\bf r}))^2}{\alpha+E-V({\bf r})}  & \quad \quad V({\bf r})<E. \\
\end{array} \right.
\end{equation}
As shown in the following figure, the threshold energy $E$ controls the portion of
the potential surface affected by the boost, while the acceleration factor
$\alpha$ determines the shape of the modified potential.
%as $\alpha$ increases, the modified potential asymptotically approaches the original potential;
%as $\alpha$ decreases, the energy surface below $E$ begins to resemble a constant potential.
Note that $\alpha$ cannot be set to zero, otherwise the derivative of the modified potential
is discontinuous.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=7cm]{figures/amd_schematic.jpg}
  \caption{Schematics of the aMD method. When the original potential (thick line) falls below a threshold energy $E$ (dashed line),
          a boost potential is added. The modified energy profiles (thin lines) have smaller barriers separating adjacent
	  energy basins.
	  %Two parameters, $E$ and $\alpha$, controls the portion of the affected potential landscape and the
	  %shape of the modified potential, respectively.
	  }
  \label{fig:amd_schematic}
\end{figure}
From an aMD simulation, the ensemble average, $\langle A \rangle$, of an observable, $A({\bf r})$, can be calculated
using the following reweighting procedure:
\begin{equation}
\langle A \rangle =\frac{\langle A({\bf r})\,\text{exp} (\beta \Delta V({\bf r})) \rangle^* }
{\langle \text{exp}  (\beta \Delta V({\bf r})) \rangle^*},
\end{equation}
in which $\beta$=$1/k_BT$, and $\langle ... \rangle$ and $\langle...\rangle^*$ represent
the ensemble average in the original and the aMD ensembles, respectively.

Currently, aMD can be applied in three modes in NAMD: aMDd, aMDT, and aMDdual~\cite{WANG2011mc}. The boost energy
is applied to the dihedral potential in the aMDd mode (the default mode), and to the total potential in the aMDT mode.
In the dual boost mode (aMDdual)~\cite{HAME2007mc}, two independent boost energies are applied, one on the dihedral potential and the other
on the (Total - Dihedral) potential.

\subsubsection{NAMD parameters}

The following parameters are used to enable accelerated MD:

\begin{itemize}

\item
\NAMDCONFWDEF{accelMD}{Is accelerated molecular dynamics active?}{{\tt on} or {\tt
off}}{{\tt off}}
{Specifies if accelerated MD is active.}

\item
\NAMDCONFWDEF{accelMDdihe}{Apply boost to dihedrals?}{{\tt on} or {\tt off}} {{\tt on}}
{Only applies boost to the dihedral potential.
By default, {\tt accelMDdihe} is turned on and the boost energy is applied to the dihedral potential of the simulated system.
When {\tt accelMDdihe} is turned off, aMD switches to the {\tt accelMDT} mode, and the boost is applied to the total potential.
}

\item
\NAMDCONF{accelMDE}{Threshold energy $E$}
{Real number}
{Specifies the threshold energy $E$ in the aMD equations.
}

\item
\NAMDCONF{accelMDalpha}{Acceleration factor $\alpha$}
{Positive real number}
{Specifies the acceleration factor $\alpha$ in the aMD equations.
}

\item
\NAMDCONFWDEF{accelMDdual}{Use dual boost mode?}{{\tt on} or {\tt off}}{{\tt off}}
{When {\tt accelMDdual} is on, aMD switches to the dual boost mode. Two independent boost potentials
will be applied: one to the dihedral potential that is controlled by the parameters {\tt accelMDE} and {\tt accelMDalpha},
and a second to the (Total - Dihedral) potential that is controlled by the  {\tt accelMDTE} and {\tt accelMDTalpha} parameters described below.
}

\item
\NAMDCONF{accelMDTE}{Threshold energy $E$ in the dual boost mode}
{Real number}
{Specifies the threshold energy $E$ used in the calculation of boost energy for the (Total - Dihedral) potential.
This option is only available when {\tt accelMDdual} is turned on.
}

\item
\NAMDCONF{accelMDTalpha}{Acceleration factor $\alpha$ in the dual boost mode}
{Positive real number}
{Specifies the acceleration factor $\alpha$ used in the calculation of boost energy for the (Total - Dihedral) potential.
This option is only available when {\tt accelMDdual} is turned on.
}

\item
\NAMDCONFWDEF{accelMDFirstStep}{First accelerated MD step}
{Zero or positive integer}{0}
{Accelerated MD will only be performed when the current step is equal to or higher than {\tt accelMDFirstStep}, and equal to or lower than {\tt accelMDLastStep}. Otherwise regular MD will be performed.
}
\item
\NAMDCONFWDEF{accelMDLastStep}{Last accelerated MD step}
{Zero or positive integer}{0}
{Accelerated MD will only be performed when the current step is equal to or higher than {\tt accelMDFirstStep}, and equal to or lower than {\tt accelMDLastStep}. Otherwise regular MD will be performed. Note that the accelMDLastStep parameter only has an effect when it is positive. When accelMDLastStep is set to zero (the default), aMD is `open-ended' and will be performed
till the end of the simulation.
}

\item
\NAMDCONFWDEF{accelMDOutFreq}{Frequency in steps of aMD output}
{Positive integer}{1}
{An aMD output line will be printed to the log file at the frequency specified by {\tt accelMDOutFreq}.
The aMD output will contain the boost potential ($dV$) at the current timestep,
the average boost potential ($dVAVG$) since the last aMD output, and various potential energy values at the current timestep.
The boost potential $dV$ can be used to reconstruct the ensemble average described earlier.
}

\end{itemize}

\subsection{Adaptive Tempering}
\label{section:adapttemp}
Adaptive tempering is akin to a single-copy replica exchange method for dynamically updating the simulation temperature. The temperature $T$ is a new random variable in the range $[Tmin,Tmax]$ that is governed by the equation $dE/dT = E-E(T)-1/T+sqrt(2)T\xi$, where $\xi$ is Gaussian white noise. The effect is that when the potential energy for a given structure is lower than the (so far calculated) average energy, the temperature is lowered. Conversely when the current energy is higher than the average energy, the temperature is raised. The effect is faster conformational sampling to find minimum energy structures. The method is implemented exactly as described by Zhang and Ma in J. Chem. Phys. 132, 244101 (2010) (using Equation 18 of their paper to calculate the average energy at a given temperature from the histogram of energies).

The dynamic temperature is realized either by changing the temperature of the Langevin thermostat or by velocity rescaling.

\subsubsection{NAMD parameters}

The following parameters are used to adaptive tempering:

\begin{itemize}

\item
\NAMDCONFWDEF{adaptTempMD}{Is adaptive tempering active?}{{\tt on} or {\tt
off}}{{\tt off}}
{Specifies whether or not adaptive tempering is used. If set to on then the following parameters are required to be set: either all of ({\tt adaptTempTmin}, {\tt adaptTempTmax}, {\tt adaptTempBins}, {\tt adaptTempDt}) or {\tt adaptTempInFile} (but not both).
}

\item
\NAMDCONFWDEF{adaptTempFreq}{steps between temperature updates}
{Positive integers}{10}
{The number of steps between temperature updates. Note that the potential energy at the current is calculated and added to the temperature-energy histogram at every step.
}

\item
\NAMDCONF{adaptTempTmin}{minimum temperature (K)}
{Positive real number}
{Sets the minimum temperature to be used in the simulation.
}

\item
\NAMDCONF{adaptTempTmax}{maximum temperature (K)}
{Positive real number}
{Sets the maximum temperature to be used in the simulation.
}

\item
\NAMDCONFWDEF{adaptTempBins}{number of temperature bins}
{Positive integer}{1000}
{Sets the number of bins to subdivide the temperature range. Each bin stores the average energy for the given temperature
}

\item
\NAMDCONFWDEF{adaptTempDt}{stepsize for temperature updates}{Positive real numbers}{$10^{-4}$}
{Integration timestep for temperature updates. This is unrelated to the simulation timestep and only scales the size of the step taken in temperature space every {\tt adaptTempFreq} steps.
}

\item
\NAMDCONF{adaptTempInFile}{adaptive tempering input filename}
{UNIX filename}
{The input file containing restart information for adaptive tempering (written out by {\tt adaptTempRestartFile}).
}

\item
\NAMDCONF{adaptTempRestartFile}{adaptive tempering restart filename}
{UNIX filename}
{The file to write out restart information for adaptive tempering.
}

\item
\NAMDCONF{adaptTempRestartFreq}{steps between writing restart file}
{Positive integer}
{Frequency of writing restart file.
}

\item
\NAMDCONFWDEF{adaptTempLangevin}{send temperature updates to langevin thermostat?}
{{\tt on} or {\tt off}}{{\tt on}}
{Setting this to on will cause the langevin thermostat to use the updated temperatures from adaptive tempering. Note that either one of adaptTempLangevin or adaptTempRescaling have to be on.
}

\item
\NAMDCONFWDEF{adaptTempRescaling}{send temperature to velocity rescaling thermostat?}
{{\tt on} or {\tt off}}{{\tt on}}
{Setting this to on will cause the veloctiy rescaling thermostat to use the updated temperatures from adaptive tempering.  Note that either one of adaptTempLangevin or adaptTempRescaling have to be on.
}

\item
\NAMDCONFWDEF{adaptTempOutFreq}{steps between printing adaptive tempering output}
{Positive integers}{10}
{The number of timesteps between printing adaptive tempering output to the log file.
}

\item
\NAMDCONFWDEF{adaptTempFirstStep}{step to start adaptive tempering}
{Non-negative integers}{0}
{The first timestep from which adaptive tempering will be run.}

\item
\NAMDCONF{adaptTempLastStep}{step to stop adaptive tempering}
{Positive integers}
{The last timestep to apply adaptive tempering.}

\item
\NAMDCONFWDEF{adaptTempCgamma}{dynamic bin averaging constant}
{Non-negative real number}{0.1}
{The calculation of the mean energy for a given bin is weighted by a factor of 1 - Cgamma / samples to damp out old statistics. Setting Cgamma to zero restores the use of a standard arithmetic mean to calculate the mean energy for each bin.}

\item
\NAMDCONFWDEF{adaptTempRandom}{assign random temperature if we step out of range?}
{{\tt on} or {\tt off}}{{\tt off}}
{If set to on and the temperature steps out of [{\tt adaptTempTmin}, {\tt adaptTempTmax}], a random temperature in that range is assigned. Otherwise the previous temperature is kept.
}

%\item
%\NAMDCONFWDEF{adaptTempDebug}{print debug output?}
%{{\tt on} or {\tt off}}{{\tt off}}
%{Print adaptive tempering debug output.
%}

\end{itemize}

\subsection{Locally enhanced sampling}
\label{section:les}

Locally enhanced sampling (LES)~\cite{ROIT91,SIMM98,SIMM00} increases
sampling and transition rates for a portion of a molecule by the use of
multiple non-interacting copies of the enhanced atoms.  These enhanced
atoms experience an interaction (electrostatics, van der Waals, and
covalent) potential that is divided by the number of copies present.
In this way the enhanced atoms can occupy the same space, while the
multiple instances and reduces barriers increase transition rates.

\subsubsection{Structure generation}

To use LES, the structure and coordinate input files must be modified to
contain multiple copies of the enhanced atoms.  \PSFGEN\ provides the
{\tt multiply} command for this purpose.  \NAMD\ supports a maximum of 255
copies, which should be sufficient.

Begin by generating the complete molecular structure and guessing
coordinates as described in Sec.~\ref{section:psfgen}.  As the last
operation in your script, prior to writing the psf and pdb files, add
the {\tt multiply} command, specifying the number of copies desired and
listing segments, residues, or atoms to be multiplied.  For example,
\verb#multiply 4 BPTI:56 BPTI:57# will create four copies of the last
two residues of segment BPTI.  You must include all atoms to be
enhanced in a single {\tt multiply} command in order for the bonded
terms in the psf file to be duplicated correctly.  Calling {\tt multiply}
on connected sets of atoms multiple times will produce unpredictable
results, as may running other commands after {\tt multiply}.

The enhanced atoms are duplicated exactly in the structure---they have
the same segment, residue, and atom names.  They are distinguished only
by the value of the B (beta) column in the pdb file, which is 0 for
normal atoms and varies from 1 to the number of copies created for
enhanced atoms.  The enhanced atoms may be easily observed in VMD with
the atom selection \verb#beta != 0#.

\subsubsection{Simulation}

In practice, LES is a simple method used to increase sampling;
no special output is generated.
The following parameters are used to enable LES:

\begin{itemize}

\item
\NAMDCONFWDEF{les}{is locally enhanced sampling active?}{{\tt on} or {\tt
off}}{{\tt off}}
{Specifies whether or not LES is active.}

\NAMDCONF{lesFactor}{number of LES images to use}
{positive integer equal to the number of images present}
{This should be equal to the factor used in {\tt multiply}
 when creating the structure.  The interaction potentials for images is
 divided by {\tt lesFactor}.
}

\item
\NAMDCONFWDEF{lesReduceTemp}{reduce enhanced atom temperature?}{{\tt on} or {\tt
off}}{{\tt off}}
{Enhanced atoms experience interaction potentials divided by {\tt lesFactor}.
This allows them to enter regions that would not normally be thermally
accessible.  If this is not desired, then the temperature of these atoms
may be reduced to correspond with the reduced potential.  This option
affects velocity initialization, reinititialization, reassignment, and
the target temperature for langevin dynamics.  Langevin dynamics is
recommended with this option, since in a constant energy simulation energy
will flow into the enhanced degrees of freedom until they reach thermal
equilibrium with the rest of the system.  The reduced temperature atoms
will have reduced velocities as well, unless {\tt lesReduceMass} is also
enabled.}

\item
\NAMDCONFWDEF{lesReduceMass}{reduce enhanced atom mass?}{{\tt on} or {\tt off}}{{\tt off}}
{Used with {\tt lesReduceTemp} to restore velocity distribution to
enhanced atoms.  If used alone, enhanced atoms would move faster than
normal atoms, and hence a smaller timestep would be required.}

\item
\NAMDCONFWDEF{lesFile}{PDB file containing LES flags}{UNIX filename} {{\tt coordinates}}
{PDB file to specify the LES image number of each atom.
If this parameter is not specified, then
the PDB file containing initial coordinates specified by
{\tt coordinates} is used.}

\item
\NAMDCONFWDEF{lesCol}{column of PDB file containing LES flags}{{\tt X}, {\tt Y}, {\tt Z}, {\tt O}, or {\tt B}}{{\tt B}}
{Column of the PDB file to specify the LES image number of each atom.
This parameter may specify any of the floating point fields of the PDB file,
either X, Y, Z, occupancy, or beta-coupling (temperature-coupling).
A value of 0 in this column indicates that the atom is not enhanced.
Any other value should be a positive integer less than {\tt lesFactor}.}

\end{itemize}

\subsection{Replica exchange simulations}

\index{replica exchange}
The {\tt lib/replica/}
directory contains Tcl scripts that implement replica exchange
both for parallel tempering (temperature exchange) and
umbrella sampling (exchanging collective variable biases).
This replaces the old Tcl server and socket connections driving a
separate NAMD process for every replica used in the simulation.

{\bf A NAMD build based on Charm++ 6.5.0 or later using one of the
``LRTS'' (low-level runtime system) machine layers is required!}
Current LRTS machine layers include mpi, netlrts, verbs (for InfiniBand),
gemini\_gni-crayxe, gni-crayxc, and pamilrts-bluegeneq.

Only temperature-exchange simulations are described below.
To employ replicas for umbrella sampling you will need to understand
this material, collective variable-based calculations (Sec.\ \ref{section:colvars}),
and basic Tcl programming to adapt the examples in {\tt lib/replica/umbrella/}
and {\tt lib/replica/umbrella2d/} until further
documentation and a tutorial are available.

This implementation is designed to be modified to implement
exchanges of parameters other than temperature or via other temperature
exchange methods.  The scripts should provide a good starting point for
any simulation method requiring a number of loosely interacting systems.

Replica exchanges and energies are recorded in the .history files
written in the output directories.  These can be viewed with, e.g.,
``{\tt xmgrace output/*/*.history}'' and processed via awk or other tools.
There is also a script to load the output into VMD and color each
frame according to replica index.  An example simulation folds
a 66-atom model of a deca-alanine helix in about 10\,ns.

{\tt replica.namd}
is the master script for replica temperature-exchange simulations.  To run:
\begin{verbatim}
          cd example
          mkdir output
          (cd output; mkdir 0 1 2 3 4 5 6 7)
          mpirun namd2 +replicas 8 job0.conf +stdout output/%d/job0.%d.log
          mpirun namd2 +replicas 8 job1.conf +stdout output/%d/job1.%d.log
\end{verbatim}

The number of MPI ranks must be a multiple of the number of replicas
(+replicas).  Be sure to increment jobX for +stdout option on command line.

{\tt show\_replicas.vmd} is a script for loading replicas into VMD;
first source the replica exchange conf file and then this script, then
repeat for each restart conf file or for example just do
``{\tt vmd -e load\_all.vmd}''.
This script will likely destroy anything else you are doing in VMD at the
time, so it is best to start with a fresh VMD.
{\tt clone\_reps.vmd} provides the {\tt clone\_reps} commmand to copy graphical
representation from the top molecule to all other molecules.

{\tt sortreplicas}, found in the namd2 binary directory, is a program to un-shuffle
replica trajectories to place same-temperature frames in the same file.
Usage:
\begin{verbatim}
  sortreplicas <job_output_root> <num_replicas> <runs_per_frame> [final_step]
\end{verbatim}
where job\_output\_root is the job specific output base path, including
\%s or \%d for separate directories as in output/\%s/fold\_alanin.job1
This will be extended with .\%d.dcd .\%d.history for input files and
.\%d.sort.dcd .\%d.sort.history for output files.  The optional final\_step
parameter will truncate all output files after the specified step,
which is useful in dealing with restarts from runs that did not complete.
Colvars trajectory files are similarly processed if they are found.

A replica exchange config file should define the following Tcl variables:
\begin{itemize}
\item {\tt num\_replicas}, the number of replica simulations to use,
\item {\tt min\_temp}, the lowest replica target temperature,
\item {\tt max\_temp}, the highest replica target temperature,
\item {\tt steps\_per\_run}, the number of steps between exchange attempts,
\item {\tt num\_runs}, the number of runs before stopping
(should be divisible by {\tt runs\_per\_frame} $\times$ {\tt frames\_per\_restart}).
\item {\tt runs\_per\_frame}, the number of runs between trajectory outputs,
\item {\tt frames\_per\_restart}, the number of frames between restart outputs,

\item {\tt namd\_config\_file}, the NAMD config file containing all parameters,
needed for the simulation except {\tt seed}, {\tt langevin},
{\tt langevinTemp}, {\tt outputEnergies},
{\tt outputname}, {\tt dcdFreq},
{\tt temperature}, {\tt bincoordinates}, {\tt binvelocities},
or {\tt extendedSystem}, which are provided by {\tt replica.namd},

\item {\tt output\_root}, the directory/fileroot for output files,
optionally including a ``\%s'' that is replaced with the replica index
to use multiple output directories,

\item {\tt psf\_file}, the psf file for {\tt show\_replicas.vmd},
\item {\tt initial\_pdb\_file}, the initial coordinate pdb file for {\tt show\_replicas.vmd},
\item {\tt fit\_pdb\_file}, the coodinates that frames are fit to by {\tt show\_replicas.vmd} (e.g., a folded structure),
\end{itemize}

The {\tt lib/replica/example/} directory contains
all files needed to fold a 66-atom model of a deca-alanine helix:
\begin{itemize}
\item {\tt alanin\_base.namd}, basic config options for NAMD,
\item {\tt alanin.params}, parameters,
\item {\tt alanin.psf}, structure,
\item {\tt unfolded.pdb}, initial coordinates,
\item {\tt alanin.pdb}, folded structure for fitting in {\tt show\_replicas.vmd},
\item {\tt fold\_alanin.conf}, config file for {\tt replica\_exchange.tcl} script,
\item {\tt job0.conf}, config file to start alanin folding for 10\,ns,
\item {\tt job1.conf}, config file to continue alanin folding another 10\,ns, and
\item {\tt load\_all.vmd}, load all output into VMD and color by replica index.
\end{itemize}

The {\tt fold\_alanin.conf} config file contains the following settings:
\begin{verbatim}
set num_replicas 8
set min_temp 300
set max_temp 600
set steps_per_run 1000
set num_runs 10000
# num_runs should be divisible by runs_per_frame * frames_per_restart
set runs_per_frame 10
set frames_per_restart 10
set namd_config_file "alanin_base.namd"
set output_root "output/%s/fold_alanin" ; # directories must exist

# the following used only by show_replicas.vmd
set psf_file "alanin.psf"
set initial_pdb_file "unfolded.pdb"
set fit_pdb_file "alanin.pdb"
\end{verbatim}

\subsection{Random acceleration molecular dynamics simulations}

The "lib/ramd" directory stores the tcl scripts and the example files for the implementation of the Random Acceleration Molecular Dynamics (RAMD) simulation method in NAMD.
The RAMD method can be used to carry out molecular dynamics (MD) simulations with an additional randomly oriented acceleration applied to the center of mass of one group of atoms (referred below as "ligand") in the system.
It can, for example, be used to identify egress routes for a ligand from a buried protein binding site.
Since its original implementation in the ARGOS (ref 1, 2) program, the method has been also implemented in AMBER 8 (ref 3), and CHARMM (ref 4).
The first implementation of RAMD in NAMD using a tcl script (available as supplementary material in ref 6) provided only limited functionality compared to the AMBER 8 implementation.

In the current implementation, the RAMD method can be performed in 2 flavors: (i) "pure RAMD simulations" in which the randomly-oriented acceleration is applied continuously, and (ii) "combined RAMD-MD simulations" in which RAMD steps alternate with standard MD steps.
Additional information is found in the README file in the "lib/ramd" directory.
The user is encouraged to carefully read this information before starting production runs.

The three required scripts are stored in "lib/ramd/scripts": (i) ramd--4.0.tcl defines the simulation parameters and passes them from the NAMD configuration file to the main script, (ii) "ramd--4.0\_script.tcl" adds the randomly oriented force and performs all related computations, and  (iii) "vectors.tcl" was borrowed from VMD and defines the vector operations used.

Two examples for running the scripts are included in the directory "lib/ramd/examples".
The user is encouraged to read the "README.examples" file provided in the same directory.

In order to turn RAMD on, the line "source /path/to/your/files/ramd--4.0.tcl" should be included in the NAMD configuration file.
Unless the user decides to store the scripts at a different location, the path
"/path/to/your/files" should point to the "lib/ramd/scripts" directory.
Otherwise, the user should make sure that the directory "/path/to/your/files" stores all three scripts described above.

The specific RAMD simulation parameters to be provided in the NAMD configuration file (listed below) should be preceded by the keyword "ramd".
The default values for these parameters are only given as guidance.
They are likely not to be suitable for other systems than those the scripts were tested on.

\begin{itemize}

\item
\NAMDCONFWDEF{ramd debugLevel}{ Set debug level of RAMD} {\tt integer value } {0} { Activates verbose output if set to an integer greater than 0. Should be used only for testing purposes because the very dense output is full of information only relevant for debugging.}

\NAMDCONFWDEF{ramd mdStart}{ Start RAMD-MD with MD or RAMD?} {{\tt yes} or {\tt no}} {{\tt no}} { Specifies if combined RAMD-MD simulation starts with MD or RAMD steps; ignored if pure RAMD simulation is performed.  Should be set to "yes" if initial MD steps are desired.}

\item
\NAMDCONFWDEF{ramd ramdSteps} { Set number steps in RAMD block} {{\tt positive integer}} {{\tt 50}} {Specifies the number of steps in 1 RAMD block; the simulations are evaluated every 'ramdSteps' steps.}

\item
\NAMDCONFWDEF{ramd mdSteps} { Set number steps in MD block} {{\tt positive integer}} {{\tt 0}} {Specifies the number of steps in 1 standard MD block; in combined RAMD-MD simulations, the RAMD blocks are evaluated every 'ramdSteps', the MD blocks every 'mdSteps' steps. Default of 0 gives pure RAMD simulation.}

\item
\NAMDCONFWDEF{ramd accel} {Set acceleration energy} {{\tt positive decimal}} {{\tt 0.25}} {Specifies acceleration in kcal/mol*A*amu to be applied during RAMD step.}

\item
\NAMDCONFWDEF{ramd rMinRamd} {Set threshold for distance traveled RAMD} {{\tt positive decimal}} {{\tt 0.01}} {Specifies a threshold value for the distance in Angstroms traveled by the ligand in 1 RAMD block. In pure RAMD simulations the direction of the acceleration is changed if the ligand traveled less than 'rMinRamd' \AA in the evaluated block. In combined RAMD-MD simulations, a switch from a RAMD block to a standard MD block is applied if the ligand traveled more than 'rMinRamd' \AA in the evaluated block.}

\item
\NAMDCONF{ramd rMinMd} {Set threshold for distance traveled in MD} {{\tt positive decimal}} {Specifies a threshold value for the distance, in Angstroms, traveled by accelerated atoms in 1 standard MD block.  In combined RAMD-MD simulations, a switch from a standard MD block to a RAMD block is applied according to the criteria described in the note below.  Required if 'mdStep' is not 0; ignored if 'mdSteps' is 0.}

\item
\NAMDCONFWDEF{ramd forceOutFreq}{Set frequency of RAMD forces output} {{\tt positive integer}, Must be divisor of both {\tt ramdSteps} and {\tt mdSteps}} {{\tt 0}} { Every 'forceOutFreq' steps, detailed output of forces will be written.}

\item
\NAMDCONFWDEF{ramd maxDist} {Set center of mass separation} {{\tt positive decimal}} {{\tt 50}} { Specifies the distance in Angstroms between the the centers of mass of the ligand and the protein when the simulation is stopped.}

\item
\NAMDCONFWDEF{ramd firstProtAtom} {First index of protein atom} {{\tt positive integer}} {{\tt 1}} { Specifies the index of the first protein atom.}

\item
\NAMDCONF{ramd lastProtAtom} {Last index of protein atom} {{\tt positive atom}} { Specifies the index of the last protein atom. }

\item
\NAMDCONF{ramd firstRamdAtom}{ First index of ligand atom } {{\tt positive integer}} {Specifies the index of the first ligand atom.}

\item
\NAMDCONF{ramd lastRamdAtom}{ Last index of ligand atom } {{\tt positive integer}} {Specifies the index of the last ligand atom. }

\item
\NAMDCONFWDEF{ramd ramdSeed}{Set RAMD seed} {{\tt positive integer}} {{\tt 14253}} {Specifies seed for the random number generator for generation of acceleration directions. Change this parameter if you wish to run different trajectories with identical parameters.}

\end{itemize}

Note:
In combined RAMD-MD simulations, RAMD blocks alternate with standard MD blocks ('ramdSteps' and 'mdSteps' input parameters). The switches between RAMD and MD blocks are decided based on the following parameters: (i) 'd' =  the distance between the protein and ligand centers of mass, (ii) 'dr' = the distance traveled by the ligand in 1 RAMD block, and (iii) 'dm' = the distance traveled by the ligand in 1 MD block. A switch from RAMD to MD is applied if 'dr' $>$ 'rRamdMin'. A switch from MD to RAMD is applied if: (i) 'dm' $<$ 'rMdMin' and 'd' $>$ 0 (acceleration direction is kept from previous RAMD block), (ii) if 'dm' $<$ 'rMdMin' and 'd' $<$ 0 (acceleration direction is changed), (iii) if 'dm' $>$ 'rMdMin' and 'd' $<$ 0 (acceleration direction is changed). In all other case, a switch is not applied.

%References:
%Luedemann, S.K., Lounnas, V. and R. C. Wade. How do Substrates Enter and Products Exit the Buried Active Site of Cytochrome P450cam ? 1. Random Expulsion Molecular Dynamics Investigation of Ligand Access Channels and Mechanisms. J Mol Biol, 303:797-811 (2000).
%Winn,P., Luedemann, S.K., Gauges,R., Lounnas, V. and R. C. Wade. Comparison of the dynamics of substrate access channels in three cytochrome P450s reveals different opening mechanisms and a new functional role for a buried arginine PNAS, 99, 5361-5366 (2002).
%Schleinkofer, K., Sudarko, Winn,P., Luedemann, S.K. and R. C. Wade. Do mammalian cytochrome P450s show multiple ligand access pathways and ligand channelling? EMBO Reports, 6, 584-589 (2005).
%Carlsson, P., Burendahl, S., Nilsson, L. Unbinding of retinoic acid from the retinoic acid receptor by random expulsion molecular dynamics. Biophys. J. 91, 3151-3161 (2006).
%Wang, T., Duan, Y. Chromophore channeling in the G-protein coupled receptor rhodopsin. J. Am. Chem. Soc. 129, 6970-6971 (2007).
%Vashisth, H., Abrams, C.F. Ligand escape pathways and (un)binding free energy calculations for the hexameric insulin-phenol complex. Biophys. J. 95, 4193-4204 (2008).
%Perakyla, M. Ligand unbinding pathways from the vitamin D receptor studied by molecular dynamics simulations. 38, 185-198 (2009).doi:10.1007/s00249-008-0369-x
%Klvana, M. et al. Pathways and Mechanisms for Product Release in the Engineered Haloalkane Dehalogenases Explored Using Classical and Random Acceleration Molecular Dynamics Simulations
J. Mol. Biol. 392, 1339-1356 (2009).
%Pavlova, M. et al. Redesigning dehalogenase access tunnels as a strategy for degrading an anthropogenic substrate Nature Chem. Biol. 5, 727-733 (2009).
%Wang, T., Duan, Y. Ligand entry and exit pathways in the beta2-adrenergic receptor. J. Mol. Biol. 392, 1102-1115 (2009).