\documentclass[a4paper,10pt]{article}
\usepackage[utf8x]{inputenc}
\usepackage{amssymb,amsmath,amsthm}
\usepackage{hyperref}
\usepackage{cleveref}

% For Python syntax highlighting.
\usepackage{minted}

\usepackage[margin=1.0in]{geometry}

% For testing wether an argument of a macro is empty.
\usepackage{xifthen}
\usepackage{bm}

% Figures and subfigures.
\usepackage{subfig}
\usepackage{graphicx}

\newcommand{\dl}{\mathrm{d}\lambda}
\setlength{\parindent}{0cm}

%opening
\title{Development Journal and Specs of the MESLAS package
}

% \input{macros}
\input{macros}

\begin{document}
\maketitle

\section{General Considerations}
The MESLAS package (\textbf{M}ulti-variate \textbf{E}xcursion \textbf{S}et \textbf{L}earning by \textbf{A}daptive \textbf{S}ampling) is a toolbox for simulation and prediction of mulitivariate Gaussian random fields.\\

The setup of the package is the following: $\gp$ is a $\no$-dimensional random
field on a domain $\nd$-dimensional
domain $D$.

Our philosophy is to always specify spatial location and response indices
together. That is, one should always specify \textbf{where} and \textbf{what}.

Spatial locations are denoted by $s$ and response indices by $\ell$. We will
use boldface (or, in the code, alternatively uppercase or plurals) to denote
vectors of such objects.

\medskip
A generalized sampling location is thus entirely defined by specifying two vectors
\begin{align*}
    \bm{s} &= \left(s_1, ..., s_n\right)\in D^n\\
    \bm{\ell} &= \left(\ell_1, ..., \ell_n\right) \in \lbrace 1, ..., \no
    \rbrace^n
\end{align*}
We will refer to $n$ as the \textit{dimension} of the generalized sampling
location and usually just talk of location, using the word \textit{spatial
location} when we want to specifically refer to points in $D$. Also, we will
use boldface $x$ as a shortcut to refer to the couple $\left(\bm{s},
\bm{\ell}\right)$ of spatial location vector and response index vector.
The shortcut notation $\gp[\bm{x}]$ thus refers to the
vector
\[
    \gp[\bm{x}]:=\left(\gp[s_1]^{\ell_1}, ..., \gp[s_n]^{\ell_n}\right) \in
    \mathbb{R}^n.
\]

\subsection{Covariance Model}
We assume a factor model which is the product of a stationary spatial component
with a response-index component
\begin{equation}
    \textrm{Cov}\left(\gp[s]^i, \gp[t]^j\right) = k\left(s - t\right)
    \gamma\left(i, j\right).
\end{equation}

This makes implementation easier, since then, to compute the covariance matrix
of a generalized observations $\left(S, L\right)$, we first compute the
pairwise distance matrix
\[
    H = \textrm{cdist}\left(S,S, p=2\right) = \begin{pmatrix}
        ||s_1 - s_1|| & \dots & ||s_1 - s_n||\\
        \vdots &  & \vdots \\
        ||s_n - s_1|| & \dots & ||s_n - s_n||\\
    \end{pmatrix}
\]
which can then be feeded to a vectorized stationary covariance function to get
$K(H)$.

For the response index part, we compute $L_1, L_2=\textrm{meshgrid}\left(L,
L\right)$ which yields
\begin{align*}
    L_1 =  \begin{pmatrix}
        l_1 & \dots & l_1\\
        \vdots &  & \vdots \\
        l_n & \dots & l_n\\
    \end{pmatrix}
    &,~
    L_2 =  \begin{pmatrix}
        l_1 & \dots & l_n\\
        \vdots &  & \vdots \\
        l_1 & \dots & l_n\\
    \end{pmatrix}
\end{align*}
and then feed it to a vectorized cross-covariance function $\gamma(L_1, L_2)$.
Finally, we get the covariance matrix by elementwise multiplication
\[
    K = K\left(H\right) \odot \gamma\left(L_1, L_2\right)
\]

\subsection{Cross-Covariance Models}
We here review different usual models for the cross-covariance part $\gamma(.,
.)$ of the covariance function. Recall that this is the part that specifies how
different components of the response vector at one fixed location interact.

\medskip
The simplest model we will consider is \textbf{uniform mixing}. In this model,
all components interact with the same coupling $\gamma_0$:
\begin{equation}
    \gamma(l, m) = \begin{cases} \sigma_l^2,~ l=m\\
        \gamma_0\sigma_l\sigma_m,~l\neq m
    \end{cases}
\end{equation}
and $\sigma_1^2, ..., \sigma_{\no}^2$ are the variances of the individual
components.

\subsection{Implementation Details}
ATTENTION: torch.meshgrid behaves differently than numpy's one.
First of all, it takes single dimensional vectors.
\[
    L=(1,2,3,4),~ \textrm{torch.meshgrid}(L,L)=
    \begin{pmatrix}
        1 & \dots & 1\\
        \vdots &  & \vdots \\
        n & \dots & n\\
    \end{pmatrix}
    ,~
    \begin{pmatrix}
        1 & \dots & n\\
        \vdots &  & \vdots \\
        1 & \dots & n\\
    \end{pmatrix}
\]

\subsection{Mean Module}
\subsection{Covariance Module}
\subsection{Gaussian Random Field Class and Sampling}
\subsection{Gridding}

\newpage

\section{Example Run}
We consider a $2$-dimensional GRF on a a $2$-dimensional $100\times 100$ regular grid on
$[0,1]^2$. The GRF has a factor covariance model, where the spatial part is a
Mat\'{e}rn $3/2$ with unit variance and lengthscale $\lambda_0 = 0.1$. The
cross-covariance is a uniform mixing with parameters
\[
    \sigma_1^2 = 0.25,~\sigma_2^2 = 0.6, \gamma_0 = 0.3
\]
and the mean function is a constant one with $\mu_0=(1, -2)$.
\medskip

The plot below shows one realization of the field on the full grid.
\begin{figure}[tbh!p]
\centering
\includegraphics[scale=0.65]{images/sample_low_correlation.png}
\caption{Simulated first (left) and second (right) component of the field.}
\end{figure}

We can also increase the cross-correlation factor $\gamma_0$ to $0.9$ to see
its effect.
\begin{figure}[tbh!p]
\centering
\includegraphics[scale=0.65]{images/sample_high_correlation.png}
\caption{Simulation of highly cross-correlated field.}
\end{figure}

\newpage
The code below shows how simple it is to sample a multivariate GRF using MESLAS.
\usemintedstyle{tango}
\inputminted{python}{example_sample.py}

\section{Co-Kriging}
Cokriging is also implemented in full generality (heterotopic) in MESLAS. The package also provides conditional simulations. Plots below show an example of cokriging and conditional simulation. The GRF is the same as in the previous section, with a high cross-correlation $\gamma_0=0.9$.

\begin{figure}[tbh!p]
\centering
\subfloat[Conditional mean]{\includegraphics[scale=0.6]{images/cond_mean_high_corr.png}}\\
\subfloat[Conditional realisation]{\includegraphics[scale=0.6]{images/cond_realisation_high_corr.png}}
\caption{Example of co-kriging a 2-dimensional GRF with MESLAS, observation locations in red.}
\label{fig:example_inv_prob}
\end{figure}

\newpage

\section{Coverage Function}
We compute the $p$-dimensional CDF using the MVNORM package of Sebastion Marmin. We re-packaged it for streamlined distribution via PiPy. The implementation is $3$ times faster than barebone PyTorch (in dimension 2). It is also vectorized over the batch dimension.

\subsection{Illustrations}
The agreed-upon goal was to produce plots to illustrate multivariate excursion
sets vs univariate.

I suggest we do that by trying to replace figure 4 in the paper.
So: sample unconditionally, illustrate various excursions, krig using this
realisation, plot coverages.

\begin{figure}[tbh!p]
\centering
\subfloat[Simulated Realisation]{\includegraphics[scale=0.6]{images/Figure4/cond_realisation.png}}\\
\subfloat[Cokriging Mean (observation locations in red)]{\includegraphics[scale=0.6]{images/Figure4/cond_mean.png}}\\
\subfloat[Coverage Function]{
	\includegraphics[scale=0.45]{images/Figure4/salinity_excu.png}
	\includegraphics[scale=0.45]{images/Figure4/temperature_excu.png}
	\includegraphics[scale=0.45]{images/Figure4/joint_excu.png}}
\caption{Diagnostics of problems in computation of the coverage function.}
\label{fig:example_inv_prob}
\end{figure}

\newpage
\section{Discrete Setup}

\subsection{Variance Reduction}
The plot clearly demonstrates that the implementation is correct (see the variance reduction of unobserved components due to correlations).

\begin{figure}[tbh!p]
\centering
	\includegraphics[scale=0.8]{images/variance_reduction.png}
\caption{Variance reduction caused by hypothetical measurements at the (generalized) locations in red.}
\end{figure}

\subsection{EIBV Criterion}
The plot below demonstrates computation of the EIBV criterion on the whole design grid. The steps are the following.
\begin{itemize}
  \item First a ground truth is generated by sampling from the GRF model.
  \item Then some data is collected (here 9 data points, following a straight northwards route).
  \item The data is used to compute kriging mean and covariances.
  \item Given those quantities, the EIBV criterion is computed for each point of the design grid, to determine which locations might be interesting for the next data collection steps.
\end{itemize}

The discretization used here is an equilateral tringular grid, with 25 nodes along one dimension (740 nodes in total). The small number of nodes gives rise to artifacts when projecting the observations locations to the grid and when plotting.

\begin{figure}[tbh!p]
\centering
\subfloat[Ground truth (observation locations in red)]{
\includegraphics[scale=0.75]{images/eibv_whole_design/ground_truth.png}}\\
\subfloat[Kriging mean]{
	\includegraphics[scale=0.75]{images/eibv_whole_design/kriging_mean.png}}\\
\subfloat[True excursion, current estimated excursion and EIBV criterion. (red where both excursion conditions are satisfied, white if only one).]{
	\includegraphics[scale=0.5]{images/eibv_whole_design/true_excursion.png}
	\includegraphics[scale=0.5]{images/eibv_whole_design/estimated_excursion.png}
	\includegraphics[scale=0.51]{images/eibv_whole_design/eibv.png}
	}
\caption{Example of computation of EIBV criterion on whole grid.}
\label{fig:example_inv_prob}
\end{figure}

\newpage
\section{Hardware/Software Specifications}
The AUV feature a Nvidia TX1 with ubuntu 16.04. Current programs use Python
2.7, but 3.7 can be used.
In order to run 3.7, we have to run
another backseat-driver (responsible for sending waypoints to the low-leve
controllers).

\end{document}


\title{Deep Learning Project 1}
\author{
        Pranav A (20478966) \\
        Big Data Institute\\
        Hong Kong University of Science and Technology
}
\date{\today}

\documentclass[12pt]{article}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{cite}
\begin{document}
\maketitle

\begin{abstract}
In this project, I have used logistic regression and multilayer perceptron as the classification of the data points.
At the end, two-layered multilayer perceptron outperformed than linear regression, achieving an accuracy of 97.3\% on the validation set.
However, the multilayer perceptron was overfitting before converging completely due to its complex architecture.
\end{abstract}

\section{Introduction}
Logistic regression is the most straightforward way to do classification.
This is visualized as the weighted-sum of the features to produce the best output.
This can also be thought of as a multilayer perceptron with no hidden layers.

MLP (Multilayer perceptron)  is the most fundamental forms of the artificial neural networks.
MLP introduces non-linearity through activation functions and extra layers to produce complex decision boundaries.

In this project, I have used these two techniques on the provided dataset.

\section{Experimental Setup}
The dataset comprises of 57 features with around 3220 items.
Since MLP are prone to overfitting, I have divided the dataset into traning and validation sets.
The training set comprises of 3000 items and the validation sets uses 220 items.

I have used Pytorch \cite{paszkepytorch} for developing this MLP architecture.

For logistic regression, I have used no hidden layers.
Thus the problem of MLP reduces down to just logistic regression.
Then I used Stochastic Gradient Descent, with a batch size of 128 and learning rate of 0.001.
Finally I used Cross-Entropy loss function with softmax as the output.

To generate a learning curve, at every iteration I plotted the loss and accuracy for training and validation set.

For multilayer perceptron, I have used two hidden layers.
Both hidden layers contain 64 neurons.
I have used Adam \cite{kingma2014adam} as the optimizer, with a batch size of 128 and learning rate of 0.001.
The activation functions used were Rectified Linear Units (ReLUs) \cite{dahl2013improving} at every layer (expect the output one).
Finally I used Cross-Entropy loss function after taking softmax of the output.

\section{Final Remarks}

\begin{figure}[]
	\centering
	\begin{minipage}{0.5\textwidth}
		\centering
		\includegraphics[width=1.1\textwidth]{Accuracy-Learning-Curve.pdf} % first figure itself
	\end{minipage}\hfill
	\begin{minipage}{0.5\textwidth}
		\centering
		\includegraphics[width=1.1\textwidth]{Loss-Learning-Curve.pdf} % second figure itself
	\end{minipage}
	\caption{Plots of the learning curve for the 2-layers MLP. On the left, accuracy is plotted while on the right, loss is plotted}
\end{figure}

\begin{table}[h]
	\centering
	\label{my-label}
	\begin{tabular}{@{}lcc@{}}
		\toprule
		& Training Accuracy & Validation Accuracy \\ \midrule
		Logistic Regression & 89.6\%           & 87.2\%             \\
		MLP with 2 layers   & \textbf{98.4\%}  & \textbf{97.3\%}    \\ \bottomrule
	\end{tabular}
	\caption{Accuracy results of the algorithms}
\end{table}

The results are shown in Table 1.
Clearly, MLP with 2 layers performs better than Logistic Regression.
I have also plotted the learning curves of the MLP experiment in the figure 1.

From the figures it is easy to see that model has not fully converged but starts to overfit after 150 iterations.
This is due to no regularization has been applied to the MLP architecture.
Maybe, a less complex MLP with less layers and dropout would have resulted in a better performance without being prone to overfitting.

\bibliographystyle{apalike}
\bibliography{ref}
\end{document}


\documentclass{warpdoc}
\newlength\lengthfigure                  % declare a figure width unit
\setlength\lengthfigure{0.158\textwidth} % make the figure width unit scale with the textwidth
\usepackage{psfrag}         % use it to substitute a string in a eps figure
\usepackage{subfigure}
\usepackage{rotating}
\usepackage{pstricks}
\usepackage[innercaption]{sidecap} % the cute space-saving side captions
\usepackage{scalefnt}
\usepackage{amsmath}
\usepackage{bm}

%%%%%%%%%%%%%=--NEW COMMANDS BEGINS--=%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newcommand{\alb}{\vspace{0.2cm}\\} % array line break
\newcommand{\mfd}{\displaystyle}
\newcommand{\nd}{{n_{\rm d}}}
\newcommand{\M}{{\bf M}}
\newcommand{\N}{{\bf N}}
\newcommand{\B}{{\bf B}}
\newcommand{\BI}{\wbar{{\bf B}}}
\newcommand{\A}{{\bf A}}
\newcommand{\C}{{\bf C}}
\newcommand{\T}{{\bf T}}
\newcommand{\Dstar}{D^{\!\star}}
\newcommand{\Fstar}{F^{\!\star}}
\newcommand{\Ustar}{U^{\!\star}}
\newcommand{\Sstar}{S^{\!\star}}
\newcommand{\Kstar}{K^{\!\star}}
\newcommand{\Ystar}{Y^{\!\star}}
\newcommand{\co}{,~~}
\newcommand{\band}{{\rm Band}}
\renewcommand{\fontsizetable}{\footnotesize\scalefont{1.0}}
\renewcommand{\fontsizefigure}{\footnotesize}
\renewcommand{\vec}[1]{\bm{#1}}
\setcounter{tocdepth}{3}
\let\citen\cite

%%%%%%%%%%%%%=--NEW COMMANDS BEGINS--=%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\setcounter{tocdepth}{3}

%%%%%%%%%%%%%=--NEW COMMANDS ENDS--=%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\author{
  Bernard Parent
}

\email{
  bernparent@gmail.com
}

\department{
  Institute for Aerospace Studies
}

\institution{
  University of Toronto
}

\title{
  Fluid Relaxation Schemes
}

\date{
  June 2001, July-August 2015
}

%\setlength\nomenclaturelabelwidth{0.13\hsize}  % optional, default is 0.03\hsize
%\setlength\nomenclaturecolumnsep{0.09\hsize}  % optional, default is 0.06\hsize

\nomenclature{

  \begin{nomenclaturelist}{Roman symbols}
   \item[$a$] speed of sound
  \end{nomenclaturelist}
}

\abstract{
abstract
}

\begin{document}
  \pagestyle{headings}
  \pagenumbering{arabic}
  \setcounter{page}{1}
%%  \maketitle
  \makewarpdoctitle
%  \makeabstract
  \tableofcontents
%  \makenomenclature
%%  \listoftables
%%  \listoffigures
\sloppy

\section{Explicit Pseudo-Time Stepping}

After adding a pseudo-time derivative, the discretized form of the governing equations, using explicit Euler
time-marching corresponds to:
%
\begin{align}
   \Gamma^n {\Delta^n {\Ustar}}
  &+  Z^n \left(\delta_t \Ustar\right)^n \nonumber\alb
  &+ \sum_{i=1}^{\nd}
      \left(  \delta_{X_i} {\Fstar_i}^{n}
            + \delta_{X_i} \left({\Dstar_i} \Ustar\right)^n
            + {\Ystar_i}^n \delta_{X_i} H^n
            - \sum_{j=1}^{\nd} \delta_{X_i} \left(  {\Kstar_{ij}}^{\!\! n} \delta_{X_j} G^{n}\right) \right)
 ={\Sstar}^{n}
\end{align}
%
where the superscript $n$ refers to the pseudo-time step,
and $\Delta^n {\Ustar}\equiv {\Ustar}^{n+1}-{\Ustar}^n$. Recall the definition of the discretized
residual:
%
\begin{align}
R_\Delta^n &\equiv  Z^n \left(\delta_t \Ustar\right)^n \nonumber\alb
  &+ \sum_{i=1}^{\nd}
      \left(  \delta_{X_i} {\Fstar_i}^{n}
            + \delta_{X_i} \left({\Dstar_i} \Ustar\right)^n
            + {\Ystar_i}^n \delta_{X_i} H^n
            - \sum_{j=1}^{\nd} \delta_{X_i} \left(  {\Kstar_{ij}}^{\!\! n} \delta_{X_j} G^{n}\right) \right)
 -{\Sstar}^{n}
\label{eqn:Rdelta}
\end{align}
%
Then, the former can be rewritten as
%
\begin{equation}
     \Delta^n{\Ustar}=   - \left(\Gamma^n\right)^{-1} R_\Delta^n
\end{equation}
%

\section{Implicit Pseudo-Time Stepping}

Despite its simplicity and the small amount of computing per iteration, explicit time stepping generally has a poor computational efficiency. This is due to the chemical reactions and diffusion terms being particularly stiff and restricting the pseudo-time step to very small values, hence leading to very slow convergence rates. This can be remedied through the use of \emph{implicit} pseudo-time stepping.

\subsection{Delta Form}

After adding a pseudo-time derivative, the discretized form of the governing equations using implicit Euler
time-marching corresponds to:
%
\begin{align}
   \Gamma^n {\Delta^n {\Ustar}}&+  Z^n \left(\delta_t \Ustar\right)^{n+1}+   \sum_{i=1}^{\nd}
      \Bigg(  \delta_{X_i} {{\Fstar_i}^{n+1}}
            + \delta_{X_i} {\Dstar_i}^{n} {\Ustar}^{n+1}
\nonumber\alb
            &+ {\Ystar_i}^n \delta_{X_i} H^{n+1}
            - \sum_{j=1}^{\nd} \delta_{X_i} \left( {\Kstar_{ij}}^{\!\! n} \delta_{X_j} G^{n+1}\right) \Bigg)
 - {\Sstar}^{n+1}=0
\label{eqn:govdisc2}
\end{align}
%
where we took some liberty in assuming that $\Gamma$, $\Kstar$, $Z$, $\Dstar$, and $\Ystar$ do not change in
pseudo-time.
Let's substract the residual evaluated at pseudo-time level $n$ as shown in Eq.\ (\ref{eqn:Rdelta}) on both sides of the latter:
%
\begin{align}
   \mfd \Gamma^n {\Delta^n {\Ustar}}
 +   \sum_{i=1}^{\nd}
      &\Bigg( \delta_{X_i} \left({{\Fstar_i}^{n+1}}-{{\Fstar_i}^{n}}\right)
          + \delta_{X_i} {\Dstar_i}^{n} \left( {\Ustar}^{n+1}-{\Ustar}^{n}\right)
          + {\Ystar_i}^n \delta_{X_i} \left(H^{n+1}-H^n \right)
              \nonumber\alb
          &- \sum_{j=1}^{\nd} \delta_{X_i}
        \left({\Kstar_{ij}}^{\!\! n} \delta_{X_j} G^{n+1}
        -{\Kstar_{ij}}^{\!\! n} \delta_{X_j} G^{n}\right)
          \Bigg)-\left({\Sstar}^{n+1} -{\Sstar}^n \right)\nonumber\alb
        &+ Z^n \left(\left(\delta_t \Ustar\right)^{n+1}-\left(\delta_t \Ustar\right)^n \right)
= -R_\Delta^n
\label{eqn:govdisc3}
\end{align}
%
Rewriting Eq.~(\ref{eqn:govdisc3}) in a more compact form, one gets the
delta form:
%
\begin{align}
   \mfd\Gamma^n{\Delta^n {\Ustar}}
   &+\sum_{i=1}^{\nd}\left( \delta_{X_i}  {\Delta^n {\Fstar_i}}
          + \delta_{X_i} \left({\Dstar_i}^{n} \Delta^n {\Ustar}\right)
          + {\Ystar_i}^n \delta_{X_i}  \Delta^n H
            - \sum_{j=1}^{\nd} \delta_{X_i}
           \left( {\Kstar_{ij}}^{\!\! n} \delta_{X_j}
                 {\Delta^{n} G}\right) \right)\nonumber\alb
  &-\Delta^n {\Sstar}+ Z^n \Delta^n(\delta_t \Ustar) = -R_{\Delta}^n
\label{eqn:deltaform}
\end{align}
%
where the operator $\Delta^n(\cdot)$ corresponds to:
%
\begin{equation}
\Delta^n(\cdot) = (\cdot)^{n+1}-(\cdot)^n
\end{equation}
%

\subsection{Block Alternate Direction Implicit (ADI)}

Equation (\ref{eqn:deltaform}) involves solving a huge banded matrix
which would require too much computer memory storage and CPU
time for multidimensional problems. One alternative is to approximate
the delta form with a multiplication of one-dimensional operators,
a technique usually referred to as approximate factorization \cite{misc:1955:peaceman,
misc:1955:douglas}. However, contrarily to the scalar approximate factorization in \cite{misc:1955:peaceman}, we here apply the approximate factorization to the matrix form of the delta form, hence resulting in a \emph{block implicit} method (see Refs.\ \cite{jcp:1980:briley} and \cite[pp.\ 318--319]{book:2001:oran} for a discussion of block implicit algorithms):
%
\begin{align}
\Bigg\{   \prod_{i=1}^{\nd}
  \Bigg[ I
        &+{(\Gamma^n)}^{-1} \delta_{X_i} \left(\frac{\partial \Fstar_i}{ \partial \Ustar}\right)^n
        +{(\Gamma^n)}^{-1} \delta_{X_i} {\Dstar_i}^n
        +{(\Gamma^n)}^{-1} {\Ystar_i}^n\delta_{X_i} \left(\frac{\partial H}{ \partial \Ustar}\right)^n
\nonumber\alb
&
        - {(\Gamma^n)}^{-1} \sum_{j=1}^{\nd} \delta_{X_i} \left( {\Kstar_{ij}}^{\!\! n}\delta_{X_j} \left(\frac{\partial G}{ \partial \Ustar}\right)^n \right)
        -\delta_{1i} {(\Gamma^n)}^{-1} \left(\frac{\partial \Sstar}{ \partial \Ustar}\right)^n \nonumber\alb
&
        +\delta_{1i} {(\Gamma^n)}^{-1} Z^n \left(\frac{\partial (\delta_t \Ustar)}{  \partial \Ustar}\right)^n
\Bigg] \Bigg\} \Delta^n {\Ustar}
        = - {(\Gamma^n)}^{-1} R_{\Delta}^n
\label{eqn:approxfact}
\end{align}
%
where $\delta_{1i}$ here refers to the Kronecker delta and not to
a discretization.
Equation (\ref{eqn:approxfact}) corresponds to Eq.~(\ref{eqn:deltaform})
if, when the multiplication is expanded, all terms involving ${(\Gamma^n)}^{-2}$
or ${(\Gamma^n)}^{-3}$ are neglected.
It is noted that the linearization matrix $\partial \Fstar_i / \partial \Ustar$ is symbolic
and stands for the Jacobian of the FDS or FVS scheme (more on this below).
The equation to solve for each node at $X_i$ for the $i$th sweep can be written as
(using the linearization matrices $\A_i$, $\B_i$, $\C_i$ listed in the appendix:
%
\begin{equation}
\begin{array}{r}
\A_i^{X_i} \Delta \widetilde{\Ustar}_i^{X_i-1} +
\B_i^{X_i} \Delta \widetilde{\Ustar}_i^{X_i} +
\C_i^{X_i} \Delta \widetilde{\Ustar}_i^{X_i+1} =
{\Delta \widetilde{\Ustar}^{X_i}_{i-1}}
\end{array}
\end{equation}
%
where ${\Delta \widetilde{\Ustar}^{X_i}_1}=- (\Gamma^{-1})^{X_i} R_{\Delta}^{X_i}$
and the total flux increment $\Delta {\Ustar}^{X_i}$ is set to ${\Delta \widetilde{\Ustar}^{X_i}_{\nd}}$.

The latter is also referred to in the litterature as Alternate-Direction-Implicit (ADI). It is noted that we are here applying the ADI scheme for the system of equations as a whole by solving block tdmas along each direction as outlined in Ref.\ \cite{jcp:1980:briley} rather than applying the ADI scheme in scalar form for each equation independently of the other.

\subsection{Block Diagonally-Dominant Alternate Direction Implicit (DD-ADI)}

The Diagonally-Dominant Alternate Direction Implicit (DD-ADI) is similar to the ADI
but the factorization is such that all terms of the original inversion matrix
are returned. It was first proposed by MacCormack \cite{aiaaconf:1997:maccormack}
as the  modified approximate factorization (MAF) and by Bardina \cite{aiaaconf:1987:bardina} as the DD-ADI.
Linearizing the delta-form of Eq.~(\ref{eqn:deltaform}),
an implicit Euler time-stepping can be expressed in any number of dimensions as:
%
\begin{equation}
  \M [ \Delta^n \Ustar ] = [- {(\Gamma^n)}^{-1} R_{\Delta}]
\end{equation}
%
with $\M$ the linearization matrix corresponding to, in three dimensions for
example:
%
\begin{equation}
\M=
 \left[
   \begin{array}{@{}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{\;\;}c@{}}
   \ddots &  0     &  0   & \ddots   &\ddots   & \ddots   & 0     &  0   &\ddots   &      &    &\ddots    &    &     &      \alb
       &\A_2^1    &  0   &     0  &\A_3^1   &\B^1      & \C_3^1  & 0    &   0   &\C_2^1  &    &    &\C_1^1 &     &      \alb
    &       &\A_2^2  &   0    &    0  &\A_3^2    & \B^2    &\C_3^2  &   0   &  0   &\C_2^2 &    &    & \C_1^2 &      \\
   \ddots &    &  &\ddots    &  0    &    0   &\ddots   &\ddots  &\ddots   &   0  &  0  &\ddots  & &  & \ddots  \alb
       &\A_1^{13} &     &    &\A_2^{13}&0 & 0 &\A_3^{13}&\B^{13} &\C_3^{13}& 0    & 0   &\C_2^{13}&     &      \alb
       &       &\A_1^{14}&   &      &\A_2^{14} &   0   &  0   &\A_3^{14}&\B^{14}   & \C_3^{14} &0&0& \C_2^{14}&     \\
       &       &     &\ddots    &      &       &\ddots   &  0   &   0   &\ddots&\ddots   & \ddots &0&0&\ddots\\
   \end{array}
 \right]
\end{equation}
%
Defining the tridiagonal matrices:
%
%
\begin{align*}
\T_3&=
 \left[
   \begin{array}{cccccc}
     \ddots&\ddots& 0   & 0   & 0           \\
     \ddots& \B^1   & \C_3^1  & 0   & 0           \alb
     0   & \A_3^2 & \B^2    & \C_3^2  & 0           \\
     0   & 0  & \A_3^3  & \B^3    & \ddots          \\
     0   & 0  & 0   & \ddots& \ddots      \alb
   \end{array}
 \right]\alb
 &=\left[ \A_3 \co \B \co \C_3 \right]^\band
\end{align*}
%
and
%
\begin{align*}
\T_2&=
 \left[
   \begin{array}{ccccccccccccc}
     \ddots    & 0  & 0   & 0   & \ddots  & 0   & 0        \alb
     0   & \B^1   & 0  & 0   & 0   & \C_2^1  & 0       \\
     0   & 0  & \B^2    & 0  & 0   & 0   & \ddots      \alb
     0   & 0  & 0  & \B^3    & 0  & 0   & 0       \\
     \ddots  & 0  & 0   & 0  & \B^4    & 0  & 0       \alb
     0   & \A_2^5 & 0   & 0   & 0  & \B^5    & 0      \\
     0   & 0  & \ddots& 0   & 0   & 0  & \ddots        \alb
   \end{array}
 \right]\alb
 &= \left[ \A_2\co 0\co  0\co  0\co  \B\co 0\co  0\co  0\co\C_2 \right]^\band
\end{align*}
%
The product $\T_2 \T_3$ gives
%
\begin{equation}
\T_2 \T_3=
 \left[
   \begin{array}{cccccccccccccc}
     \ddots   & \ddots  & 0     & \ddots     & \ddots   & \ddots   & 0        & 0 \\
     \ddots & \B^1 \B^1    & \B^1 \C_3^1 & 0         & \C_2^1 \A_3^5& \C_2^1 \B^5     & \C_2^1 \C_3^5 & 0 \\
     0     & \B^2 \A_3^2  & \B^2 \B^2   & \B^2 \C_3^2     & 0      & \C_2^2 \A_3^6   & \C_2^2 \B^6  & \ddots  \\
     \ddots  &  0     & \B^3 \A_3^3 & \B^3 \B^3       & \B^3 \C_3^3  & 0         & \C_2^3 \A_3^7 & \ddots \\
     \ddots  & \A_2^4 \C_3^0& 0     & \B^4 \A_3^4     & \B^4 \B^4    & \B^4 \C_3^4     & 0      & \ddots  \\
     \ddots  & \A_2^5 \B^1 & \A_2^5 \C_3^1& 0     & \B^5 \A_3^5     & \B^5 \B^5    & \B^5 \C_3^5     & 0  \\
     0     & \A_2^6 \A_3^2& \A_2^6 \B^2 & \A_2^6 \C_3^2& 0     & \B^6 \A_3^6     & \B^6 \B^6      & \ddots \\
     0     &  0    & \ddots   & \ddots  & \ddots   & 0     & \ddots    & \ddots \\
   \end{array}
 \right]
\end{equation}
%
or,
%
\begin{equation}
\T_2 \T_3=\left[
 \A_2 \A_3^{-4} \co
 \A_2 \B^{-4} \co
 \A_2 \C_3^{-4} \co
 0 \co
 \B \A_3     \co
 \B \B   \co
 \B \C_3 \co
 0 \co
 \C_2 \A_3^{+4} \co
 \C_2 \B^{+4} \co
 \C_2 \C_3^{+4}
\right]^\band
\end{equation}
%
while the product $\T_2 \BI \T_3$ gives (with $\BI \equiv \B^{-1}$):
%
\begin{equation}
\T_2 \BI \T_3=
 \left[
   \begin{array}{c@{\;}c@{\;}c@{\;}c@{\;}c@{\;}c@{\;}c@{\;}c@{\;}c@{\;}c@{\;}c@{\;}c@{\;}c@{\;}c}
     \ddots  & \ddots  & 0     & \ddots     & \ddots   & \ddots   & 0        & 0 \\
     \ddots  & \B^1     & \C_3^1 & 0         & \C_2^1 \BI^5 \A_3^5& \C_2^1     & \C_2^1 \BI^5 \C_3^5 & 0 \\
     0     & \A_3^2  & \B^2   & \C_3^2     & 0      & \C_2^2 \BI^6 \A_3^6   & \C_2^2  & \ddots  \\
     \ddots  &  0     & \A_3^3 & \B^3       & \C_3^3  & 0         & \C_2^3 \BI^7 \A_3^7 & \ddots \\
     \ddots  & \A_2^4 \BI^0 \C_3^0& 0     &  \A_3^4     &  \B^4    &  \C_3^4     & 0      & \ddots  \\
     \ddots  & \A_2^5  & \A_2^5 \BI^1 \C_3^1& 0     & \A_3^5     & \B^5    & \C_3^5     & 0  \\
     0     & \A_2^6 \BI^2 \A_3^2& \A_2^6  & \A_2^6 \BI^2 \C_3^2& 0     &  \A_3^6     & \B^6       & \ddots \\
     0     &  0    & \ddots   & \ddots  & \ddots   & 0     & \ddots    & \ddots \\
   \end{array}
 \right]
\end{equation}
%
or,
%
\begin{align}
\T_2 \BI \T_3=\left[
 \A_2 \BI^{-4} \A_3^{-4} \co
 \A_2 \co
 \A_2 \BI^{-4} \C_3^{-4} \co
 0 \co
 \A_3     \co
 \B    \co
 \C_3 \co
 0 \co
 \C_2 \BI^{+4} \A_3^{+4} \co
 \C_2 \co \right.\nonumber\alb
\left.
 \C_2 \BI^{+4} \C_3^{+4}
\right]^\band
\end{align}
%
Defining the matrix
%
\begin{equation}
\T_1=
 \left[
   \begin{array}{ccccccccccccccc}
   \ddots &0       & 0    &\ddots    &0    & 0    & 0     \alb
   0    &\B^1      & 0    &\cdots    &\C_1^1 & 0    & 0     \alb
   0    &0       & \B^2   &\cdots    &0    & \C_1^2 & 0     \\
   \ddots &\vdots    & \vdots &\ddots    &\vdots & \vdots & \ddots  \alb
   0    &\A_1^{13} & 0    &\cdots    &\B^{13}& 0    & 0     \alb
   0    &0       &\A_1^{14}& \cdots  &0    &\B^{14} & 0     \\
   0    &0       &0     &\ddots    &0    &0     &\ddots   \alb
   \end{array}
 \right]
\end{equation}
%
then,
%
\begin{displaymath}
\begin{array}{l}
\T_1 \BI \T_2 \BI \T_3=  \alb
  \begin{array}{l}
   \left[
   \A_1 \BI^{-12}\A_2^{-12} \BI^{-16} \A_3^{-16}\co
   \A_1 \BI^{-12}\A_2^{-12}\co
   \A_1 \BI^{-12}\A_2^{-12} \BI^{-16} \C_3^{-16}\co
   0\co
   \A_1 \BI^{-12} \A_3^{-12}\co \right. \alb
   \A_1\co
   \A_1 \BI^{-12} \C_3^{-12}\co
   0\co
   \A_1 \BI^{-12}\C_2^{-12} \BI^{-8} \A_3^{-8}\co
   \A_1 \BI^{-12}\C_2^{-12}\co
   \A_1 \BI^{-12}\C_2^{-12} \BI^{-8} \C_3^{-8}\co
   0\co \alb
   \A_2 \BI^{-4} \A_3^{-4}\co
   \A_2\co
   \A_2 \BI^{-4} \C_3^{-4}\co
   0\co
   \A_3\co
   \B\co
   \C_3\co
   0\co
   \C_2 \BI^{+4} \A_3^{+4}\co
   \C_2\co
   \C_2 \BI^{+4} \C_3^{+4}\co
   0\co\alb

   \C_1 \BI^{+12}\A_2^{+12} \BI^{+8} \A_3^{+8}\co
   \C_1 \BI^{+12}\A_2^{+12}\co
   \C_1 \BI^{+12}\A_2^{+12} \BI^{+8} \C_3^{+8}\co
   0\co
   \C_1 \BI^{+12} \A_3^{+12}\co
   \C_1\co\alb
   \left.
   \C_1 \BI^{+12} \C_3^{+12}\co
   0\co
   \C_1 \BI^{+12}\C_2^{+12} \BI^{+16} \A_3^{+16}\co
   \C_1 \BI^{+12}\C_2^{+12}\co
   \C_1 \BI^{+12}\C_2^{+12} \BI^{+16} \C_3^{+16}
   \right]^\band\alb
  \end{array}
\end{array}
\end{displaymath}
%
where the decomposition error terms can be identified by comparing the latter
to matrix $\M$:
%
\begin{displaymath}
  \begin{array}{r}
  \M=
   \left[ 0\co
   0\co
   0\co
   0\co
   0\co
   \A_1\co
   0\co
   0\co
   0\co
   0\co
   0\co
   0\co
   0\co
   \A_2\co
   0\co
   0\co
   \A_3\co
   \B\co
   \C_3\co \right. \alb
   \left.
   0\co
   0\co
   \C_2\co
   0\co
   0\co
   0\co
   0\co
   0\co
   0\co
   0\co
   \C_1\co
   0\co
   0\co
   0\co
   0\co
   0 \right]^\band
  \end{array}
\end{displaymath}
%
The decomposition error matrix, $\N$ can then be written as:
%
\begin{equation}
  \N\equiv \T_1 \BI \T_2 \BI \T_3- \M
\end{equation}
%
\begin{displaymath}
\begin{array}{l}
\N=  \alb
  \begin{array}{l}
   \left[
   \A_1 \BI^{-12}\A_2^{-12} \BI^{-16} \A_3^{-16}\co
   \A_1 \BI^{-12}\A_2^{-12}\co
   \A_1 \BI^{-12}\A_2^{-12} \BI^{-16} \C_3^{-16}\co
   0\co
   \A_1 \BI^{-12} \A_3^{-12}\co \right. \alb
   0 \co
   \A_1 \BI^{-12} \C_3^{-12}\co
   0\co
   \A_1 \BI^{-12}\C_2^{-12} \BI^{-8} \A_3^{-8}\co
   \A_1 \BI^{-12}\C_2^{-12}\co
   \A_1 \BI^{-12}\C_2^{-12} \BI^{-8} \C_3^{-8}\co
   0\co \alb
   \A_2 \BI^{-4} \A_3^{-4}\co
   0\co
   \A_2 \BI^{-4} \C_3^{-4}\co
   0\co
   0\co
   0\co
   0\co
   0\co
   \C_2 \BI^{+4} \A_3^{+4}\co
   0\co
   \C_2 \BI^{+4} \C_3^{+4}\co
   0\co\alb

   \C_1 \BI^{+12}\A_2^{+12} \BI^{+8} \A_3^{+8}\co
   \C_1 \BI^{+12}\A_2^{+12}\co
   \C_1 \BI^{+12}\A_2^{+12} \BI^{+8} \C_3^{+8}\co
   0\co
   \C_1 \BI^{+12} \A_3^{+12}\co
   0\co\alb
   \left.
   \C_1 \BI^{+12} \C_3^{+12}\co
   0\co
   \C_1 \BI^{+12}\C_2^{+12} \BI^{+16} \A_3^{+16}\co
   \C_1 \BI^{+12}\C_2^{+12}\co
   \C_1 \BI^{+12}\C_2^{+12} \BI^{+16} \C_3^{+16}
   \right]^\band\alb
  \end{array}
\end{array}
\end{displaymath}
%
The advantage of DD-ADI versus standard ADI is that all the original terms in
$\M$ are present in $\T_1 \BI \T_2 \BI \T_3$.
Recalling that $\M [ \Delta^n \Ustar ] = [- \Gamma^{-1} R_{\Delta}]$ (for a direct inversion),
and that $\M \approx \T_1 \BI \T_2 \BI \T_3$
the DD-ADI algorithm can be implemented in three dimensions as:
%
\begin{displaymath}
\begin{array}{rcl}
  {\rm step~1} & ~~ & \T_1 [\Delta^n \Ustar_1] = [- \Gamma^{-1} R_{\Delta}]\alb
  {\rm step~2} & ~~ & \T_2 [\Delta^n \Ustar_2] = [\B]^\band [\Delta^n \Ustar_1] \alb
  {\rm step~3} & ~~ & \T_3 [\Delta^n \Ustar] = [\B]^\band [\Delta^n \Ustar_2]
\end{array}
\end{displaymath}
%
and in two dimensions as:
%
\begin{displaymath}
\begin{array}{rcl}
  {\rm step~1} & ~~ & \T_1 [\Delta^n \Ustar_1] = [- \Gamma^{-1} R_{\Delta}]\alb
  {\rm step~2} & ~~ & \T_2 [\Delta^n \Ustar] = [\B]^\band [\Delta^n \Ustar_1] \alb
\end{array}
\end{displaymath}
%

\subsection{Block Iterative DD-ADI Algorithm: Removal of Decomposition Error}

The DD-ADI can be further improved through an iterative method to reduce the decomposition error as suggested by MacCormack in Ref.\ \cite{caf:2001:maccormack}.

The direct inversion can be written as:
%
\begin{displaymath}
  \begin{array}{c}
  \M [ \Delta^n \Ustar ] = [- \Gamma^{-1} R_{\Delta}] \alb
{\rm or~~}  \M [ \Delta^n \Ustar ] + \N [ \Delta^n \Ustar ]= [- \Gamma^{-1} R_{\Delta}] + \N [ \Delta^n \Ustar ]\alb
{\rm or~~}  \T_1 \BI \T_2 \BI \T_3 [ \Delta^n \Ustar ] = [- \Gamma^{-1} R_{\Delta}] + \N [ \Delta^n \Ustar ]
  \end{array}
\end{displaymath}
%
since $\N+\M=\T_1 \BI \T_2 \BI \T_3$.
The idea behind removing the decomposition error is in performing
$k$ times the approximate factorization on the same residual, each
time using a more updated decomposition error term on the RHS:
%
\begin{equation}
  \T_1 \BI \T_2 \BI \T_3 [ \Delta^n {\Ustar}^{(k)} ] = [- \Gamma^{-1} R_{\Delta}] + \N [ \Delta^n {\Ustar}^{(k-1)} ]
\end{equation}
%
where $k=1,2,3,4,5,..,\infty$ and where it is assumed that
$\N [ \Delta^n {\Ustar}^{(k-1)} ] \rightarrow 0$ as $k \rightarrow \infty$.
The latter can also be written as:
%
\begin{equation}
    \T_1 \BI \T_2 \BI \T_3 [ \Delta^n {\Ustar}^{(k)} ]
  = [- \Gamma^{-1} R_{\Delta}] + \T_1 \BI \T_2 \BI \T_3 [ \Delta^n {\Ustar}^{(k-1)} ]
    - \M [ \Delta^n {\Ustar}^{(k-1)} ]
\end{equation}
%
with $\Delta^n {\Ustar}^{(0)}=0$. Step by step, this becomes:
%
%
\begin{displaymath}
 \begin{array}{rcl}
   k=1 & ~~&
    \T_1 \BI \T_2 \BI \T_3 [ \Delta^n {\Ustar}^{(1)} ]
    = [- \Gamma^{-1} R_{\Delta}]  \alb
   k=2 & ~~&
    \T_1 \BI \T_2 \BI \T_3 [ \Delta^n {\Ustar}^{(2)} ]
    = [- 2 \Gamma^{-1} R_{\Delta}] - \M [ \Delta^n {\Ustar}^{(1)} ] \alb
   k=3 & ~~&
    \T_1 \BI \T_2 \BI \T_3 [ \Delta^n {\Ustar}^{(3)} ]
    = [- 3 \Gamma^{-1} R_{\Delta}] - \M [ \Delta^n {\Ustar}^{(1)} + \Delta^n {\Ustar}^{(2)} ]  \alb
   k=4 & ~~&
    \T_1 \BI \T_2 \BI \T_3 [ \Delta^n {\Ustar}^{(4)} ]
    = [- 4 \Gamma^{-1} R_{\Delta}] - \M [ \Delta^n {\Ustar}^{(1)} + \Delta^n {\Ustar}^{(2)} + \Delta^n {\Ustar}^{(3)}]
 \end{array}
\end{displaymath}
%

\appendix

\section{Linearization Matrices for FDS First Order Scheme}

We here list the linearization matrices for the first-order Roe flux difference splitting (FDS) scheme.
The matrices $\A_i$, $\B$ and $\C_i$ are defined as, assuming a linearization
of the Roe scheme with the Roe Jacobian frozen:
%
\begin{align}
\A_i =
  \Gamma^{-1}
  \Bigg\{
    &-K^{X_i-\frac{1}{2}}_{ii} \left(\frac{\partial G}{\partial \Ustar}\right)^{X_i-1}
    -\mfd\frac{\left( {\Omega} |A| \right)_i^{X_i-\frac{1}{2}}}{2 {\Omega}^{X_i-1}}
                         -\frac{A^{X_i-1}}{2}\nonumber\alb
    &-\big({\Dstar_i}^+\big)^{X_i-1}
     -\big({\Ystar_i}^+\big)^{X_i-1/2} \left(\frac{\partial H}{\partial \Ustar}\right)^{X_i-1}
  \Bigg\}
\end{align}
%
\begin{align}
\B_i &=
I+\Gamma^{-1}
 \Bigg\{
     -\delta_{1i} \left(\frac{\partial \Sstar}{\partial \Ustar}\right)^{X_i}
    +\delta_{1i} Z^{X_i} \left(\frac{\partial (\delta_t \Ustar)}{\partial \Ustar}\right)^{X_i} \nonumber\alb
    &+\left( K^{X_i-\frac{1}{2}}_{ii}+K^{X_i+\frac{1}{2}}_{ii} \right)\left(\frac{\partial G}{\partial \Ustar}\right)^{X_i}
    + \mfd\frac{\left( {\Omega} |A| \right)_i^{X_i-\frac{1}{2}}+
           \left( {\Omega} |A| \right)_i^{X_i+\frac{1}{2}}}{2 {\Omega}^{X_i}} \nonumber\alb
    &+\big({\Dstar_i}^+\big)^{X_i}-\big({\Dstar_i}^-\big)^{X_i}
     +\big({\Ystar_i}^+\big)^{X_i-1/2} \left(\frac{\partial H}{\partial \Ustar}\right)^{X_i}
     -\big({\Ystar_i}^-\big)^{X_i+1/2} \left(\frac{\partial H}{\partial \Ustar}\right)^{X_i}
  \Bigg\}
\end{align}
%
\begin{align}
\B &=
I+\Gamma^{-1}
 \Bigg\{
    -\left(\frac{\partial \Sstar}{\partial \Ustar}\right)^{X_i}
    + Z^{X_i} \left(\frac{\partial (\delta_t \Ustar)}{\partial \Ustar}\right)^{X_i}
   \nonumber\alb
    &+ \sum_{j=1}^\nd \Bigg[
     \left( K^{X_j-\frac{1}{2}}_{jj}+K^{X_j+\frac{1}{2}}_{jj} \right)\left(\frac{\partial G}{\partial \Ustar}\right)^{X_j}
    +\frac{\left( {\Omega} |A| \right)_j^{X_j-\frac{1}{2}}+
           \left( {\Omega} |A| \right)_j^{X_j+\frac{1}{2}}}{2 {\Omega}^{X_j}}
   \nonumber\alb
    &+\big({\Dstar_j}^+\big)^{X_j}-\big({\Dstar_j}^-\big)^{X_j}
     +\big({\Ystar_j}^+\big)^{X_j-1/2} \left(\frac{\partial H}{\partial \Ustar}\right)^{X_j}
     -\big({\Ystar_j}^-\big)^{X_j+1/2} \left(\frac{\partial H}{\partial \Ustar}\right)^{X_j}
    \Bigg]
  \Bigg\}
\end{align}
%
\begin{align}
\C_i =
  \Gamma^{-1}
  \Bigg\{
    &-K^{X_i+\frac{1}{2}}_{ii} \left(\frac{\partial G}{\partial \Ustar}\right)^{X_i+1}
    -\mfd\frac{\left( {\Omega} |A| \right)_i^{X_i+\frac{1}{2}}}{2 {\Omega}^{X_i+1}}
                           +\frac{A^{X_i+1}}{2}\nonumber\alb
    &+\big({\Dstar_i}^-\big)^{X_i+1}
    +\big({\Ystar_i}^-\big)^{X_i+1/2} \left(\frac{\partial H}{\partial \Ustar}\right)^{X_i+1}
  \Bigg\}
\end{align}
%
where $|A|=L^{-1}  |\Lambda|  L$, where $A=\partial \Fstar_i/\partial \Ustar$, where ${\Ystar}^\pm=\frac{1}{2}(\Ystar\pm|\Ystar|)$, and where $\Omega$ is the inverse of the metrics Jacobian.

  \bibliographystyle{warpdoc}
  \bibliography{all}

\end{document}


\documentclass[a4paper,twocolumn]{article}

\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{graphicx}
\usepackage{fullpage}

\usepackage[small]{titlesec}
% patch titlesec bug of not showing (sub)title numbering http://tex.stackexchange.com/a/300259
\usepackage{etoolbox}
\makeatletter
\patchcmd{\ttlh@hang}{\parindent\z@}{\parindent\z@\leavevmode}{}{}
\patchcmd{\ttlh@hang}{\noindent}{}{}{}
\makeatother

\makeatletter
\newcommand{\thickhline}{%
    \noalign {\ifnum 0=`}\fi \hrule height 1pt
    \futurelet \reserved@a \@xhline
}
\makeatother

\newcommand*\mean[1]{\overline{#1}}

\title{Convolutional Neural Fabrics $-$ summary}
\author{Matěj Nikl}

\begin{document}
\maketitle
\noindent
Convolutional Neural Fabrics (CNFs) is a architecture for embedding an exponentially large number of CNN architectures (via massive parameter sharing). It is intended as a step aside from the conventional CNN model architecture selection problem $-$  it is not trying to solve it, instead it offers the opportunity for the fabrics to learn/select the most appropriate model (or even ensemble of models) within itself by standard back-propagation algorithm, leaving only the hyper-parameters of number of layers and channels up to us.
Other hyper-parameters, which are being ``set'' automatically by embedding in the fabrics are:
\begin{itemize}
    \item filter size per layer
    \item stride per layer
    \item number of pooling vs. convolutional layers
    \item type of pooling operator per layer
    \item size of the pooling regions
    \item ordering of pooling and convolutional layers
    \item channel connectivity pattern between layers
    \item type of activation (ReLU vs MaxOut)
\end{itemize}

% \section{Defining the Fabric}

\begin{figure}[!h]
    \includegraphics[width=\columnwidth]{fabrics1.png}
    \caption{Trellis embedding of two seven-layer CNNs. Trellis nodes receiving the input and producing output are encircled. All edges are oriented to the right, down in the first layer, and towards the output in the last layer. The channel dimension of the 3D trellis is omitted for clarity.}
\end{figure}

\section{Making sense of the Fabrics}
The Fabrics consist of a 3D trellis that connects response maps at different layers, scales, and channels with a sparse homogeneous local connectivity pattern.
Each node in the trellis represents a response map with the same number of dimensions as the input signal.

\subsection{The trellis structure}
It is spanned by three axes:
\begin{enumerate}
    \item a \textbf{layer} axis $-$ along which all edges advance, is analogous to the depth axis of a CNN
    \item a \textbf{scale} axis $-$ along which response maps of different resolutions are organized from fine to coarse
    \item a \textbf{channel} axis $-$ along which different response maps of the same scale and layer are organized
\end{enumerate}

Each node is connected to a 3 x 3 scale-channel neighborhood in the previous layer, i.e. channel $c$ at scale $s$ receives input from channels $\{c-1, c, c+1\}$ at scales $\{s-1, s, s+1\}$. Input from a finer scale is obtained via strided convolution, and input from a coarser scale by convolution, after upsampling by padding zeros around the activations at the coarser level.

\subsection{Embedded architectures}
\paragraph{Re-sampling operators}
Fine-to-coarse change of the resolution of response maps is done using stride-two convolutions. Larger strides can be obtained by following multiple such edges.

\textit{Average pooling} across small regions is also a strided convolution with uniform filter weights, thus available in the trellis. Average pooling over larger areas is also available.

\textit{Bi-linear interpolation} is commonly used in deconvolutional networks. Factor-two interpolation can be represented on coarse-to-fine edges by using a filter with $1$ in the center, $1/4$ on corners, and $1/2$ elsewhere. Larger powers of two can be obtained by repetition.

\begin{figure*}[t]
    \centering
    \includegraphics[width=0.375\textwidth]{fabrics2.png}
    \hskip 1cm
    \includegraphics[width=0.5\textwidth]{fabrics3.png}
    \caption{Visualization of mean-squared filter weights (mean along weights and channels) in models learned for Part Labels (left) and MNIST (right). Layers are laid out horizontally, and scales vertically.}
\end{figure*}

{
\renewcommand{\arraystretch}{1.25}
\begin{table*}[ht]
\centering
    \begin{tabular}{l c c r c}
        \thickhline
                                          & Year & Augmentation & \# Params. & Error (\%) \\
        \hline

        Chang et al.                      & 2015 & N            & 447 K      & 0.24 \\
        Lee et al.                        & 2015 & N            &            & 0.31 \\
        Grid LSTM                         & 2016 & T            &            & 0.32 \\
        Dropconnect                       & 2013 & T+F          & 379 K      & 0.32 \\
        CKN                               & 2014 & N            &  43 K      & 0.39 \\
        Maxout                            & 2013 & N            & 420 K      & 0.45 \\
        Network in Network                & 2013 & N            &            & 0.47 \\
        \hline
        Ours: CNF-sparse (L = 16, C = 32) &      & T            & 249 K      & 0.48 \\
        Ours: CNF-dense (L = 8, C = 64)   &      & T            & 5.3 M      & 0.33 \\
        \thickhline
    \end{tabular}
    \caption{Comparison of results with the state of the art on MNIST. Data augmentation with translation and flipping is denoted by T and F respectively, N denotes no data augmentation.}
\end{table*}
}

\paragraph{Filter sizes}
A 5 x 5 filter can be implemented by computing nine intermediate channels to obtain a vectorized version of the 3 x 3 neighborhood at each pixel. A second 3 x 3 convolution can then aggregate values across the original 5 x 5 patch, and output the desired convolution. Repetition allows to implement every filter of any desired size.

\paragraph{Ordering convolution and re-sampling}
It is up to the back-propagation algorithm to choose the desired path(s) to use $-$ by setting all, except one paths weights to zero, a particular sequence of convolutions and re-sampling operators is obtained.

\paragraph{Channel connectivity pattern}
This sparsely connected among the channel axis trellis suffices to emulate densely connected convolutional layers by copying channels, convolving them, and locally aggregating them.

\section{Principles of modularization}
The CNF architecture does not contain any standalone modules. The network itself can utilize some of its parts more, and by weight visualization it can be seen as modules being instationed, maybe even for certain purposes, however they still make up a integral, inseparable part of the network.

\section{Principles of growing}
The CNF architecture does not allow for any more growing as a conventional (C)NN does. Its purpose is not in growing, but in dynamically utilizing its trellis structure, whilst hiding away multiple hyper-parameters.

\end{document}