\documentclass[review]{elsarticle}

% Packages and macros go here
\usepackage{amsfonts}
\usepackage{amsmath}
\usepackage{lineno,hyperref}
\modulolinenumbers[5]
\usepackage{graphicx}
\usepackage{epstopdf}
\usepackage{algorithmic}
\usepackage{subcaption}
\captionsetup{font=small}
\captionsetup[sub]{font=footnotesize}
\usepackage{nicefrac}
\usepackage{units}
\usepackage[acronyms]{glossaries}

\journal{Elsevier}

% \title{Data recovery in computational fluid dynamics through deep image priors}
% \author[main]{M.~T.~Henry de Frahan\corref{cor1}}
% \ead{marc.henrydefrahan@nrel.gov}
% \author[main]{R.~W.~Grout}
% \ead{ray.grout@nrel.gov}
% \cortext[cor1]{Corresponding author}
% \address[main]{High Performance Algorithms and Complex Fluids, Computational Science Center, National Renewable Energy Laboratory, 15013 Denver W Pkwy, ESIF301, Golden, CO 80401, USA}

%% `Elsevier LaTeX' style
%\bibliographystyle{elsarticle-num}
\bibliographystyle{elsarticle-num-names}
\makeatletter
\providecommand{\doi}[1]{%
  \begingroup
  \let\bibinfo\@secondoftwo
  \urlstyle{rm}%
  \href{http://dx.doi.org/#1}{%
    doi:\discretionary{}{}{}%
    \nolinkurl{#1}%
  }%
  \endgroup
}
\makeatother

%=================================================================================
% Abbreviations
\newacronym{dns}{DNS}{direct numerical simulations}
\newacronym{cfd}{CFD}{computational fluid dynamics}
\newacronym{pod}{POD}{proper orthogonal decomposition}
\newacronym{gpr}{GPR}{Gaussian process regression}
\newacronym{gp}{GP}{Gaussian processes}
\newacronym{gpu}{GPU}{graphics processing unit}
\makeglossaries
\glsdisablehyper

\begin{document}
\begin{frontmatter}

\title{Data recovery in computational fluid dynamics through deep image priors}

\author[main]{M.~T.~Henry de Frahan\corref{cor1}}
\ead{marc.henrydefrahan@nrel.gov}

\author[main]{R.~W.~Grout}
\ead{ray.grout@nrel.gov}

\cortext[cor1]{Corresponding author}

\address[main]{High Performance Algorithms and Complex Fluids, Computational Science Center, National Renewable Energy Laboratory, 15013 Denver W Pkwy, ESIF301, Golden, CO 80401, USA}

\begin{abstract}

  One of the challenges encountered by computational simulations at
  exascale is the reliability of simulations in the face of hardware
  and software faults. These faults, expected to increase with the
  complexity of the computational systems, will lead to the loss of
  simulation data and simulation failure and are currently addressed
  through a checkpoint-restart paradigm. Focusing specifically on
  \gls{cfd} simulations, this work proposes a method that uses a deep
  convolutional neural network to recover simulation data. This data
  recovery method (i) is agnostic to the flow configuration and
  geometry, (ii) does not require extensive training data, and (iii) is
  accurate for very different physical flows. Results indicate that
  the use of deep image priors for data recovery is more accurate than
  standard recovery techniques, such as the \gls{gpr}, also known as
  Kriging. Data recovery is performed for two canonical fluid flows:
  laminar flow around a cylinder and homogeneous isotropic
  turbulence. For data recovery of the laminar flow around a cylinder,
  results indicate similar performance between the proposed method and
  \gls{gpr} across a wide range of mask sizes. For homogeneous
  isotropic turbulence, data recovery through the deep convolutional
  neural network exhibits an error in relevant turbulent quantities
  approximately three times smaller than that for the
  \gls{gpr}. Forward simulations using recovered data illustrate that
  the enstrophy decay is captured within $10\%$ using the deep
  convolutional neural network approach. Although demonstrated
  specifically for data recovery of fluid flows, this technique can be
  used in a wide range of applications, including particle image
  velocimetry, visualization, and computational simulations of
  physical processes beyond the Navier-Stokes equations.

\end{abstract}

\begin{keyword}
data recovery \sep fault tolerance \sep Gaussian process regression \sep deep convolutional neural network \sep computational fluid dynamics
\end{keyword}

\end{frontmatter}

\glsresetall

\section{Introduction}
As modern computational efforts reach exascale, hardware and software
faults will increasingly cause difficulty in completing
simulations~\cite{Brown2010}. Current research in hardware systems and
software frameworks~\cite{Gropp2004, Hoemmen2011, Teranishi2014,
  Cappello2014, Gamell2015, Grout2017} continues to develop techniques
to detect and anticipate system failures; however, assuming that a
fault has been detected and signaled, the data loss from these
failures will require data recovery processes~\cite{Lee2015}. This
work addresses this challenge. Current computational codes rely
on a checkpoint and restart paradigm to recover from faults, requiring
either significant memory consumption for frequent checkpoints or
large resimulation efforts. The ability to recover the missing data
without resorting to data checkpoints has the potential to increase
simulation resilience.

In the context of \gls{cfd}, data recovery has been explored using a
variety of machine learning approaches. Gappy \gls{pod} has shown
particular success in reconstructing missing
data~\cite{Everson1995,Tan2003,Venturi2004}. This approach relies on
combining \gls{pod} with least-squares estimates~\cite{Yates1933,
  Little2002} and data from \acrlong{dns} snapshots. Venturi and
Karniadakis~\cite{Venturi2004} expanded on the methods proposed by
Everson and Sirovich~\cite{Everson1995} and used this technique to
reconstruct missing data of unsteady flow past a cylinder. Other
approaches rely on \gls{gpr}, often referred to as Kriging in
geophysics, which is a a reconstruction technique that uses the mean and
covariance of the \acrlong{gp} prior. The prior's covariance is determined
by a kernel whose hyperparameters are optimized using training
data. Gunes et al.~\cite{Gunes2006} compare \gls{pod}-based and
\gls{gpr}-based solution reconstruction and show that \gls{gpr}
interpolations are particularly effective for unsteady flows
(including instability regions), whereas \gls{pod}-based methods are
advantageous when the temporal resolution is high. In addition to
these methods, Lee et al.~\cite{Lee2015} proposed a ``resimulation''
method in which the missing data region is resimulated using
appropriate initial and boundary conditions. This new method is
evaluated for lid-driven cavity flows and flows past a cylinder at low
Reynolds numbers. Lee et al.~\cite{Lee2017} combined the gap-tooth
algorithm, previously developed for dynamic systems~\cite{Gear2003},
multiresolution information fusion, and auxiliary data to construct a
general framework for fault-tolerant \gls{cfd}. The method is
demonstrated to work well for simulations of the heat equation and
lid-driven cavity flow.

Recently, the deep learning community has been proposing methods for
data recovery in the field of inverse image reconstruction problems,
which include denoising, inpainting, and
super-resolution,~\cite{Goodfellow2014,Burger2012,Dosovitskiy2015,Lefkimmiatis2016,Ledig2017,Tai2017,Lai2017}. Deep
convolutional neural networks, particularly generative adversarial
networks, have been very successful at solving this class of
problems~\cite{Goodfellow2014}. Inpainting is of particular relevance
to our objective of reconstructing flow solutions. The objective of
inpainting is to fill in missing portions of a damaged image such that
the result is indistinguishable from the original image. Various
generative adversarial neural networks have been proposed for image
inpainting with notable
success~\cite{Yeh2016,Denton2016,Pathak2016,Li2017a,Sasaki2017}. Generally
speaking, these approaches have relied on training deep neural
networks with an extensive and large data set of images such that the
network learns image priors that it can use in other
configurations. The deep learning methodology used in this work was
first developed by Ulyanov et al.~\cite{Ulyanov2017} for various
inverse image reconstruction problems, including inpainting. In
contrast with previous image reconstruction solution with deep neural
networks, Ulyanov et al.~\cite{Ulyanov2017} showed that ``contrary to
the belief that learning is necessary for building good image priors,
a great deal of image statistics are captured by the structure of a
convolutional image generator independent of learning.''  Instead of
training the neural networks with a large database of images, the
authors use untrained neural networks to fit single degraded images,
using the network weights as parameters for solving the image
reconstruction problem.

In this work, we use deep convolutional neural networks, such as those
proposed by Ulyanov et al.~\cite{Ulyanov2017}, for spatial reconstruction of the flow
solution for simulations wherein some type of fault led to loss of
data, e.g.,\,processor failure. In contrast to gappy
\gls{pod}~\cite{Venturi2004}, we assume that the current gappy data are
the only available data for the reconstruction procedure. This
assumption is relevant to large simulations where it is
computationally expensive to reload data residing on the file system
and the reconstruction process is restricted to data already in
memory. One advantage of using deep convolutional neural networks is
that this approach avoids eigenmode decompositions for solution
reconstruction, which could restrict the applicability or translation of
the method to new configurations. As illustrated in this work, the
method proposed here is not specific to the flow configuration and
does not require multiple training data samples.

This paper is organized as follows. In Section\,\ref{sec:formulation}, we
present the problem formulation and define the objective function for
the data recovery problem. In Section\,\ref{sec:nn}, we detail the
architecture of the deep convolutional neural network used to perform
the data recovery process for fluid flows. In Section\,\ref{sec:results}, we
present our results by evaluating the neural network's ability to
perform data recovery for two canonical flows: laminar flow past a
cylinder, Section\,\ref{sec:cylinder}, and homogeneous isotropic turbulence,
Section\,\ref{sec:hit}. These results are compared with data recovery
performed through \gls{gpr}. Finally, conclusions and future work are
presented in Section\,\ref{sec:ccl}.

\section{Problem formulation}\label{sec:formulation}

In this work, we evaluate the performance of deep convolutional neural
networks for data recovery in \gls{cfd}. Deep convolutional neural
networks have shown particular success for solving the image
reconstruction problem~\cite{Goodfellow2014}. Image reconstruction is
analogous to data recovery because they share a similar objective to
provide synthetic data that closely match the missing data. The
image reconstruction problem can be cast as an optimization problem:
\begin{align}
  \min_x{E(x;x_0) + R(x)}
\end{align}
where $x$ is the original image that needs to be recovered, $x_0$ is
the corrupted image, $E(x;x_0)$ is the task-dependent data term, and
$R(x)$ is the image prior. In the case of inpainting, the
task-dependent data term is:
\begin{align}
  E(x;x_0) = || (x-x_0) \circ m||^2
\end{align}
where $\circ$ is the Hadamart product, $m \in \{0,1\}^{h\times w}$
represents the binary mask, and $h$ and $w$ are the image height and
width. The image prior is usually captured through the training of
convolutional neural networks using a large image database. In the
approach proposed by Ulyanov et al.~\cite{Ulyanov2017}, $R(x)$ is replaced by a
parameterization such that the optimization problem becomes:
\begin{align}
  \min_\theta{E(f_\theta(z); x_0)} \label{equ:loss}
\end{align}
where $f$ represents the convolutional neural network with parameters
$\theta$ that is initialized randomly, and $z$ is a fixed input. The
fixed input for the neural network can take many forms but is usually
chosen to be random uniform noise or smoothly varying data.
Note that the neural network input is \textit{fixed}. Given a
deteriorated image, the neural network effectively learns, by
backpropagation and network parameter tuning, the encoding necessary
to map the fixed input to an output, i.e.,\,the \textit{recovered}
image, which minimizes the loss function,~Equation\,\ref{equ:loss}.

We emphasize that physical constraints are not explicitly included in
the data recovery process. This has the advantage of enabling a
reconstruction technique that does not depend on the physical nature
of the problem. Higher fidelity can be achieved, however, by
incorporating physical constraints, as suggested in~\cite{Raissi2017,
  Sirignano2018}. The work presented here focuses on two-dimensional
reconstruction, though there is no inherent methodological limitation
to reconstructing three-dimensional data directly.

\section{Neural network architecture}\label{sec:nn}

The network chosen for this work is a convolutional neural network
that exhibits an encoder-decoder architecture with approximately 2
million tunable parameters and no skip connections,
Figure\,\ref{fig:encoder_decoder}. This architecture enables the
network to encode the input in the latent space and then decode the
latent space representation into the reconstructed image. The
nonlinear activation function used in the network is LeakyReLU
\cite{He2015}. Downsampling was performed through simple striding in
the convolution procedure, Figure\,\ref{fig:downsample}, and
upsampling was done through nearest-neighbor upsampling,
Figure\,\ref{fig:upsample}. The number of filters in the downsampling
and upsampling layers, $n_f$, was kept fixed at 128, and the kernel
size, $k$, was fixed at 3. Experiments showed that using a fixed
smoothly varying input $z$ for the neural network imposes a smoothness
prior, which is beneficial for data recovery for fluid flows. The
optimization process was performed using Adam~\cite{Kingma2014}. The
implementation was done in PyTorch~\cite{Paszke2017}, and the learning
process was computed on a Tesla V100 \acrlong{gpu}. The number of
iterations for all the experiments was 2000, thereby reducing
the loss function by three orders of magnitude.

\begin{figure}[!tbp]%
  \centering%
  \begin{subfigure}[t]{1\textwidth}%
    \includegraphics[width=\textwidth]{./figs/encoder_decoder.pdf}%
    \caption{Encoder-decoder architecture.}\label{fig:encoder_decoder}%
  \end{subfigure}\\[0.3cm]%
  \begin{subfigure}[b]{0.33\textwidth}%
    \includegraphics[width=0.92\textwidth]{./figs/downsample.pdf}%
    \caption{Downsampling unit.}\label{fig:downsample}%
  \end{subfigure}%
  \hfill%
  \begin{subfigure}[b]{0.33\textwidth}%
    \includegraphics[width=\textwidth]{./figs/upsample.pdf}%
    \caption{Upsampling unit.}\label{fig:upsample}%
  \end{subfigure}%
  \hfill%
  \begin{subfigure}[b]{0.15\textwidth}%
    \includegraphics[width=\textwidth]{./figs/legend.pdf}%
    \vspace*{0.4cm}%
    \caption{Legend.}\label{fig:legend}%
  \end{subfigure}%
  \caption{Deep convolutional neural network used for data recovery.}\label{fig:arch}%
\end{figure}%

\section{Results}\label{sec:results}

To demonstrate the efficacy of using deep neural networks for data
recovery, we investigate two types of flows: laminar flow over a
cylinder for data recovery of large flow scales and homogeneous
isotropic turbulence for data recovery of flows spanning a wide range
of scales. We compare the deep convolutional neural network results
with \gls{gpr}. The sample points used to train the \gls{gpr} are
located in the region surrounding the mask with a depth of 10 cells,
similar to~\cite{Lee2015}. Beyond a depth of 10 cells surrounding
the masked regions, training the regressor becomes computationally
intractable because the \gls{gpr} complexity is $\mathcal{O}(n^3)$, where
$n$ is the number of training points. A radial basis function is used
as the \gls{gpr} kernel.

\subsection{Laminar flow around a cylinder}\label{sec:cylinder}
The first numerical tests of the data recovery process are performed
for the laminar flow around a cylinder ($Re = 200$). The simulation is
performed using Nalu-Wind, a low Mach Navier-Stokes solver leveraging
the Trilinos
libraries\footnote{\url{https://github.com/Exawind/nalu-wind}}; and the
$t=234\,\unit{s}$ snapshot is used for the numerical tests, at which
time the vortices behind the cylinder were fully developed. Masks
simulating data loss because of processor failure are generated in the
cylinder wake. To capture typical domain decomposition methods for
structured grids, the masks are square boxes and vary in size
depending on the number of processors used for the simulation. The
mask box length, $L_m$, ranged from $0.5D$ to $5D$, where $D$ is the
cylinder diameter, and the masks are located at 40 random locations in
the cylinder wake, leading to 240 unique masks to be applied to the
simulation data. For the reconstruction process, reflection padding is
used for the boundary conditions.

An example reconstruction is presented in
Figure\,\ref{fig:illustrated_cyl} for $L_m=2D$, where the deep
convolutional neural network presents a slightly better reconstructed
velocity field than \gls{gpr}. Specifically, the partially masked
vortex is more accurately reconstructed using the deep convolutional
neural network. The average $L_2$ error norm for the velocity fields
as a function of $L_m$ is presented in
Figure\,\ref{fig:cyl_error}. Both reconstruction techniques, \gls{gpr}
and the deep convolutional neural network, present similar error
profiles. At higher $L_m$ the neural network performs slightly better
than \gls{gpr} for the $x$-direction velocity, whereas it performs
similarly for all other lengths. Given the structured nature of the
flow field, it is unsurprising that \gls{gpr} performs well at
moderate mask sizes, given previously published
results~\cite{Gunes2006}.

\begin{figure}[!tbp]%
  \centering%
  \begin{subfigure}[t]{0.45\textwidth}%
    \includegraphics[width=\textwidth]{./figs/cyl_umag0.png}%
    \caption{Original data.}\label{fig:cyl_umag0}%
  \end{subfigure}%
  \hfill%
  \begin{subfigure}[t]{0.45\textwidth}%
    \includegraphics[width=\textwidth]{./figs/cyl_umag0_masked.png}%
    \caption{Deteriorated data.}\label{fig:cyl_umag0_masked}%
  \end{subfigure}\\%
  \begin{subfigure}[t]{0.45\textwidth}%
    \includegraphics[width=\textwidth]{./figs/cyl_umagr.png}%
    \caption{Deep convolutional neural network.}\label{fig:cyl_umagr}%
  \end{subfigure}%
  \hfill%
  \begin{subfigure}[t]{0.45\textwidth}%
    \includegraphics[width=\textwidth]{./figs/cyl_umagi.png}%
    \caption{\gls{gpr}.}\label{fig:cyl_umagi}%
  \end{subfigure}%
  \caption{Velocity magnitude for laminar flow around a cylinder where the mask box length is twice the cylinder diameter, $L_m=2D$.}\label{fig:illustrated_cyl}%
\end{figure}%

\begin{figure}[!tbp]%
  \centering%
  \begin{subfigure}[t]{0.48\textwidth}%
    \includegraphics[width=\textwidth]{./figs/cyl_error_u.png}%
    \caption{$x$-direction velocity.}\label{fig:cyl_error_u}%
  \end{subfigure}%
  \hfill%
  \begin{subfigure}[t]{0.48\textwidth}%
    \includegraphics[width=\textwidth]{./figs/cyl_error_v.png}%
    \caption{$y$-direction velocity.}\label{fig:cyl_error_v}%
  \end{subfigure}%
  \caption{Average $L_2$ error norm as a function of mask box length,
    $L_m$, for laminar flow past a cylinder ($Re=200$). Red squares:
    deep convolutional neural network; green diamonds:
    \gls{gpr}.}\label{fig:cyl_error}%
\end{figure}%

\subsection{Homogeneous isotropic turbulence}\label{sec:hit}

For these numerical tests, we use two-dimensional slices of
homogeneous isotropic turbulence with a Taylor microscale Reynolds
number $Re_\lambda= \nicefrac{\rho_0 u' \lambda}{\mu}=133$, where
$\rho_0$ is the reference density,
$u' = \sqrt{\nicefrac{\overline{u_i u_i}}{3}}$ is the initial mean
fluctuating velocity,
$\lambda = \nicefrac{\overline{u_1^2}}{\overline{\left(\frac{\partial
        u_1}{\partial x_1}\right)^2}}$ is the Taylor microscale, and
$\mu$ is the dynamic viscosity; a turbulent Mach number
$M_t = \nicefrac{u_0}{c_s} = 0.1$, where $c_s$ is the speed of sound;
and a Prandtl number $Pr = \nicefrac{\mu c_p}{k} = 0.71$, where $c_p$
is the heat capacity at constant pressure, and $k$ is the thermal
conductivity. The reference temperature and pressure are 300~K and 1
atmosphere and the ideal gas equation of state is used to relate the
thermodynamic quantities. The domain ranges from $[0,2\pi]$ with
periodic boundary conditions. In this work, we use
PeleC\footnote{\url{https://github.com/AMReX-Combustion/PeleC}}, an
explicit compressible Navier-Stokes flow solver based on the AMReX
library\footnote{\url{https://amrex-codes.github.io}}, to demonstrate
the data recovery process. For the reconstruction process, periodic,
i.e.,\,wrapped, padding was used for the boundary conditions.

Initial two-dimensional data slices are generated by slicing in each
direction a numerical simulation of homogeneous isotropic turbulence
at a resolution of 64 cells in each direction, leading to 192 unique
slices. The velocities in each direction are assigned an input channel
for the neural network. Masks simulating data loss because of processor
failure are generated independently, following typical domain
decomposition. We explore two different parameters associated with the
mask generation process. The first is the total percentage of missing
data, $f$, ranging from $6.25\%$ to $25\%$. The second is the length
scale associated with each block of missing data, $L_m$, ranging from
$3.125\%$ to $50\%$ of the domain length, or $0.74\lambda$ to
$11.87\lambda$. For each pair of parameters $f$ and $L_m$, we randomly
generate ten different masks, resulting in 130 unique masks. These are
randomly applied to 100 initial slices, resulting in 1300 slices
requiring reconstruction. For each of these deteriorated slices, the
neural network parameters are tuned to optimize the reconstruction
loss function, Equation\,(\ref{equ:loss}). The data from the resulting
recovered slice are then used for comparison with the original
slice. The data recovery process is illustrated for one slice in
Figure\,\ref{fig:illustrated_hit}.

\begin{figure}[!tbp]%
  \centering%
  \begin{subfigure}[t]{0.32\textwidth}%
    \includegraphics[width=\textwidth]{./figs/umag0.png}%
    \caption{Original.}\label{fig:original}%
  \end{subfigure}%
  \hfill%
  \begin{subfigure}[t]{0.32\textwidth}%
    \includegraphics[width=\textwidth]{./figs/umag0_masked.png}%
    \caption{Deteriorated.}\label{fig:masked}%
  \end{subfigure}%
  \hfill%
  \begin{subfigure}[t]{0.32\textwidth}%
    \includegraphics[width=\textwidth]{./figs/umagr.png}%
    \caption{Recovered.}\label{fig:result}%
  \end{subfigure}%
  \caption{Velocity magnitude in homogeneous isotropic turbulence
    illustrating the data recovery process, where 25\% of the original
    data is missing and the length scale associated to each missing
    block is 6.25\% of the domain
    ($1.5\lambda$).}\label{fig:illustrated_hit}%
\end{figure}%

The average error in $u'$ and $\lambda$ for all the reconstructions is
approximately three times larger for the \gls{gpr} process compared to
the deep convolutional neural network. For all the slices, individual
energy spectra are calculated for the original data and the data
recovered through the deep convolutional neural network and
\gls{gpr}. The average energy spectrum is presented in
Figure\,\ref{fig:spectra}. The average error from the \gls{gpr}
reconstruction increases at high wavenumbers, indicating that it not
able to accurately capture the smallest scales of turbulence,
Figure\,\ref{fig:error_spectra}. This behavior is not exhibited with
the deep neural network reconstruction, and the error increases
slightly as a function of wavenumber.

\begin{figure}[!tbp]%
  \centering%
  \begin{subfigure}[t]{0.48\textwidth}%
    \includegraphics[width=\textwidth]{./figs/spectra.png}%
    \caption{Average energy spectrum.}\label{fig:spectra}%
  \end{subfigure}%
  \hfill%
  \begin{subfigure}[t]{0.48\textwidth}%
    \includegraphics[width=\textwidth]{./figs/error_spectra.png}%
    \caption{Normalized error, $e_{E_k} = \frac{|E^h_k - E_k|}{E_k}$.}\label{fig:error_spectra}%
  \end{subfigure}%
  \caption{Energy spectrum and error as a function of wavenumber $k$. Solid red: original data; dashed green: deep convolutional neural network; dot-dashed blue: \gls{gpr}.}\label{fig:spec}%
\end{figure}%

Velocities in slices of the original and recovered data are used as
initial conditions for two-dimensional decay simulations. In these
simulations performed in PeleC, the initial conditions for the
velocity fields are taken from the (i) original data, (ii)
recovered data from the deep convolutional neural network, and (iii)
recovered data from the \gls{gpr}. The final time for the simulations
is $5\tau$, where $\tau =
\nicefrac{\lambda}{u'}$. Figure\,\ref{fig:hit_enstrophy} illustrates
the decay of normalized enstrophy,
$\omega = \frac{\lambda^2}{u'^2 V}\int_V \left( \nabla \times u
\right)^2 \mathrm{d}V$, for simulations where $f=25\%$ and
$L_m \in [0.74\lambda, 11.87\lambda]$, or $3.125\%$ to $50\%$ of the
domain length. \gls{gpr} reconstruction exhibits a significantly
different enstrophy decay from the original data,
Figure\,\ref{fig:enstrophy_gp}. For the deep convolutional neural
network, the enstrophy decay is well captured at all mask sizes, with
slightly less accuracy for large $L_m$,
Figure\,\ref{fig:enstrophy_dl}. Across the range of mask sizes, the
normalized error at $t=1\tau$ in kinetic energy and enstrophy is less than
10\% for the deep convolutional neural network and around 20\% for the
\gls{gpr} reconstruction. These differences in reconstruction
procedures are attributed to the deep convolutional neural network's
ability to preserve the energy spectra and accurately represent all
the length scales.

\begin{figure}[!tbp]%
  \centering%
  \begin{subfigure}[t]{0.48\textwidth}%
    \includegraphics[width=\textwidth]{./figs/enstrophy_result.png}%
    \caption{Deep convolutional neural network.}\label{fig:enstrophy_dl}%
  \end{subfigure}%
  \hfill%
  \begin{subfigure}[t]{0.48\textwidth}%
    \includegraphics[width=\textwidth]{./figs/enstrophy_interp.png}%
    \caption{\gls{gpr}.}\label{fig:enstrophy_gp}%
  \end{subfigure}%
  \caption{Normalized enstrophy as a function of time where 25\% of
    the original data is missing. Solid black: original data; solid
    red: $L_m=0.74\lambda$; dashed green: $L_m=1.48\lambda$;
    dot-dashed blue: $L_m=2.97\lambda$; dotted orange:
    $L_m=5.94\lambda$; dot-dot-dashed purple:
    $L_m=11.87\lambda$.}\label{fig:hit_enstrophy}%
\end{figure}%

\section{Conclusion}\label{sec:ccl}

This work evaluated the use of deep convolutional neural networks for
data recovery of fluid flows in the context of data loss because of
hardware or software faults. The method proposed here leverages an
encoder-decoder deep convolutional neural network to transform a fixed
input to a recovered output using only deteriorated data and
eschewing a training database. Comparisons were performed with
\acrlong{gpr}, a standard data recovery algorithm often referred to as
Kriging. Data recovery was performed on two different canonical flow
configurations: laminar flow around a cylinder and homogeneous
isotropic turbulence. For data recovery of the laminar flow around a
cylinder, results indicate similar performance between the proposed
method and \gls{gpr} across a wide range of mask sizes. For homogeneous
isotropic turbulence, data recovery through the deep convolutional
neural network exhibits an error in relevant turbulent quantities
approximately three times smaller than that for the \gls{gpr}. Forward
simulations using recovered data illustrate that the enstrophy decay
is accurately captured using the deep convolutional neural network
approach.

We emphasize that the work presented here is not necessarily beholden
to the specific inpainting technique used. The deep learning community
has developed many different methods for image inpainting that can be
used for data recovery, and it is expected that state-of-the-art
methodologies would perform comparably well. Rather, the use of deep
image priors as first proposed by Ulyanov et al.~\cite{Ulyanov2017}
and investigated here provides a convenient framework to perform data
recovery of fluid flows because it does not require pretraining the neural
network to construct image priors for different flows. It is therefore
agnostic to the specific flow configuration, and the same framework can
be used for very different flows. This technique, however, does
necessitate the solution of an optimization problem for each data
recovery task. Future work will investigate alleviating this through
partial pretraining, transfer learning, and perceptual loss
functions.

This work --- including data sets, demonstration notebooks, analysis
scripts, and figures --- can be publicly accessed at the project's GitHub
page.\footnote{\url{https://github.com/NREL/deep-image-prior-cfd}} Traditional
machine learning algorithms were implemented through
scikit-learn~\cite{Pedregosa2011} and the deep learning algorithms
through PyTorch~\cite{Paszke2017}.

\section*{Acknowledgments}
This work was authored by the National Renewable Energy Laboratory, operated by Alliance for Sustainable Energy, LLC, for the U.S. Department of Energy (DOE) under Contract No. DE-AC36-08GO28308. Funding provided by U.S. Department of Energy Office of Science and National Nuclear Security Administration. The views expressed in the article do not necessarily represent the views of the DOE or the U.S. Government. The U.S. Government retains and the publisher, by accepting the article for publication, acknowledges that the U.S. Government retains a nonexclusive, paid-up, irrevocable, worldwide license to publish or reproduce the published form of this work, or allow others to do so, for U.S. Government purposes.

This research was supported by the Exascale Computing Project (ECP), Project Number: 17-SC-20-SC, a collaborative effort of two DOE organizations -- the Office of Science and the National Nuclear Security Administration -- responsible for the planning and preparation of a capable exascale ecosystem -- including software, applications, hardware, advanced system engineering, and early testbed platforms -- to support the nation's exascale computing imperative.

\section*{References}
\bibliography{library}

\end{document}


\XtoCBlock{Sin3Gen}
\label{block:Sin3Gen}
\begin{figure}[H]\includegraphics{Sin3Gen}\end{figure}

\begin{XtoCtabular}{Inports}
A & Amplitude\tabularnewline
\hline
f & Frequency\tabularnewline
\hline
\end{XtoCtabular}

\begin{XtoCtabular}{Outports}
u & Sine wave output phase u\tabularnewline
\hline
v & Sine wave output phase v\tabularnewline
\hline
w & Sine wave output phase w\tabularnewline
\hline
\end{XtoCtabular}

\begin{XtoCtabular}{Mask Parameters}
fmax & Maximum Frequency in Hz\tabularnewline
\hline
Offset & Offset\tabularnewline
\hline
ts\_fact & Multiplication factor of base sampling time (in integer format)\tabularnewline
\hline
\end{XtoCtabular}

\subsubsection*{Description:}
Generation of a 3 sine waves with amplitude (A) and frequency (f).

% include optional documentation file
\InputIfFileExists{\XcHomePath/Library/General/Doc/Sin3Gen_Info.tex}{\vspace{1ex}}{}

\subsubsection*{Implementations:}
\begin{tabular}{l l}
\textbf{FiP8} & 8 Bit Fixed Point Implementation\tabularnewline
\textbf{FiP16} & 16 Bit Fixed Point Implementation\tabularnewline
\textbf{FiP32} & 32 Bit Fixed Point Implementation\tabularnewline
\textbf{Float32} & 32 Bit Floating Point Implementation\tabularnewline
\textbf{Float64} & 64 Bit Floating Point Implementation\tabularnewline
\end{tabular}

\XtoCImplementation{FiP8}
\index{Block ID!432}
\nopagebreak[0]
% Implementation details
\begin{tabular}{l l}
\textbf{Name} & FiP8 \tabularnewline
\textbf{ID} & 432 \tabularnewline
\textbf{Revision} & 1.0 \tabularnewline
\textbf{C filename} & Sin3Gen\_FiP8.c \tabularnewline
\textbf{H filename} & Sin3Gen\_FiP8.h \tabularnewline
\end{tabular}
\vspace{1ex}

8 Bit Fixed Point Implementation

\begin{XtoCtabular}{Controller Parameters}
delta\_phi & Angle increment\tabularnewline
\hline
offset & Amplitude offset\tabularnewline
\hline
phi & Current angle\tabularnewline
\hline
\end{XtoCtabular}

% Implementation data structure
\XtoCDataStruct{Data Structure:}
\begin{lstlisting}
typedef struct {
     uint16        ID;
     int8          *A;
     int8          *f;
     int8          u;
     int8          v;
     int8          w;
     int8          delta_phi;
     int8          offset;
     int8          phi;
} SIN3GEN_FIP8;
\end{lstlisting}

\ifdefined \AddTestReports
\InputIfFileExists{\XcHomePath/Library/General/Doc/Test_Sin3Gen_FiP8.tex}{}{}
\fi
\XtoCImplementation{FiP16}
\index{Block ID!433}
\nopagebreak[0]
% Implementation details
\begin{tabular}{l l}
\textbf{Name} & FiP16 \tabularnewline
\textbf{ID} & 433 \tabularnewline
\textbf{Revision} & 1.0 \tabularnewline
\textbf{C filename} & Sin3Gen\_FiP16.c \tabularnewline
\textbf{H filename} & Sin3Gen\_FiP16.h \tabularnewline
\end{tabular}
\vspace{1ex}

16 Bit Fixed Point Implementation

\begin{XtoCtabular}{Controller Parameters}
delta\_phi & Angle increment\tabularnewline
\hline
offset & Amplitude offset\tabularnewline
\hline
phi & Current angle\tabularnewline
\hline
\end{XtoCtabular}

% Implementation data structure
\XtoCDataStruct{Data Structure:}
\begin{lstlisting}
typedef struct {
     uint16        ID;
     int16         *A;
     int16         *f;
     int16         u;
     int16         v;
     int16         w;
     int16         delta_phi;
     int16         offset;
     int16         phi;
} SIN3GEN_FIP16;
\end{lstlisting}

\ifdefined \AddTestReports
\InputIfFileExists{\XcHomePath/Library/General/Doc/Test_Sin3Gen_FiP16.tex}{}{}
\fi
\XtoCImplementation{FiP32}
\index{Block ID!434}
\nopagebreak[0]
% Implementation details
\begin{tabular}{l l}
\textbf{Name} & FiP32 \tabularnewline
\textbf{ID} & 434 \tabularnewline
\textbf{Revision} & 1.0 \tabularnewline
\textbf{C filename} & Sin3Gen\_FiP32.c \tabularnewline
\textbf{H filename} & Sin3Gen\_FiP32.h \tabularnewline
\end{tabular}
\vspace{1ex}

32 Bit Fixed Point Implementation

\begin{XtoCtabular}{Controller Parameters}
delta\_phi & Angle increment\tabularnewline
\hline
offset & Amplitude offset\tabularnewline
\hline
phi & Current angle\tabularnewline
\hline
\end{XtoCtabular}

% Implementation data structure
\XtoCDataStruct{Data Structure:}
\begin{lstlisting}
typedef struct {
     uint16        ID;
     int32         *A;
     int32         *f;
     int32         u;
     int32         v;
     int32         w;
     int32         delta_phi;
     int32         offset;
     int32         phi;
} SIN3GEN_FIP32;
\end{lstlisting}

\ifdefined \AddTestReports
\InputIfFileExists{\XcHomePath/Library/General/Doc/Test_Sin3Gen_FiP32.tex}{}{}
\fi
\XtoCImplementation{Float32}
\index{Block ID!435}
\nopagebreak[0]
% Implementation details
\begin{tabular}{l l}
\textbf{Name} & Float32 \tabularnewline
\textbf{ID} & 435 \tabularnewline
\textbf{Revision} & 0.1 \tabularnewline
\textbf{C filename} & Sin3Gen\_Float32.c \tabularnewline
\textbf{H filename} & Sin3Gen\_Float32.h \tabularnewline
\end{tabular}
\vspace{1ex}

32 Bit Floating Point Implementation

\begin{XtoCtabular}{Controller Parameters}
delta\_phi & Angle increment\tabularnewline
\hline
offset & Amplitude offset\tabularnewline
\hline
phi & Current angle\tabularnewline
\hline
\end{XtoCtabular}

% Implementation data structure
\XtoCDataStruct{Data Structure:}
\begin{lstlisting}
typedef struct {
     uint16        ID;
     float32       *A;
     float32       *f;
     float32       u;
     float32       v;
     float32       w;
     float32       delta_phi;
     float32       offset;
     float32       phi;
} SIN3GEN_FLOAT32;
\end{lstlisting}

\ifdefined \AddTestReports
\InputIfFileExists{\XcHomePath/Library/General/Doc/Test_Sin3Gen_Float32.tex}{}{}
\fi
\XtoCImplementation{Float64}
\index{Block ID!436}
\nopagebreak[0]
% Implementation details
\begin{tabular}{l l}
\textbf{Name} & Float64 \tabularnewline
\textbf{ID} & 436 \tabularnewline
\textbf{Revision} & 0.1 \tabularnewline
\textbf{C filename} & Sin3Gen\_Float64.c \tabularnewline
\textbf{H filename} & Sin3Gen\_Float64.h \tabularnewline
\end{tabular}
\vspace{1ex}

64 Bit Floating Point Implementation

\begin{XtoCtabular}{Controller Parameters}
delta\_phi & Angle increment\tabularnewline
\hline
offset & Amplitude offset\tabularnewline
\hline
phi & Current angle\tabularnewline
\hline
\end{XtoCtabular}

% Implementation data structure
\XtoCDataStruct{Data Structure:}
\begin{lstlisting}
typedef struct {
     uint16        ID;
     float64       *A;
     float64       *f;
     float64       u;
     float64       v;
     float64       w;
     float64       delta_phi;
     float64       offset;
     float64       phi;
} SIN3GEN_FLOAT64;
\end{lstlisting}

\ifdefined \AddTestReports
\InputIfFileExists{\XcHomePath/Library/General/Doc/Test_Sin3Gen_Float64.tex}{}{}
\fi


\section{Conclusion}\label{sec:conclusion}

Price response functions provide quantitative information on the deviation from
Markovian behavior. They measure price changes resulting from execution of
market orders. We used these functions in big data analysis for spot foreign
exchange markets. Such a study was, to the best of our knowledge, never done
before.

We analyzed price response functions in spot foreign exchange markets for
different years and different time scales. We used trade time scale and
physical time scale to compute the price response functions for the seven major
foreign exchange pairs for three different years. These major pairs are highly
relevant in the dynamics of the market. The use of different time scales and
calendar years in the work had the intention to display the different behaviors
the price response function could take when the time parameters differ.

The price response functions were analyzed according to the time scales. On
trade time scale, the signals were noisier. For both time scales we observe
that the signal for all the pairs increases to a maximum and then starts to
slowly decrease. However, for the year 2008 the shape of the signals is not as
well defined as in the other years. The increase-decrease behavior observed in
the spot foreign exchange market was also reported in correlated financial
markets \cite{my_paper_response_financial,Wang_2016_avg}. These results show
that the price response functions conserve their behavior in different years
and in different markets. The shape of the price
response functions is qualitatively explained considering an initial increase
caused by the autocorrelated transaction flow. To assure diffusive prices,
price response flattens due to market liquidity adapting to the flow in the
initial increase.

On both scales, the more liquid pairs have a smaller price response function
compared with the non-liquid pairs. As the liquid pairs have more trades during
the market time, the impact of each trade is reduced. Comparing years and
scales, the price response signal is stronger in past than in recent years. As
algorithmic trading has gained great relevance, the quantity of trades has
grown in recent years, and in consequence, the impact in the response has
decreased.

Finally, we checked the pip bid-ask spread impact in price response functions
for three different years. We used 46 foreign exchange pairs and grouped them
depending on the conditions of the corresponding year analyzed. We employ the
year average pip bid-ask spread of every pair for each year. For all the year
and time scales, the price response function signals were stronger for the
groups of pairs with larger pip bid-ask spreads and weaker for the group of
pairs with smaller bid-ask spreads. For the average of the price response
functions, it was only possible to see the increase-maximum-decrease behavior
in the year 2015 in both scales, and in the year 2019 on trade time scale.
Hence, the noise in the cross and exotic pairs due to the lack of trading
compared with the majors seems stronger. A general average price response
behavior for each year and time scale was spotted for the groups, suggesting a
market effect on the foreign exchange pairs in each year.

Comparing the response functions in stock and spot currency exchange markets
from a more general viewpoint, we find a remarkable similarity. It triggers the
conclusion that the order book mechanism generates in a rather robust fashion
the observed universal features in these two similar, yet different subsystems
within the financial system.


\subsection{Prior and posterior confidence}

\subsubsection{Introduction}

If we have a new fact, and a prior, we can create a posterior condfidence on the fact.


    \filetitle{blazer}{Reorder steady-state equations into block-recursive structure}{model/blazer}

	\paragraph{Syntax}\label{syntax}

\begin{verbatim}
[NameBlk,EqtnBlk] = blazer(M,...)
\end{verbatim}

\paragraph{Input arguments}\label{input-arguments}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{M} {[} model {]} - Model object.
\end{itemize}

\paragraph{Output arguments}\label{output-arguments}

\begin{itemize}
\item
  \texttt{M} {[} model {]} - Model object with variables and
  steady-state equations regrouped to create block-recursive structure.
\item
  \texttt{NameBlk} {[} cell {]} - Cell of cellstr with variable names in
  each block.
\item
  \texttt{EqtnBlk} {[} cell {]} - Cell of cellstr with equations in each
  block.
\end{itemize}

\paragraph{Description}\label{description}

The reordering algorithm first identifies equations with a single
variable in each, and variables occurring in a single equation each, and
then uses a combination of column and row approximate minimum degree
permutations (\texttt{colamd}) followed by a Dulmage-Mendelsohn
permutation (\texttt{dmperm}).

The output arguments \texttt{NameBlk} and \texttt{EqtnBlk} are 1-by-N
cell arrays, where N is the number of blocks, and each cell is a 1-by-Kn
cell array of strings, where Kn is the number of variables and equations
in block N.

\paragraph{Example}\label{example}


\section{User Guide}
This section contains descriptions of how the gravity effector code works and includes descriptions of and notes on variables. It should be helpful to users who wish to use the gravity effector module.

\subsection{Code Diagram}
The diagram in Fig. \ref{img:codeFlow} demonstrates the basic iterative logic of the gravity effector module. There is extensive additional code that deals with things from the messaging system to transforming the spacecraft position from one frame to another. In general, the inputs shown must be given for each gravity body to be considered. There are, however, convenience functions which add the standard values of these inputs for common celestial bodies. An example is simIncludeGravbody.createEarth() in the test\_scenarioOrbitManeuver.py tutorial.

After the inputs are given for each gravity body, the computeGravityInertial() method calculates the $0^{\textrm{th}}$ degree gravity term for each body and its effects on the spacecraft. If useSphericalHarmParams is True for a given body, then computeField() is called to calculate higher degree spherical harmonics terms for that body.

\begin{figure}[H]
	\centering \includegraphics[height=1.0\textwidth, keepaspectratio]{Figures/codeFlow.pdf}
	\caption{A pseudo-code diagram demonstrating the flow of inputs and outputs in the gravity effector module.}
	\label{img:codeFlow}
\end{figure}

\subsection{Variable Definition and Code Description}
The variables in Table \ref{tabular:vars} are available for user input. Variables used by the module but not available to the user are not mentioned here. Variables with default settings do not necessarily need to be changed by the user, but may be.
\begin{table}[H]
	\caption{Definition and Explanation of Variables Used.}
	\label{tab:errortol}
	\centering \fontsize{10}{10}\selectfont
	\begin{tabular}{ | m{3cm}| m{3cm} | m{3cm} | m{6cm} |} % Column formatting,
		\hline
		\textbf{Variable}   							& \textbf{LaTeX Equivalent} 	&		\textbf{Variable Type} & \textbf{Notes}			  \\ \hline
		spherHarm.maxDeg					&$l_{\text{max}}$		 	  & double & Default setting: 0"inertial\_state number of degree to use when calculating gravity effects using sperical harmonics.\\ \hline
		radEquator			   & $R_{\mathrm{ref}}^{l}$			& double & [m] Default setting: 0.0. 	This is the reference radius of the gravity body.\\ \hline
		muBody					& $\mu$ 		& double & [m3/s2] Default setting: 0.0f. This is the gravitational parameter of the body. Required Input to get any non-zero values out of the code.\\ \hline
		isCentralBody & N/A & bool & Default setting: False. Determines whether the body in question is the central body and if initial spacecraft position and velocity are determined as relative or inertial.\\ \hline
		isDisplayBody & N/A & bool & Default setting: False. Determines whether the body in question is the focus of the visualization\\ \hline
		ephemTime & N/A & double & [s] Default setting: 0. The ephemeris time for the body in question \\ \hline
		ephIntTime & N/A & double & [s] Default setting: 0. Required Input. The integration time associated with the ephem data. \\ \hline
		bodyInMsgName & N/A & string & Required Input. The name of the message with gravity body data in the body frame. \\ \hline
		outputMsgName & N/A & string & Required Input. The name of the message containing ephemeris information in display frame \\ \hline
		planetEphemName & N/A & string & Required Input. An ephemeris name for the planet (user-named). \\ \hline
		\label{tabular:vars}
	\end{tabular}
\end{table}

\subsection{Using Central Bodies and Relative Dynamics}
\subsubsection{Using Central Bodies}
In simulations with multiple planetary bodies, using dynamics relative to a central body can improve accuracy. Generally, this is the right thing to do rather than using an absolute coordinate set. If a user has a gravBody called \verb|earth|, the central body flag should be set to True.
\verb|earth.isCentralBody = True|
The dynamics will then take care of themselves, but the user needs to be careful to input initial position and velocity values as \textit{relative to} the central body. This can be input from a set of Keplerian orbital elements using \verb|orbitalMotion.elem2rv| as in\\ \verb|Basilisk/tests/scenarios/scenarioBasicOrbit.py|.

The user should be aware that if spacecraft position and velocity are read back from a message log or plotted that the absolute position and velocity will be returned. It will take additional work to convert the outputs back to a relative form by subtracting out the central body positions and velocities. No rotation will be needed, though.
It is critical that the relative position and velocities are given in a frame which is linearly translated but \textbf{not rotated} from the simulation inertial frame. There is no handling of rotated relative frames within the dynamics. The orbital element to position and velocity conversion in the section below can be used for relative dynamics inputs, as well.

\subsubsection{Not Using Central Bodies}
If no planets are designated as central bodies, an absolute initial position and velocity must be given. Again, \verb|orbitalMotion.elem2rv| can be used if the orbital elements are given in a frame not rotated from the simulation inertial frame. However, now, the initial position and velocity of the central body must be accounted for. These can be retrieved from spice via the planetStates utility:\\\\
\verb|oe = om.ClassicElements()|\\
\verb|oe.a = orbit_a * 1000 #m, orbit semi-major axis|\\
\verb|oe.e = orbit_e #eccentricity|\\
\verb|oe.i = radians(orbit_i) #inclination, radians.|\\
\verb|oe.Omega = radians(orbit_O) # orbit RAA, radians|\\
\verb|oe.omega = radians(orbit_o) #orbit argument of periapsis, radians|\\
\verb|oe.f = radians(orbit_f) # orbit true anomaly, radians|\\
\verb|r_sc_E, v_sc_E = om.elem2rv(muEarth, oe) #get xyz coordinates from keplerian elements|\\\\
\verb|ephemerides = spice_interface.SpiceInterface()|\\
\verb|ephemerides.ModelTag = "SpiceInterfaceData"|\\
\verb|ephemerides.SPICEDataPath = splitPath[0] + '../supportData/EphemerisData/'|\\
\verb|ephemerides.outputBufferCount = 2|\\
\verb|ephemerides.planetNames = spice_interface.StringVector(["earth", "sun"])|\\
\verb|ephemerides.UTCCalInit = simStart #pick a UTC string|\\
\verb|earthPos_N, earthVel_N = planetStates.planetPositionVelocity('EARTH', simStart)|\\
\verb|r_sc_N = array(r_sc_E).flatten() + array(earthPos_N).flatten()|\\
\verb|v_sc_N = array(v_sc_E).flatten() + array(earthVel_N).flatten()|\\
\verb|scObject.hub.r_CN_NInit = array(r_sc_N)|\\
\verb|scObject.hub.v_CN_NInit = array(v_sc_N)|\\\\
Of course, if a user has initial positions and velocities directly, those should be used. See scenarioCentralBody.py for a working example.\\

\subsubsection{Reference Frames}
An understanding of spice reference frames will help to explain the code above. The spice inertial frame is the ICRF. The ICRF is coplaner with the Earth's equator. Generally, the Earth Centered Inertial system one would give Keplerian elements in is aligned with ICRF. ICRF is referred to within spice as "j2000" for legacy reasons and because the J2000 system is only rotated from the ICRF by a few milliarcseconds.


\documentclass[10pt]{asme2ej}

% Edited as https://www.overleaf.com/project/5f8228e5060d380001e71b21

\usepackage{epsfig} %% for loading postscript figures
\usepackage{placeins}
\usepackage[all]{xy}
\usepackage{dsserif}
\usepackage{listings}
\usepackage{xcolor}
\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codegray}{rgb}{0.5,0.5,0.5}
\definecolor{codepurple}{rgb}{0.58,0,0.82}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}
\lstdefinestyle{mystyle}{
    backgroundcolor=\color{backcolour},
    commentstyle=\color{codegreen},
    keywordstyle=\color{magenta},
    numberstyle=\tiny\color{codegray},
    stringstyle=\color{codepurple},
    basicstyle=\ttfamily\footnotesize,
    breakatwhitespace=false,
    breaklines=true,
    captionpos=b,
    keepspaces=true,
    numbers=left,
    numbersep=5pt,
    showspaces=false,
    showstringspaces=false,
    showtabs=false,
    tabsize=2
}

\lstset{style=mystyle}

\newcommand{\catname}[1]{\textbf{#1}}
\newcommand{\Set}{\catname{Set}}
\newcommand{\Rel}{\catname{Rel}}

%% The class has several options
%  onecolumn/twocolumn - format for one or two columns per page
%  10pt/11pt/12pt - use 10, 11, or 12 point font
%  oneside/twoside - format for oneside/twosided printing
%  final/draft - format for final/draft copy
%  cleanfoot - take out copyright info in footer leave page number
%  cleanhead - take out the conference banner on the title page
%  titlepage/notitlepage - put in titlepage or leave out titlepage
%
%% The default is oneside, onecolumn, 10pt, final

\title{Intuitionistic Logic Implemented in Scala\\
Submitted for Publication at SAI Computing Conference, London, 2021}

%%% first author
\author{Vlad Patryshev}

\begin{document}
\UseRawInputEncoding

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{abstract}
{\it
An implementation of intuitionistic (Grothendieck topos-based, for a finite site) is described, with a variety of examples.

This paper demonstrates how non-Boolean logic can be implemented, to enable using all the power of Intuitionism. For that, a library was developed that works with topos logic. Sample code shows the logic for a variety of sites.
}
\end{abstract}

\section{Introduction}

 In this paper I present a Scala implementation of topos logic (that is, an intuitionistic logic). Intuitionism includes all kinds of fuzzy and temporal logic (see TLA+). Having a code package modeling Intuitionism, one can experiment with different kinds of fuzziness and different shape s of time flow. The toposes used are Grothendieck toposes, and the sites are finite.

 Two categories (toposes) are chosen over which Grothendieck toposes are built. These categories are $Set$ and $Setf$, that is, a category of all (available in the code) sets, and the category of all finite sets available in the code. Since Scala does not exactly implement sets as are defined in set theories, a couple of new classes are introduced to resolve this issue.

This paper demonstrates how non-Boolean logic can be implemented, to enable using all the power of Intuitionism. For that, a library was developed that works with topos logic. Sample code shows the logic for a variety of sites.
The code is located at
\verb+https://github.com/vpatryshev/Categories+.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Non-Boolean Logic}
Let us first introduce the idea of non-Booleanness, in a strict mathematical sense of the term.

\subsection{The Meaning of Non-Booleanness}

Traditionally, as soon as we switch from the complicated logic of the real world into the realm of formal software design, we assume that just two outcomes are possible for any statement: it is either true or false.
This is not a feature of mathematical discourse but rather an interesting feature of the modern way of thinking.
In real life, we admit that there are more possibilities: something we may not know today, we may learn tomorrow.
We may assume something to be true with a certain probability.
Similarly, in science we may not know the answer, and the answer may be something we don't expect - like with the case of Continuum Hypothesis.

In intuitionistic logic, the double negation law does not have to hold. If it does, the logic is Boolean. Here we will show a way to eliminate the double negation law while keeping the remaining rules and axioms that don't depend on double negation.
Of course, if the double negation law is included, we still have an intuitionistic logic that is also Boolean.

Also a logic may have more than two logical constants, but that does not make it non-Boolean.
The double negation rule may still be applicable in such a logic.

\subsubsection{Example 1. Boolean Logic, but not 2-Valued}

Consider bytes and their operations: bitwise conjunction, bitwise disjunction, and negation. There are 256 different values, but we know
that double negation in this bitwise logic is an identity.
So, this 256-valued logic is Boolean.\\

\subsection{Dropping Booleanness}

Remove the rule stating that either $P$ or $\neg P$ is true.
Formally, that $\vdash P \lor \neg P$. Once we remove this rule,
we cannot apply it anymore in general settings, although it may still work in some cases.

We immediately bump into a problem: this rule was used to define
implication: $P\ \rightarrow\ Q\ \equiv\ \neg\ P\ \lor\ Q$.
We must define implication differently. Here's a reasonable alternative solution:

$$(P \land Q)\vdash R \equiv P \vdash ( Q \rightarrow R)$$

Informally, saying that $R$ can be deduced from a conjunction $P\land Q$
is the same as saying that we can deduce the implication $Q\rightarrow R$ from $P$.
Imagine $Q$ is $\top$. Then, from the formula
above, it follows that $Q\rightarrow R$ is the same as R. On the other hand,
if $Q$ is closer to the bottom, the situation may change.
For example, if $Q$ is “below” $R$, the implication $Q\rightarrow R$ is true.
We can interpret the implication defined this way as the level of dependency of $R$ on $Q$.\\

As you see, we can define implication via conjunction, and negation is not involved.\\

Does this remind you currying? Given a function $f(P,Q):R$, we
transform it to a function $f_c(P): Q \rightarrow R$.\\

Now that implication is defined, we can define negation as
$\neg P = P\rightarrow \bot$. Its properties make it
similar to classical negation, but the double negation law is not generally applicable anymore. Assume we use the definition above.
In the table below, we list some properties that are valid for this definition.
\FloatBarrier
\begin{table}[!htbp]
\begin{center}
\label{table_Sample_Statements}
\begin{tabular}{c l l}
& & \\
\hline
Statement & Meaning \\
\hline
$P \land \neg P \vdash \bot$ & Negation of $P$ is incompatible with $P$.\\
  $\neg \neg \neg P \vdash \neg P$ & Triple negation is the same as single negation.\\
  $P \vdash \neg \neg P$ & Double negation of $P$ is weaker than $P$.\\
  $\neg P \lor \neg Q \vdash \neg (P \land Q)$ &  If not $P$ or not $Q$, then we can’t have both $P$ and $Q$\\
  $\neg (P \lor Q) \vdash \neg P \land \neg Q$ &  If disjunction of $P$ and $Q$ is not true, then neither $P$ nor $Q$ is true.\\
  $\neg P \land \neg Q \vdash \neg (P \lor Q)$ & If neither $P$ nor $Q$ is true, their disjunction is not true.\\
\hline
\end{tabular}
\end{center}
\end{table}

Some properties do not apply in this kind of logic:
\begin{table}[!htbp]
\begin{center}
\label{table_Sample_Wrong_Statements}
\begin{tabular}{c l l}
& & \\
\hline
Wrong Statement & What’s wrong with it? \\
\hline
$\neg \neg P \vdash P$   &                     Negation of negation of $P$ is not strong enough to give us $P$.\\

  $\neg (P \land Q) \vdash \neg P\lor \neg Q$ &   Even if we can’t have both at the same time,
  this does not mean that one of them is always wrong.\\
\hline
\end{tabular}
\end{center}
\end{table}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Example 2. Three Logical Values (“Ternary Logic”)}

Start with three logical values, $\top$, $\bot$, and $?$. Define operations for them:

\begin{center}
\begin{tabular}{ |c|c|c| }
\hline
$\textbf{x}$ & $\textbf{\neg x}$ & $\textbf{\neg \neg x}$ \\
\hline
$\top$ & $\bot$ & $\top$ \\
$?$    & $\bot$ & $\top$ \\
$\bot$ & $\top$ & $\bot$ \\
  \hline
\end{tabular}
\end{center}

\begin{center}
\begin{tabular}{ |c|c|c|c|c| }
\hline
$\textbf{x}$ & $\textbf{y}$ & ${\textbf{x} {\land} \textbf{y}}$  & $\textbf{x} \lor \textbf{y}$ & $\textbf{x} \rightarrow \textbf{y}$   \\
\hline
$\top$ & $\top$ & $\top$ & $\top$ & $\top$ \\
$\top$ &   $?$ &    $?$ &  $\top$ & $?$ \\
$\top$ & $\bot$ & $\bot$ & $\top$ & $\bot$ \\
 $?$ &  $\top$ &   $?$ &  $\top$ & $\top$ \\
 $?$ &    $?$ &    $?$ &    $?$ & $\top$ \\
 $?$ &  $\bot$ & $\bot$ &   $?$ & $\bot$ \\
$\bot$ & $\top$ & $\bot$ & $\top$ & $\top$ \\
$\bot$ &   $?$ &  $\bot$ &   $?$ & $\top$ \\
$\bot$ & $\bot$ & $\bot$ & $\bot$ & $\top$ \\
  \hline
\end{tabular}
\end{center}

This is the simplest intuitionistic logic, and it is a good tool for testing our statements.

The value ‘?,’ or ‘unknown,’ is somewhere between ‘truth’ $\top$ and ‘false’ $\bot$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Example 3. Infinite Number of Logical Values}

Take $[0,1]$, the set of all real numbers between $0$ and $1$, including
$0$ and $1$. We can turn it into logic by defining:

-   $\bot = 0$

-   $\top =1$

-   $a \land b = glb(a,b)$

-   $a \lor b = lub(a,b)$

In this example, implication is defined, as always, via the equivalence:
$(x~\land~y)~\leq~z~\equiv~x~\leq~(y~\rightarrow~z)$.

We consider the following two cases separately:

First, when $y\leq z$,
$(x~\land~y)~\leq~z$ is always true, for any $x$. Hence $y~\rightarrow~z$
cannot be smaller than any $x$, that is, it must be the top element, $1$.

In the opposite case, when $y$ is bigger than $z$ (we are talking about numbers),
$( x \land y \leq z) \equiv ( x \leq z )$, so now we will have
$x~\leq~z~\equiv~x~\leq~(y~\rightarrow~z)$, which means that $y~\rightarrow~z~=~z$.

Remember that we defined negation, $\neg x$, as $x\rightarrow 0$, and we have
$\neg x\ \equiv\ if\ (x = 0)\ 1\ else\ 0$. You can also check whether double
negation maps $0$ to $0$ and any non-zero value to $1$.

In this example, we started with a set of values that was popular in "fuzzy logic," and we were able to build
a pretty sound logic, except that it could not be made Boolean: double negation is not an identity.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Sets and Categories}
 \\

\subsection{Sets}
 \\
Scala programming language uses a pretty loose notion of sets; they have too many methods (like \verb+head()+), and functions are not defined as "special sets of pairs", as they are defined in ZFC. So we had to introduce a special class for working with sets. Class $BigSet$ is the most general class defining sets; it does not even have to be enumerable.

\begin{lstlisting}[language=Scala]
abstract class BigSet[T] extends Set[T] {
 override def size: Int = Sets.InfiniteSize
...
\end{lstlisting}

This abstract "container" covers all possible sets.
Remember, sets are not typed (in a set theory).

\begin{lstlisting}[language=Scala]
object Sets {
 type set = Set[Any]

 implicit class untype[T](s: Set[T]) {
   def untyped: set = s.asInstanceOf[set]
 }

 def isFinite(s: Set[_]) Boolean = s.size != InfiniteSize
 val Empty: set = Set.empty[Any]
 val Unit: set = Set(Empty)
 val FiniteSets = BigSet.comprehension(isFinite)
 ...
\end{lstlisting}

We can now define non-enumerable sets (thus banning all those Scala/Java methods that enumerate):

\begin{lstlisting}[language=Scala]
trait NonEnumerableSet[T] extends Set[T] {
  private def notEnumerable = throw Stone(...)
  override def isEmpty: Boolean = notEnumerable
  def iterator: Iterator[T] = notEnumerable
  override def toArray[S >: T : ClassTag] = notEnumerable
}
\end{lstlisting}

These classes will be used to properly define Grothendieck toposes over sites.

\section{Categories}

Before defining \verb+Category+, let's define \verb+Graph+:

\begin{lstlisting}[language=Scala]
trait Graph { graph =>
  type Node
  type Arrow
  def nodes: Set[Node]
  def arrows: Set[Arrow]
  def d0(f: Arrow): Node
  def d1(f: Arrow): Node
...
\end{lstlisting}

\verb+trait Graph+ implements a pretty big number of useful methods, but what's important is that it specifies \verb+Node+s, \verb+Arrow+s, and their relationships.\\

Having a type of \verb+Graph+s, we can define a \verb`Category` in Scala like this:

\begin{lstlisting}[language=Scala]
abstract class Category(override val name: String) extends Graph {
  type Obj = Node
  def id(o: Obj): Arrow
  def m(f: Arrow, g: Arrow): Option[Arrow]  // composition

  lazy val op: Category = {
    val src = this
    new Category(s"~$name") {
      override def id(o: Obj): Arrow = src.id(o)
      override def m(f: Arrow, g: Arrow) = src.m(f, g)
    }
  }
\end{lstlisting}

Note that composition of arrows is a partial function lifted to \verb+Option+.\\

A category does not necessarily have to be finite: here is an example of an infinite one:

\begin{lstlisting}[language=Scala]
object N extends BigSet[BigInt] with EnumerableSet[BigInt] {
  override def iterator: Iterator[BigInt] = new Iterator[BigInt] {...}
  override def contains(n: BigInt): Boolean = n >= 0
}
object PoSet {
  lazy val ofNaturalNumbers: PoSet[BigInt] = new PoSet(N, comparator) {...}
}
lazy val NaturalNumbers: Category = fromPoset("N", PoSet.ofNaturalNumbers)
\end{lstlisting}

We should never attempt to materialize such a category. \\

That was a countable category; we can even build an uncountable one:

\begin{lstlisting}[language=Scala]
class SetCategory(objects: BigSet[Set[Any]])
  extends Category(name="Sets", graphOfSets(objects)) {
  type Node = set
  type Arrow = SetFunction

  override def d0(f: SetFunction): set = f.d0
  override def d1(f: SetFunction): set = f.d1
  override def m(f: Arrow, g: Arrow): Option[Arrow] = f compose g
  override def id(s: set): SetFunction = SetFunction.id(s)
  override def toString: String = "Category of All Sets"
...
object Setf extends SetCategory(FiniteSets) {
  override def toString: String = "Category of Finite Sets"
}
\end{lstlisting}

But what is \verb+SetFunction+?

\begin{lstlisting}[language=Scala]
case class SetFunction(
  override val tag: String,
  override val d0: set,
  override val d1: set,
  mapping: Any => Any) extends SetMorphism { self =>
...
def compose(g: SetFunction): Option[SetFunction] =
  if (d1 == g.d0) {
    Some(new SetFunction(newTag, d0, g.d1, (x: Any) => g(self(x))))
  } else None
\end{lstlisting}

\subsection{Samples of Categories}
Below are some example of categories, built using these tools:

\FloatBarrier
\begin{table}[!htbp]
\begin{center}
\label{table_Sample_Categories}
\begin{tabular}{|c|c|l|}
\hline
$\mathbb{ 0 }$ & &
\begin{lstlisting}[language=Scala]
val _0_ : Cat = segment(0)
\end{lstlisting}
\\
\hline
$\mathbb{ 1 } $ & • &
\begin{lstlisting}[language=Scala]
val _1_ : Cat = segment(1)
\end{lstlisting}
\\
\hline
$\mathbb{1}+\mathbb{1}$ & • • &
\begin{lstlisting}[language=Scala]
val _1plus1_ : Cat = discrete(Set("a", "b"))
\end{lstlisting}
\\
\hline
$\mathbb{2}$ &
\xymatrix{0 \ar[r] & 1}
&
\begin{lstlisting}[language=Scala]
val _2_ : Cat = segment(2)
\end{lstlisting}
\\
\hline
$\mathbb{3}$ &
\xymatrix{&1 \ar[dr] \\
0 \ar[ur] \ar[rr] && 2  }
&
\begin{lstlisting}[language=Scala]
val _3_ : Cat = segment(3)
\end{lstlisting}
\\
\hline
$\mathbb{4}$ &
\xymatrix{
&1 \ar[drr] \ar[r] & 2 \ar[dr]\\
0 \ar[ur] \ar[urr] \ar[rrr] &&& 3
}
&
\begin{lstlisting}[language=Scala]
val _4_ : Cat = segment(4)
\end{lstlisting}
\\
\hline
$ParallelPair$ &
\xymatrix{
0 \ar@<2pt>[r]^a \ar@<-2pt>[r]_b & 1
}
&
\begin{lstlisting}[language=Scala]
category"ParallelPair:({0,1}, {a:0->1, b:0->1})"
\end{lstlisting}
\\
\hline
$Pullback$ &
\xymatrix{
& b \ar[d]^{bd} \\
c \ar[r]_{cd} & d
}
&
\begin{lstlisting}[language=Scala]
category"Pullback:({b,c,d}, {bd:b->d, cd:c->d})"
\end{lstlisting}
\\
\hline
$Square$ &
\xymatrix{
a \ar[d]_{ac} \ar[r]^{ab} \ar[dr]|{ad} & b \ar[d]^{bd} \\
c \ar[r]_{cd} & d
}
&
\begin{lstlisting}[language=Scala]
category"Square:({a,b,c,d},
{ab:a->b, ac:a->c, bd:b->d, cd:c->d, ad:a->d},
{bd o ab = ad,cd o ac = ad})"
\end{lstlisting}
\\
\hline
\end{tabular}
\end{center}
\end{table}

\subsection{Functors and Natural Transformations}

Definitions are trivial:

\begin{lstlisting}[language=Scala]
abstract class Functor(
  val d0: Category, val d1: Category
) extends Morphism {
  def objectsMapping(x: d0.Obj): d1.Obj
  def arrowsMapping(a: d0.Arrow): d1.Arrow
  def compose(next: Functor): Option[Functor] = {
    if (this.d1 != next.d0) None else Some { //...
\end{lstlisting}

\begin{lstlisting}[language=Scala]
abstract class NaturalTransformation
  extends Morphism[Functor, Functor] {

  val domain: Category = d0.d0
  val codomain: Category = d1.d1 // == d0.d1

  def transformPerObject(x: domain.Obj): codomain.Arrow
\end{lstlisting}

In the picture, you see a natural transformation for category "Pullback":

\xymatrix{
& F[b] \ar[d] \ar@{.>}[drr] \\
F[c] \ar[r] \ar@{.>}[drr] & F[d] \ar@{.>}[drr] && G[b]  \ar[d]\\
&&G[c] \ar[r] &G[d]
}

\section{Copresheaves (aka Diagrams)}

\subsection{Definition}
A $Copresheaf$, also known as $Diagram$, of category \catname{C} in category \catname{D} is a functor from \catname{C} to \catname{D}.

We will, further on, be dealing with copresheaves in \Set, and call them $diagrams$.

\begin{lstlisting}[language=Scala]
abstract class Diagram(val domain: Category)
  extends Functor(tag, topos.domain, Setf) { diagram =>

  def isElementOf(other: Diagram): Boolean =
    d0.objects.forall { o => other(o) contains (this(o)) }

  def isContainedIn(other: Diagram): Boolean =
    d0.objects.forall { o => this(o) subsetOf other(o) }
\end{lstlisting}

\subsection{Example 5, Diagrams over $\mathbb{N}$}
A functor $\mathbb{N} \rightarrow \textbf{Sets}$ consists of sets $F[i]$ and functions $F[i]$→$F[i+1]$. (Other functions are compositions of these).
These diagrams model discreet “time flow” : discrete, \cite{lamp}.
That’s the basic type of TLA+ book.

\xymatrix{
F[0] \ar[r] &F[1] \ar[r] &F[2] \ar[r] &{...}
}\\

Below is a table of some shapes of time that we encounter in practice.

\FloatBarrier

\begin{table}[!htbp]
\begin{center}
\label{table_Sample_Diagram Categories}
\begin{tabular}{|c|l|}
%\hline
%$\textbf{Site}$ & $\textbf{Concept}$ \\
\hline
\xymatrix{
F[0] \ar[r] &F[1] \ar[r] &F[2] \ar[r] &{...}
} & TLA+ \\
\hline
$\bullet$ & Just sets\\
\hline
\xymatrix{0 \ar[r] & 1}
& Single transition (today and tomorrow)
\\
\hline
\xymatrix{
& b \ar[d]^{bd} \\
c \ar[r]_{cd} & d
}
& Git Merge
\\
\hline
\xymatrix{
a \ar[d]_{ac} \ar[r]^{ab} \ar[dr]|{ad} & b \ar[d]^{bd} \\
c \ar[r]_{cd} & d
}
& Eventual Consistency
\\
\hline
\xymatrix{
{Feb1} \ar[r] & {Feb 2} \ar@(ur,ul)[]_{6:00} \ar[r] & {Feb 3}
}
& Groundhog day
\\
\hline
\end{tabular}
\end{center}
\end{table}

\FloatBarrier

\section{Logic in Grothedieck Topos}

Now that we have a category of diagrams (it's a Grothendieck topos), we can dive deeper and build logic in it. We will be using points of objects to illustrate logic. Generally speaking, a Grothendieck topos is not well-pointed, though.

\subsection{Point of a Diagram}
A point is any arrow \xymatrix{1 \ar[r] & D}, where $1$ is a terminal object of the category of diagrams.

\begin{lstlisting}[language=Scala]
class Point(val tag: Any, val mapping: Any => Any)
\end{lstlisting}

\subsection{Subobject Classifier}
This is a special object in a category of diagrams contains all the logic of a topos; we just have to discover it.

\xymatrix{&A \ar@{>->}[d]_f \ar[r] & 1 \ar@{>->}[d]^{true} \\
&B \ar[r]_{\chi_f} & \Omega
}

How can we build it?

$\Omega(x) \equiv hom(hx, \Omega)$ - by Yoneda lemma

$hom(hx, \Omega)  \equiv {p \subset hx}$ - by definition of $\Omega$, see above.

\begin{lstlisting}[language=Scala]
case class Representable(x: domain.Obj)
  extends Diagram(s"hom($x, _)", domain)
//...

object Omega extends Diagram("Omega", this, domain) {
  val False: Point = points.head named "false"
  val True: Point = points.last named "true"
  val conjunction: DiagramArrow = ... // OmegaxOmega(x) => Omega(x)
  val disjunction: DiagramArrow = ... // ΩxΩ(x) => Omega(x)
al implication: DiagramArrow = ... // ΩxΩ(x) => Omega(x)
  ...
}
val OmegaxOmega = product2(Omega, Omega)

val DiagonalOfOmega: DiagramArrow =
    buildArrow("Diagonal", Omega, OmegaxOmega, _ => (s: Any) => (s, s)

\end{lstlisting}

\subsubsection{How to Build Conjunction in $\Omega$}

\xymatrix{
&1 \ar@{>->}[d]_{(true,true)} \ar[rr] && 1 \ar@{>->}[d]^{true} \\
& \Omega\times\Omega \ar[rr]_{\land} && \Omega
}

\subsubsection{How to Build Implication in $\Omega$}

Start with building a subobject $\Omega_1$ defining a partial order on $\Omega$:

\xymatrix{
\Omega_1 \ar@{>->}[r] & \Omega\times\Omega \ar@<1ex>[r]^{∧} \ar@<-1ex>[r]_{p_1}
& \Omega
}

Now, implication is the arrow that classifies $\Omega_1$:

\xymatrix{
 \Omega_1 \ar[d] \ar[rr] && 1 \ar[d]^{true}  \\
 \Omega \times \Omega \ar[rr]_{implication} && \Omega
}

\begin{lstlisting}[language=Scala]
lazy val implication: DiagramArrow = {
  val inclusion: DiagramArrow = inclusionOf(Omega_1) in OmegaxOmega iHope

  classifies(inclusion, "=>")
}
\end{lstlisting}

\subsection{Predicate Logic}

First, what is a predicate in a topos? It's just any arrow
$D\ \rightarrow\ \Omega$.\\

Logical operations on $\Omega$ provide operations on predicates:

\begin{lstlisting}[language=Scala]
trait Predicate extends DiagramArrow { p: DiagramArrow =>
  val d0: Diagram
  val d1: Diagram = Omega
  def and(q: Predicate): Predicate = binaryOp(q, "&", Omega.conjunction)
  def or(q: Predicate): Predicate = binaryOp(q, "|", Omega.disjunction)
  def ==>(q: Predicate): Predicate = binaryOp(q, "=>", Omega.implication)
  ...
}

def not(p: Predicate): Predicate = p ==> FalsePredicate
\end{lstlisting}

As an example, here's a test case from a unittest for this code:

\begin{lstlisting}[language=Scala]
def check(c: Category): MatchResult[Any] = {
 val topos = new CategoryOfDiagrams(c)
 import topos._

  for { p <- Omega.points map (_.asPredicate) } {
    (True  ==> p)    === p
    (False ==> p)    === True
    (p     ==> p)    === True
    (p     ==> True) === True
    not(not(not(p))) === not(p)

    for { q <- Omega.points map (_.asPredicate)
          r <- Omega.points map (_.asPredicate) } {
        (p and q ==> r) === (p ==> (q ==> r))
      }
    }
}
\end{lstlisting}

\xymatrix{}

\begin{lstlisting}[language=Scala]
\end{lstlisting}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
Neither Scala nor Java was designed to deal with problems like these: typeless sets, categories, infinite collections.
Also, the higher order type system in Scala 2 is not very convenient for
the problems like the ones I encountered implementing categories and toposes; dependent types would make more sense.
There's a hope that Scala 3 will help dealing with it properly.
Strangely, implementing it in Haskell was even more challenging.

In addition to intuitionistic logic, Lawvere topologies can be implemented based on this package; and the implementation is now available in the same repository.
Using an older version of this software, we managed to enumerate all Lawvere topologies over finite sets.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% The bibliography is stored in an external database file
% in the BibTeX format (file_name.bib).  The bibliography is
% created by the following command and it will appear in this
% position in the document. You may, of course, create your
% own bibliography by using the bibliography environment as in
%
% \begin{thebibliography}{12}
% ...
% \bibitem{itemreference} D. E. Knudsen.
% {\em 1966 World Bnus Almanac.}
% {Permafrost Press, Novosibirsk.}
% ...
% \end{thebibliography}

% Here's where you specify the bibliography style file.
% The full file name for the bibliography style file
% used for an ASME paper is asmems4.bst.
\bibliographystyle{asmems4}

% Here's where you specify the bibliography database file.
% The full file name of the bibliography database for this
% article is asme2e.bib. The name for your database is up
% to you.
\bibliography{london2021}

\cite{bartosz, pierce, tt, sheaves, brief, nlab1, repo, ctgd}

\end{document}


\documentclass[]{AVSSimReportMemo}
\usepackage{AVS}
\usepackage{colortbl}

\newcommand{\ModuleName}{inertial3D}
\newcommand{\subject}{Guidance Module to Perform an Inertially Fixed Pointing}
\newcommand{\status}{Initial Version}
\newcommand{\preparer}{M. Cols}
\newcommand{\summary}{Generate the reference attitude trajectory for a general 3D inertial pointing.  A corrected body frame will align with the desired reference frame.     }

\begin{document}

\makeCover

%
%	enter the revision documentation here
%	to add more lines, copy the table entry and the \hline, and paste after the current entry.
%
\pagestyle{empty}
{\renewcommand{\arraystretch}{2}
\noindent
\begin{longtable}{|p{0.5in}|p{4.5in}|p{1.14in}|}
\hline
{\bfseries Rev}: & {\bfseries Change Description} & {\bfseries By} \\
\hline
Draft & initial copy & M. Cols \\
\hline

\end{longtable}
}

\newpage
\setcounter{page}{1}
\pagestyle{fancy}

\tableofcontents
~\\ \hrule ~\\

\section{Module Input and Output}
Table \ref{tab:inputTable} shows the input Configuration Data of the module Inertial 3D Point.
\begin{table}[h!]
	\centering
	\caption{Input Configuration Data}
	\begin{tabular}{|l|l|l|p{3in}|}
		\hline
		\rowcolor{BrickRed}
		\textcolor{white}{Name} & \textcolor{white}{Type} &
		\textcolor{white}{Length} &
		\textcolor{white}{Description}  \\ \hline
		$\sigma_{R_0/N}$ & double [] & 3 &
		MRP attitude set of the desired reference frame with respect to the inertial frame . \\ \hline
	\end{tabular}
	\label{tab:inputTable}
\end{table}

Table \ref{tab:outputTable} shows the Attitude Reference output message of the module Inertial 3D Point.
\begin{table}[h!]
	\centering
	\caption{Output Attitude Reference Message}
	\begin{tabular}{|l|l|l|p{3in}|}
		\hline
		\rowcolor{BrickRed}
		\textcolor{white}{Name} & \textcolor{white}{Type} &
		\textcolor{white}{Length} &
		\textcolor{white}{Description}  \\ \hline
		$\sigma_{R/N}$ & double [] & 3 &
		MRP attitude set of the reference frame with respect to the inertial frame. \\ \hline
		$\leftexp{N} \omega_{R/N}$ & double [] & 3 &
		Angular rate vector of the reference frame with respect to the inertial expressed in inertial frame components. \\ \hline
		$\leftexp{N} {\dot{\omega}_{R/N}}$ & double [] & 3 &
		Angular acceleration vector of the reference frame with respect to the inertial expressed in inertial frame components. \\ \hline
	\end{tabular}
	\label{tab:outputTable}
\end{table}
\newpage

\section{Introduction}
This technical note discusses the guidance mathematics to compute a reference frame $\mathcal{R}$ that is aligned the with an inertially fixed  frame  $\mathcal{R}_0$, as shown in Figure~\ref{fig:Fig1}.
\begin{figure}[htb]
	\centerline{
	\includegraphics{Figures/Fig3}
	}
	\caption{Illustration of the input inertially fixed frame $\mathcal{R}_{0}:\{ \hat e_{1}, \hat e_{2}, \hat e_{3} \}$, the generated reference frame $\mathcal{R}: \{ \hat r_{1}, \hat r_{2}, \hat r_{3}\}$ and the inertial frame $\mathcal{N}:\{ \hat{\bm n}_{1}, \hat{\bm n}_{2}, \hat{\bm n}_{3} \}$}
	\label{fig:Fig1}
\end{figure}

\section{Reference Frame Generation}
The modules requires the desired reference orientation in terms of the MRP set $\bm{\sigma}_{R_{0}N}$. This input is only set once and does not have to be changed.
Let us designate $\mathcal{R}$ as the output generated reference frame. Since the fixed-pointing is inertial:
\begin{equation}
	\bm{\sigma}_{RN} = \bm{\sigma}_{R_{0}N}
\end{equation}
\begin{equation}
	\bm{\omega}_{RN} = \dot{\bm{\omega}}_{RN} = 0
\end{equation}

\bibliographystyle{unsrt}
\bibliography{references}

\end{document}


    \filetitle{bb}{IRIS serial date number for bimonthly date}{dates/bb}

	\paragraph{Syntax}\label{syntax}

\begin{verbatim}
Dat = bb(Y)
Dat = bb(Y,B)
\end{verbatim}

\paragraph{Input arguments}\label{input-arguments}

\begin{itemize}
\item
  \texttt{Y} {[} numeric {]} - Years.
\item
  \texttt{B} {[} numeric {]} - Bimonth; if omitted, first bimonth
  (January-February) is assumed.
\end{itemize}

\paragraph{Output arguments}\label{output-arguments}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{Dat} {[} numeric {]} - IRIS serial date numbers representing
  the bimonthly date.
\end{itemize}

\paragraph{Description}\label{description}

\paragraph{Example}\label{example}


\subsection{Euler's theorem}


\documentclass{article}
\usepackage{graphicx}
\usepackage{titletoc}
\usepackage{titlesec}
\usepackage{geometry}
\usepackage{fontspec, xunicode, xltxtra}
\usepackage{float}
\usepackage{cite}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{listings}
\usepackage{titletoc}
\usepackage{bm}
\usepackage{multirow}

\geometry{left=3cm,right=3cm,top=3cm,bottom=3cm}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\expec}{E}

\begin{document}
\title{\textsf{Course Project for Pattern Recognition}}
\author{Fan JIN\quad (2015011506)}
\maketitle

\tableofcontents
\clearpage

\section{Introduction and Preprocessing}
{
    Here is a glance at the datasets.
    Columns with only one unique value are removed, as they have no effect as observations.
    The training set is roughly 9 times as much size as the testing set.

    \begin{table}[!hbp]
        \centering
        \begin{tabular}{|c|c|c|c|}
        \hline
        File & sample size (nrow) & number of features (ncol) & number features in effect \\
        \hline
        2ctrainX.txt & 2298 & 25133 & 17951 \\
        \hline
        2ctestX.txt & 256 & 25133 & 17951 \\
        \hline
        10ctrainX.txt & 16074 & 25133 & 24307 \\
        \hline
        10ctestX.txt & 1787 & 25133 & 24307 \\
        \hline
        \end{tabular}
        \caption{Shape of datasets}
    \end{table}

    \begin{itemize}
        \item For two-category classification, the response ``2ctrainY.txt'' takes values 1 and 2.
        \item For ten-category one, the response ``10ctrainY.txt'' takes values ranging from 1 to 10.
    \end{itemize}
}

\section{Two-category classification}
{
    \subsection{Feature selection}
    {
        Feature selection is neccesary in this case, for we have much more features than data points, referred to as the ``large p, small n'' problem.

        Methods of feature selection include:
        \begin{itemize}
            \item Non-wrapping: Fisher-based
            \item Non-wrapping: Correlation-based
            \item Non-wrapping: Entropy-based
            \item Non-wrapping: Statistical t-distribution
            \item Non-wrapping: Principal Component Analysis (PCA)
            \item Wrapping: Foreward-backward recursive
            \item Stochastic: Genetic algorithm
        \end{itemize}

        Here we use non-wrapping methods for feature selection. We chose Fisher-based and PCA.

        \subsubsection{Fisher criterion}
        {
            The fisher criterion is defined as
            $$F = \frac{(\mu_1 - \mu_2)^2}{s_1^2 + s_2^2},$$
            where $\mu_1$, $\mu_2$ are the means of two categories, and $s_1^2$, $s_2^2$ are the varainces, in terms of some feature.

            The larger the Fisher criterion, the more significant the feature is. We select 2000 features with the largest Fisher criteria.
        }

        \subsubsection{the PCA}
        {
            The Principal Component Analysis performs eigendecomposition to the correlation matrix of the data points, and retains the components with principal eigenvalues.

            We retain 99\% of total variance in raw data, which gives $2202$ features in total.
        }
    }

    \subsection{Classifier}
    {
        \subsubsection{Linear SVM}
        {
            The support vector machine, or SVM, is a powerful method for both classification and regression, particularly in case of ``large p, small n''.

            It is crucial to choose a kernel that balances between prediction accuracy and overfitting aversion.
            We tried linear, quadratic, and RBF kernels. The linear kernel turns out the best.
        }

        \subsubsection{Fully connected NN}
        {
            The neural network has advantages over the linear SVM in case of nonlinearity.

            Since we have only limited data points, we cannot train a deep network.
            We took a simple perceptron network, which has one hidden layer with 1500 nodes.
            We tuned the hidden layer size, and $1000$ proves the best hyperparameter.
        }
    }

    \subsection{Experiments}
    {
        \begin{table}[!hbp]
            \centering
            \begin{tabular}{|c|c|c|c|c|c|c|c|}
            \hline
            Dim. Reduction & Classifier & \multicolumn{5}{|c|}{5-fold cross validation} & Avg. Accuracy \\
            \hline
            Fisher & Linear SVM & 0.9848 & 0.9934 & 0.9869 & 0.9869 & 0.9956 & 0.9896 \\
            \hline
            PCA & Linear SVM & 0.9544 & 0.9434 & 0.9629 & 0.9586 & 0.9498 & 0.9539 \\
            \hline
            Fisher & FC NN & 0.9891 & 0.9978 & 0.9891 & 1.0000 & 0.9956 & 0.9943 \\
            \hline
            PCA & FC NN & 0.8828 & 0.8652 & 0.8823 & 0.9019 & 0.8758 & 0.8816 \\
            \hline
            \end{tabular}
            \caption{Comparison}
        \end{table}

        The Fisher criterion has better performance in dimensionality reduction, compared with the PCA.

        It is worth noting that the neural network does not always perform better than the SVM.
        It does when we use Fisher criterion for dimensionality reduction, while it does not when the PCA is used.
        This could be explained by the weak performance of the PCA, as it performs a linear transformation on the raw data.
        The data might not be normally distributed, and thus the correlation matrix fails to identify all the principal components.

        We tuned the hyperparamters, and found the best network size is around 1000. This attains high accuracy while avoiding overfitting.

        \begin{table}[!hbp]
            \centering
            \begin{tabular}{|c|c|c|c|c|c|c|}
            \hline
            Size of hidden layer & \multicolumn{5}{|c|}{5-fold cross validation} & Avg. Accuracy \\
            \hline
            200 & 0.9934 & 0.9978 & 0.9934 & 0.9847 & 0.9934 & 0.9925 \\
            \hline
            500 & 0.9891 & 0.9978 & 0.9891 & 1.0000 & 0.9934 & 0.9939 \\
            \hline
            1000 & 0.9891 & 0.9978 & 0.9891 & 1.0000 & 0.9956 & 0.9943 \\
            \hline
            1500 & 0.9891 & 0.9978 & 0.9891 & 1.0000 & 0.9956 & 0.9943 \\
            \hline
            \end{tabular}
            \caption{Finding the proper network size}
        \end{table}
    }

    \subsection{Prediction}
    {
        Since it has the highest accuracy in 5-fold cross validation, the fully connected NN with Fisher-based dimensionality reduction is chosen for prediction.
        The prediction is stored in ``2ctestY.txt'' in the root directory.
    }
}

\section{Ten-category classification}
{
    \subsection{Feature selection}
    {
        The dataset is too large for SVD (singular value decomposition), so it would be diffucult to perform full PCA on it.
        We tried using ``arpack'' instead, but the script ran out of memory on a GPU server.

        Considering this, only Fisher criterion was used in this task.
        We chose 2000 features out of 24307, and trained the reduced dataset using SVM and FCNN.
    }

    \subsection{Experiments}
    {
        \begin{table}[!hbp]
            \centering
            \begin{tabular}{|c|c|c|c|c|c|c|c|}
            \hline
            Dim. Reduction & Classifier & \multicolumn{5}{|c|}{5-fold cross validation} & Avg. Accuracy \\
            \hline
            Fisher & Linear SVM & 0.9916 & 0.9878 & 0.9850 & 0.9903 & 0.9875 & 0.9884 \\
            \hline
            Fisher & FC NN & 0.9891 & 0.9950 & 0.9937 & 0.9925 & 0.9934 & 0.9927 \\
            \hline
            \end{tabular}
            \caption{Comparison}
        \end{table}

        The fully connected NN attains higher accuracy as the linear SVM does.

        \begin{table}[!hbp]
            \centering
            \begin{tabular}{|c|c|c|c|c|c|c|}
            \hline
            Size of hidden layer & \multicolumn{5}{|c|}{5-fold cross validation} & Avg. Accuracy \\
            \hline
            200 & 0.9934 & 0.9925 & 0.9875 & 0.9940 & 0.9931 & 0.9921 \\
            \hline
            500 & 0.9546 & 0.9928 & 0.9931 & 0.9934 & 0.9866 & 0.9841 \\
            \hline
            1000 & 0.9953 & 0.9944 & 0.9944 & 0.9937 & 0.9937 & 0.9943 \\
            \hline
            1500 & 0.9959 & 0.9940 & 0.9944 & 0.9947 & 0.9940 & 0.9946 \\
            \hline
            \end{tabular}
            \caption{Finding the proper network size}
        \end{table}

        The optimal hyperparameter for the hidden layer size is around 1000.

        Compared with two-category classification, the ten-category one attains slightly lower accuracy in training and cross validation.

    }

    \subsection{Prediction}
    {
        Since it has the highest accuracy in 5-fold cross validation, the fully connected NN with Fisher-based dimensionality reduction is chosen for prediction.
        The prediction is stored in ``10ctestY.txt'' in the root directory.
    }
}

\section{Visualization and Clustering}
{
    \subsection{PCA}
    {
        \begin{figure}[H]
            \centering
            \includegraphics[width = 1\linewidth]{./src3/pca.png}
            \caption{Dimensionality reduction using PCA}
        \end{figure}
    }

    \subsection{t-SNE}
    {
        \begin{figure}[H]
            \centering
            \includegraphics[width = 1\linewidth]{./src3/tsne.png}
            \caption{Dimensionality reduction using t-SNE}
        \end{figure}
    }

    \subsection{Clustering based on t-SNE}
    {
        It is astouding to find the result of PCA, although with good linearity, does not split up all the 10 categories in the 2d space.
        Only one category is distributed away from others. In comparison, the t-SNE works fine. We will use t-SNE for clustering.

        Since the number of categories is known, k-means seems a good choice. We fix $k=10$.
        The birch clustering algorithm is also tested. The Birch algorithm turns out better than K-means.

        \begin{table}[!hbp]
            \centering
            \begin{tabular}{|c|c|}
            \hline
            Method & Accuracy \\
            \hline
            K-means & 0.6926 \\
            \hline
            Birch & 0.7343 \\
            \hline
            \end{tabular}
            \caption{Comparison}
        \end{table}

        \begin{figure}[H]
            \centering
            \includegraphics[width = 0.9\linewidth]{./src3/kmeans.png}
            \includegraphics[width = 0.9\linewidth]{./src3/birch.png}
            \caption{K-means (top); Birch (bottom)}
        \end{figure}
    }
}

\section{Source Code}
{
    \subsection{Root directory}
    {
        \begin{itemize}
            \item ``2ctestY.txt'': Two-category prediction. Use this file for evaluation.
            \item ``10ctestY.txt'': Ten-category prediction. Use this file for evaluation.
            \item ``report.pdf'': Project report.
        \end{itemize}
    }

    \subsection{``data1'' directory}
    {
        \begin{itemize}
            \item ``2ctrainX.txt'': Raw dataset.
            \item ``2ctrainY.txt'': Raw dataset.
            \item ``2ctestY.txt'': Raw dataset.
            \item ``data1.npz'': Preprocessed file. (invalid columns removed)
        \end{itemize}
    }

    \subsection{``data2'' directory}
    {
        \begin{itemize}
            \item ``10ctrainX.txt'': Raw dataset.
            \item ``10ctrainY.txt'': Raw dataset.
            \item ``10ctestY.txt'': Raw dataset.
            \item ``data2.npz'': Preprocessed file. (invalid columns removed)
        \end{itemize}
    }

    \subsection{``src1'' directory}
    {
        \begin{itemize}
            \item ``preproc.py'': Preprocessing, removing identical columns.
            \item ``feature\_selection'': Feature selection, generating ``fisher\_2000.npz'' and ``pca\_99.npz''.
            \item ``SVM.py'': Training a SVM and use it for prediction.
            \item ``FCNN'': Training a fully connected NN and use it for prediction.
        \end{itemize}
    }

    \subsection{``src2'' directory}
    {
        \begin{itemize}
            \item ``preproc.py'': Preprocessing, removing identical columns.
            \item ``feature\_selection'': Feature selection, generating ``fisher\_2000.npz''.
            \item ``SVM.py'': Training a SVM and use it for prediction.
            \item ``FCNN'': Training a fully connected NN and use it for prediction.
        \end{itemize}
    }

    \subsection{``src3'' directory}
    {
        \begin{itemize}
            \item ``pca.py'': Dimensionality reduction by PCA, generating ``pca.npz''.
            \item ``tsne.py'': Dimensionality reduction by t-SNE, generating ``tsne.npz''.
            \item ``visualization'': Visualization after loading ``pca.pna'' and ``tsne.npz''.
            \item ``kmeans.py'': Clustering by K-means.
            \item ``birch.py'': Clustering by Birch.
        \end{itemize}
    }
}

\clearpage
\end{document}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{pgfplotstable}
\pgfplotsset{compat=1.15}
\usepackage{fancyvrb}

\title{}
\author{}
\date{}

\begin{document}

\maketitle

\section{Introduction}

Using our DenseNet-BC we have approximately $634k$ 32FP-ops and $626k$ FP32-params. This gives us a score of approximately (rounding up) $0.078$ in total, where for the ops, we obtain $0.06$ as our score, and for the parameters, we obtain $0.018$ as our score.

To test a pre-trained network and check the score, just run profile.py. To train a network, just run train.py and then re-run profile.py.

We describe our methods and computations in the following:

\section{Data Augmentation}
\label{data_aug}

We first use standard Data augmentation such as flip and random crop. In addition, we propose to add other different data augmentations. We use CutMix with $\alpha = 1$ for all training dataset, and we drop $64$ pixels randomly for each input training image.

\section{Architecture}

We use a DenseNet-BC architecture with $K = 8$, depth $D = 196$ and reduction $R = 0.5$.

\section{Computing the Number of Parameters}

To get the number of parameters of the DenseNet-BC architecture, we compute the number of parameters of each layer (or each Bottleneck layer that contains 2 Batch norm layers, 2 Convolutional layers and 2 Relu), as described in Table~\ref{table:params}

\begin{table}[h]
    \centering
    \begin{tabular}{c|c}
    Layer     & Number of parameters  \\
    \hline
    Convolution     &      $F_{in}F_{out}S_c^2$ \\
    \hline
    Batch norm & $4F_{in}$ \\
    \hline
    $Bottleneck_i$ & $ 4[F_{in} + K(4 + F_{in} + i + K(1 + i)]$\\
    \hline
    Linear & $F_{in}F_{out}$\\
    \hline
    Transition & $F_{in}(4 + RF_{in})$ \\
    \hline
    \end{tabular}
    \caption{The number of parameters for each layer. Note that $F_{in}$ represents the number of input feature maps, $F_{out}$ represents the number of output feature maps, $S_c^2$ represents the kernel size of a convolutional operation, $i$ represents the depth of the bottleneck inside a block that contains $\frac{D-4}{6}$ bottleneck layers.}
    \label{table:params}
\end{table}

\section{Computing the Number of Operations}

to get the number of operations required to process one input through the  DenseNet-BC network, we compute the number of operations of each layer (or each Bottleneck layer that contains 2 Batch norm layers, 2 Convolutional layers and 2 Relu), as described in Table~\ref{table:Flops}

\begin{table}[h]
    \centering
    \begin{tabular}{c|c}
    Layer     & Number of operations  \\
    \hline
    Convolution     &  $[\frac{F_{in}F_{out}S_c^2}{2} + (F_{in}S_c^2 -1)F_{out}]P $ \\
    \hline
    Batch norm & $3F_{in}P$ \\
    \hline
    Relu & $F_{in}P$\\
    \hline
    Average pooling & $S_p^2PF_{in} $ \\
    \hline
    Linear & $1.5F_{in}F_{out}$ \\
    \hline
    Transition & $4F_{in}P + [\frac{F_{in}F_{out}S_c^2}{2} + (F_{in}S_c^2 -1)F_{out}]P + S_p^2PF_{out}$ \\
    \hline
    $Bottleneck_i$ & Batch norm1 + Relu1 + Batch norm2 + Relu2 + Conv1 + Conv2\\
    & Batch norm1 = $3P(F_{in} + iK)$\\
    & Relu1 = $P(F_{in} + iK)$\\
    & Batch norm2 = $12P_{in}K$\\
    & Relu2 = $4P_{in}K$\\
    & Conv1 = $P[\frac{(F_{in} + iK)4K}{2} + (F_{in} + ik -1)4K]$\\
    & Conv2 = $[\frac{36K^2}{2} + (36K-1)K]P$\\
    \hline

    \hline

    \hline
    \end{tabular}
    \caption{The number of operations for each layer. Note that $F_{in}$ represents the number of input feature maps, $F_{out}$ represents the number of output feature maps, $S_c^2$ represents the kernel size of a convolutional operation, $S_p^2$ represents the kernel size of an average pooling operation, $i$ represents the depth of the bottleneck inside a block that contains $\frac{D-4}{6}$ bottleneck layers, $P$ represents the number of pixels of an input feature map.}
    \label{table:Flops}
\end{table}

\section{Training procedure}

We train the network with the data augmentation described on Section~\ref{data_aug} for 200 epochs, using a batch size of 32 and a softmax temperature of 6. Our learning rate starts at 0.1 and is divided by 10 at epochs 100 and 150. We then fine-tune the network by training it for an extra 5 epochs without data-augmentation.

\newpage

\input{architecture.tex}

\end{document}