% !TEX root = ParticleFilter.tex
\section{Experimental Results\label{experiments}}

\subsection{Outline of the Experiments}

Recall that the aim of the paper is to \aim. By examining the error (i.e. the difference between the particle filter estimate of the system state and the pseudo-truth data) under different conditions, the paper will present some preliminary estimates of the potential for the use of particle filtering in real-time crowd simulation. In particular, the paper will estimate the minimum number of particles ($N_p$) that are needed to model a system that has a given number of agents ($N_a$). Although the dynamics of the model used here are simpler than those of more advanced crowd models, and hence the particle filter is tasked with an easier problem than it would have to solve in a real-world scenario, these experiments provide valuable insight into the general \textit{potential} of the method for crowd simulation.

It is well known that one of the main drawbacks with a particle filter is that the number of particles required can explode as the complexity of the system increases~\citep[e.g.][]{snyder_obstacles_2008, carrassi_data_2018}. In this system, the complexity of the model is determined by the number of agents. As outlined in Section~\ref{method}, randomness is introduced through agent interactions.  Therefore the fewer the number of agents, the smaller the chance of interactions occurring and the lower the complexity. As the number of agents increases,we find that the number of interactions increases exponentially (Figure~\ref{fig:collision_count}, discussed in the following section, will illustrate this). The paper will also experiment with the amount of particle noise that needs to be included to reduce the problem of particle deprivation ($\sigma_p$), but this is not a focus of the experiments.

%Figure~\ref{fig:PF_flowchart} outlined the modelling process. The particle filter runs in parallel to a single instance of the StationSim model, which is used to create pseudo-truth data (observations) representing the the locations of each agent at a given iteration. In each data assimilation `window', the truth model and all particles are stepped forward a number of iterations. At the end of the window the particle filter receives pseudo-true observations and uses these to calculate the particle weights (i.e. the error associated with each particle).

To quantify the `success' of each parameter configuration -- i.e. the number of particles $N_p$ and amount of particle noise $\sigma_p$ that allow the particle filter to reliably represent a system of $N_a$ agents -- an estimate of the overall error associated with a particular particle filter configuration is required. There are a number of different ways that this error could be calculated. Here, we calculate the mean weight of each particle after resampling in every data assimilation window (the weights vary during an experiment because in early windows there are few agents and hence very little stochasticity so particle errors are low). Then we take the mean of these individual particle errors. In addition, because the results from an individual experiment can vary slightly, each particle configuration is executed a number of times, $M$, and the median error across experiments is calculated.

Formally if $\nu(t,j)$ is the error of particle $j$ in a data assimilation window $t$, and the total number of windows is $T$, then the total error of that particle filter configuration across $M$  experiments is:
\begin{equation}
  E_{N_p,\sigma_p,N_a} =  {\rm median}_M \left( \sum_{j=1}^{N_p}  \frac{\sum_{t=1}^{T}\nu(t)}{T} \right)
\end{equation}
where $median_M()$ calculates the median over $M$ experiments.

In summary, the Table~\ref{tab:experiment_parameters} outlines the parameters that are used in the experiments. There are a number of other model and particle filter parameters that can vary, but ultimately these do not influence the results outlined here and are not experimented with. The parameters, and code, are available in full from the project repository \cite{stationsimgit}.
% \begin{quote}\url{XXXX github url}\end{quote}

\begin{table*}[ht] \caption{Main parameters used in the experiments}
	\begin{center}
		\begin{tabular}{l l l }
		    \hline Parameter & Symbol & Value / Range \\
			\hline
			Number of agents & $N_a$ & $[ 2, 40 ]$ \\
			Number of particles & $N_p$ & $[ 1, 10000 ]$ \\
			Particle noise & $\sigma_p $ & $[0.25, 0.5]$ \\
			Measurement noise & $\sigma_m$ &  $1.0$ \\
			Number of experiments (repetitions) & M & 20 \\
			Model iterations in each data assimilation window & - & 100 \\
			\hline
	\end{tabular} \end{center} \label{tab:experiment_parameters}
\end{table*}%

\subsection{Results}

\subsubsection{Overall Error}

Figure~\ref{fig:median_abs_error} plots the median of the mean error over all experiments to show how it varies with differing numbers of agents and particles. Due to the computational difficulty in running experiments with large numbers of particles and the need for an exponential increase in the number of particles with the number of agents (discussed in detail below), there are fewer experiments with large numbers of particles and hence the the experiments are not distributed evenly across the agents/particles space. Thus the error is presented in the form of an interpolated heat map. Broadly there is a reduction in error from the bottom-right corner (few particles, many agents) to the top left corner (many particles, few agents). The results illustrate that, as expected, there is a larger error with increasing numbers of agents but this can be mediated with larger numbers of particles. Note the logarithmic scale used on the vertical axis; an exponentially greater number of particles is required for each additional agent included in the simulation.

\begin{figure}[ht]
	\centering
	\includegraphics[width=0.7\textwidth]{figures/results/median_abs_error}
	\caption{Median of the mean errors after resampling} \label{fig:median_abs_error}
\end{figure}

There are two reasons for this exponential increase in the number of particles required. Firstly, as the number of agents increases so does the dimensionality of the state space. Also, and perhaps more importantly, with additional agents the chances of collisions, and hence stochastic behaviour, increases exponentially. This is illustrated by Figure~\ref{fig:collision_count} which presents the total number of collisions that occur across a number of simulations with a given number of agents.

\begin{figure}[ht]
	\centering
	\includegraphics[width=0.7\textwidth]{figures/results/wiggle_count}
	\caption{The total number of collisions that occur given the number of agents. The blue line represents a polynomial regression model of order 2 with 99\% confidence intervals.} \label{fig:collision_count}
\end{figure}

On its own, the overall particle filter error (Figure~\ref{fig:median_abs_error}) reveals little information about which configurations would be judged `sufficiently reliable' to be used in practice. Therefore it is illuminating to visualise some of the results of individual particle filter runs to see how the estimates of the individual agents' locations vary, and what might be considered an `acceptable' estimate of the true state. Figure~\ref{fig:ani-10agents-10particles} illustrates the state of a particle filter with 10 particles and 10 agents at the end of its first data assimilation window (after resampling). With only ten agents in the system there are few, if any, collisions and hence very little stochasticity; all particles are able to represent the locations of the agents accurately.

\begin{figure}[ht]
	\centering
	\includegraphics[width=0.45\textwidth]{figures/results/ani-10agents-10particles-window100}
	\includegraphics[width=0.45\textwidth]{figures/results/ani-10agents-10particles-window200}
	\caption{State of a particle filter with 10 agents and 10 particles after 100 and 200 iterations (after resampling at the end of the first and second data assimilation windows). Black squares illustrate the (pseudo) real locations of the agents and red circles represent the locations of those agents as predicted by individual particles (in this case all particles predict the agents' locations accurately and hence the red circles overlap).} \label{fig:ani-10agents-10particles}
\end{figure}

As the number of agents increases, collisions become much more likely and the chance of a particle diverging from the pseudo-truth state increases considerably. It becomes more common that no single particle will correctly capture the behaviours of \textit{all} agents. Therefore even after resampling there are some agents whose locations the particle filter is not able to accurately represent. Figure~\ref{fig:ani-50agents-10particles} illustrates a filter running with 40 agents and still only 10 particles. The long black lines show the locations of pseudo-real agents and the corresponding agents in the individual particles; it is clear that for some agents none of the particles have captured their pseudo-real locations. This problem can be mediated by increasing the number of particles. Figure~\ref{fig:median_abs_error} showed that with approximately 10,000 particles the error for simulations with 40 agents drops to levels that are comparable to the simulations of 10 agents. Hence a rule of thumb is that any particle filter with an overall error that is comparable to a clearly successful filter (i.e. Figure~\ref{fig:ani-10agents-10particles}) are reliably estimating the state of the system.

\begin{figure}[ht]
	\centering
	\includegraphics[width=0.45\textwidth]{figures/results/ani-40agents-10particles-window100}
	\includegraphics[width=0.45\textwidth]{figures/results/ani-40agents-10particles-window200}
	\caption{State of a particle filter with 50 agents and 10 particles. Lines connecting black squares (the pseudo-real agent locations) to red circles (particle estimate of the agent location) show that some particle estimates are a long way from the true agent locations.} \label{fig:ani-50agents-10particles}
\end{figure}

\subsubsection{Impact of Resampling}

Resampling is the process of weighting all particles according to how well they represent the pseudo-truth data; those with higher weights are more likely to be sampled and used in the following data assimilation window. This is important to analyse because it is the means by which the particle filter improves the overall quality of its estimates. Figure~\ref{fig:resampling} illustrates the impact of resampling on the error of the particle filter. With fewer than 10 agents in the system resampling is unnecessary because all particles are able to successfully model the state of the system. With more than 10 agents, however, it becomes clear that the population of particles will rapidly diverge from the pseudo-truth system state and resampling is essential for the filter manage to limit the overall error.

\begin{figure}[ht]
	\centering
	\includegraphics[width=0.9\textwidth]{figures/results/resampling}
	\caption{Median of the mean errors before and after resampling}\label{fig:resampling}
\end{figure}

\subsubsection{Particle Variance}

As discussed in Section~\ref{particle_sampling}, the variance of the population of particles can be a reliable measure for estimating whether particle deprivation is occurring. If there is very little variance in the particles, then it is likely that they have converged close to a single point in the space of possible states. This needs to be avoided because, in such situations, it is extremely unlikely that any particles will reliably represent the state of the real system. Figure~\ref{fig:variance_results} illustrates this by visualising the mean error of all particles, $\nu(t)$ (defined in Equation~\ref{eqn:particle_error}), in each data assimilation window, $t$, and their variance under different numbers of agents and particles. Note that each agent/particle configuration is executed 10 times and the results are visualised as boxplots. Also, simulations with larger numbers of agents are likely to run for a larger number of iterations, but the long-running models usually have very few agents in them in later iterations (most have left, leaving only a few slow agents). Hence only errors up to 600 iterations, where most of the agents have left the environment in most of the simulations, are shown. The graphs in Figure~\ref{fig:variance_results} can be interpreted as follows:

\begin{figure}[ht]
	\centering
	\includegraphics[width=0.9\textwidth]{figures/results/variance_results}
	\caption{Mean error and variance of particles under different combinations of the number of particles and number of agents}\label{fig:variance_results}
\end{figure}

\begin{itemize}
    \item With \textbf{5 agents and 10 particles} there is very low error and low variance (so low that the box plots are difficult to make out on Figure~\ref{fig:variance_results}). This suggests that particle deprivation is occurring, but there are so few agents that the simulation is largely deterministic so the particles are likely to simulate the pseudo-truth observations accurately regardless.
    \item When the number of agents is increased to \textbf{30 agents and 10 particles}, the errors are much larger. The increased non-linearity introduced by the greater number of agents (and hence greater number of collisions) means that the population of particles, as a whole, is not able to simulate the pseudo-truth data. Although particle variance can be high, none of the particles are successfully simulating the target.
    \item Finally, with \textbf{30 agents and 10,000 particles}, the errors are relatively low in comparison, especially after the first few data assimilation windows.
\end{itemize}

\subsubsection{Particle Noise}

The number of particles ($N_p$) is the most important hyper-parameter, but the amount of noise added to the particles ($\sigma_p$) is also important as this is the means by which particle deprivation is prevented. However, the addition of too much noise will push a particle a long way away from the true underlying state. Under these circumstances, although particle deprivation is unlikely none of the particles will be close to the true state. To illustrate this, Figure~\ref{fig:median_abs_error-noise2.0} presents the median error with a greater amount of noise ($\sigma_p=0.5$). The overall pattern is similar to the equivalent graph produced with $\sigma_p=0.25$ in Figure~\ref{fig:median_abs_error} -- as the number of agents in the simulation ($N_a$) increases so does the number of particles required to maintain low error ($N_p$)  -- but the overall error in each $N_a$/$N_p$ combination is substantially larger for the experiments with additional noise. Future work could explore the optimal amount of noise in more detail; $\sigma=0.25$ was found to be the most reliable through trial and error.

\begin{figure}[ht]
	\centering
	\includegraphics[width=0.5\textwidth]{figures/results/median_abs_error-noise2_0}
	\caption{Median of the mean errors after resampling with additional noise ($\sigma=0.5$).} \label{fig:median_abs_error-noise2.0}
\end{figure}


\section{Introduction}

% notion of realism, the things that we want to show. Say that there is no
% single way on how to describe water and that it is all approximations. Talk
% about the PhD thesis on the water rendering

Although water covers most of the earth's surface\footnote{Approximately 71\%
\autocite{howard2016how}} it still remains an open problem in both mathematics
and physics. There is no single solution available and approximations have to be
made. The topic is broad and complex involving subjects ranging form fast
Fourier transforms to Lagrangian and Eulerian simulations. The purpose of this
report is to present interesting water models we found in the literature and
explain how we implemented one. This document also gives our vision for further
work, namely which model should be used or avoided. In order to read this
document the reader should be familiar with the rendering pipeline and its
programmable parts. More than enough background information can be found in
\autocite{RTR3}.

We begin this report by presenting the different water models and candidate
applications in \autoref{sec:water_models}. Then we explain how we implemented
the \textit{Gerstner Wave} model in \autoref{sec:simple_gerstner_waves} and
provide performance and visual results in
\autoref{sec:performance_visual_results}. These are analyzed and discussed in
\autoref{sec:discussion}. Finally we conclude this report and give our vision
for further work in \autoref{sec:conclusion}.

\autoref{fig:anno} shows an example of an unknown real-time water model
implemented in the upcoming game \textit{Anno 1800}, developed by Blue Byte.

\begin{figure}[ht]
    \centering
    \includegraphics[height=5cm]{anno_1800.jpg}
    \caption{Harbour view from \textit{Anno 1800}. Source:
    \href{https://forum.quartertothree.com/t/anno-1800-city-building-in-the-industrial-revolution/131265/12}{\url{forum.quartertothree.com}}}\label{fig:anno}
\end{figure}


\chapter{Shape-from-Shading}
\label{ch:sfs}

ASP provides a tool, named \texttt{sfs}, that can improve the level of
detail of DEMs created by ASP or any other source using
\textit{shape-from-shading} (SfS). The tool takes as input one or more
camera images, a DEM at roughly the same resolution as the images, and
returns a refined DEM.

\texttt{sfs} works only with ISIS cub images. It has been tested
thoroughly with Lunar LRO NAC datasets, and some experiments were done
with Mars HiRISE images and with pictures from Charon, Pluto's moon. As
seen later in the text, it returns reasonable results on the Moon as far
as $85^\circ$ and even $89.6^\circ$ South.

Currently, \texttt{sfs} is computationally expensive, and is practical
only for DEMs whose width and height are several thousand pixels. It can be
sensitive to errors in the position and orientation of the cameras, the
accuracy of the initial DEM, and to the value of the two weights it uses.
Yet, with some effort, it can work quite well.

A tool named \texttt{parallel\_sfs} is provided (section \ref{psfs})
that parallelizes \texttt{sfs} using multiple processes (optionally on
multiple machines) by splitting the input DEM into tiles with padding,
running \texttt{sfs} on each tile, and then blending the results.

The \texttt{sfs} program can model position-dependent albedo, different
exposure values for each camera, shadows in the input images, and regions
in the DEM occluded from the Sun. It can refine the positions and orientations
of the cameras.

The tool works by minimizing the cost function
\begin{equation}\label{cost}
  % \begin{multline}\label{cost}
  \int\!\! \int \! \sum_k \left[ I_k(\phi)(x, y) - T_k A(x, y)
    R_k(\phi)(x, y) \right]^2\,
  % R_k(\phi)(x, y) \right]^2\,  \\
  + \mu \left\|\nabla^2 \phi(x, y) \right\|^2
  + \lambda  \left[ \phi(x, y) - \phi_0(x, y) \right]^2
  \, dx\, dy.
\end{equation}
% \end{multline}

Here, $I_k(\phi)(x, y)$ is the $k$-th camera image interpolated at
pixels obtained by projecting into the camera 3D points from the terrain
$\phi(x, y)$, $T_k$ is the $k$-th image exposure, $A(x, y)$ is the
per-pixel albedo, $R_k(\phi)(x, y)$ is the reflectance computed from the
terrain for $k$-th image, $\left\|\nabla^2 \phi(x, y) \right\|^2 $ is the sum
of squares of all second-order partial derivatives of $\phi$, $\mu > 0$
is a smoothing term, and $\lambda > 0$ determines how close we should
stay to the input terrain $\phi_0$ (smaller $\mu$ will show more detail
but may introduce some artifacts, and smaller $\lambda$ may allow for
more flexibility in optimization but the terrain may move too far from the input).

We use either the regular Lambertian reflectance model,
or the Lunar-Lambertian model \cite{mcewen1991photometric}, more specifically as given in
\cite{lohse2006derivation} (equations (3) and (4)).
Also supported is the Hapke model,
\cite{johnson2006spectrophotometric}, \cite{fernando2013surface},
\cite{hapke2008bidirectional}, \cite{hapke1993opposition}.
Custom values for the coefficients of these models can be passed to the program.

\section{How to get good test imagery}

We obtain the images from
\url{http://wms.lroc.asu.edu/lroc/search} (we search for EDR images of
type NACL and NACR).

A faster (but not as complete) interface is provided by
\url{http://ode.rsl.wustl.edu/moon/indexproductsearch.aspx}. The related
site
\url{http://ode.rsl.wustl.edu/moon/indextools.aspx?displaypage=lolardr}
can provide LOLA datasets which can be used as (sparse) ground truth.

We advise the following strategy for picking images. First choose a small
longitude-latitude window in which to perform a search for imagery. Pick
two images that are very close in time and with a big amount of overlap
(ideally they would have consecutive orbit numbers).
Those can be passed to ASP's \texttt{stereo} tool to create an initial DEM. Then, search for
other images close to the center of the maximum overlap of the first
two images. Pick one or more of those, ideally with different illumination
conditions than the first two. Those (together with one of the first two images) can be used for SfS.

To locate the area of spatial overlap, the images can be map-projected (either
with \texttt{cam2map} with a coarse resolution) or with
\texttt{mapproject}, using for example the LOLA DEM as the terrain to project onto,
or the DEM obtained from running \texttt{stereo} on those images. Then the images can be overlayed
in \texttt{stereo\_gui}. A good sanity check is to examine the shadows in
various images. If they point in different directions in the images and perhaps
also have different lengths, that means that illumination conditions are
different enough, which will help constrain the \texttt{sfs} problem better.

\section{Running sfs at 1 meter/pixel using a single image}

In both this and the next sections we will work with LRO NAC images taken
close to the Lunar South Pole, at a latitude of $85^\circ$ South (the tool was
tested on equatorial regions as well). We will use four images,
M139939938LE, M139946735RE, M173004270LE, and M122270273LE.

We first retrieve the data sets.
\begin{verbatim}
  wget http://lroc.sese.asu.edu/data/LRO-L-LROC-2-EDR-V1.0/\
LROLRC_0005/DATA/SCI/2010267/NAC/M139939938LE.IMG
  wget http://lroc.sese.asu.edu/data/LRO-L-LROC-2-EDR-V1.0/\
LROLRC_0005/DATA/SCI/2010267/NAC/M139946735RE.IMG
  wget http://lroc.sese.asu.edu/data/LRO-L-LROC-2-EDR-V1.0/\
LROLRC_0009/DATA/SCI/2011284/NAC/M173004270LE.IMG
  wget http://lroc.sese.asu.edu/data/LRO-L-LROC-2-EDR-V1.0/\
LROLRC_0002/DATA/MAP/2010062/NAC/M122270273LE.IMG
\end{verbatim}

Then we convert them to ISIS cubes, initialize the SPICE kernels, and
perform radiometric calibration and echo correction. Here are the steps,
illustrated on the first image:
\begin{verbatim}
  lronac2isis from = M139939938LE.IMG     to = M139939938LE.cub
  spiceinit   from = M139939938LE.cub
  lronaccal   from = M139939938LE.cub     to = M139939938LE.cal.cub
  lronacecho  from = M139939938LE.cal.cub to = M139939938LE.cal.echo.cub
\end{verbatim}
We rename, for simplicity, the obtained four processed datasets to
A.cub, B.cub, C.cub, and D.cub.

The first step is to run stereo to create an initial guess DEM. We
picked for this the first two of these images. These form a stereo pair,
that is, they have a reasonable baseline and sufficiently close times of
acquisition (hence very similar illuminations). These conditions are
necessary to obtain a good stereo result.
\begin{verbatim}
parallel_stereo --job-size-w 1024 --job-size-h 1024 A.cub B.cub    \
                --left-image-crop-win 0 7998 2728 2696             \
                --right-image-crop-win 0 9377 2733 2505            \
                --threads 16 --corr-seed-mode 1  --subpixel-mode 3 \
                run_full1/run
\end{verbatim}
Next we create a DEM at 1 meter/pixel, which is about the resolution
of the input images. We use the stereographic projection since this
dataset is very close to the South Pole. Then we crop it to the region
we'd like to do SfS on.
\begin{verbatim}
  point2dem -r moon --stereographic --proj-lon 0 \
    --proj-lat -90 run_full1/run-PC.tif
  gdal_translate -projwin -15471.9 150986 -14986.7 150549  \
    run_full1/run-DEM.tif run_full1/run-crop-DEM.tif
\end{verbatim}
This creates a DEM of size $456 \times 410$ pixels.

Then we run \texttt{sfs}:
\begin{verbatim}
  sfs -i run_full1/run-crop-DEM.tif A.cub -o sfs_ref1/run           \
     --reflectance-type 1                                           \
    --smoothness-weight 0.08 --initial-dem-constraint-weight 0.0001 \
    --max-iterations 10 --use-approx-camera-models                  \
    --crop-input-images
\end{verbatim}

The smoothness weight is a parameter that needs tuning. If it is too
small, SfS will return noisy results, if it is too large, too much
detail will be blurred. Here we used the Lunar Lambertian model. The
meaning of the other \texttt{sfs} options can be looked up in section
\ref{sfs}.

We show the results of running this program in figure
\ref{fig:sfs1}. The left-most figure is the hill-shaded original DEM,
which was obtained by running:
\begin{verbatim}
  hillshade --azimuth 300 --elevation 20 run_full1/run-crop-DEM.tif \
    -o run_full1/run-crop-hill.tif
\end{verbatim}
The second image is the hill-shaded DEM obtained after running
\texttt{sfs} for 10 iterations.

The third image is, for comparison, the map-projection of A.cub onto the
original DEM, obtained via the command:
\begin{verbatim}
  mapproject --tr 1 run_full1/run-crop-DEM.tif A.cub A_map.tif --tile-size 128
\end{verbatim}
The forth image is the colored absolute difference between the original
DEM and the SfS output, obtained by running:
\begin{verbatim}
  geodiff --absolute sfs_ref1/run-DEM-final.tif run_full1/run-crop-DEM.tif
  colormap --min 0 --max 2 --colormap-style binary-red-blue \
    run-DEM-final__run-crop-DEM-diff.tif
\end{verbatim}
\begin{figure}[h!]
  \begin{center}
    \includegraphics[width=7in]{images/sfs1.jpg}
    \caption[sfs]{An illustration of \texttt{sfs}. The images are, from
      left to right, the original hill-shaded DEM, the hill-shaded DEM obtained
      from \texttt{sfs}, the image A.cub map-projected onto the original DEM,
      and the absolute difference of the original and final DEM, where the brightest
      shade of red corresponds to a 2 meter height difference.}
    \label{fig:sfs1}
  \end{center}
\end{figure}

It can be seen that the optimized DEM provides a wealth of detail and
looks quite similar to the input image. It also did not diverge
significantly from the input DEM. We will see in the next section that
SfS is in fact able to make the refined DEM more accurate than the
initial guess (as compared to some known ground truth), though that is
not guaranteed, and most likely did not happen here where just one image
was used.

\section{SfS with multiple images in the presence of shadows}

In this section we will run \texttt{sfs} with multiple images. We would
like to be able to see if SfS improves the accuracy of the DEM rather
than just adding detail to it. We evaluate this using the following
(admittedly imperfect) approach. We resample the original images by a
factor of 10, run stereo with them, followed by SfS using the stereo
result as an initial guess and with the resampled images. As ground
truth, we create a DEM from the original images at 1 meter/pixel, which
we bring closer to the initial guess for SfS using
\texttt{pc\_align}. We would like to know if running SfS brings us even
closer to this ``ground truth'' DEM.

The most significant challenge in running SfS with multiple images is
that shape-from-shading is highly sensitive to errors in camera position
and orientation. The \texttt{sfs} tool can improve these by floating
them during optimization and by using a coarse-to-fine scheme, where
the problem is first solved using subsampled images and terrain then
it is successively refined.

If possible, it may still be desirable to bundle-adjust the cameras first
(section \ref{bundleadjust}). It is important to note that bundle adjustment may
fail if the images have sufficiently different illumination, as it will
not be able to find matches among images. A solution to this is discussed
in section \ref{sfs-lola}.

To make bundle adjustment and stereo faster, we first crop the images,
such as shown below (the crop parameters can be determined via
\texttt{stereo\_gui}).
\begin{verbatim}
  crop from = A.cub to = A_crop.cub sample = 1 line = 6644 nsamples = 2192 nlines = 4982
  crop from = B.cub to = B_crop.cub sample = 1 line = 7013 nsamples = 2531 nlines = 7337
  crop from = C.cub to = C_crop.cub sample = 1 line = 1 nsamples = 2531 nlines = 8305
  crop from = D.cub to = D_crop.cub sample = 1 line = 1 nsamples = 2531 nlines = 2740
\end{verbatim}
Then we bundle-adjust and run stereo
\begin{verbatim}
  bundle_adjust A_crop.cub B_crop.cub C_crop.cub D_crop.cub    \
    --min-matches 1 -o run_ba/run
  stereo A_crop.cub B_crop.cub run_full2/run --subpixel-mode 3 \
    --bundle-adjust-prefix run_ba/run
\end{verbatim}

This will result in a point cloud, \verb#run_full2/run-PC.tif#, which will
lead us to the ``ground truth'' DEM. As mentioned
before, we'll in fact run SfS with images subsampled by a factor of
10. Subsampling is done by running the ISIS \texttt{reduce} command
\begin{verbatim}
  for f in A B C D; do
    reduce from = ${f}_crop.cub to = ${f}_crop_sub10.cub sscale = 10 lscale = 10
  done
\end{verbatim}

We run bundle adjustment and stereo with the subsampled images using
commands analogous to the above:
\begin{verbatim}
  bundle_adjust A_crop_sub10.cub B_crop_sub10.cub C_crop_sub10.cub D_crop_sub10.cub \
    --min-matches 1 -o run_ba_sub10/run --ip-per-tile 100000
 stereo A_crop_sub10.cub B_crop_sub10.cub run_sub10/run --subpixel-mode 3           \
   --bundle-adjust-prefix run_ba_sub10/run
\end{verbatim}
We'll obtain a point cloud named \verb#run_sub10/run-PC.tif#.

We'll bring the ``ground truth'' point cloud closer to the initial guess
for SfS using \texttt{pc\_align}:
\begin{verbatim}
  pc_align --max-displacement 200 run_full2/run-PC.tif run_sub10/run-PC.tif \
    -o run_full2/run --save-inv-transformed-reference-points
\end{verbatim}
This step is extremely important. Since we ran two bundle adjustment
steps, and both were without ground control points, the resulting clouds
may differ by a large translation, which we correct here. Hence we would like to
make the ``ground truth'' terrain aligned with the datasets on which we
will perform SfS.

Next we create the ``ground truth'' DEM from the aligned high-resolution
point cloud, and crop it to a desired region:
\begin{verbatim}
  point2dem -r moon --tr 10 --stereographic --proj-lon 0 --proj-lat -90 \
    run_full2/run-trans_reference.tif
  gdal_translate -projwin -15540.7 151403 -14554.5 150473               \
    run_full2/run-trans_reference-DEM.tif run_full2/run-crop-DEM.tif
\end{verbatim}
We repeat the same steps for the initial guess for SfS:
\begin{verbatim}
  point2dem -r moon --tr 10 --stereographic --proj-lon 0 --proj-lat -90 \
    run_sub10/run-PC.tif
  gdal_translate -projwin -15540.7 151403 -14554.5 150473               \
    run_sub10/run-DEM.tif run_sub10/run-crop-DEM.tif
\end{verbatim}
After this, we run \texttt{sfs} itself. Since our dataset has many shadows, we found
that specifying the shadow thresholds for the tool improves the
results. The thresholds can be determined using \texttt{stereo\_gui}.
This can be done by turning on shadow-threshold mode from the GUI menu,
and then clicking on a few points in the shadows.  Then the thresholded images
can be visualized/updated from the menu as well, and this process can be iterated.
\begin{verbatim}
  sfs -i run_sub10/run-crop-DEM.tif A_crop_sub10.cub C_crop_sub10.cub \
    D_crop_sub10.cub -o sfs_sub10_ref1/run --threads 4                \
    --smoothness-weight 0.12 --initial-dem-constraint-weight 0.0001   \
    --reflectance-type 1 --float-exposure                             \
    --float-cameras --use-approx-camera-models                        \
    --max-iterations 10  --crop-input-images                          \
    --bundle-adjust-prefix run_ba_sub10/run                           \
    --shadow-thresholds "0.00162484 0.0012166 0.000781663"
\end{verbatim}
We compare the initial guess to \texttt{sfs} to the ``ground truth'' DEM
obtained earlier and the same for the final refined DEM using
\texttt{geodiff} as in the previous section. Before SfS:

\begin{verbatim}
  geodiff --absolute run_full2/run-crop-DEM.tif run_sub10/run-crop-DEM.tif
  gdalinfo -stats run-crop-DEM__run-crop-DEM-diff.tif | grep Mean=
\end{verbatim}
and after SfS:
\begin{verbatim}
  geodiff --absolute run_full2/run-crop-DEM.tif sfs_sub10_ref1/run-DEM-final.tif
  gdalinfo -stats run-crop-DEM__run-DEM-final-diff.tif | grep Mean=
\end{verbatim}

The mean error goes from 2.64~m to 1.29~m, while the standard deviation
decreases from 2.50~m to 1.29~m. Visually the refined DEM looks more detailed
as well as seen in figure \ref{fig:sfs2}. The same experiment can be
repeated with the Lambertian reflectance model (reflectance-type 0), and
then it is seen that it performs a little worse.

We also show in this figure the first of the images used for SfS,
\verb#A_crop_sub10.cub#, map-projected upon the optimized DEM. Note that
we use the previously computed bundle-adjusted cameras when
map-projecting, otherwise the image will show as shifted from its true
location:
\begin{verbatim}
  mapproject sfs_sub10_ref1/run-DEM-final.tif A_crop_sub10.cub A_crop_sub10_map.tif \
    --bundle-adjust-prefix run_ba_sub10/run
\end{verbatim}
\begin{figure}[h!]
  \begin{center}
    \includegraphics[width=7in]{images/sfs2.jpg}
    \caption[sfs]{An illustration of \texttt{sfs}. The images are, from
      left to right, the hill-shaded initial guess DEM for SfS, the hill-shaded DEM obtained
      from \texttt{sfs}, the ``ground truth'' DEM, and the first of the
      images used in SfS map-projected onto the optimized DEM.}
    \label{fig:sfs2}
  \end{center}
\end{figure}

\section{Dealing with large camera errors and LOLA comparison}
\label{sfs-lola}

SfS is very sensitive to errors in camera positions and
orientations. These can be optimized as part of the problem, but if they
are too far off, the solution will not be correct. In the previous
section we used bundle adjustment to correct these errors, and then we
passed the adjusted cameras to \texttt{sfs}. However, bundle adjustment may
often fail, simply because the illumination conditions can be very
different among the images, and interest point matching may not succeed.

The option \texttt{-\/--coarse-levels \it{int}} can be passed to
\texttt{sfs}, to solve for the terrain using a multi-resolution
approach, first starting at a coarse level, where camera errors have
less of an impact, and then jointly optimizing the cameras and
the terrain at ever increasing levels of resolution. Yet, this may still
fail if the terrain does not have large and pronounced features on the
scale bigger than the errors in the cameras.

The approach that we found to work all the time is to find interest points
in the images, first automatically, and if that fails then manually, as
the human eye is much more skilled at identifying a given landmark in
multiple images, even when the lightning changes drastically. Picking
about 4 landmarks in each image (if doing it manually) is
sufficient. Ideally they should be positioned far from each other, to
improve the accuracy.

Below is one example of how we select interest points, run SfS,
and then compare to LOLA, which is an independently acquired
sparse dataset of 3D points on the Moon. According to
\cite{smith2011results}, the LOLA accuracy is on the order of 1~m. To
ensure a meaningful comparison of stereo and SfS with LOLA, we resample
the LRO NAC images by a factor of 4, making them nominally 4
m/pixel. This is not strictly necessary, the same exercise can be
repeated with the original images, but it is easier to see the
improvement due to SfS when comparing to LOLA when the images are
coarser than the LOLA error itself.

We work with the same images as before. They are resampled as follows:
\begin{verbatim}
  for f in A B C D; do
    reduce from = ${f}_crop.cub to = ${f}_crop_sub4.cub sscale=4 lscale=4
  done
\end{verbatim}

The \texttt{stereo} and \texttt{point2dem} tools are run to get a first-cut DEM. Bundle adjustment is not done at this stage yet.
\begin{verbatim}
  stereo A_crop_sub4.cub B_crop_sub4.cub run_stereo_noba_sub4/run --subpixel-mode 3
  point2dem --stereographic --proj-lon -5.7113451 --proj-lat -85.000351 \
    run_stereo_noba_sub4/run-PC.tif --tr 4
\end{verbatim}

We would like now to automatically or manually pick interest points for
the purpose of doing bundle adjustment. It much easier to compute these
if the images are first mapprojected, which brings them all into the
same perspective. This approach is described in section \ref{mapip}, and
here just the relevant commands are shown.

\begin{verbatim}
  for f in A B C D; do
    mapproject --tr 4 run_stereo_noba_sub4/run-DEM.tif ${f}_crop_sub4.cub \
      ${f}_crop_sub4_v1.tif --tile-size 128
  done
\end{verbatim}

(Optional manual interest point picking in the mapprojected images can happen here.)

\begin{verbatim}
  P='A_crop_sub4_v1.tif B_crop_sub4_v1.tif' # to avoid long lines below
  Q='C_crop_sub4_v1.tif D_crop_sub4_v1.tif run_stereo_noba_sub4/run-DEM.tif'
  bundle_adjust A_crop_sub4.cub B_crop_sub4.cub C_crop_sub4.cub D_crop_sub4.cub  \
    -o run_ba_sub4/run --mapprojected-data  "$P $Q"                              \
    --min-matches 1
\end{verbatim}

An illustration is shown in Figure \ref{fig:sfs3}.

\begin{figure}[t!]
  \begin{center}
    \includegraphics[width=7in]{images/sfs3.jpg}
    \caption[sfs]{An illustration of how interest points are picked manually for the purpose of bundle adjustment and then SfS.}
    \label{fig:sfs3}
  \end{center}
\end{figure}

A good sanity check to ensure that at this stage cameras are aligned
properly is to map-project using the newly obtained camera adjustments
and then overlay the obtained images in the GUI.  The features in all
images should be perfectly on top of each other.
\begin{verbatim}
  for f in A B C D; do
    mapproject --tr 4 run_stereo_noba_sub4/run-DEM.tif ${f}_crop_sub4.cub  \
     ${f}_crop_sub4_v2.tif --tile-size 128 --bundle-adjust-prefix run_ba_sub4/run
  done
\end{verbatim}
This will also show where the images overlap, and hence on what portion of the DEM we
can run SfS.

Then we run stereo, followed by SfS.
\begin{verbatim}
  stereo A_crop_sub4.cub B_crop_sub4.cub run_stereo_yesba_sub4/run             \
    --subpixel-mode 3 --bundle-adjust-prefix run_ba_sub4/run
  point2dem --stereographic --proj-lon -5.7113451 --proj-lat -85.000351        \
    run_stereo_yesba_sub4/run-PC.tif --tr 4
  gdal_translate -srcwin 138 347 273 506 run_stereo_yesba_sub4/run-DEM.tif     \
    run_stereo_yesba_sub4/run-crop1-DEM.tif
  sfs -i run_stereo_yesba_sub4/run-crop1-DEM.tif A_crop_sub4.cub               \
    C_crop_sub4.cub D_crop_sub4.cub -o sfs_sub4_ref1_th_reg0.12_wt0.001/run    \
    --shadow-thresholds '0.00149055 0.00138248 0.000747531'                    \
    --threads 4 --smoothness-weight 0.12 --initial-dem-constraint-weight 0.001 \
    --reflectance-type 1 --float-exposure --float-cameras --max-iterations 20  \
    --use-approx-camera-models --crop-input-images                             \
    --bundle-adjust-prefix run_ba_sub4/run
\end{verbatim}

We fetch the portion of the LOLA dataset around the current DEM from the
site described earlier, and save it as
\verb#RDR_354E355E_85p5S84SPointPerRow_csv_table.csv#. It is necessary
to align our stereo DEM with this dataset to be able to compare
them. We choose to bring the LOLA dataset into the coordinate system
of the DEM, using:
\begin{verbatim}
  pc_align --max-displacement 280 run_stereo_yesba_sub4/run-DEM.tif             \
    RDR_354E355E_85p5S84SPointPerRow_csv_table.csv -o run_stereo_yesba_sub4/run \
    --save-transformed-source-points
\end{verbatim}

Then we compare to the aligned LOLA dataset the input to SfS and its output:
\begin{verbatim}
  geodiff --absolute -o beg --csv-format '1:lon 2:lat 3:radius_km' \
    run_stereo_yesba_sub4/run-crop1-DEM.tif run_stereo_yesba_sub4/run-trans_source.csv
  geodiff --absolute -o end --csv-format '1:lon 2:lat 3:radius_km' \
    sfs_sub4_ref1_th_reg0.12_wt0.001/run-DEM-final.tif             \
    run_stereo_yesba_sub4/run-trans_source.csv
\end{verbatim}

We see that the mean error between the DEM and LOLA goes down, after SfS,
from  1.14~m to 0.90~m, while the standard deviation decreases from
1.18~m to 1.06~m.

\section{Running SfS with an external initial guess DEM and extreme illumination}

Here we will illustrate how SfS can be run in a very difficult
situation. We chose a site very close to the Lunar South Pole, at around
$89.7^\circ$ South. We used an external DEM as an initial guess terrain,
in this case the LOLA gridded DEM, as such a DEM has values in in
permanently shadowed regions. The terrain size is 5 km by 5 km at 1
meter/pixel (we also ran a 10 km by 10 km region in the same location).

Here the topography is very steep, the shadows are very long and vary
drastically from image to image, and some portions of the terrain show
up only in some images. All this makes it very hard to register images
to each other and to the ground. We solved this by choosing very carefully
a large set of representative images.

The first step is to fetch the underlying LOLA DEM. We use the 20
meter/pixel one, resampled to 1 meter/pixel, creating a DEM
named \texttt{ref.tif}.

\begin{verbatim}
   wget http://imbrium.mit.edu/DATA/LOLA_GDR/POLAR/IMG/LDEM_80S_20M.IMG
   wget http://imbrium.mit.edu/DATA/LOLA_GDR/POLAR/IMG/LDEM_80S_20M.LBL
   pds2isis from = LDEM_80S_20M.LBL to = ldem_80s_20m.cub
   image_calc -c "0.5*var_0" ldem_80s_20m.cub -o ldem_80s_20m_scale.tif
   gdal_translate -projwin -7050.500 -5759.500 -1919.500 -10890.500 \
     ldem_80s_20m_scale.tif ldem_80s_20m_scale_crop.tif
   gdalwarp -r cubicspline -tr 1 1 ldem_80s_20m_scale_crop.tif ref.tif
\end{verbatim}

Note that we scaled its heights by 0.5 per the information in the LBL file.

By far the hardest part of this exercise is choosing the images. We
downloaded several hundred of them by visiting the web site noted
earlier and searching by the longitude-latitude bounds. The .IMG images
were converted to .cub as before, and they were mapprojected onto the
reference DEM, initially at a lower resolution to get a preview of
things.

The mapprojected images were mosaicked together using
\texttt{dem\_mosaic} with the option \texttt{-\/-block-max}, with a
large value of \texttt{-\/-block-size} (larger than the image size), and
using the \texttt{-\/-t\_projwin} option to specify the region of
interest (in \texttt{stereo\_gui} one can find this region by selecting
it with Control-Mouse).  When the mosaicking tool runs, the sum of
pixels in the current region for each image will be printed to the
screen. Images with a positive sum of pixels are likely to contribute to
the desired region.

The obtained subset of images should be sorted by the Sun azimuth (this
angle is printed when running \texttt{sfs} with the \texttt{-\/-query}
option on the .cub files). Out of those, the following were kept:

\begin{verbatim}
  M114859732RE.cal.echo.cub       73.1771
  M148012947LE.cal.echo.cub       75.9232
  M147992619RE.cal.echo.cub       78.7806
  M152979020RE.cal.echo.cub       96.895
  M117241732LE.cal.echo.cub       97.9219
  M152924707RE.cal.echo.cub       104.529
  M150366876RE.cal.echo.cub       104.626
  M152897611RE.cal.echo.cub       108.337
  M152856903RE.cal.echo.cub       114.057
  M140021445LE.cal.echo.cub       121.838
  M157843789LE.cal.echo.cub       130.831
  M157830228LE.cal.echo.cub       132.74
  M157830228RE.cal.echo.cub       132.74
  M157809893RE.cal.echo.cub       135.604
  M139743255RE.cal.echo.cub       161.014
  M139729686RE.cal.echo.cub       162.926
  M139709342LE.cal.echo.cub       165.791
  M139695762LE.cal.echo.cub       167.704
  M142240314RE.cal.echo.cub       168.682
  M142226765RE.cal.echo.cub       170.588
  M142213197LE.cal.echo.cub       172.497
  M132001536LE.cal.echo.cub       175.515
  M103870068LE.cal.echo.cub       183.501
  M103841430LE.cal.echo.cub       187.544
  M142104686LE.cal.echo.cub       187.765
  M162499044LE.cal.echo.cub       192.747
  M162492261LE.cal.echo.cub       193.704
  M162485477LE.cal.echo.cub       194.662
  M162478694LE.cal.echo.cub       195.62
  M103776992RE.cal.echo.cub       196.643
  M103776992LE.cal.echo.cub       196.643
\end{verbatim}

(the Sun azimuth is shown on the right, in degrees). These were then mapprojected
onto the reference DEM at 1 m/pixel using the South Pole stereographic
projection specified in this DEM. The \texttt{parallel\_bundle\_adjust}
tool is employed to co-register the images and correct camera errors.

\begin{verbatim}
     parallel_bundle_adjust --processes 8 --ip-per-tile 1000 --overlap-limit 30   \
       --num-iterations 10 --num-passes 2 --min-matches 1                         \
       <images> --mapprojected-data '<mapprojected images> ref.tif' -o ba/run
\end{verbatim}

For bundle adjustment we in fact used even more images that overlap
with this area, but likely this set is sufficient, and it is this set
that was used later for shape-from-shading.
Here more bundle
adjustment iterations are desirable, but this step takes too long. And a large
\texttt{-\/-ip-per-tile} can make a difference in images with rather
different illumination conditions.

It is very important to keep a lot of images in bundle adjustment, to ensure
that there are enough overlaps and sufficiently similar illumination conditions
among them for bundle adjustment to succeed. Later, just a subset can be used
for shape-from-shading, enough to cover the entire region, preferable with
multiple illumination conditions at each location.

A very critical part of the process is aligning the obtained cameras to the ground:

\begin{verbatim}
    pc_align --max-displacement 400 --save-transformed-source-points              \
      --compute-translation-only --csv-format '1:lon 2:lat 3:height_above_datum'  \
      ref.tif ba/run-final_residuals_no_loss_function_pointmap_point_log.csv      \
      -o ba/run
\end{verbatim}

The value of \texttt{-\/-max-displacement} could be too high perhaps, it
is suggested to also experiment with half of that and keep the result
that has the smaller error.

The flag \texttt{-\/-compute-translation-only} turned out to be
necessary as \texttt{pc\_align} was introducing a bogus rotation.

The obtained alignment transform can be applied to the cameras to make them aligned
to the ground:

\begin{verbatim}
    mkdir -p ba_align
    /bin/cp -fv ba/run* ba_align
    bundle_adjust --skip-matching --num-iterations 0 --force-reuse-match-files \
      --num-passes 1 --initial-transform ba/run-transform.txt                  \
      --input-adjustments-prefix ba/run <images> -o ba_align/run
\end{verbatim}

The images should now be projected onto this DEM as:

\begin{verbatim}
   mapproject --tr 1 --bundle-adjust-prefix ba_align/run \
     ref.tif image.cub image.map.tif
\end{verbatim}

It is very important to overlay the obtained images and ensure that they
are precisely on top of each other and on top of the LOLA DEM.

There are occasions in which the alignment transform is still not accurate
enough. Then, one can refine the cameras using the reference terrain as
a constraint in bundle adjustment:

\begin{verbatim}
    mkdir -p ba_align_ref
    /bin/cp -fv ba_alin/run* ba_align_ref
    bundle_adjust --skip-matching --num-iterations 5 --force-reuse-match-files   \
      --num-passes 1 --input-adjustments-prefix ba_align/run <images>            \
      --camera-weight 0 --heights-from-dem ref.tif --heights-from-dem-weight 0.1 \
      -o ba_align_ref/run
\end{verbatim}

After mapprojecting with the newly refined cameras in \texttt{ba\_align\_ref}
any residual alignment errors should go away. The value used for
\texttt{-\/-heights-from-dem-weight} may need some experimentation. The reference
DEM has vertical error as well, so of this value is too high, it may bind too
tightly to this somewhat inaccurate DEM. Yet, making it too low may not
constrain sufficiently the uncertainty that exists in the height of triangulated
points after bundle adjustment, which is rather high since LRO NAC is mostly looking
down so the convergence angle among any rays going through matching interest points
is small.

It is suggested that the user examine the file
\begin{verbatim}
  ba_align_ref/run-final_residuals_no_loss_function_pointmap_point_log.csv
\end{verbatim}

to see if the reprojection errors (column 4) are reasonably small, say mostly
on the order of 0.1 pixels. The triangulated point cloud
from this file should also hopefully be close to the reference DEM. Their
difference is found as:
\begin{verbatim}
  geodiff --absolute                                                         \
    --csv-format '1:lon 2:lat 3:height_above_datum' ref.tif                  \
    ba_align_ref/run-final_residuals_no_loss_function_pointmap_point_log.csv \
    -o ba_align_ref/run
\end{verbatim}

Some of the differences that will be saved to disk are likely outliers,
but mostly they should be small, perhaps on the order of 1 meter.

If, even after this step, the mapprojected images fail to be perfectly
on top of each other, or areas with poor coverage exist, more images
with intermediate illumination conditions and more terrain coverage
should be added and the process should be restarted. As a last resort,
any images that do not overlay correctly must be removed from
consideration for the shape-from-shading step.

Next, SfS follows:

\begin{verbatim}
   parallel_sfs -i ref.tif <images> --shadow-thresholds '<thresholds>'  \
     --bundle-adjust-prefix ba_align_ref/run -o sfs/run                 \
     --use-approx-camera-models --crop-input-images                     \
     --blending-dist 10 --min-blend-size 100 --threads 4                \
     --smoothness-weight 0.12 --initial-dem-constraint-weight 0.0001    \
     --reflectance-type 1 --max-iterations 5  --save-sparingly          \
     --tile-size 200 --padding 50 --num-processes 20                    \
     --nodes-list <machine list>
\end{verbatim}

The shadow thresholds were chosen here to be 0.003.

The first step that will happen when this is launched is computing the
exposures. That one can be a bit slow, and can be done offline, using
the flag \texttt{-\/-compute-exposures-only} in this tool, and then the
computed exposures can be passed to the command above via the
\texttt{-\/-image-exposures-prefix} option.

The obtained shape-from-shading terrain should be studied carefully to
see if it shows any systematic shift or rotation compared to the initial
LOLA gridded terrain. If necessary, it can be re-aligned to it using
\texttt{pc\_align} (perhaps using only the compute translation flag).

If SfS is run on multiple overlapping terrains, it is unavoidable that
the output terrains will be offset in respect to each other, hopefully
by no more than several meters or tens of meters. After aligning them to
LOLA as well as possible, they can also be aligned to each other if they
have enough overlap, using the \texttt{pc\_align} option
\texttt{-\/-initial-transform-from-hillshading} (whose value should be
either 'translation' or 'rigid', unless a scale change is suspected).
One should consider using zero iterations here or disabling the
rotations in case it appears that this tool is overeager and the aligned
DEM moves in unacceptable ways.

Lastly, the \texttt{geodiff} tool can be deployed to see how the SfS DEM
compares to the initial guess or to the raw ungridded LOLA measurements.

\section{Insights for getting the most of SfS}
\label{sfs:insights}

Here are a few suggestions we have found helpful when running \texttt{sfs}:

\begin{itemize}{}

\item First determine the appropriate smoothing weight $\mu$ by running a
  small clip, and using just one image. A value between 0.06 and 0.12 seems to work
  all the time with LRO NAC, even when the images are subsampled. The other weight, $\lambda,$ can
  be set to something small, like $0.0001.$ This can be increased to $0.001$ if noticing
  that the output DEM strays too far.

\item As stated before, more images with more diverse illumination conditions
  result in more accurate terrain. Ideally there should be at least 3 images, with the shadows
  being, respectively, on the left, right, and then perhaps missing or small.

\item Bundle-adjustment for multiple images is crucial, to eliminate
  camera errors which will result in \texttt{sfs} converging to a local
  minimum. This is described in section \ref{sfs-lola}.

\item Floating the albedo (option \texttt{-\/-float-albedo}) can
  introduce instability and divergence, it should be avoided unless
  obvious albedo variation is seen in the images.

\item Floating the DEM at the boundary (option
  \texttt{-\/-float-dem-at-boundary}) is also suggested to be avoided.

\item Overall, the best strategy is to first use SfS for a single image
  and not float any variables except the DEM being optimized, and then
  gradually add images and float more variables and select whichever
  approach seems to give better results.

\item If an input DEM is large, it may not be completely covered by a
  single set of imagery with various illumination conditions.  It should
  then be broken up into smaller regions (with overlap), the SfS problem
  can be solved on each region, and then every output terrain can be
  transformed using \texttt{pc\_align} into LOLA's global coordinate
  system, where they can be mosaicked together using
  \texttt{dem\_mosaic}. Or, \texttt{sfs} can be run not on one clip, but
  on an entire collection of clips covering this area to get the
  adjustments, and then \texttt{parallel\_sfs} can be run as described in
  the previous section.

  The easier case is when at least the two images in the stereo pair
  cover the entire terrain. Then, portions of this terrain can be used
  as an initial guess for each SfS sub-problem (even as the other images
  used for SfS change), the results can be mosaicked, and the alignment
  to LOLA can happen just once, after mosaicking. This approach is
  preferable, if feasible, as alignment to LOLA is more accurate if the
  terrain to align is larger in extent.

\item The \texttt{mapproject} program can be used to map-project each image
  onto the resulting SfS DEM (with the camera adjustments solved using
  SfS). These orthoimages can be mosaicked using \texttt{dem\_mosaic}. If the
  \texttt{-\/-max} option is used with this tool, it create a mosaic with
  the most illuminated pixels from this image. If during SfS the camera
  adjustments were solved accurately, this mosaic should have little or no
  blur.

\item For challenging datasets it is suggested to first work at 1/4th of
the full resolution (the resolution of an ISIS cube can be changed using
the \texttt{reduce} command, and the DEM can be made coarser with
\texttt{gdalwarp} or by converting it to a cloud with \texttt{pc\_align}
with zero iterations and then regenerated with \texttt{point2dem}).
This should make the whole process perhaps an order of magnitude
faster. Any obtained camera adjustment files are still usable at the
full resolution (after an appropriate rename), but it is suggested that
these adjustments be reoptimized using the full resolution cameras,
hence these should be initial guesses for \texttt{bundle\_adjust}'s
\texttt{-\/-input-adjustments-prefix} option.

\end{itemize}


%! Author = tstreule

\section{Optical Microscopy}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Reflection and Refraction}
%
\formbox{Law of Reflection}{\theta\ped{inc} = \theta_2}
\formbox{Law of Refraction}{\frac{n\ped{inc}}{n_2} = \frac{\lambda_0/\lambda\ped{inc}}{\lambda_0/\lambda_2} = \frac{\sin\theta_2}{\sin\theta\ped{inc}}}
\quad for $\theta\ped{inc}<\theta_c$
\formula{Total reflection}{\sin\theta_c = n_2/n\ped{inc}}
\quad \textbf{always total} if $n\ped{inc}<n_2$

\formula{Paraxial approx.}{\theta \simeq \sin\theta \simeq \tan\theta, \;\;\; \theta\ll 1}
\formula{Thin lens approx.}{R \ll S_o,\,S_i}
\formbox{Lens makers formula}{\frac{1}{S_o} + \frac{1}{S_i} = \frac{1}{f} = \frac{n_\textrm{lens}-n}{n} \left( \frac{1}{R_1} - \frac{1}{R_2} \right)}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Ray Tracing}
%
%	\begin{itemize}
%		\item Rays passing \textbf{through the optical center} of a lens continue in a straight line
%		\item Rays traveling \textbf{parallel to the optical axis} pass through the focal point after refraction and vice versa
%		\item \textbf{Parallel rays} pass through the same point in the focal plane after refraction and vice versa
%	\end{itemize}
\begin{minipage}{\linewidth}
    \includegraphics[width=.45\columnwidth]{Optical_Lens}
    \qquad
    \includegraphics[width=.4\columnwidth]{Optical_Lens2}
\end{minipage}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{minipage}{\linewidth}
    \begin{minipage}[t]{.5\columnwidth-.5\columnsep}
        \subsubsection{Simple Microscope}
        %
        \includegraphics[width=\columnwidth]{Optical_Simple_Microscope}
    \end{minipage}%
    \hspace{\columnsep}%
    \begin{minipage}[t]{.5\columnwidth-.5\columnsep}
        \subsubsection{Beam Expander}
        %
        \hfill for light source

        \includegraphics[width=\columnwidth]{Optical_Beam_Expander}
    \end{minipage}

    \formbox{Magnification}{M = h_2/h_1 \overset{*}{=} f_2/f_1}
    \quad * since $h_1 = f_1\sin\theta$
\end{minipage}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Simple ``modern'' Microscope}
%
\parbox{.5\columnwidth}{(uniform!) Illumination:}
\parbox{.5\columnwidth}{Camera: ($M = f_{tube} / f_{obj}$)}

\includegraphics[width=.49\columnwidth]{Optical_Modern_Illumination}
\hfill
\includegraphics[width=.49\columnwidth]{Optical_Modern_Camera}

\parbox{.5\columnwidth}{\centering\small(Objective lens)}
\parbox{.5\columnwidth}{\centering\small(Tube lens)}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Fundamental Limits of Lenses \& Resulution}
%
Due to a \textbf{finite aperture} will points presented as the PSF (point-spread).

\begin{minipage}{.3\columnwidth}
\includegraphics[width=\columnwidth]{Optical_PSF}
\end{minipage}%
\hspace{1.5\boxmargin}%
\begin{minipage}{.7\columnwidth-1.5\boxmargin}
First zero occurs at: \quad
$\theta \overset{(\theta\ll1)}{\simeq} \sin\theta \simeq 1.22\frac{\lambda_0}{d}$\\
$d$: radius of the aperture\par
$\bullet$ $\sin\theta \simeq \tan\theta = x/L$\\
$\bullet$ $r_1 - r_2 = d\cdot x/L$
\end{minipage}

\formtex{\textbf{Rayleigh criterion}}{maximum of the one is on the minimum of the}
\formtex{~}{other curve (FWHM)}
\formbox{Resolution}{\Delta r \simeq f\sin\theta \times M \simeq 1.22\frac{\lambda_0f}{D} \times M = 0.61\frac{\lambda_0n}{\textrm{NA}} \times M \vspace{-1mm}}
\formbox{Numerical Aperture}{\textrm{NA} = n\sin\theta \simeq n\,\frac{D}{2f}}
\quad $n$: refractive index,
\formula{~}{\text{$D$: aperture size of lens, $f$: focal length of lens}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Fluorescence Microscopy}
%
Emitted photon has \textbf{less energy}.
$\to$ Absorption $\neq$ emission spectr.

\includegraphics[width=.5\columnwidth]{Optical_Excitation_Emission}
\includegraphics[width=.5\columnwidth]{Optical_Fluorescence_Setup}


% ---------------------------------------------------------------------
% "Estimating Mixed Hitting-Time Models" - mhte.tex
% ---------------------------------------------------------------------
% by Jaap H. Abbring and Tim Salimans
% ---------------------------------------------------------------------

\documentclass[fleqn,12pt]{article}
\usepackage{comment}
\usepackage{natbib}
\usepackage{amsthm,amsfonts,amsopn,amsmath,amssymb}
\usepackage{vmargin,setspace}
\usepackage{pgf,pgfplots,tikz}
\usepackage{booktabs}

\RequirePackage[pagebackref=false]{hyperref}
\RequirePackage{hypernat}
%\usepackage{float}

\usetikzlibrary{arrows,calc,fit,matrix,positioning,shapes.multipart,shapes.symbols}
\hypersetup{colorlinks=true,linkcolor=cyan,urlcolor=cyan,citecolor=darkgray,
    pdftitle=Replication of The Likelihood of Mixed Hitting Times,
    pdfauthor=Jaap Abbring and Tim Salimans,
    pdfsubject=JEL Codes C14 C41,
    pdfkeywords=duration analysis first passage time identification Laplace transform Levy process maximum likelihood Mellin inverse formula mixture optimal stopping strike duration,
    pdfdisplaydoctitle=true}

\setpapersize{A4} \setmargrb{25mm}{18mm}{25mm}{23mm}
%\doublespace

\newcommand{\cites}[1]{\citeauthor{#1}'s (\citeyear{#1})}

\begin{document}
% ---------------------------------------------------------------------
%  TITLE PAGE
% --------------------------------------------------------------------
\title{Replication of `The Likelihood of Mixed Hitting Times'}

\author{Jaap H. Abbring\thanks{%
Department of Econometrics \& OR, Tilburg University, P.O. Box 90153, 5000 LE Tilburg, The Netherlands; and CEPR. E-mail:
\href{mailto:jaap@abbring.org}{jaap@abbring.org}. Web: \href{http://jaap.abbring.org}{jaap.abbring.org}.}\and Tim Salimans\thanks{Google Brain, Amsterdam, The Netherlands. E-mail:
\href{mailto:salimanstim@gmail.com}{salimanstim@gmail.com}. Web: \href{http://github.com/TimSalimans}{github.com/TimSalimans}.
\newline The research of Jaap Abbring is financially supported by the Netherlands Organisation for Scientific Research (NWO) through Vici grant 453-11-002.
}}

\date{\today}

\maketitle

{\em This document reports on the replication of \citet{jem21:abbringsalimans}. It was generated by running {\tt make} in the current version of the replication package \citep{zenodo21:abbringsalimans}.}

\section{Main Results}

\begin{figure}[h]
\caption{Approximation Error of the Log Likelihood for Various $M$\label{fig:llherr}}
\medskip
\begin{centering}
\begin{tikzpicture}
\tikzstyle{every axis y label}=[yshift=10pt]
\tikzstyle{every axis x label}=[yshift=-5pt]
   \begin{axis}[width=15cm,height=7.5cm,
        name=llherr,
        ylabel={Average absolute error in $\ell_N(\alpha)$},
        xlabel={$M$},
        axis x line=bottom,
        every outer x axis line/.append style={-,color=gray,line width=0.75pt},
        axis y line=left,
        every outer y axis line/.append style={-,color=gray,line width=0.75pt},
        xtick={5,10,15,20,25,30},
        xticklabels={$5$,$10$,$15$,$20$,$25$,$30$},
        ytick={-10,-5,0,5},
        yticklabels={$10^{-10}$,$10^{-5}$,$10^0$,$10^5$},
        major tick length=2.5mm,
        every tick/.append style={line width=0.75pt},
        ymin=-10,
        ymax=5,
       xmin=0,
       xmax=30,
        scaled ticks=false,
        /pgf/number format/precision=2,
        /pgf/number format/set thousands separator={}]
        \addplot[color=blue!75,smooth,mark=*,line width=0.75pt] table[x=M,y=llherr,col sep=comma]{fig1.csv} ;
         \draw[color=black!75,line width=0.75pt,dashed] (axis cs:0,-5) -- (axis cs:30,-5);
         \draw[color=black!75,line width=0.75pt,dashed] (axis cs:0,0) -- (axis cs:30,0);
\end{axis}
\end{tikzpicture}
\end{centering}

\footnotesize{\input{fig1times}}
\end{figure}

\begin{figure}[h]
\caption{Approximation Error of the Log Inverse Gaussian Density Function\label{fig:proberr}}
\medskip
\begin{centering}
\begin{tikzpicture}
\tikzstyle{every axis y label}=[yshift=10pt]
\tikzstyle{every axis x label}=[yshift=-5pt]
   \begin{axis}[width=15cm,height=7.5cm,
        name=llherr,
        ylabel={Absolute error in $\ln f_{\mathrm{BM}}(t|X)$},
        xlabel={$\ln f_{\mathrm{BM}}(t|X)$},
        axis x line=bottom,
        every outer x axis line/.append style={-,color=gray,line width=0.75pt},
        axis y line=left,
        every outer y axis line/.append style={-,color=gray,line width=0.75pt},
        xtick={-25,-20,-15,-10,-5,0},
        xticklabels={$-25$,$-20$,$-15$,$-10$,$-5$,$0$},
        ytick={-10,-5,0},
        yticklabels={$10^{-10}$,$10^{-5}$,$10^0$},
        major tick length=2.5mm,
        every tick/.append style={line width=0.75pt},
        ymin=-12,
        ymax=2,
       xmin=-28,
       xmax=0,
        scaled ticks=false,
        /pgf/number format/precision=2,
        /pgf/number format/set thousands separator={}]
        \addplot[color=blue!75,only marks,mark=*,mark size=1pt] table[x=lap,y=logerr,col sep=comma]{fig2.csv} ;
         \draw[color=black!75,line width=0.75pt,dashed] (axis cs:-28,0) -- (axis cs:0,0);
         \draw[color=black!75,line width=0.75pt,dashed] (axis cs:-28,-5) -- (axis cs:0,-5);
         \draw[color=black!75,line width=0.75pt,dashed] (axis cs:-28,-10) -- (axis cs:0,-10);
\end{axis}
\end{tikzpicture}
\end{centering}
\end{figure}

\begin{figure}[h]
\caption{Approximate Probability Density and Histogram of Simulated Values of $\ln T$ for a Specification With Shocks and Heterogeneity\label{fig:histogram}\label{fig:approxshock}}
\medskip
\begin{centering}
\begin{tikzpicture}
\tikzstyle{every axis y label}=[yshift=10pt]
\tikzstyle{every axis x label}=[yshift=-5pt]
   \begin{axis}[width=15cm,height=7.5cm,
        name=llherr,
        ylabel={Density of $\ln T$},
        xlabel={$\ln t$},
        axis x line=bottom,
        every outer x axis line/.append style={-,color=gray,line width=0.75pt},
        axis y line=left,
        every outer y axis line/.append style={-,color=gray,line width=0.75pt},
        major tick length=2.5mm,
        every tick/.append style={line width=0.75pt},
        scaled ticks=false,
        /pgf/number format/precision=2,
        /pgf/number format/set thousands separator={}]
        \addplot[color=green!75,ybar interval,line width=0.50pt] table[x=logtlow,y=hist,col sep=comma]{fig3hist.csv} ;
        \addplot[color=blue!75,smooth,line width=0.75pt] table[x=logt,y=invlt,col sep=comma]{fig3invlap.csv} ;
    %    \addplot[color=red!75,dashed,line width=0.75pt] table[x=logt,y=smooth,col sep=comma]{fig3invlap.csv} ;
         \draw[color=black!75,line width=0.75pt,dashed] (axis cs:-28,0) -- (axis cs:0,0);
         \draw[color=black!75,line width=0.75pt,dashed] (axis cs:-28,-5) -- (axis cs:0,-5);
         \draw[color=black!75,line width=0.75pt,dashed] (axis cs:-28,-10) -- (axis cs:0,-10);
   \end{axis}
\end{tikzpicture}
\end{centering}

\footnotesize{\input{fig3times}}
\end{figure}

\clearpage
\input{tab1}
\clearpage
\input{tab1lowM}
\clearpage
\input{tab1mig}
\clearpage
\input{fig4}

\noindent The numbers in Column IV imply that there are four unobserved types of labor conflict, on average commanding respectively \vone, \vtwo, \vthree, and \vfour\ strike weeks.\\  \ \\
\input{tab1times}\\ \ \\
The estimation procedure for a version of the model in Column VI with a single gamma shock stops at a similar estimate of $\lambda$ (\gammalambda) and huge estimates of $\tau$ (\gammatau) and $\omega$ (\gammaomega). At these estimated values of $\tau$ and $\omega$, the gamma shock distribution is effectively degenerate at $-\tau/\omega=$\gammamintdivo. As this is very close to the estimated size of the discete shock in Column VI, the log likelihood is virtual identical: \gammallh.\\ \ \\
Both the MHT and the MPH models fit the empirical hazard well, but the MPH model's log likelihood, at \mphllh, is \diffllh\ points lower.

\begin{figure}[t]
\caption{Aggregate Strike End Hazard Rates\label{fig:strikehazard}}
\begin{centering}
\begin{tikzpicture}
\tikzstyle{every axis x label}=[yshift=-5pt]
   \begin{axis}[width=15cm,height=7.2cm,
        name=strikes,
        ylabel={Hazard rate per week},
        xlabel={Strike duration in weeks},
        axis x line=bottom,
        every outer x axis line/.append style={-,color=gray,line width=0.75pt},
        axis y line=left,
        every outer y axis line/.append style={-,color=gray,line width=0.75pt},
        xtick={4,8,12,16,20,24,28,32},
        xticklabels={$4$,$8$,$12$,$16$,$20$,$24$,$28$,$32$},
        ytick={0,0.1,0.2,0.3,0.4,0.5,0.6},
        yticklabels={$0$,$0.1$,$0.2$,$0.3$,$0.4$,$0.5$,$0.6$},
        major tick length=2.5mm,
        every tick/.append style={line width=0.75pt},
        ymin=0,
        ymax=0.6,
        xmin=0,
        xmax=38,
        scaled ticks=false,
        /pgf/number format/precision=2,
        /pgf/number format/set thousands separator={}]
        %Hazard paths.
        \addplot[color=red!75,smooth,line width=0.75pt] table[x=t,y=mht,col sep=comma]{fig4.csv} (axis cs: 35.3,0.35) node[color=red!75]{MHT};
        \addplot[color=blue!75,smooth,line width=0.75pt] table[x=t,y=mph,col sep=comma]{fig4.csv} (axis cs: 35.3,0.31) node[color=blue!75]{MPH};
        \addplot[color=green!75,smooth,line width=0.75pt] table[x=t,y=data,col sep=comma]{fig4.csv} (axis cs: 35.3,0.5) node[color=green!75]{Data};
\end{axis}
\end{tikzpicture}
\end{centering}
\end{figure}

\clearpage
\section{Other Checks}
\setcounter{table}{0}
\renewcommand{\thetable}{R\arabic{table}}

\input{chckgrad}

\clearpage

\begin{thebibliography}{}
\bibitem[\protect\citeauthoryear{Abbring and Salimans}{Abbring and Salimans}{2021a}]{jem21:abbringsalimans}
Abbring, J.~H. and T.~Salimans (2021a).
\newblock The likelihood of mixed hitting times.
\newblock {\em Journal of Econometrics\/}~{\em XX\/}, forthcoming.
\newblock \href{https://arxiv.org/abs/1905.03463}{arXiv:1905.03463} [econ.EM].

\bibitem[\protect\citeauthoryear{Abbring and Salimans}{Abbring and Salimans}{2021b}]{zenodo21:abbringsalimans}
Abbring, J.~H. and T.~Salimans (2021b).
\newblock The likelihood of mixed hitting times: Replication package.
\newblock \href{https://doi.org/10.5281/zenodo.4663359}{doi:10.5281/zenodo.4663359}.

\bibitem[\protect\citeauthoryear{Kennan}{Kennan}{1985}]{jem85:kennan}
Kennan, J. (1985).
\newblock The duration of contract strikes in {U.S.} manufacturing.
\newblock {\em Journal of Econometrics\/}~{\em 28}, 5--28.

\end{thebibliography}
\end{document}


\chapter{Background \& Review}
Everything mentioned in this chapter should already be familiar to you from other math classes.
These topics span a few major areas: algebra, pre-calculus, and trigonometry.
Ideas from these topics will often be used implicitly or without a passing reference.\bigskip

If you are unfamiliar with anything mentioned, you can use many of the great online resources like Khan Academy to familiarize yourself before moving forward.

\input{../common/algebraPreCalc/algebraPreCalc.tex} % Algebra and Pre-Calc


\chapter{General Conclusions}
\label{Conclusions}
There are many real world applications of organic semiconductors \cite{Anthony2008Jan, Dimitrakopoulos2001Jan, Sasabe2011Feb} and accurate models of charge transport are important to facilitate new materials discovery and characterisation. However, due to mobilities falling within an intermediate region where neither band theories nor hopping theories are applicable non-adiabatic atomistic simulations must be used \cite{oberhofer_charge_2017}. Among the litany of techniques proposed there are no single silver bullets. As always, the user must make the compromise between accurate dynamics and computational cost. Two of the most popular mixed-quantum classical techniques are trajectory surface hopping (TSH) and Ehrenfest. However, these both suffer from well known problems such as over-coherent nuclear-electronic dynamics  no branching of the nuclear wavefunction in Ehrenfest and lack of a first-principles grounding in TSH.
\\\\
To overcome these challenges a new technique, coupled-trajectory mixed-quantum classical molecular dynamics (CTMQC), has been proposed \cite{agostini_semiclassical_2015} to more rigorously account for decoherence, branching of the nuclear wavefunction and to provide a technique based in first principles physics. This technique, derived from the exact factorisation of the molecular wavefunction \cite{abedi_exact_2010}, appears as a `corrected' Ehrenfest scheme where the correction comes from 2 new terms -an adiabatic time-integrated force and a quantum momentum.
\\\\
In this report I have outlined an implementation of CTMQC paired with a fragment-orbital based (FOB) technique to produce an efficient FOB-CTMQC propagator capable of simulating hundreds of organic molecules. The FOB method is based on the assumption that the electronic couplings (off-diagonal Hamiltonian elements) are proportional to the overlap between singly occupied molecular orbitals (SOMOs). This approximation has been validated in many organic semiconductors and provides a significant speed-up when compared to using density functional theory.
\\\\
The implementation process is still under progress. However, initial results are promising. Some key tests have been discussed including Rabi oscillation, energy conservation, norm conservation and the fulfilment of a fundamental equation \eqref{eq:S26}. The tests have been mostly positive. However, due a denominator in the equation for calculating the quantum momentum approaching zero -causing the quantum momentum to spike. The norm was not well conserved and a smoothing tanh$^2$ function was used to fix this.
\\\\
To build on this work I would now like to apply FOB-CTMQC to more realistic systems and to eventually compare with experimental results. However, a number of tasks must be completed before this is possible. These include implementing a sensible algorithm for calculating the nuclear width parameter, $\sigma_{\nu}^{I}$, used in the calculation of the quantum momentum. This determines the width of the gaussians that combine to give the quantum momentum. I am currently using a frozen width of $\sqrt{2}$ bohr. The norm conservation and the spikes in the quantum momentum should be monitored also, if these get worse an improved smoothing algorithm will have to be implemented. Over the next year I would like to also test whether detailed balance is reached in CTMQC as well as comparing CTMQC with our in-house FOB-SH algorithm as well as with more accurate methods. CTMQC should be more accurate than the surface hopping (FOB-SH) algorithm and handle decoherence in a better way. For smaller systems CTMQC can be benchmarked against more accurate results such as the multi-configuration time-dependent Hartree method discussed here \cite{cattarius_all_2001}. Finally many optimisations will have to be implemented before moving to realistic systems consisting of hundreds of molecules or more. One of the most important of these is the parallelisation of the code in order to efficiently use multiple processors.


\documentclass[times, utf8, seminar, numeric]{fer}
\usepackage{booktabs}
\usepackage{float}
\usepackage[symbol]{footmisc}

\begin{document}

% TODO: Navedite naslov rada.
\title{Fast edit distance and sequence alignment}

% TODO: Navedite vaše ime i prezime.
\author{Kristijan Biščanić \\ Luka Hrabar \\ Ela Marušić}

\maketitle

\tableofcontents

\chapter{Introduction}
The string edit distance is defined as the minimal cost to transform one character string into the other. The operations allowed to achieve this transformation are:
\begin{itemize}
\item delete - a character is deleted from the first string, from any location
\item insert - a character is inserted into the second string, at any location
\item replace - a character in the first string is replaced with some other character
\end{itemize}
Each of these operations has some, usually integer, cost.

The edit script is defined as the actual sequence of operations used to transform one string into the other, with the sum of operation costs equal to the edit distance. These topics are extensively researched in many areas of computer science and our work uses them in the context of bioinformatics.

Sequence alignment is a process of arranging the symbolic representations of DNA, RNA or protein sequences so that their most similar elements are juxtaposed. Such alignment is useful to identify regions of similarity and many bioinformatics tasks depend upon successful alignments.

We have implemented, tested and analyzed a faster algorithm for computing string edit distances and sequence alignments. This algorithm was published by Masek and Paterson \cite{masek1980faster} and is inspired by the Four Russians Algorithm. Implemented algorithm will be compared to and tested against Fischer-Wagner algorithm \cite{needleman1970general}, which is based on dynamic programming.

\chapter{Algorithm}
\section{The basic algorithm}
Dynamic programming problems can, more often than not, be reduced to a matrix of solutions to subproblems where the value of a cell ($i$, $j$) depends only on the cells with coordinates ($u\leq i$, $v\leq j$). This is the case with edit distance and sequence alignment problems. An example is shown in table \ref{tab:a}, with the final solution in cell ($N$, $M$).
\begin{table}[H]
\centering
\begin{tabular}{c|ccccc}
  & - & G & C & A & T\\
\hline
- & $0$ & $1$ & $2$ & $3$ & $4$ \\
G & $1$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
A & $2$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
T & $3$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
T & $4$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
\end{tabular}
\quad\quad
\begin{tabular}{c|ccccc}
  & - & G & C & A & T\\
\hline
- & $0$ & $1$ & $2$ & $3$ & $4$ \\
G & $1$ & $0$ & $1$ & $2$ & $3$ \\
A & $2$ & $1$ & $2$ & $1$ & $2$ \\
T & $3$ & $2$ & $3$ & $2$ & $1$ \\
T & $4$ & $3$ & $4$ & $3$ & $2$ \\
\end{tabular}
\caption{String edit matrix $E$}\label{tab:a}
\end{table}

The first row and column of the matrix are computed using the operation costs for the corresponding character. All other values in the matrix are computed as:
\begin{equation}
E_{i,j} = min(E_{i-1, j-1} + R, E_{i-1, j} + I, E_{i, j - 1} + D)
\end{equation}
where $R$, $I$, $D$ are the costs of the (matrix movement) operations described in the introduction.
Due to the fact that each value can be computed in $O(1)$ time in the presented manner, both the time and space complexities of this matrix calculation equal $O(N \times M)$, where $N$ and $M$ denote the length of the strings.

\section{The basic algorithm with optimized space complexity}
Lets say we traverse and fill the matrix from earlier in a row-major fashion. If we take a closer look we'll notice that the values from row $i$ depend only on the values from the previous row, $i-1$, and on the values we've already calculated in this row. Therefore, we can forget the values from row $i-2$ and earlier, effectively reducing the memory required for the algorithm to two arrays of length equal to the number of columns. In other terms, the space complexity is reduced to $O(min(N, M))$ if we set the number of columns using the shorter of the two strings in question.

\section{The Four Russians method}
We know that a problem matrix is represented using two strings and two initial vectors of values as shown earlier. In fact, every subproblem of our problem is represented using such strings and vectors:

\begin{figure}[H]
\centering
\includegraphics[scale=.25]{s1}
\caption{Submatrices - the bolded submatrix is completely defined by the strings s1, s2 and two vectors of initial values in its first row and first column}\label{pic:s1}
\end{figure}

If we split the matrix into smaller submatrices (\ref{pic:s1}) and precalculate the solutions to these subproblems we can combine them to solve the large matrix. This is possible because the final vectors of some submatrix are the initial vectors of some other submatrices as seen in (\ref{pic:s1}). The final solution is yet again in the bottom-right cell of the large matrix.

The space and time complexities using this technique would be reduced to \\
$O(min(N, M) / t)$ and $O(N * M / t^2)$, respectively. The $t$ denotes the dimension of the submatrices used. But, in order for it to work, we need to precalculate every submatrix that might occur in our problem.

\section{Enumerating the submatrices}
The number of initial submatrix setup combinations is determined using the following expression:
\begin{equation}
  combinations = |\sigma|^2t \cdot |\lambda|^2t
\end{equation}
The expression is derived from the fact that every subproblem is determined using two initial vectors of costs and two strings. $|\sigma|$ denotes the size of the alphabet used for the strings, $t$ represents the submatrix dimension, and $|\lambda|$ is the size of the set that contains every value that can occur in the initial vectors.

\section{The step observation}
The set of possible values from earlier can obviously be pretty large and we need to find a way to reduce the number of precalculations. Fortunately, we can make the following observation - the values in these initial vectors, i.e. the values in general, tend to increase as we move towards the bottom-right corner of the matrix.

Before we continue, we will fix the operation costs as follows, in the context of string alignment:
\begin{itemize}
\item move right in the matrix, i.e. use the next character of the top string, will have cost $1$
\item move down in the matrix, i.e. use the next character of the left string, will have cost $1$
\item move diagonally down-right, i.e. use both strings' next characters will have cost $0$ if the characters match, $2$ otherwise
\end{itemize}
With these costs, the difference between any two adjacent matrix elements is restricted to \{$-1$, $0$, $1$\} (for proof see \citep{gusfield1997algorithms}).

The next step is to code all matrix values using these differences. We have two expressions, one for vertical steps and one for horizontal steps (for more details see \citep{masek1980faster}):
\begin{equation}
\delta_{i,j} - \delta_{i-1,j} = min\{R_{i,j} - (\delta_{i-1,j}-\delta_{i-1,j-1}), D, I + (\delta_{i,j-1} - \delta_{i-1, j-1})\}
\end{equation}
\begin{equation}
\delta_{i,j} - \delta_{i,j-1} = min\{R_{i,j} - (\delta_{i,j-1}-\delta_{i-1,j-1}), D + (\delta_{i-1,j} - \delta_{i-1, j-1}), I\}
\end{equation}

$D$ and $I$ are the costs of deletion and insertion, in other words matrix "movement", which are $1$ in our case. $R_{i,j}$ denotes the cost of replacing the $i$-th character of the first string with the $j$-th character of the second and is equal to $0$ if the two characters match ($2$ otherwise).

We are now able to reduce the number of submatrices we have to precalculate using vectors of steps, one horizontal and one vertical, instead of vectors of matrix values to:
\begin{equation}
combinations = |\sigma|^2t \cdot 3^2t
\end{equation}

To save space we will be storing only the final vectors for each submatrix, mapped to the initial vector and string combination for that submatrix. The time complexity to calculate a single submatrix's final step vectors is $O(t^2)$ because we have to traverse the entire submatrix.

\section{Combining everything}
Once we have computed all the possible submatrices, we can use them to generate the edit matrix of steps. More precisely, since we have two types of steps (those moving horizontally and those moving vertically), we will be generating two edit matrices of steps.

The first row and column of the matrix are initialized as demonstrated in the example below. The initial cost in the top left cell is 0, and the cells in the column and row are filled with the cost of deletion and insertion, respectively.

\begin{table}[H]
\centering
\begin{tabular}{c|ccccc}
  & - & G & C & A & T\\
\hline
- & $0$ & $1$ & $1$ & $1$ & $1$ \\
G & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
A & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
T & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
T & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
\end{tabular}
\quad\quad
\begin{tabular}{c|ccccc}
  & - & G & C & A & T\\
\hline
- & $0$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
G & $1$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
A & $1$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
T & $1$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
T & $1$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
\end{tabular}
\caption{Matrices of horizontal and vertical steps}
\end{table}

The rest of the matrix can now be filled with submatrices we have already calculated. We know the first row and column and the substrings for the top-left submatrix, so we can simply retrieve the precalculated final row and column.

\begin{table}[H]
\centering
\begin{tabular}{c|ccccc}
  & - & G & C & A & T\\
\hline
- & $0$ & $1$ & $1$ & $1$ & $1$ \\
G & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
A & $0$ & $-1$ & $1$ & $\cdot$ & $\cdot$ \\
T & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
T & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
\end{tabular}
\quad\quad
\begin{tabular}{c|ccccc}
  & - & G & C & A & T\\
\hline
- & $0$ & $\cdot$ & $0$ & $\cdot$ & $\cdot$ \\
G & $1$ & $\cdot$ & $-1$ & $\cdot$ & $\cdot$ \\
A & $1$ & $\cdot$ & $1$ & $\cdot$ & $\cdot$ \\
T & $1$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
T & $1$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
\end{tabular}
\end{table}

Since the submatrices in our matrix overlap, the last column of a submatrix is also the first column of the submatrix to the right (and the last row is also the first row of the submatrix below). Knowing this, we can continue to determine the last row and column of submatrices in a rowwise manner.

\begin{table}[H]
\centering
\begin{tabular}{c|ccccc}
  & - & G & C & A & T\\
\hline
- & $0$ & $1$ & $1$ & $1$ & $1$ \\
G & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
A & $0$ & $-1$ & $1$ & $-1$ & $1$ \\
T & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ & $\cdot$ \\
T & $0$ & $-1$ & $1$ & $-1$ & $-1$ \\
\end{tabular}
\quad\quad
\begin{tabular}{c|ccccc}
  & - & G & C & A & T\\
\hline
- & $0$ & $\cdot$ & $0$ & $\cdot$ & $0$ \\
G & $1$ & $\cdot$ & $-1$ & $\cdot$ & $-1$ \\
A & $1$ & $\cdot$ & $1$ & $\cdot$ & $-1$ \\
T & $1$ & $\cdot$ & $1$ & $\cdot$ & $-1$ \\
T & $1$ & $\cdot$ & $1$ & $\cdot$ & $1$ \\
\end{tabular}
\end{table}

The edit distance is the sum of steps along some path from the top left cell to the bottom right. For example, if we sum the steps along the first column and the last row, we get an edit distance of $2$. The sequence of edit operations can be retrieved, provided that we have kept in memory all of the step rows and columns. We can now backtrack through the matrix by regenerating the submatrices which are crossed by the edit path. This is possible since the value in the top left cell of each submatrix can be computed in the same manner as the final edit distance, and the steps along the top row and left column are known.

\section{Complexity analysis}
If we set the submatrix dimension as $t = log_{3|\sigma|}N$, the total time complexity is equal to:
\begin{itemize}
\item $O(N \cdot log_{3|\sigma|}^2N)$ for submatrix precalculation
\item $O(N^2 / log_{3|\sigma|}^2N)$ for submatrix combination
\end{itemize}

The second complexity assumes the submatrices are stored in such a fashion that they can be retrieved in constant time. This is possible because the total number of possible submatrices can be precomputed easily so we can allocate enough space. For more details on the matrix indexing see the implementation. For more details on the complexities involved see \citep{gusfield1997algorithms}.

\chapter{Test results}
Our algorithm was tested on synthetic sequences. Pairs of sequence reads were simulated using the wgsim tool from a reference chromosome of Escherichia coli bacteria, and then converted into the FASTA file format which is used as an input file format. The output file format of our program was MAF (\textit{Multiple Alignment Format}) containing score, which is equal to string edit distance, and sequence alignment for algorithms that support it.

Pairs of sequence reads were generated with lengths of 100, 200, 500, 1 000, 5 000, 10 000, 50 000, 100 000, 500 000 and 1 000 000 characters. Every pair was then tested on our algorithm, as well as the Fischer-Wagner algorithm for reference. Every test was timed and the maximal memory usage was observed for each of the algorithms. Fischer-Wagner algorithm was tested on sequences up to 100 000 characters because of its large time complexity. Both algorithms were tested in the same conditions on the same computer running inside a Bio-Linux-8 virtual machine with access to 2 CPU cores and 3GB RAM.
\begin{table}[h]
\centering
\begin{tabular}{rccc}
$N$ & Fischer-Wagner & Masek-Paterson & Masek-Paterson (alignment)\\
\hline
$      100 $&$ < 0.01$ s&$ <0.01$ s&$ <0.01 $ s \\
$      200 $&$ < 0.01$ s&$ 0.07$ s&$ 0.07$ s \\
$      500 $&$ < 0.01$ s&$ 0.07$ s&$ 0.07$ s\\
$    1 000 $&$ 0.03$ s&$ 0.07$ s&$ 0.09$ s\\
$    5 000 $&$ 0.78$ s&$ 0.21$ s&$ 0.55$ s\\
$   10 000 $&$ 3.12$ s&$ 0.65$ s&$ 1.99$ s\\
$   50 000 $&$ 94.94$ s&$ 22.75$ s&$ - $\footnotemark[2]\\
$  100 000 $&$ 380.24$ s&$ 41.93$ s&$ - $\footnotemark[2]\\
$  500 000 $&$ 2 - 3$ h\footnotemark[1] &$ 724.78$ s&$ - $\footnotemark[2]\\
$1 000 000 $&$ 9 - 10$ h\footnotemark[1] &$ 49.73$ min&$ - $\footnotemark[2]\\
\end{tabular}
\caption{Time comparison}
\end{table}
\begin{table}[h]
\centering
\begin{tabular}{rccc}
$N$ & Fischer-Wagner & Masek-Paterson & Masek-Paterson (alignment) \\
\hline
$      100 $&$ 6976$ KB &$ 6320$ KB &$ 6832$ KB \\
$      200 $&$ 7008$ KB &$ 7920$ KB &$ 8448$ KB \\
$      500 $&$ 7024$ KB &$ 7936$ KB &$ 10.75$ MB \\
$    1 000 $&$ 7056$ KB &$ 7952$ KB &$ 19.5$ MB\\
$    5 000 $&$ 7296$ KB &$ 8176$ KB &$ 295.5$ MB\\
$   10 000 $&$ 7616$ KB &$ 8416$ KB &$ 1.13$ GB\\
$   50 000 $&$ 9824$ KB &$ 335.7$ MB&$ - $\footnotemark[2]\\
$  100 000 $&$ 12.42$ MB &$ 356.9$ MB&$ - $\footnotemark[2]\\
$  500 000 $&$ -$\footnotemark[1]&$ 370.6$ MB&$ - $\footnotemark[2]\\
$1 000 000 $&$ -$\footnotemark[1]&$ 392.8$ MB&$ - $\footnotemark[2]\\
\end{tabular}
\caption{Maximal memory consumption comparison}
\end{table}
\footnotetext[1]{too much time required for execution}
\footnotetext[2]{too much memory required for execution}

\chapter{Conclusion}
We've presented an implementation of fast string alignment as described in \cite{masek1980faster}. Using the Four Russians algorithm combined with the proposed edit matrix reduction system we've achieved a significant improvement over the quadratic edit distance algorithm of Fischer-Wagner. In terms of time complexity, our implementation is faster by a factor of $log^2_{3|\sigma|}(N)$ where $|\sigma|$ denotes the alphabet size. When discussing space complexity, we use less memory (by the same factor) but the required memory is still a bottleneck when dealing with long strings. If we wanted to calculate the string alignment for strings up to a million characters in length we'd still need a small supercomputer.

The method we explored leaves barely any room for space complexity optimization. With that in mind, the next potential improvement would be to research and implement a more memory-efficient idea, for example \cite{kundeti2008extending}.

\bibliography{literatura}
\bibliographystyle{fer}

\nocite{*}
\end{document}


\unnumberedchapter{Abstract}
\chapter*{Abstract}
\subsection*{\thesistitle}

This body of work examines the non-equilibrium dynamics of vortex lattice carrying Bose--Einstein condensates. We solve the mean-field Gross--Pitaevskii equation for a two-dimensional pancake geometry, in the co-rotating frame within the limit of high rotation frequencies. The condensate responds to this by creating a large periodic lattice of vortices with 6-fold triangular symmetry. By applying two distinct perturbations to this lattice, we examine the resulting effects on the vortices during time evolution. The first perturbation involves applying an optical potential with matching geometry to the vortex lattice. We observe the appearance of interference fringes, and we show that these can be described by moir\'e interference theory. This is backed up by a decomposition of the kinetic energy spectra of the condensate. The applied perturbation only modifies the condensate density, with the vortex positions largely unaffected. From this we conclude that the vortex lattice is very stable and robust against phononic disturbances.

Next, by removing vortices at predefined positions in the lattice using phase imprinting techniques, we examine the resulting order of the lattice. By performing this we generate stable topological defects in the crystal structure.  The resulting lattice remains highly ordered in the presence of low numbers of these defects, where crystal structure and order of the lattice shows to be highly robust. By varying the type of imprinted phases we can create controllable degrees of disorder in the lattice. This disorder is analysed using orientational correlations, Delaunay triangulation, and Voronoi diagrams of the vortex lattice, and demonstrates a method for examining order and generating disorder in vortex lattices in Bose--Einstein condensates.

All work described makes extensive use of GPU computing techniques, and allows for the simulation of these systems to be realised in short times. The implementation of the calculations using GPU computing are also discussed, where the software is shown to be the fastest of its kind out of the independently tested software suites.


\documentclass[a4paper,11pt]{article}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{graphicx}

\title{Long Short Term Memory}
\author{Marco Marini}

\begin{document}

\maketitle
\tableofcontents

\begin{abstract}
Long Short Term Memory (LSTM)
\end{abstract}

\section{General}

The LSTM is a recurrent neural network built up  by LSTM cell.

Several neurons compose the LSTM cell:

\begin{enumerate}
	\item input neuron
	\item input gate
	\item CEC (Constant Error Carrousels)
	\item forget gate
	\item output activation function
	\item output gate
\end{enumerate}

The input neuron and each gate receive values from input signals and the previuos outputs (RNN).

The output signal of LSTM is

\[
	y(t) = y^h(t) \cdot y^f(t)
\]
where
\[
	y^f(t) = f\left( z^f(t) \right)
\]
\[
	z^f(t) = \sum w^f_i I_i(t)
\]
\[
	y^h(t) = h\left( z^h(t) \right)
\]
\[
	z^h(t) = y^g(t) \cdot y^s(t) + z^h(t-1) \cdot y^r(t)
\]
\[
	y^r(t) = r \left( z^r(t) \right)
\]
\[
	z^r(t) = \sum w^r_i I_i(t)
\]
\[
	y^g(t) = g \left( z^g(t) \right)
\]
\[
	z^g(t) = \sum w^g_i I_i(t)
\]
\[
	y^s(t) = s \left( z^s(t) \right)
\]
\[
	z^s(t) = \sum w^s_i I_i(t)
\]

\section{Back propagation}

Let's now derive the back propagation error

\[
	\frac{\partial y(t)}{\partial w^f_i} = y^h(t) \frac{\partial y^f(t)}{\partial w^f_i}
\]
\[
	\frac{\partial y^f(t)}{\partial w^f_i} =
		\frac{\partial f\left( z^f(t) \right)}{\partial z^f(t)}
		\cdot \frac{\partial z^f(t) }{\partial  w^f_i}
		= f'(t) \cdot  I_i(t)
\]

\[
\begin{array}{c}
	\frac{\partial y(t)}{\partial w^r_i} = y^f(t) \frac{\partial y^h(t)}{\partial w^r_i} =
\\
	= y^f(t) \cdot h'(t) \frac{\partial z^h(t)}{\partial w^r_i} =
\\
	= y^f(t) \cdot h'(t) z^h(t-1) \cdot \frac{\partial y^r(t)}{\partial w^r_i} =
\\
	= y^f(t) \cdot h'(t) z^h(t-1) \cdot r'(t) \cdot \frac{\partial z^r(t)}{\partial w^r_i}
\\
	= y^f(t) \cdot h'(t) z^h(t-1) \cdot r'(t) \cdot I_i(t)
\end{array}
\]

\[
\begin{array}{c}
	\frac{\partial y(t)}{\partial w^s_i} = y^f(t) \frac{\partial y^h(t)}{\partial w^s_i} =
\\
	= y^f(t) \cdot h'(t) \frac{\partial z^h(t)}{\partial w^s_i} =
\\
	= y^f(t) \cdot h'(t) \cdot y^g(t) \frac{\partial y^s(t)}{\partial w^s_i} =
\\
	= y^f(t) \cdot h'(t) \cdot y^g(t) s'(t) \cdot \frac{\partial z^s(t)}{\partial w^s_i} =
\\
	= y^f(t) \cdot h'(t) \cdot y^g(t) \cdot s'(t) \cdot I_i(t)
\end{array}
\]

\begin{eqnarray}
	\frac{\partial y(t)}{\partial w^f_i} = y^h(t) \cdot f'(t) \cdot I_i(t)
\\
	\frac{\partial y(t)}{\partial w^r_i} = y^f(t) \cdot h'(t) z^h(t-1) \cdot r'(t) \cdot I_i(t)
\\
	\frac{\partial y(t)}{\partial w^s_i} = y^f(t) \cdot h'(t) \cdot y^g(t) \cdot s'(t) \cdot I_i(t)
\\
	\frac{\partial y(t)}{\partial w^g_i} = y^f(t) \cdot h'(t) \cdot y^s(t) \cdot g'(t) \cdot I_i(t)
\end{eqnarray}

\section{Error}

Let be \[ X_{ijk} \] the input signals $ j = 1, \cdots, m$ of sample $ i = 1, \cdots, n$ at time $ k = 1, \cdots, t_i$ and \[ Y_{ik} \] the expected output signal of sample $ i = 1, \cdots, n$ at time $ k = 1, \cdots, t_i$

The total error of network is

\[
	\delta = \frac{1}{2} \sum_{ik} \left[ Y_{ik} - y(k, I_{ijk})\right] ^ 2
\]

\section{Test}

The test is composed by a set of samples with 2 input signals.

The output is a JK memory latch

\begin{tabular}{|c|c|c|}
	\hline J & K & Y \\
	\hline 0 & 0 & Y \\
	\hline 0 & 1 & 0 \\
	\hline 1 & 0 & 1 \\
	\hline 1 & 1 & !Y \\
	\hline
\end{tabular}

The activation functions are

\[
\begin{array}{l}
f(x) = tanh(x) \\
g(x) = tanh(x) \\
h(x) = x \\
r(x) = tanh(x) \\
s(x) = \tanh(x) \\
\end{array}
\]

Gradients are:

\begin{eqnarray}
	\frac{\partial y(t)}{\partial w^f_i} = y^h(t) \cdot \left[ 1 - y^f(t)\right] ^2 \cdot I_i(t)
\\
	\frac{\partial y(t)}{\partial w^r_i} = y^f(t) \cdot z^h(t-1) \cdot
	\left[ 1 - y^r(t) \right]^2  \cdot I_i(t)
\\
	\frac{\partial y(t)}{\partial w^s_i} = y^f(t) \cdot y^g(t) \cdot \left[ 1 - y^s(t) \right]^2 \cdot I_i(t)
\\
	\frac{\partial y(t)}{\partial w^g_i} = y^f(t) \cdot y^s(t) \cdot \left[ 1 - y^g(t) \right]^2 \cdot I_i(t)
\end{eqnarray}

\end{document}