\documentclass[12pt, titlepage]{article}
\usepackage[english]{babel}
\usepackage[utf8]{inputenc}

\usepackage{graphicx} %graphics files inclusion
% \usepackage{subfig} %subfigures
\usepackage{amsmath} %advanced maths
\usepackage{amssymb} %additional math symbols
\usepackage{dsfont}  %contains \mathds{1}
\usepackage{algpseudocode}
\usepackage{url}

\usepackage{dirtree} %directory tree visualisation

\usepackage[draft]{todonotes}   % notes showed

\usepackage[ruled,vlined]{algorithm2e}
\newcommand\mycommfont[1]{\footnotesize\ttfamily\textcolor{blue}{#1}}
\SetCommentSty{mycommfont}

\usepackage{xcolor, colortbl}
\usepackage{pdfpages}

\newcommand{\lnorm}[1]{\left\lVert#1\right\rVert^2}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}

\algnewcommand\algorithmicforeach{\textbf{for each:}}
\algnewcommand\ForEach{\item[ \algorithmicforeach]}
\author{Radek Bartyzal}

\begin{document}
\begin{titlepage}
    \centering
    \vfill
    {\bfseries\Huge
        Adam: a method for stochastic optimization \\
    }
        \vskip1cm

    {\bfseries\Large
    Czech Technical University in Prague\\
    Faculty of Informatics\\
    \vskip1cm
    Radek Bartyzal\\
    }
    \vskip1cm
    October 21, 2018
    \vfill

    \vfill
    \vfill
\end{titlepage}

\tableofcontents

\section{Prerequisites}
The following text assumes that the reader is familiar with these terms:

\begin{itemize}
\item Neural Networks: Chapter 1 of \cite{cit:nn}
\item Backpropagation: Chapter 2 of \cite{cit:nn}
\end{itemize}

\section{Optimization of neural networks}\label{sec:optim}
Training of a neural network consists of minimization of its loss function $L$. An example of such loss function is:

\begin{tabbing}
$w$ = weights \hspace{40mm} \= $b$ = biases\\
$n$ = number of inputs \> $x$ = one input\\
$a$ = output of network for $x$\\
$y(x)$ = desired output for $x$
\end{tabbing}
$$
L(w,b) = \frac{1}{2n} \sum_x{ \| y(x) - a \|^2}
$$

By using the backpropagation algorithm we are able to calculate the gradient of the loss function with regards to every parameter. We can use this gradient to update the parameters.
They are updated by a taking a small step in the opposite direction of the gradient since that is the direction in which the loss function decreases the most in its value. The size of the step is controlled by a parameter $\eta$ called \textit{learning rate}. This is a general description of a \textit{Gradient Descent} algorithm, however there are many different ways how to calculate the actual step and we will present the ones most relevant to this work.

\section{Gradient Descent variants}\label{sec:sgd_variants}
There are three variants of the basic gradient descent algorithm and all of them share the following inputs:

\begin{equation}
\begin{aligned}
\theta_0 &\quad \text{Initial parameters}\\
N &\quad \text{Number of training examples}\\
x_i,\ \forall i <  N &\quad \text{Training examples}\\
\hat{y}_i,\ \forall i < N &\quad \text{Labels for the training examples}\\
T &\quad \text{Number of epochs}
\end{aligned}
\end{equation}

\begin{itemize}
\item \textbf{Batch Gradient Descent (BGD)} shown as Algorithm \ref{alg:BGD} calculates the step as an average of gradients over all the training examples. This approach unfortunately does not scale well because from a certain point you cannot load the whole dataset into memory making each single update extremely slow. Another problem is that it does not allow \textit{online learning} meaning we cannot simply continue training with newly arrived data points.

\begin{algorithm}
  \label{alg:BGD}
  \caption{Batch Gradient Descent}
  \DontPrintSemicolon
  \For{$t\gets 1$ \KwTo $T$}
  {
  $g = 0$\;
  \ForEach{$i \in \text{dataset}$}
    {
     $g = g + \nabla L(\hat{y}_i,f(\theta_{t-1}, x_i))$\;
    }
  $g = \frac{1}{N} g$\;
  $\theta_t = \theta_{t-1} - \eta g$
  }
\end{algorithm}

\item \textbf{Stochastic Gradient Descent (original SGD)} shown as Algorithm \ref{alg:SGD} updates the parameters with the gradient of every training example sequentially. The training data is shuffled between epochs to add an element of randomness resulting in increased chances of finding a better local minimum. This approach alleviates both mentioned problems of BGD, unfortunately it introduces its own one. Due to the frequent updates, the global loss tends to have a very high variance which may help it escape local minima but it also complicates convergence.

\begin{algorithm}
  \label{alg:SGD}
  \caption{Stochastic Gradient Descent}
  \DontPrintSemicolon
  $\theta = \theta_0$\;
  \For{$t\gets 1$ \KwTo $T$}
  {
  $\mathit{shuffle(dataset)}$\;
  \ForEach{$i \in \text{dataset}$}
    {
     $g = \nabla L(\hat{y}_i,f(\theta, x_i))$\;
     $\theta = \theta - \eta g$\;
    }
  }
\end{algorithm}

\item \textbf{Mini-batch Stochastic Gradient Descent (SGD)} described in Algorithm \ref{alg:mini_SGD} is the logical combination of the two mentioned methods. By updating the parameters after each mini batch of size $B$ the algorithm achieves significantly less variance of the loss while keeping the advantage of frequent updates. It also leverages the fast matrix operations available on current graphical processors. The new hyperparameter $B$ can be selected based on the size of the dataset and available memory to strike a balance between speed and variance. This algorithm is generally referred to as SGD because due to the strong disadvantages of both BGD and original SGD they are very rarely used.

\begin{algorithm}
  \label{alg:mini_SGD}
  \caption{Mini-batch Stochastic Gradient Descent}
  \DontPrintSemicolon
  $\theta = \theta_0$\;
  \For{$t\gets 1$ \KwTo $T$}
  {
  $\mathit{shuffle(dataset)}$\;
  \ForEach{$\text{mini\_batch} \in \text{dataset}$}
  {
  $g = 0$\;
  \ForEach{$i \in \text{mini\_batch}$}
    {
     $g = g + \nabla L(\hat{y}_i,f(\theta, x_i))$\;
    }
  $g = \frac{1}{B} g$\;
  $\theta = \theta - \eta g$
  }
  }
\end{algorithm}
\end{itemize}

Even though the SGD fixes the mentioned imperfections, it still has two significant problems:

\begin{itemize}
\item Finding a good learning rate can prove to be difficult, but we can offload this task to a \textit{learning rate scheduler} that changes it during training based both on elapsed time steps and past performance  \cite{darken1992learning}. Although the adaptable learning rate performs much better than a constant one, the fact that it is identical for all parameters causes problems in situation where each example while having a high dimension has only few non-zero features. Therefore some features occur more frequently than others which is not reflected in the applied learning rate. That results in slower updates to the less frequent features.

\item The loss function of a deep neural network is undoubtedly very complex which brings many challenges to optimization. The most profound difficulty in optimizing such a high dimensional non-convex function is however believed to stem from an extensive number of saddle points surrounded by large plateaus with high error \cite{dauphin2014identifying}. This is where the loss function increases in some dimensions while it is constant or decreasing in other dimensions. A simple three dimensional example can be a slowly descending valley with steep slopes on both sides. The problem stems from the small decrease in value in the one dimension that actually leads to a minimum. An intuitive explanation of what will happen is that the SGD will keep jumping across the valley while moving very slowly in the desired direction.
\end{itemize}

The following methods alleviate one or both of the described issues \cite{ruder2016overview}.

\section{AdaGrad}
AdaGrad is an adaptive gradient method attempting to solve the issue of some features appearing less frequently than others \cite{duchi2011adaptive}. The general idea is for the learner to give larger weight to infrequent features when they appear.

This is implemented by taking note of the past gradient updates and using the sum of the squared past gradients to divide the actual learning rate.

We start with a gradient of the loss function $L$ with respect to a parameter $i$ at time $t$:

\begin{equation}\label{eq:gradient}
g_{t,i} = \nabla_{\theta} L(\theta_{t,i})
\end{equation}

The classic mini-batch SGD update would look like this:

\begin{equation}\label{eq:sqd_update}
\theta_{t+1,i}^{SGD} = \theta_{t,i} - \eta g_{t,i}
\end{equation}

However the AdaGrad update leverages the past gradient update information to adaptively change the learning rate for each of the parameters separately:

\begin{equation}\label{eq:adagrad_update}
\theta_{t+1,i}^{AdaGrad} = \theta_{t,i} - \frac{\eta}{\sqrt{ \sum_{\tau=1}^t{g_{\tau,i}^2}} + \epsilon} g_{t,i}
\end{equation}

The epsilon is used to prevent division by an extremely small number. Even though this new update rule nicely adapts to each parameter, their learning rates keep getting smaller with increasing time steps. This is caused by the sum of squared gradients that can only increase with time resulting in a smaller and smaller effective learning rate, possibly reaching zero and stopping the training entirely. Another issue is the sensitivity to the initial setting of the learning rate. If the gradients are too large at the beginning, the parameter updates will be small for the rest of the training \cite{zeiler2012adadelta}.

\section{RMSProp and AdaDelta}
Both RMSProp and AdaDelta have been invented around the same time to solve the mentioned issue of AdaGrad's diminishing learning rate. The RMSProp has been introduced by G. Hinton in his course at University of Toronto \cite{rmsprop}. It is slightly simpler than AdaDelta \cite{zeiler2012adadelta} while using the same idea which is why we will discuss it here.

The central idea is to use a decaying running average of past squared gradients representing gradients from a certain time window instead of using all of them. This ensures that the training will not slow down after a large number of updates.

The running average of past gradients can be effectively calculated as:

\begin{equation}\label{eq:running_avg}
\mathbb{E}[g^2]_t = \gamma \mathbb{E}[g^2]_{t-1} + (1 - \gamma) g_t^2
\end{equation}

Then we just replace the summation term in the AdaGrad update rule with this running average estimate:

\begin{equation}\label{eq:rmsprop_update}
\theta_{t+1}^{RMSProp} = \theta_{t} - \frac{\eta}{\sqrt{\mathbb{E}[g^2]_t} + \epsilon} g_{t}
\end{equation}

The decay rate $\gamma$ is recommended to be set to $0.9$ while a good default learning rate is $0.001$. These methods are generally much less sensitive to a different initial learning rates making hyper-parameter tuning easier \cite{zeiler2012adadelta}.

\section{Adam}
The Adam (adaptive moment estimator) method is an improvement of the previously mentioned RMSProp with an addition of an estimate of the first order momentum \cite{kingma2014adam}.

Just as the RMSProp the Adam calculates the decaying running average of the squared past gradients, here called $v_t$. It also calculates the running average of the gradients themselves, called $m_t$ the same way. The $m_t$ and $v_t$ estimate the first and second order moments of the gradient corresponding to the mean and the uncentered variance.

\begin{equation}\label{eq:adam_moments}
\begin{aligned}
m_t &= \beta_1 m_{t-1} + (1 - \beta_1) g_t\\
v_t &= \beta_2 v_{t-1} + (1 - \beta_2) g_t^2
\end{aligned}
\end{equation}

The $m_t$ and $v_t$ are unfortunately biased toward zero at the start of training due to their initialization to zero vectors. To correct that bias the Adam algorithm divides the moment estimates with a time sensitive term approaching $1$ with increasing number of time steps:

\begin{equation}\label{eq:adam_moments}
\begin{aligned}
\hat{m_t} &= \frac{m_t}{(1 - \beta_1^t)}\\
\hat{v_t} &= \frac{v_t}{(1 - \beta_2^t)}
\end{aligned}
\end{equation}

These bias corrected moment estimates are then used in the actual update rule similarly to the RMSProp and AdaDelta algorithms:

\begin{equation}\label{eq:adam_update}
\theta_{t+1}^{Adam} = \theta_{t} - \frac{\eta}{\sqrt{v_t} + \epsilon} m_t
\end{equation}

The recommended default values for the hyper-parameters are $\beta_1 = 0.9$, $\beta_2 = 0.999$, $\epsilon = 10^{-8}$.
The positive aspects of the second order moment dividing the learning rate have been explained in the previous sections. The addition of the first order moment can be understood as a momentum term. It aims to solve the problem SGD has with getting out of saddle points.
The effects of momentum can be presented on the example with the slowly descending valley given at the end of Section \ref{sec:sgd_variants}.

If we imagine a ball without momentum in such valley, it will keep going up the opposing sides while slowly moving through the valley. If we add momentum to the ball, it will dampen its oscillation while increasing the speed of movement in the direction of consistent decrease in function value. We can compare that to adding weight to the ball.

Transferring the example to the effects on optimization, the updates in the dimensions where the gradient directions keep changing will be smaller while the updates in the dimensions that are consistently going a certain direction will become incrementally larger.

A follow-up research into Adam has uncovered several convergence issues and possible areas of improvement which led to the introduction of new versions such as:

\begin{itemize}
\item \textbf{NAdam:} Adapt the momentum term with the Nesterov accelerated gradient method \cite{dozat2016incorporating}.
\item \textbf{AdamW:} Claims to fix the weight decay calculation \cite{loshchilov2017fixing}.
\item \textbf{AmsGrad:} Finds errors in the proof of Adam's convergence and claims to improve it by introducing AmsGrad algorithm \cite{reddi2018convergence}.
\end{itemize}

To the contrary of the numerous papers claiming to improve the original Adam algorithm, experimental results show that it in many cases works better or at least as well as its newer variants. While some of them look promising, there is no single variant dominating others at all tested optimization tasks \cite{adam-variants}.

\section{Conclusion}

The Adam algorithm remains as one of the best ways to train neural networks. However the mini batch SGD is also still extensively used because it is well understood and achieves very good results with a properly tuned learning rate schedule.

\begin{thebibliography}{0}
  \bibitem[1]{cit:nn} Nielsen, Michael A. Neural networks and deep learning. Vol. 25. USA: Determination press, 2015.

  \bibitem[2]{darken1992learning} Darken, Christian, Joseph Chang, and John Moody. "Learning rate schedules for faster stochastic gradient search." Neural Networks for Signal Processing [1992] II., Proceedings of the 1992 IEEE-SP Workshop. IEEE, 1992.

  \bibitem[3]{dauphin2014identifying} Dauphin, Yann N., et al. "Identifying and attacking the saddle point problem in high-dimensional non-convex optimization." Advances in neural information processing systems. 2014.

  \bibitem[4]{ruder2016overview} Ruder, Sebastian. "An overview of gradient descent optimization algorithms." arXiv preprint arXiv:1609.04747 (2016).

  \bibitem[5]{zeiler2012adadelta} Zeiler, Matthew D. "ADADELTA: an adaptive learning rate method." arXiv preprint arXiv:1212.5701 (2012).

  \bibitem[6]{kingma2014adam} Kingma, Diederik P., and Jimmy Ba. "Adam: A method for stochastic optimization." arXiv preprint arXiv:1412.6980 (2014).

  \bibitem[7]{dozat2016incorporating} Dozat, Timothy. "Incorporating nesterov momentum into adam." (2016).

  \bibitem[8]{loshchilov2017fixing} Loshchilov, Ilya, and Frank Hutter. "Fixing weight decay regularization in adam." arXiv preprint arXiv:1711.05101 (2017).

  \bibitem[9]{reddi2018convergence} Reddi, Sashank J., Satyen Kale, and Sanjiv Kumar. "On the convergence of adam and beyond." (2018).

  \bibitem[10]{adam-variants} FastAI’s experimental results of Adam variants [online]. \url{http:
//www.fast.ai/2018/07/02/adam-weight-decay/}, [Online; accessed
2018-October-21].

  \bibitem[11]{rmsprop} G. Hinton’s lecture introducing RMSProp [online]. \url{http:
//www.cs.toronto.edu/˜tijmen/csc321/slides/lecture_slides_
lec6.pdf}, [Online; accessed 2018-October-21].

  \bibitem[12]{duchi2011adaptive} Duchi, John, Elad Hazan, and Yoram Singer. "Adaptive subgradient methods for online learning and stochastic optimization." Journal of Machine Learning Research 12.Jul (2011): 2121-2159.

  \end{thebibliography}

\end{document}


\chapter{Objects of Arrays}

\index{array!of cards}

In the previous chapter, we defined a class to represent cards and used an array of \java{Card} objects to represent a deck.
In this chapter, we take additional steps toward object-oriented programming.

First we define a class to represent a deck of cards.
Then we present algorithms for shuffling and sorting decks.
Finally, we introduce \java{ArrayList} from the Java library and use it to represent collections of cards.

%While reading the following sections, we recommend that you create a {\tt Deck.java} file and paste in all the examples.
%You will need {\tt Card.java} from the previous chapter for it to compile.

%The code for this chapter is in {\tt Card.java} and {\tt Deck.java}, which are in the directory {\tt ch13} in the repository for this book.
%Instructions for downloading this code are on page~\pageref{code}.

\section{Decks of Cards}
\label{deck}

\index{Deck}
\index{class!Deck}

Here is the beginning of a \java{Deck} class that encapsulates an array of \java{Card} objects:

\begin{code}
public class Deck {
    private Card[] cards;

    public Deck(int n) {
        this.cards = new Card[n];
    }

    public Card[] getCards() {
        return this.cards;
    }
}
\end{code}

\index{constructor}
\index{memory diagram}
\index{diagram!memory}

The constructor initializes the instance variable with an array of \java{n} cards, but it doesn't create any \java{Card} objects.
Figure~\ref{fig.deckobject} shows what a \java{Deck} looks like with no cards.

\begin{figure}[!ht]
\begin{center}
\includegraphics{figs/deckobject.pdf}
\caption{Memory diagram of an unpopulated \java{Deck} object.}
\label{fig.deckobject}
\end{center}
\end{figure}

We'll add another constructor that creates a standard 52-card array and populates it with \java{Card} objects:

\begin{code}
public Deck() {
    this.cards = new Card[52];
    int index = 0;
    for (int suit = 0; suit <= 3; suit++) {
        for (int rank = 1; rank <= 13; rank++) {
            this.cards[index] = new Card(rank, suit);
            index++;
        }
    }
}
\end{code}

This method is similar to the example in Section~\ref{cardarray}; we just turned it into a constructor.
We can use it to create a complete \java{Deck} like this:

\begin{code}
Deck deck = new Deck();
\end{code}

\index{printDeck}

Now that we have a \java{Deck} class, we have a logical place to put methods that pertain to decks.
Looking at the methods we have written so far, one obvious candidate is \java{printDeck} from Section~\ref{cardarray}.
Here's how it looks, rewritten as an instance method of \java{Deck}:

\begin{code}
public void print() {
    for (Card card : this.cards) {
        System.out.println(card);
    }
}
\end{code}

%\begin{code}
%public void print() {
%    for (int i = 0; i < this.cards.length; i++) {
%        System.out.println(this.cards[i]);
%    }
%}
%\end{code}

Notice that when we transform a static method into an instance method, the code is shorter.

Here's how we invoke it:

\begin{code}
deck.print();
\end{code}

\section{Shuffling Decks}
\label{shuffle}

\index{shuffle}

For most card games, you have to shuffle the deck; that is, put the cards in a random order.
In Section~\ref{random} we saw how to generate random numbers, but it is not obvious how to use them to shuffle a deck.

One possibility is to model the way humans shuffle; for example, we could divide the deck in two halves and then choose alternately from each one.
Since humans usually don't shuffle perfectly, after about seven iterations the order of the deck is pretty well randomized.

But a computer program would have the annoying property of doing a perfect shuffle every time, which is not very random.
In fact, after eight perfect shuffles, you would find the deck back in the order you started in!
For more on this, see \url{https://en.wikipedia.org/wiki/Faro_shuffle}.

\index{pseudocode}

A better shuffling algorithm is to traverse the deck one card at a time, and at each iteration, choose two cards and swap them.
To outline this algorithm, we'll use a combination of Java statements and English comments.
This technique is sometimes called {\bf pseudocode}.

\index{shuffle}

\begin{code}
public void shuffle() {
    for each index i {
        // choose a random number between i and length - 1
        // swap the ith card and the randomly-chosen card
    }
}
\end{code}

\index{helper method}
\index{method!helper}

The nice thing about pseudocode is that it often makes clear what other methods you are going to need.
In this case, we need a method that chooses a random integer in a given range and a method that takes two indexes and swaps the cards at those positions.

\begin{code}
private static int randomInt(int low, int high) {
    // return a random number between low and high,
    // including both
}

private void swapCards(int i, int j) {
    // swap the ith and the jth cards in the array
}
\end{code}

\index{randomInt}
\index{swapCards}

Methods like \java{randomInt} and \java{swapCards} are called {\bf helper methods}, because they help you solve parts of the problem.
Helper methods are often \java{private}, because they are only used by methods in the class, and are not needed by client users of the class. Clients need not know about these methods - they are private to the \java{Deck} class.

\index{top-down design}
\index{design process}

The process of writing pseudocode first and then writing helper methods to make it work is a kind of {\bf top-down design} (see \url{https://en.wikipedia.org/wiki/Top-down_and_bottom-up_design}).
It is an alternative to ``incremental development'' and ``encapsulation and generalization'', the other design processes you have seen in this book.

One of the exercises at the end of the chapter asks you to write the helper methods \java{randomInt} and \java{swapCards}, and use them to implement \java{shuffle}.

When you do the exercise, notice that \java{randomInt} is a class method and \java{swapCards} is an instance method.
Do you understand why?

\section{Selection Sort}
\label{sorting}

\index{selection sort}
\index{sort!selection}

Now that we have shuffled the deck, we need a way to put it back in order.
There is an algorithm for sorting that is ironically similar to the algorithm for shuffling.
It's called {\bf selection sort}, because it works by traversing the array repeatedly and selecting the lowest (or highest) remaining card each time.

During the first iteration, we find the lowest card and swap it with the card in the 0th position.
During the $i$th iteration, we find the lowest card to the right of $i$ and swap it with the $i$th card.
Here is pseudocode for selection sort:

\begin{code}
public void selectionSort() {
    for each index i {
        // find the lowest card at or to the right of i
        // swap the ith card and the lowest card found
    }
}
\end{code}

Again, the pseudocode helps with the design of the helper methods.
For this algorithm we can reuse \java{swapCards} from the previous section, so we only need a method to find the lowest card; we'll call it \java{indexLowest}.

\begin{code}
private int indexLowest(int low, int high) {
    // find the lowest card between low and high
}
\end{code}

%slr: 1-14-20
The method \java{indexLowest} is an \textbf{instance} method which means the instance \java{this} is available (in scope).  Thus the \java{Card} array is available inside the method as \java{this.cards}.
%slr: end 1-14-20

One of the exercises at the end of the chapter asks you to write \java{indexLowest}, and then use it and \java{swapCards} to implement \java{selectionSort}.

\section{Merge Sort}
\label{mergesort}

\index{efficiency}

Selection sort is a simple algorithm, but it is not very efficient.
To sort $n$ items, it has to traverse the array $n-1$ times.
Each traversal takes an amount of time proportional to $n$.
The total time, therefore, is proportional to $n^2$.

\index{merge sort}
\index{sort!merge}

We will develop a more efficient algorithm called {\bf merge sort}.
To sort $n$ items, merge sort takes time proportional to $n \log_2 n$.
That may not seem impressive, but as $n$ gets big, the difference between $n^2$ and $n \log_2 n$ can be enormous.

For example, $\log_2$ of one million is around 20.
So if you had to sort a million numbers, merge sort would require 20 million steps.
But selection sort would require one trillion steps!

The idea behind merge sort is this: if you have two decks, each of which has already been sorted, you can quickly merge them into a single, sorted deck.
Try this out with a deck of cards:

\begin{enumerate}

\item Form two decks with about 10 cards each, and sort them so they are face up with the lowest cards on top.
Place the decks in front of you.

\item Compare the top card from each deck and choose the lower one.
Flip it over and add it to the merged deck.

\item Repeat step 2 until one of the decks is empty.
Then take the remaining cards and add them to the merged deck.

\end{enumerate}

%slr: 1-14-20  maybe embed video?

The result should be a single sorted deck.
In the next few sections, we'll explain how to implement this algorithm in Java.

\section{Subdecks}

\index{subdeck}
\label{subdeck}

The first step of merge sort is to split the deck into two ``subdecks'', each with about half of the cards.
So we need a method that takes a deck, and a range of indexes, and returns a new deck that contains the specified subset of cards.

\begin{code}
public Deck subdeck(int low, int high) {
    Deck sub = new Deck(high - low + 1);
    for (int i = 0; i < sub.cards.length; i++) {
        sub.cards[i] = this.cards[low + i];
    }
    return sub;
}
\end{code}

%slr: 1-14-20 added last two sentences to paragraph
The first line creates an unpopulated \java{Deck} object that contains an array of \java{null} references.
Inside the \java{for} loop, the subdeck gets populated with references to \java{Card} objects.
The expression \java{sub.cards.length} uses the 'dot operator' to access \java{sub}'s \java{cards} instance variable which yields an array of \java{Card}.  Then, using the dot operator again, we obtain the array's length.

\index{off-by-one}

The length of the subdeck is \java{high - low + 1}, because both the low card and the high card are included.
This sort of computation can be confusing, and forgetting the ``\java{+ 1}'' often leads to {\bf off-by-one} errors.
Drawing a picture is usually the best way to avoid them.

%For example, to select the middle three of five values in an array, we need \java{3 - 1 + 1} values.
%
%\begin{center}
%\begin{tabular}{ccccc}
%\hline
%\multicolumn{1}{|c|}{} & \multicolumn{1}{c|}{X} & \multicolumn{1}{c|}{X} & \multicolumn{1}{c|}{X} & \multicolumn{1}{c|}{} \\
%\hline
%0                      & 1                      & 2                      & 3                      & 4                     \\
%\end{tabular}
%\end{center}

\index{constructor}
\index{overload}

Figure~\ref{fig.subdeck} is a memory diagram of a subdeck with \java{low = 0} and \java{high = 4}.
The result is a hand with five cards that are {\em shared} with the original deck; that is, they are aliased.

\begin{figure}[!ht]
\begin{center}
\includegraphics{figs/subdeck.pdf}
\caption{Memory diagram showing the effect of \java{subdeck}.}
\label{fig.subdeck}
\end{center}
\end{figure}

\index{aliasing}
\index{reference}

Note that the items of \java{sub} are aliases of corresponding items of \java{cards}. Aliasing might not be a good idea, because changes to shared cards would be reflected in multiple decks.
But since \java{Card} objects are immutable, this kind of aliasing is not a problem.
And it saves some memory because we don't create duplicate \java{Card} objects.

\section{Merging Decks}

\index{merge}

The next helper method we need is \java{merge}, which takes two sorted subdecks and returns a new deck containing all cards from both decks, in order.
Here's what the algorithm looks like in pseudocode, assuming the subdecks are named \java{d1} and \java{d2}:

\begin{code}
private static Deck merge(Deck d1, Deck d2) {
    // create a new deck, d3, big enough for all the cards

    // use the index i to keep track of where we are at in
    // the first deck, and the index j for the second deck
    int i = 0;
    int j = 0;

    // the index k traverses the result deck
    for (int k = 0; k < d3.length; k++) {
        // if d1 is empty, use top card from d2
        // if d2 is empty, use top card from d1
        // otherwise, compare the top two cards

        // add lowest card to the new deck at k
        // increment i or j (depending on deck)
    }
    // return the new deck
}
\end{code}

An exercise at the end of the chapter asks you to implement \java{merge}.
It's a little tricky, so be sure to test it with different subdecks.
Once your \java{merge} method is working, you can use it to write a simplified version of merge sort:

\begin{code}
public Deck almostMergeSort() {
    // divide the deck into two subdecks
    // sort the subdecks using selectionSort
    // merge the subdecks, return the result
}
\end{code}

If you have working versions of \java{subdeck}, \java{selectionSort}, and \java{merge}, you should have no trouble getting this method working.
But it is still not very efficient, because it uses \java{selectionSort} to sort the subdecks.
We can make it more efficient if we use \java{mergeSort} instead, but that means we have to make it recursive!

\section{Adding Recursion}

To make \java{mergeSort} work recursively, you have to add a base case; otherwise it repeats forever.

The simplest base case is a subdeck with 1 cards.
If there is only one card, it can't be out of order, so we consider it sorted.
And if it is already sorted, we can just return it.

And it will turn out to be convenient if we handle another base case, a subdeck with 0 cards.
By the same logic, if there are no cards, they can't be out of order.
So we consider an empty deck to be sorted, and return it.

With these base cases, a recursive version of \java{mergeSort} looks like this:

\begin{code}
public Deck mergeSort() {
    // if the deck has 0 or 1 cards, return it
    // otherwise, divide the deck into two subdecks
    // sort the subdecks using mergeSort
    // merge the subdecks
    // return the result
}
\end{code}

\index{leap of faith}

%slr:  comment out 'leap of faith' and related discussion in next three paragraphs
%As usual, there are two ways to think about recursive programs: you can follow the flow of execution, or you can make the ``leap of faith'' (see Section~\ref{leap_of_faith}).
%This example should encourage you to make the leap of faith.

%When you use \java{selectionSort} to sort the subdecks, you don't feel compelled to follow the flow of execution.
%You assume it works because you already debugged it.
%When you make \java{mergeSort} recursive, you just replace one sorting algorithm with another.
%There is no reason to read the program differently.
%
%Well, almost.
%You have to think about the base cases and make sure that you reach them.
%But other than that, writing the recursive version should be no problem.
%The most difficult part of merge sort is the \java{merge} method, and that part is not recursive.

%As an exercise at the end of this chapter, you'll have a chance to finish it off.

\section{Static Context}

Figure~\ref{fig.deck} shows a UML class diagram for \java{Deck}, including the instance variable, \java{cards}, and the methods we have so far.
In UML diagrams, \java{private} attributes and methods begin with a minus sign (\java{-}) and \java{static} methods are underlined.

\index{UML}
\index{class diagram}
\index{diagram!class}

\begin{figure}[!ht]
\begin{center}
\includegraphics{figs/deck.pdf}
\caption{UML diagram for the \java{Deck} class.}
\label{fig.deck}
\end{center}
\end{figure}

The helper methods \java{randomInt} and \java{merge} are \java{static}, because they do not read or write any instance variables.
All other methods are instance methods, because they access the instance \java{this} and its instance variable \java{cards}.

When you have static methods and instance methods in the same class, it is easy to get them confused.

To invoke an instance method, you need an instance:

\begin{code}
Deck deck = new Deck();
deck.print();  // correct
\end{code}

\java{Deck} with a capital \java{D} is a class, and \java{deck} with a lowercase \java{d} is an object.

If you try to invoke \java{print} like this:

\begin{code}
Deck.print();  // wrong!
\end{code}

% DW suggested that at some point we should warn students
% about using \java{this} in a static method

\index{static context}
\index{this}

You get an compiler error like this:

\begin{stdout}
Non-static method print() cannot be referenced from a
static context.
\end{stdout}

By ``static context'', the compiler means you are trying to invoke a method in a context that requires a static method.

On the other hand, if you have a \java{Deck} object, you can use it to invoke a static method:

\begin{code}
Deck deck = new Deck();
int i = deck.randomInt(0, 51);  // legal, but not good style
\end{code}

This is legal, but it is not considered good style, because someone reading this code would expect \java{randomInt} to be an instance method.

%slr:  1-14-20
A much more common and better way of accessing static methods if by using the class itself:

\begin{code}
int i = Deck.randomInt(0, 51);  // invoke static method via the class
\end{code}

Another common error is to use \java{this} in a static method.
For example, if you write something like this:

\begin{code}
private static Deck merge(Deck d1, Deck d2) {
    return this.cards;  // wrong!
}
\end{code}

You get a compiler error like this:

\begin{stdout}
Non-static variable this cannot be referenced from a
static context.
\end{stdout}

The problem is that \java{cards} is an instance variable, so it is ``non-static'', so you can't access it from a static method.
In general, you can't use \java{this} in a static method, because a static method is not invoked on an object.

For beginners, error messages about non-static context can be confusing and frustrating.
We hope this section helps.

%\index{sort!Arrays}
%\index{array!sorting}

%Normally we wouldn't implement two different sorting algorithms in the same class.
%Our goal with \java{Deck} was to demonstrate static methods and different ways of solving the same problem.
%In practice, we could just write a single \java{sort} method that uses \java{java.util.Arrays}.
%
%\begin{code}
%public void sort() {
%    Arrays.sort(this.cards);
%}
%\end{code}

\section{Piles of Cards}

\index{War (card game)}

Now that we have classes that represent cards and decks, let's use them to make a game.
One of the simplest card games that children play is called ``War'' (see \url{https://en.wikipedia.org/wiki/War_(card_game)}).

Initially, the deck is divided evenly into two piles, one for each player.
During each round, each player takes the top card from their pile and places it, face up, in the center.
Whoever has the highest ranking card, ignoring suit, takes the two cards and adds them to the bottom of their pile.
The game continues until one player has won the entire deck.

We could use the \java{Deck} class to represent the individual piles.
However, our implementation of \java{Deck} uses a \java{Card} array, and the length of an array can't change.
As the game progresses, we need to be able to add and remove cards from the piles.

\index{ArrayList}
\index{collection}

We can solve this problem with an \java{ArrayList}, which is in the \java{java.util} package.
An \java{ArrayList} is a {\bf collection}, which is an object that contains other objects.
It provides methods to add and remove elements, and it grows and shrinks automatically.

%The Java library includes many other collections (see \url{https://thinkjava.org/collections}).
%For our purposes, \java{ArrayList} is a good choice because it provides methods to add and remove elements, and it grows and shrinks automatically.

\index{Pile}
\index{class!Pile}

We define a new class named \java{Pile} to represent a pile of cards.
It uses an \java{ArrayList} to store \java{Card} objects.

\begin{code}
public class Pile {
    private ArrayList<Card> cards;

    public Pile() {
        this.cards = new ArrayList<Card>();
    }
}
\end{code}

\index{angle brackets}
\index{brackets!angle}
\index{\textless\textgreater\ angle brackets}

When you declare an \java{ArrayList}, you specify the type it contains in angle brackets (\java{<>}).
This declaration says that \java{cards} is not just an \java{ArrayList}; it's an \java{ArrayList} of \java{Card} objects.

The constructor initializes \java{this.cards} with an empty \java{ArrayList}.

%Java collections can only store objects, not primitives like \java{int}.
%But you can use wrapper classes, for example \java{ArrayList<Integer>}.

Now let's think about the methods we need to play the game.

At the beginning of each round, each player draws a card from the top of their pile.
So we define a method to do that:

%TODO: ML suggests renaming this drawCard.  I am inclined to agree.

\begin{code}
public Card popCard() {
    return this.cards.remove(0);  // from the top of the pile
}
\end{code}

\java{popCard} removes the \java{Card} at the beginning of the \java{ArrayList}, which we think of as the top of the pile.

Because we use \java{ArrayList.remove}, it automatically shifts the remaining cards to fill the gap.

At the end of each round, the winner adds cards to the bottom of their pile.
So we define a method to do that:

\begin{code}
public void addCard(Card card) {
    this.cards.add(card);        // to the bottom of the pile
}
\end{code}

\java{ArrayList} provides a method, \java{add}, that adds an element to the end of the collection, which we think of as the bottom of the pile.

To know when to stop the game, have to check if one of the piles is empty.
Here's a method to do that:

\begin{code}
public boolean isEmpty() {
    return this.cards.isEmpty();
}
\end{code}

So far, these methods don't do very much; they just invoke methods on the instance variable, \java{cards}.
Methods like these are called {\bf wrapper methods} because they wrap one method with another.

% TODO: Change the code to get rid of size (unless we need it somewhere else) and change empty to isEmpty

%\begin{code}
%public int size() {
%    return this.cards.size();
%}
%\end{code}

Finally, to start the game, we need to divide the deck into two equal parts.
We can do that with \java{subdeck} from Section~\ref{subdeck} and a new method, \java{addDeck}:

\begin{code}
public void addDeck(Deck deck) {
    for (Card card : deck.getCards()) {
        this.cards.add(card);
    }
}
\end{code}

\java{addDeck} takes a \java{Deck} object, loops through the cards, and adds them to the \java{Pile}.
Notice that it does not remove the cards from the \java{Deck}, so the \java{Deck} and the \java{Pile} share cards.
But that won't be a problem because cards are immutable.

\section{Playing War}

Now we can use \java{Deck} and \java{Pile} to implement the game.
We'll start by creating a deck and shuffling:

\begin{code}
Deck deck = new Deck();
deck.shuffle();
\end{code}

Then we divide the \java{Deck} into two piles:

\begin{code}
Pile p1 = new Pile();
p1.addDeck(deck.subdeck(0, 25));

Pile p2 = new Pile();
p2.addDeck(deck.subdeck(26, 51));
\end{code}

The game itself is a loop that repeats until one of the piles is empty.
At each iteration, we draw a card from each pile and compare their ranks.

\begin{code}
while (!p1.isEmpty() && !p2.isEmpty()) {
    // pop a card from each pile
    Card c1 = p1.popCard();
    Card c2 = p2.popCard();

    // compare the cards ignoring suit
    int diff = c1.getRank() - c2.getRank();
    if (diff > 0) {
        p1.addCard(c1);
        p1.addCard(c2);
    } else if (diff < 0) {
        p2.addCard(c1);
        p2.addCard(c2);
    } else {
        // it's a tie
    }
\end{code}

If the two cards have the same rank, it's a tie.
In that case, each player draws four more cards.
Whoever has the higher fourth card takes all cards in play.
If there's another tie, they draw another four cards, and so on.

One of the exercises at the end of this chapter asks you to implement the \java{else} block when there's a tie.

After the \java{while} loop ends, we display the winner based on which pile is not empty.

\begin{code}
if (p2.isEmpty()) {
    System.out.println("Player 1 wins!");
} else {
    System.out.println("Player 2 wins!");
}
\end{code}

\java{ArrayList} provides many other methods that we didn't use for this example.
Take a minute to read the documentation, which you can find by doing a web search for ``Java ArrayList''.

\section{Vocabulary}

\begin{description}

\term{pseudocode}
A way of designing programs by writing rough drafts in a combination of English and Java.

\term{helper method}
A method that implements part of a more complex algorithm; often it is not particularly useful on its own.

\term{top-down design}
Breaking down a problem into sub-problems, and solving each sub-problem one at a time.

\term{selection sort}
A simple sorting algorithm that searches for the smallest or largest element $n$ times.

\term{merge sort}
A recursive sorting algorithm that divides an array into two parts, sorts each part (using merge sort), and merges the results.

\term{off-by-one}
A common programming mistake that results in iterating one time too many, or too few.

\term{static context}
The parts of a class that run without reference to a specific instance of the class.

\term{collection}
A Java library class, like \java{ArrayList}, that represents a group of objects.

\term{wrapper method}
A method that calls another method without doing much additional work.

%\term{insertion sort}
%Another sorting algorithm that inserts elements into place, one at a time.

\end{description}

\section{Exercises}

The code for this chapter is in the {\tt ch13} directory of {\tt ThinkJavaCode2}.
See page~\pageref{code} for instructions on how to download the repository.
Before you start the exercises, we recommend that you compile and run the examples.

\begin{exercise}  %%V6 Ex13.5

Write a \java{toString} method for the \java{Deck} class.
It should return a single string that represents the cards in the deck.
When it's printed, this string should display the same results as the \java{print} method in Section~\ref{deck}.

\index{StringBuilder}
\index{efficiency}

{\it Hint:} You can use the \java{+} operator to concatenate strings, but it is not very efficient.
Consider using \java{java.lang.StringBuilder} instead; see Section~\ref{stringbuilder}.

\end{exercise}

\begin{exercise}  %%V6 Ex13.2
\label{ex.shuffle}

The goal of this exercise is to implement the shuffling algorithm from this chapter.

\begin{enumerate}

% TODO: Do we need to clarify the instructons for finding the starter code?

\item In the repository for this book, you should find the file named {\tt Deck.java}.
Check that you can compile it in your environment.

\item Implement the \java{randomInt} method.
You can use the \java{nextInt} method provided by \java{java.util.Random}, which we saw in Section~\ref{random}.

{\it Hint:} To avoid creating a \java{Random} object every time \java{randomInt} is invoked, consider defining a class variable.

\item Write a \java{swapCards} method that takes two indexes and swaps the cards at the given locations.

\item Fill in the \java{shuffle} method using the algorithm in Section~\ref{shuffle}.

\end{enumerate}

\end{exercise}

\begin{exercise}  %%V6 Ex13.3

The goal of this exercise is to implement the sorting algorithms from this chapter.
Use the {\tt Deck.java} file from the previous exercise, or create a new one from scratch.

% TODO: Again, we could provide clearer instructions for setting up the exercises.

\begin{enumerate}

\item Implement the \java{indexLowest} method.
Use the \java{Card.compareTo} method to find the lowest card in a given range of the deck, from \java{lowIndex} to \java{highIndex}, including both.

\item Fill in \java{selectionSort} using the algorithm in Section~\ref{sorting}.

\item Using the pseudocode in Section~\ref{mergesort}, implement the \java{merge} method.
The best way to test it is to build and shuffle a deck.
Then use \java{subdeck} to form two small subdecks, and use selection sort to sort them.
Finally, pass the two halves to \java{merge} and see if it works.
\index{testing}

\item Fill in \java{almostMergeSort}, which divides the deck in half, then uses \java{selectionSort} to sort the two halves, and uses \java{merge} to create a new, sorted deck.
You should be able to reuse code from the previous step.

\item Implement \java{mergeSort} recursively.
Remember that \java{selectionSort} is a modifier and \java{mergeSort} is a pure method, which means that they get invoked differently:

\begin{code}
deck.selectionSort();      // modifies an existing deck
deck = deck.mergeSort();   // replaces old deck with new
\end{code}

\end{enumerate}

\end{exercise}

\begin{exercise}

%%V6 Ex13.1
You can learn more about the sorting algorithms presented in this chapter at \url{https://www.toptal.com/developers/sorting-algorithms}.
This site provides explanations of the algorithms, along with animations that show how they work.
It also includes an analysis of their efficiency.

%%V6 Ex13.4
For example, ``insertion sort'' is an algorithm that inserts elements into place, one at a time.
Read about it on the website and play the animations.
Then write a method named \java{insertionSort} that implements this algorithm.

One goal of this exercise is to practice top-down design.
Your solution should use a helper method, named \java{insert}, that implements the inner loop of the algorithm.
\java{insertionSort} should invoke this method $n-1$ times.

\end{exercise}

\begin{exercise}  %%V6.5 NEW

%TODO: Clarify setup instructions

Find and open the file \java{War.java} in the repository.
The \java{main} method contains all the code from the last section of this chapter.
Check that you can compile and run this code before proceeding.

The program is incomplete; it does not handle the case when two cards have the same rank.
Finish implementing the \java{main} method, beginning at the line that says: \java{// it's a tie}.

When there's a tie, draw three cards from each pile and store them in a collection, along with the original two.
Then draw one more card from each pile and compare them.
Whoever wins the tie takes all ten of these cards.

If one pile does not have at least four cards, the game ends immediately.
If a tie ends with a tie, draw three more cards, and so on.

Notice that this program depends on \java{Deck.shuffle} so you might have to do Exercise~\ref{ex.shuffle} first.

\end{exercise}


\documentclass[onecolumn]{paper}
\usepackage{tabularx}
\usepackage{color}
\begin{document}

		\section{Unequal Probabilities After Crash and Encryption}

		Consider following scenario:

		1) There are two equivalent states: Both logs are empty but residual blocks are $b_1$ and $b_2$.

		2) User attempts to write $b_1$.

		3) Crash happens after write but before sync.

		4) After reboot, new header is persisted but the new block didn't.

		5) After recovery, one state has empty log, and other has a valid transaction.\\

		{\bf Why didn't we encounter this during proofs?}

		This dichotomy will manifest itself in refinement of abstract tokens.

		First state will require abstract token to be CrashAfter since transaction is committed after recovery.

		Second state will require abstract token to be CrashBefore since transaction is rolled back after recovery.

		This conflict would appear in the proof of two abstract tokens refined from two related states being the same.

		I didn't encounter this because I didn't do that proof yet. If I attempted, it probably wouldn't go through.\\

		{\bf How can we solve this?}\\
		What needs to be fixed is that, with the same reboot function, both states either should roll back or keep the transaction.
		Above requirement means that, there shouldn't be a case where newly written block to log is exactly equal to the already existing block.
		There are two ways to achieve this:\\

		1) Design a log that {\bf guarantees} that requirement.

		I did it back in the day but it was very complicated. I expect it to be practically infeasible to reason about that log. That is why we didn't pursue this.
		Could be an interesting side project though.\\

		2) Design a log where this case is an {\bf exponentially unlikely} event.
		This makes it reasonable to exclude such cases from our proofs.

		This is the approach we take with hash collisions and encryption achieves this.
		If we encrypt each transaction with a fresh key, above situation only happens when $encrypt(k_1, b_1) = encrypt (k_2, b_2)$ where $k_1 \ne k_2$. This is exponentially unlikely as far as I am aware.\\

		{\bf How can we capture this?}\\
		We can't introduce an axiom that says

		$\forall\ k_1\ k_2\ b_1\ b_2,\ k_1 \ne k_2 \rightarrow encrypt(k_1, b_1) \ne encrypt(k_2, b_2)$\\

		At first glance it looks fine because key and block are abstract types.
		Therefore you can't construct an example where $k_1 \ne k_2\ \wedge\ encrypt(k_1, b_1) = encrypt(k_2, b_2)$ which would allow you to prove False.

		Still, this isn't kosher because it makes our model not fit the world. There is no such encrypt function that satisfies the above axiom. Assuming it would make our proofs meaningless, since they are not about our world anymore.\\

		Another way to achieve it is how we achieved it with hashing. Keep a record of each encryption done, when you see a collision like above, let program get stuck. This is the implementation I had before, which I scrapped recently.

		This was also the reason why our GetKey was weird. We needed to know that there won't be a collision when we encrypt our data with the generated key. I could be able to get rid of that weirdness thanks to we moving to simulations from bisimulations.
		Bisimulations required total correctness, simulations require partial correctness.\\

		Although above approach plays nice with 95\% of the project, it has a problem with write/extend special case NI proofs' transfer.
		One requirements of the transfer lemma for spacial case is "if $s_1$ and $s_2$ are equivalent states and you can run $write(a, b_1)$ from $s_1$, then you can run $write(a, b_2)$ from $s_2$ with the same oracle."

		To show that you can run $write(a, b_2)$ from $s_2$, you need to prove that there won't be any collision when you encrypt $b_2$ with a newly generated key. It is the same key that is generated in $write(a, b_1)$, since the oracles are the same.

		Success of first execution gives you the fact that there is no collision when you encrypt $b_1$. However, there is no way to prove from the stated fact that there won't be a collision when you encrypt $b_2$. This will prevent you from proving the desired property.\\

		{\bf I currently have no solution to this. I will keep thinking and keep you updated about it. If anything comes to your mind, please let me know.}\\

		{\color{red} \bf Nickolai:} Thanks, I think the problem makes more sense now.
		In terms of how to solve it..  What about putting the ciphertext into the oracle when encrypt is invoked?
		What I mean is..  When the application calls encrypt(), you choose a ciphertext non-deterministically (i.e., through the oracle).  This gives you the freedom to, in the proof, manipulate the ciphertext chosen for any particular encryption.
		If you had this degree of freedom, then, when proving this case, you could demonstrate a second execution by showing another choice of encrypted result in the other oracle, so that the execution of the recovery code goes down the same path.
		That is, if in execution 1 you saw encrypt(k1, b1) = encrypt(k2, b2), then you can make it also be equal in execution 2.
		And similarly if you saw the two encryptions be non-equal in execution 1, you can make them be non-equal in execution 2.
		The only constraint on this new oracle feature is that you can't choose colliding ciphertexts.
		This means that, to give you this necessary degree of freedom, you have to choose fresh keys often enough so that you're never constrained too much by prior encryptions.\\

		{\color{blue} \bf Atalay:} This may not work due to following problem.

		Framework requires oracles to be independent of the confidential data. This means you should be able to execute $write(a, b_1)$ from $s_1$, and $write(a, b_2)$ from $s_2$ with the same oracle. If oracle supplies the fresh key and the ciphertext, then it will be encrypt(k, b1) = encrypt(k, b2). Only way this being true is that, somehow, encrypt function of $s_1$ is different than one in $s_2$. I am not entirely sure what this entails and can this difference be a source of leakage.

		\subsection{Oracle Chosen Ciphertext Approach}
		The system (lowest layer) allows encrypting at most one block (plaintext) with a particular key.  The state tracks key $\rightarrow$ option (plaintext * ciphertext) for this purpose.  If this mapping contains None for some key, then calling encrypt(key, plaintext) chooses a random ciphertext (supplied by the oracle) and puts Some (plaintext, ciphertext) into that state under the corresponding key.

		If this mapping contains Some \_, it's not legal to call encrypt anymore for that key.  So, for that matter, keygen() and encrypt() can be combined, so that this combined function returns the fresh key and the ciphertext all at once.

		decrypt(key, ciphertext) can be called only if that mapping contains precisely that ciphertext for that key, in which case it returns the corresponding plaintext.  Otherwise, calling decrypt is not allowed (if the mapping is None or contains some other ciphertext).

		The implication of this model of encryption is that two blocks (b1 and b2) will have precisely the same ciphertext (supplied by the identical oracle) in the two executions, if these blocks are encrypted with the same key.

		To achieve the constraint of at most one block (plaintext) per key, I'm thinking the file system generates a fresh key every time it needs to encrypt a block.  Whenever the file system needs to write some block to the log, it encrypts it with a fresh key, and stores the key wherever it's already storing the address that goes along with this block.  On recovery, decrypting blocks is straightforward: just use each block's key, after validating that these are indeed the correct blocks (i.e., hash matches).

		In practice, it might be a bit silly to generate a fresh key for each block, so a realistic implementation would probably use the same key but a fresh IV (based on a monotonically increasing counter) for each block.

		It might even be worth modeling that IV-based optimization, because it would allow the file system to not store a separate key for each block in the log, and instead just have the header store the key and the first sequence number for the logged blocks.  This way, on recovery, the file system code will decrypt using sequence numbers starting from the first one, and it doesn't require additional per-block overhead for a key or IV.
		But I think this "in practice" comment is just an optimization if the idea above makes sense.\\

		{\bf All possible histories are related problem}

		Think our FS but abstractions are "collapsed". So there is base layers (disk, crypto etc.) And a single layer of abstraction above the entire implementation (file disk).

		Since file disk doesn't have a notion of encryption, base layer states with different "encryption histories" will refine the same file disk. More precisely, any combination of "chosen ciphertexts" as the history will be a valid refinement.

		This means no matter what the oracle chooses as ciphertext for the result of the new encryption, there will be two related states where it will cause a collision in only one of them.

		I think above example is a strong evidence of why "ciphertext choosing" approaches may not work.
		No matter what the choice is, there will be a related state where it is used.

		\subsection{Excluding Colliding Oracles Approach}
		Another approach we can take is excluding series of events that leads to a collision in encryption from our theorems. Similar to hashing, our theorems will have a condition that say "if there is no collisions, ..." To justify this approach, probability of it happening should be exponentially low.
		{\color{red} I don't know the probability of it yet.}

		If probability is not low enough, then we can add a scheme where encryption history only consists of last (2 * \# of blocks in the log) encryption results. It is an upper bound on how many blocks can be on the disk at any given time. Since this is much much smaller than possible combinations, I think it would be justifiable to assume such collision won't appear on the log.

		Apart from that, plan is same with hashing. System keeps track of encryption results and if it sees a collision, it gets stuck.

		Our premise will look like this:
		\begin{verbatim}
			(forall k, In (NewKey k) oracle -> (encrypt k v2) doesn't collide in s2) ->
			NI statement for (write a v1, write a v2) holds.
		\end{verbatim}

\end{document}


\section{Theory: Angle-Informed Importance Maps for CADIS and FW-CADIS}
\label{sec:methodtheory}

There exist methods to generate
variance reduction parameters
for deep penetration radiation transport problems with strong anisotropy in the
flux. These methods have shown to have varying success, and may not be fully
automated. The solution proposed in this dissertation is a formulation that we
have named the $\Omega$-CADIS-methods. This section will commence with a brief
discussion of the foundational research
on which the $\Omega$-CADIS-methods are built.
That discussion serves as a primer for the subsequent section, which
is an introduction to the $\Omega$-CADIS-methods and a discussion on how they differ
from their predecessors.

\subsection{Previous Work}
\label{sec:omegabknd}

As discussed in Sections \ref{sec:CADIS}
through \ref{sec:AngleVR}, the existing gold standard for automatically
generating variance reduction parameters for deep penetration
fixed-source radiation transport problems are
CADIS and FW-CADIS. Both of these methods are very effective at
generating variance reduction parameters for local and global solutions,
respectively. However, CADIS and FW-CADIS have only been implemented
to perform variance reduction in
space and energy, not angle. As a result, solutions for problems with
strong anisotropy in the flux are not always optimized with these methods,
resulting in slow convergence times and low FOM values.
Problems with strong anisotropies in the flux require more than just
space- and energy- variance reduction techniques.
A number of angle-informed variance reduction methods have been
investigated, most notably AVATAR, LIFT, and a modified version of
CADIS using AVATAR-type angular parameters.

LIFT, AVATAR, and Simple Angular CADIS all showed that by including angular
information into Monte Carlo variance reduction parameters the FOM can be
improved. However, none of these methods used the actual angular flux to
calculate the variance reduction parameters for the problem they were
optimizing. Without explicitly using the angular flux solutions they were
limited in which types of problems they were applicable, because some assumption
of the degree of anisotropy of the flux was made. Further, LIFT and
Simple Angular CADIS showed that by including substantial angular biasing in the
weight windows in problems where the approximation to the angular flux is not
sufficient, the FOM can decrease not unsubstantially, defeating the purpose of using
these methods.

\subsection{The $\Omega$ Methods}
\label{sec:omegaintro}

The foundation of the $\Omega$-methods is built upon CADIS and FW-CADIS. As with
both methods,
the $\Omega$-methods will use a version of the adjoint scalar flux to
consistently bias a Monte Carlo problem with the intention of reducing the
variance. In Section \ref{sec:Importance} the concept of importance was
introduced. Notably, it was shown that the adjoint flux is a good marker
for the likelihood of particles to contribute to a tally, which is the
particle's importance. It was also shown that the
product of the forward and adjoint fluxes generates a pseudo-particle flux called
the contributon flux, where contributons are ``importance particles''.
These importance particles can be used to show preferential flow paths from a
source to a tally or desired location.

By using a version of the adjoint scalar flux that has been formulated with the
contributon flux, the direction of particle flow will be incorporated into the
importance map and, consequently, the
variance reduction parameters. By using this variant of the adjoint scalar flux,
the method, like traditional CADIS, will show increasing importance as the
particles travel near the adjoint source. However, because this variant of the adjoint
flux incorporates directionality of the particle flow, not all regions near the
adjoint source are equally important. In this way, the adjusted flux
incorporates features from both the adjoint- and contributon- fluxes.

The adjusted adjoint scalar flux quantity, or the $\Omega$-adjoint
scalar flux, is
%
\begin{equation}
  \phi^{\dagger}_{\Omega}(\vec {r} ,E)  = \frac{\int_{\Omega}{\psi^{\dagger}
                             (\vec{r}, E, \hat\Omega)
                             \psi(\vec{r}, E, \hat\Omega)} d\hat{\Omega}}
                             {\int_{\Omega}\psi(\vec{r}, E, \hat\Omega)
                             d\hat{\Omega}}.
\label{eq:omega_basic}
\end{equation}
%
The $\Omega$-flux is a hybridization of the adjoint scalar flux and the contributon
flux. It is both a normalized contributon flux and a forward-weighted
adjoint flux. As a result, it should inherit some of the advantages of
each of the traditional
adjoint and the contributon fluxes. Because it maintains dimensionality of the
traditional adjoint scalar flux, it can be used in place of the standard adjoint
scalar flux in both
CADIS and FW-CADIS variance reduction parameter generation.
This means that the method can capitalize on existing infrastructure used to
generate variance reduction parameters for CADIS and FW-CADIS, and only the
software handling the transport and flux-generation requires modification.

\subsubsection{CADIS-$\Omega$}
\label{sec:cadomega}

As with CADIS, CADIS-$\Omega$ consistently biases a problem's source and
particle weights according to their importance. However, CADIS-$\Omega$
uses the
$\Omega$-adjoint scalar flux rather than the standard adjoint scalar flux
to generate the biased source distribution, weight windows,
and the particle birth weights. Furthermore, because $\phi_{\Omega}^{\dagger}$
is used to calculate these values in CADIS-$\Omega$,
the consistent-biasing hallmark for
which CADIS is known is maintained.
The adjusted
formulation of CADIS using the $\Omega$ fluxes is given by Eqs.
\eqref{eq:CADISomegamethod}.
%
\begin{subequations}
\label{eq:CADISomegamethod}
%
The biased source distribution used by CADIS-$\Omega$ is formulated just as it
is in CADIS, except the adjusted adjoint fluxes are used:
\begin{equation}
\begin{split}
  \hat{q}_{\Omega}  & = \frac{\phi_{\Omega}^{\dagger}(\vec {r} ,E)q(\vec {r} ,E)}
               {\iint\phi_{\Omega}^{\dagger}(\vec {r} ,E)
               q(\vec {r} ,E) dE d\vec{r}} \\
               & = \frac{\phi_{\Omega}^{\dagger}(\vec {r} ,E)q(\vec {r},E)}
               {R_{\Omega}}.
\end{split}
\end{equation}
%
The  starting weights of the particles sampled from the
biased source distribution, $\hat{q}$ are given by
\begin{equation}
\begin{split}
  w_{0, \Omega}  & = \frac{q}{\hat{q}_{\Omega}} \\
  & = \frac{R_{\Omega}}{\phi_{\Omega}^{\dagger}(\vec {r} ,E)} ,
\end{split}
\end{equation}
%
and the new target weights for the particle are
\begin{equation}
  \hat{w}_{\Omega} = \frac{R_{\Omega}}{\phi_{\Omega}^{\dagger}(\vec{r} ,E)} .
\end{equation}
\label{eq:cadomega_eqns}
\end{subequations}

\subsubsection{FW-CADIS-$\Omega$}
\label{sec:fwcadomega}

FW-CADIS differs from CADIS in that it requires a forward deterministic
calculation to generate $q^{\dagger}$, which is used as the source distribution
in the adjoint deterministic problem (recall that CADIS sets
$q^{\dagger}=\sigma_d$). Depending on the type of global
response desired, FW-CADIS runs a deterministic forward calculation to
approximate the global response in the problem. The inverse of these responses
is then used to generate the biased adjoint source distribution for the adjoint
deterministic run. Therefore, the behavior of FW-CADIS-$\Omega$
in the forward biasing
portion of the calculation will remain unchanged from FW-CADIS.
%
\begin{subequations}
The generalized form for the adjoint source definition is given by the fraction
of the response in a region of phase space, $P$, over the total response in the
problem, or
\begin{equation*}
  { q^{\dagger}_{\Omega}} (P) = { q^{\dagger}} (P)=\frac{\sigma_d(P)}{R} .
\end{equation*}
%
When applied to the spatially-dependent global dose, $\int
\phi(\vec{r},E)\sigma_d(\vec{r},E) dE$, the adjoint source will be
\begin{equation*}
  { q^{\dagger}_{\Omega} }(\vec { r } ,E) = { q^{\dagger} }(\vec { r } ,E)
  =\frac { \sigma _{ d }(\vec { r } ,E) }{ \int {
  \sigma _{ d }(\vec { r } ,E)\psi (\vec { r } ,E,) } dE } .
\end{equation*}
%
The adjoint source for the spatially-dependent total flux $\int \phi(\vec{r},E)
dE $ is
\begin{equation*}
  { q^{\dagger}_{\Omega} }(\vec { r }) = { q^{\dagger} }(\vec { r })
  =\frac { 1 }{ \int { \phi (\vec { r } ,E) } dE } .
\end{equation*}
%
The adjoint source for the energy- and spatially-dependent flux
$\phi(\vec{r},E)$ is
\begin{equation*}
  { q^{\dagger}_{\Omega} }(\vec { r } ,E) = { q^{\dagger} }(\vec { r } ,E)=\frac { 1 }{\phi (\vec { r } ,E) } .
\end{equation*}
\label{eq:fwcadomega_eqns}
\end{subequations}

One advantage of FW-CADIS-$\Omega$ is that, from a transport perspective, the
$\Omega$-method is no more expensive than standard FW-CADIS. Because both
versions require a forward and adjoint deterministic calculation, an extra
transport step is not required as it is for CADIS-$\Omega$. This is attractive,
but the nature of FW-CADIS might not be the most well-suited for the
$\Omega$-methods. Because FW-CADIS attempts to evenly distribute particles
throughout the problem using the forward-biased adjoint fluxes,
the additional forward normalization with the $\Omega$-methods will likely skew
the particle distribution in the problem in the forward direction,
and it may place too great of
importance on the forward-moving particles in generating the variance reduction
parameters.


% !TeX root = ../main.tex

\section{Results}
\subsection{Data set}
\begin{frame}
    \frametitle{Epinions data set}
    \vspace{-2cm}
    \begin{columns}
    \column{0.5\textwidth}
    \begin{table}
        \centering
        \caption{Epinions Sample}
        \small
        \begin{tabular}{ |c|c|c| }
            \hline
            \textbf{user} & \textbf{item} & \textbf{Rating} \\
            \hline
            36153 & 62461 & 5 \\
            \hline
            427 & 38005 & 5  \\
            \hline
            751 & 53361 & 4 \\
            \hline
            11001 & 118950 & 4 \\
            \hline
            1169 & 66176 & 5 \\
            \hline
            9808 & 84459 & 2 \\
            \hline
            85 & 7446 & 4 \\
            \hline
            14717 & 3397 & 2 \\
            \hline
        \end{tabular}
    \end{table}
    \column{0.5\textwidth}
    \begin{table}
        \centering
        \caption{Epinions Descriptive}
        \small
        \begin{tabular}{ |c|c|c|c| }
            \hline
            &\textbf{Ratings Matrix} & \textbf{Train} & \textbf{Test}\\
            \hline
            count & 664824 & 520203 & 144621\\
            \hline
            mean & 3.9917 & 3.99 & 3.9975\\
            \hline
            std & 1.2068 & 1.2072 & 1.2053\\
            \hline
            min & 1 & 1 & 1\\
            \hline
            25\% & 3 & 3 & 3\\
            \hline
            50\% & 4 & 4 & 4\\
            \hline
            75\% & 5 & 5 & 5\\
            \hline
            max & 5 & 5 & 5\\
            \hline
        \end{tabular}
    \end{table}
\end{columns}
\end{frame}
\subsection{Evaluation Metrics}
\begin{frame}[t]
    \vspace{-1cm}
    \hspace{15mm}\underline{\textbf{Single Model}}  \hspace{5cm}\underline{\textbf{Combined Models}}
    \frametitle{Evaluation Metrics}
    \begin{columns}
    \column{0.4\textwidth}
    \begin{flalign*}
            RMSE &= \sqrt{\frac{\sum_{(u,i) \in \mathcal{T}(r_{u,i} - \hat{r}_{u,i})^2}}{n}} \\
            MAE  &= \frac{\sum_{(u,i) \in \mathcal{T}\left|r_{u,i} - \hat{r}_{u,i}\right|}}{n} \\
            RMSUE &= \frac{1}{n}\sum_{u \in \mathcal{T}}\sqrt{\frac{\sum_{i \in \mathcal{I}_u(r_{u,i} - \hat{r}_{u,i})^2}}{n_u}} \\
            MAUE &= \frac{1}{n}\sum_{u \in \mathcal{T}}\frac{\sum_{i \in \mathcal{I}_u\mathopen|r_{u,i} - \hat{r}_{u,i}\mathclose|}}{n_u}\\
    \end{flalign*}
    \vspace{3cm}
    \column{0.6\textwidth}
    \small
    \begin{flalign*}
        RMSE_{Total} &= \sqrt{\frac{n_{KNN}*RMSE_{KNN}^2 + n_{R-KNN}*RMSE_{R-KNN}^2}{n_{KNN} + n_{R-KNN}}}\\
        MAE_{Total} &= \frac{n_{KNN}*MAE_{KNN} + n_{R-KNN}*MAE_{R-KNN}}{n_{KNN} + n_{R-KNN}}
    \end{flalign*}
    \centering
    \footnotesize
    \begin{itemize}
        \item $\mathcal{T}$ is the test set
        \item $r_{u,i}$ is the truth value of a rating for $user_u$ to $item_i$
        \item $\hat{r_{u,i}}$ is the prediction value of a rating for $user_u$ to $item_i$
        \item $n$ is the number of rating predictions
    \end{itemize}
    \vspace{5cm}
    \end{columns}
\end{frame}

\subsection{Results}
\begin{frame}
    \frametitle{Volume of Predictions}
    \vspace{-1.6cm}
    \begin{table}
\centering
\caption{Ratings Predicted with KNN and Recursive-KNN}
\footnotesize
\begin{tabular}{ccc|ccc|c}
\multicolumn{3}{c|}{\textbf{USERS}}            & \multicolumn{3}{c|}{\textbf{ITEMS}}            &                          \\ \cline{1-6}
\textbf{KNN} & \textbf{R-KNN} & \textbf{TOTAL} & \textbf{KNN} & \textbf{R-KNN} & \textbf{TOTAL} & \textbf{SIMILARITY}      \\ \hline
88379        & 34392          & 122771         & 89800        & 33315          & 123115         & Adjusted Cosine          \\
100311       & 24122          & 124433         & 100311       & 24122          & 124433         & Cosine                   \\
100311       & 24122          & 124433         & 100311       & 24122          & 124433         & Jaccard                  \\
93924        & 29747          & 123671         & 94528        & 29203          & 123731         & MAD                      \\
93924        & 29747          & 123671         & 94528        & 29203          & 123731         & MSD                      \\
88379        & 34392          & 122771         & 89800        & 33315          & 123115         & Modified Adjusted Cosine \\
100311       & 24122          & 124433         & 100311       & 24122          & 124433         & Modified Cosine          \\
59017        & 40470          & 99487          & 54644        & 34096          & 88740          & Modified Pearson 1       \\
89936        & 28183          & 118119         & 84511        & 24357          & 108868         & Modified Pearson 2       \\
89936        & 28183          & 118119         & 84511        & 24357          & 108868         & Pearson
\end{tabular}
\end{table}
\end{frame}
\begin{frame}[t]
    \frametitle{User-based KNN and Total RMSE}
    \vspace{-0.7cm}
    \begin{columns}
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{User_RMSE_KNN.eps}
    \end{figure}
    \centering
    \tiny
    Modified cosine at K=100, RMSE=1.1479835373
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{Total}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{evaluation_user_rmse.eps}
    \end{figure}
    \centering
    \tiny
    Modified cosine at K=100 \& M=3, RMSE=1.1604146071
\end{columns}
\end{frame}
\begin{frame}[t]
    \frametitle{User-based KNN and Total MAE}
        \vspace{-0.7cm}
        \begin{columns}
            \column{0.5\textwidth}
            \centering
            \underline{\textbf{KNN}}
        \begin{figure}
        \centering
        \includegraphics[width=1\textwidth,height=0.6\textheight]{User_MAE_KNN.eps}
        \end{figure}
        \centering
        \tiny
        Jaccard coefficient at K=50, MAE=0.8518433005
            \column{0.5\textwidth}
            \centering
            \underline{\textbf{Total}}
        \begin{figure}
        \centering
        \includegraphics[width=1\textwidth,height=0.6\textheight]{evaluation_user_mae.eps}
        \end{figure}
        \centering
        \tiny
        Jaccard coefficient at K=50 \& M=3, MAE=0.854066176
    \end{columns}
\end{frame}
\begin{frame}[t]
    \frametitle{User-based KNN and Recursive-KNN RMSUE}
    \vspace{-0.7cm}
    \begin{columns}
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{User_RMSUE_KNN.eps}
    \end{figure}
    \centering
    \tiny
    Modified cosine at K=100, RMSUE=1.0031145695
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{Recursive-KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{evaluation_user_rmsue.eps}
    \end{figure}
    \centering
    \tiny
    Modified cosine at K=100 \& M=3, RMSUE=0.9089525549
\end{columns}
\end{frame}
\begin{frame}[t]
    \frametitle{User-based KNN and Recursive-KNN MAUE}
    \vspace{-0.7cm}
    \begin{columns}
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{User_MAUE_KNN.eps}
    \end{figure}
    \centering
    \tiny
    Cosine similarity at K=50, MAUE=0.8819077974
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{Recursive-KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{evaluation_user_maue.eps}
    \end{figure}
    \centering
    \tiny
    Adjusted cosine at K=75 \& M=20, MAUE=0.8558869566
\end{columns}
\end{frame}
\begin{frame}[t]
    \frametitle{Item-based KNN and Total RMSE}
    \vspace{-0.7cm}
    \begin{columns}
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{Item_RMSE_KNN.eps}
    \end{figure}
    \centering
    \tiny
    Adjusted cosine at K=95, RMSE=1.2984970982
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{Total}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{evaluation_item_rmse.eps}
    \end{figure}
    \centering
    \tiny
    Adjusted cosine at K=100 \& M=3, RMSE=1.3155259043
\end{columns}
\end{frame}
\begin{frame}[t]
    \frametitle{Item-based KNN and Total MAE}
        \vspace{-0.7cm}
        \begin{columns}
            \column{0.5\textwidth}
            \centering
            \underline{\textbf{KNN}}
        \begin{figure}
        \centering
        \includegraphics[width=1\textwidth,height=0.6\textheight]{Item_MAE_KNN.eps}
        \end{figure}
        \centering
        \tiny
        Modified adjusted cosine at K=15, MAE=0.9499219326
            \column{0.5\textwidth}
            \centering
            \underline{\textbf{Total}}
        \begin{figure}
        \centering
        \includegraphics[width=1\textwidth,height=0.6\textheight]{evaluation_item_mae.eps}
        \end{figure}
        \centering
        \tiny
        Adjusted cosine at K=100 \& M=3, MAE=0.9707211649
    \end{columns}
\end{frame}
\begin{frame}[t]
    \frametitle{Item-based KNN and Recursive-KNN RMSUE}
    \vspace{-0.7cm}
    \begin{columns}
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{Item_RMSUE_KNN.eps}
    \end{figure}
    \centering
    \tiny
    Adjusted cosine at K=100, RMSUE=1.0328968801
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{Recursive-KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{evaluation_item_rmsue.eps}
    \end{figure}
    \centering
    \tiny
    Adjusted cosine at K=100 \& M=3, RMSUE=1.0367245756
\end{columns}
\end{frame}
\begin{frame}[t]
    \frametitle{Item-based KNN and Recursive-KNN MAUE}
    \vspace{-0.7cm}
    \begin{columns}
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{Item_MAUE_KNN.eps}
    \end{figure}
    \centering
    \tiny
    Modified adjusted cosine at K=30, MAUE=0.9318609947
        \column{0.5\textwidth}
        \centering
        \underline{\textbf{Recursive-KNN}}
    \begin{figure}
    \centering
    \includegraphics[width=1\textwidth,height=0.6\textheight]{evaluation_item_maue.eps}
    \end{figure}
    \centering
    \tiny
    Adjusted cosine at K=90 \& M=3, MAUE=0.9864005988
\end{columns}
\end{frame}


%!TEX root = ../data-imputation.tex
\section{Discussion}
\label{sec:discussion}

We investigated the performance of classical and modern imputation approaches on a large number of heterogeneous data sets under realistic conditions. In the following, we highlight some of the key findings.

\subsection{Simpler Imputation Methods Yield Competitive Results}
%
When evaluating imputation quality, our results demonstrate that simple supervised learning methods achieve competitive results, and in many cases, outperform modern generative deep learning-based approaches. In particular, in the MCAR and MAR setting, we see in Figures \ref{fig:fully_observed_impute_rank_boxplot} and \ref{fig:corrupted_impute_rank_boxplot} that $k$-NN, random forest, and the discriminative DL approach are, for at least $50\%$ of the cases, among the better ranks one, two, or three. Random forest tends to achieve the best rank more often. This effect is largely independent of whether the imputation methods are trained on complete or incomplete data.

This finding is in line with \cite{Imputation_Benchmark_3, Imputation_Benchmark_2, Imputation_Benchmark_4}. In these previous studies, the authors report that $k$-NN imputation is the best choice in most situations. However, \cite{Imputation_Benchmark_2, Imputation_Benchmark_4} did not incorporate a random forest imputation method. Other comparisons show a slight advantage of discriminative deep learning methods over random forests \citep{biessmann2019datawig}, but these experiments were conducted on a much smaller selection of data sets.

For categorical columns (see Figure \ref{fig:fully_observed_impute_rank_boxplot} and \ref{fig:corrupted_impute_rank_boxplot}, upper row) in the more challenging imputation settings MAR or MNAR with large missingness fractions, the mean/mode imputation tends to achieve better ranks. This effect can be attributed to the fact that the sets of observed categorical values often have small cardinality. Especially for skewed distributions, using the most frequent value to substitute missing values is a good approximation of the ground truth. If the training data contains a large fraction of missing values, the underlying dependencies exploited by learning algorithms are difficult to capture. For this reason, mean/mode scores for higher MNAR missing values in $75\%$ of the cases on rank two or better (visualized in Figure \ref{fig:corrupted_impute_rank_boxplot}). \cite{Imputation_Benchmark_3} did not explicitly calculate the ranks but their plots show the same tendency.

Since GAIN failed in about $33\%$ of settings when training data was complete, this could be a reason why, in most cases, GAIN achieves the worst ranks (see Figure \ref{fig:fully_observed_impute_rank_boxplot}). This is supported by the fact that GAIN does not fail for settings with incomplete training data, and often shows better ranks (see Figure \ref{fig:corrupted_impute_rank_boxplot}).

All in all, using random forest, discriminate DL, or $k$-NN are good choices in most experimental settings and promise the best imputation quality. However, incorporating the model's training and inference time, presented in Table \ref{tab:time}, shows that the discriminative DL approach is substantially slower for training and inference than the other two methods. This is because we used the expensive default model optimization of AutoKeras. Exploring fewer hyperparameters could decrease its imputation performance drastically. The training duration's high variance indicates that trying a large number of hyperparameters is necessary for good performance because early stopping would finish the training if the model converges. $k$-NN's standard deviation for inference is in contrast to random forest very high. This is expected as the inference time grows exponentially with the number of training data points. We conclude that given the similar performance of $k$-NN and random forests when the training data set is large, random forests (or similar methods) should be preferred over naive $k$-NN implementations. Alternatively, one might use appropriate speedups for the nearest neighbor search, such as $kd$-trees or approximate nearest neighbor search.

To summarize, the best performing imputation approach is random forest. It not only ranks best in most experimental settings, but it also shows a good balance of training, including optimizing hyperparameters and inference time that is not influenced by the training set size. However, when coping with data sets that miss $30\%$ or more values of the pattern MNAR, imputing categorical columns with their mode is very often the best choice.

\subsection{Substantial Downstream Improvements when Imputation Method was Trained on Complete Data}
%
Our results show that imputation can have a substantial positive impact on predictive performance in downstream ML tasks. We observe improvements in the downstream task of 10\% to 20\% in more than 75\% of our experiments. This holds for most imputation methods; we did not observe a clear advantage for an imputation method overall. Taking into account the considerable differences in wall-clock run time, our results indicate that also when choosing an imputation method that is both fast and improves downstream predictive performance random forests would be the preferred imputation method.

The positive impact of imputation on downstream performance is most pronounced when the imputation methods were trained on fully observed data. When imputation methods were trained on incomplete data, the positive impact of imputing missing values in the test data was substantially lower, sometimes even negative. While this might seem a disadvantage we emphasize that in many application use cases we can ensure that the training data be fully observed, for instance by acquiring more data before training the imputation as well as the downstream ML model.

\subsection{Limitations}
\label{sec:limitations}
%
Because one of the main goals of this study is a comprehensive comparison of imputation methods on a large number of data sets and missingness conditions, we made some decisions that limit our results.

Firstly, the used data sets consist of a maximum of $25$ features and $100k$ observations. For this reason, we can not conclude from our experiments how the imputation methods perform on large-scale data sets. Further, our data sets only contain numerical or categorical columns and no image- or text-based data, e.g., used in other deep learning-based imputation approaches \citep{Biessmann2018a}. However, in that work, the authors only considered text data as an input field to an imputation method, not as a column that could be imputed. Generally, most modern ML applications that involve text data are based on rather sophisticated natural language models. Combinations of such models with tabular data are an important field of research \citep{Yin2020} but beyond the scope of most imputation research so far.

Secondly, to measure the imputation impact on the downstream performance, we discarded and imputed values in only a single column. Therefore, the impact depends heavily on the chosen column's importance (e.g., see \cite{Jenga}). Generally, the impact when using an imputation model could vary when multiple columns are affected by missing values.


\documentclass[nofootinbib,amssymb,amsmath]{revtex4}
\usepackage{mathtools}
\usepackage{amsthm}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{lmodern}
\usepackage{graphicx}
\usepackage{color}

%Put an averaged random variable between brackets
\newcommand{\ave}[1]{\left\langle #1 \right\rangle}
\newcommand{\HC}{\texttt{HaplotypeCaller}}
\newcommand{\Mutect}{\texttt{Mutect}}
\newcommand{\code}[1]{\texttt{#1}}
\newcommand{\mc}[1]{\mathcal{#1}}

\newtheorem{lemma}{Lemma}
\newtheorem{corollary}{Corollary}

\def\SL#1{{\color [rgb]{0,0,0.8} [SL: #1]}}
\def\DB#1{{\color [rgb]{0,0.8,0} [DB: #1]}}

\begin{document}

\title{Pair HMM probabilistic realignment in HaplotypeCaller and Mutect}
\author{David Benjamin\footnote{The author took no part in development of the methods described below -- credit belongs to several others on the GATK team. }}
\email{davidben@broadinstitute.org}
\affiliation{Broad Institute, 75 Ames Street, Cambridge, MA 02142}

\date{\today}

\begin{abstract}
After generating candidate haplotypes, the GATK tools \HC~and \Mutect~realign reads against these haplotypes to obtain a matrix of likelihoods for each read to be derived from each haplotype.  Here we describe the probabilistic model specifying this likelihood as well as its computational implementation.  We do not describe the translation of this implementation into native code optimized for vectorized architectures.
\end{abstract}

\maketitle

\section{The Pair HMM model}
We want to calculate the probability $P( \mc{R} | \mc{H})$ of read $\mc{R}$ to be sequenced from haplotype $\mc{H}$, where the haplotypes are sufficiently long that reads are contained within them.  This likelihood is the sum of likelihoods of all possible alignments $\mc{A}$ of $\mc{R}$ to $\mc{H}$:

\begin{equation}
P(\mc{R} | \mc{H}) = \sum_{\mc{A}} P(\mc{R}, \mc{A} | \mc{H})
\end{equation}

We represent alignments as sequences
\begin{equation}
{\rm alignment} = \{ (i_1, j_1, s_1), (i_2, j_2, s_2) \ldots (i_N, j_N, s_N) \},
\end{equation}
where $i$ and $j$ represent positions within the read and haplotype, respectively, and $s_n \in \{ M, I, D \}$ represents the states of match, insertion, and deletion of the read relative to the haplotype.  For example, an alignment $\{ (1,10, M), (2,11, M), (3,12, M), (4,13, M), (4,14, D), (4,15, D), (5,16, M), (6,17, M), (7,17, I), (8,18, M) \}$ means that positions 1 - 4 of the read match positions 10 - 13 of the haplotype, followed by a two-base deletion (advancing from 13 to 15 in the haplotype without advancing in the read), followed by a match of read positions 5 - 6 with haplotype positions 16 - 17, followed by an insertion at read position 7, followed by a match.  The allowable transitions $(i_n, j_n) \rightarrow (i_{n+1}, j_{n+1})$ are
\begin{itemize}
\item $(i, j, M/D/I) \rightarrow (i+1, j+1, M)$: match of read position $i+1$ with haplotype position $j + 1$
\item $(i, j, M/D) \rightarrow (i, j+1, D)$: deletion after read position $i$ -- haplotype position $j+1$ is deleted
\item $(i, j, M/I) \rightarrow (i + 1, j, I)$: insertion at read position $i + 1$
\end{itemize}
Note that the state label $s$ seems redundant because it can be reconstructed from the sequence of $i$ an $j$.  While this is true, our model treats indel starts differently from indel continuations, and by including the state label we can distinguish these conveniently\footnote{That is, we can tell which type of transition it is by looking back one unit instead of two.  Thus we have a first-order Markov model instead of a second-order Markov model.}.  These transitions are illustrated as a finite state machine in Figure \ref{fig:fsm}.

\begin{figure}
\center
\includegraphics[scale=0.35]{finite_state.png}
\caption{Finite state machine of transitions among match, insertion, and deletion alignment states.  In the absence of per-base BQSR indel qualities transition probabilities are parametrized  by two constants, the indel start probability $\delta$ and the indel continuation probability $\epsilon$, such that $T_{MI} = T_{MD} = \delta$, $T_{MM} = 1 - 2 \delta$, $T_{II} = T_{DD} = \epsilon$, $T_{IM} = T_{DM} = 1 - \epsilon$, and $T_{ID} = T_{DI} = 0$.}
\label{fig:fsm}
\end{figure}

The read-alignment likelihood has two components.  First is the probability of the sequence of match, insertion, and deletion states, which is
\begin{equation}
P(\mc{A}) = \prod_k T_{s_k, s_{k+1}},
\end{equation}
where $T$ is a matrix of state transition probabilities\footnote{The gap continuation probability corresponds to a phred-scaled quality that is set by the \code{gcpHMM} argument, which is 10 by default, implying that $T_{DD} = T_{II} = 10^{-10/10} = 1/10$ and $T_{DM} = T_{IM} = 1 - T_{II}$.  The indel start transitions are derived from the read's BQSR base insertion and base deletion qualities, if they exist, or a phred-scaled indel start quality of 45 if they do not.  In production at the Broad Institute bams do not have BQSR indel qualities, hence the constant default is used.  That is, $T_{MD} = T_{MI} = 10^{-4.5}$ and $T_{MM} = 1 - 2 \times 10^{-4.5}$.}\footnote{The elements of $T$ are either empirical or, (sometimes) in the case of indel start transitions, derived from BQSR.  However in principle all elemnts of $T$ could be learned by applying the Baum-Welch algorithm to some known training data.  For example, one could use a haploid cell line, for which any assembly region has a single haplotype.}.  The index $k$ runs from 1 to the number of states in the alignment, that is, the read length plus the number of deleted reference bases.  Next is the emission probability of the read bases given the haplotype bases they align to and the base qualities:
\begin{equation}
P(\mc{R} | \mc{A}, \mc{H}) = \prod_k P(r_{i_k} | h_{j_k}, q_{i_k})^{{\rm I}[s_k = M]},
\end{equation}
where $r_m$ and $h_n$ are the $m$th read bases and $n$th haplotype base and $q_m$ is the quality of the read's $m$th base.  Note that this only includes alignments in the match state.  The per-base emission is given directly from the definition of base quality:
\begin{equation}
P(b_2 | b_1, q) = \left\{ \begin{array}{cc} \epsilon(q)/3 & (b_1 \ne b_2) \\ 1 - \epsilon(q) & (b_1 = b_2)  \end{array} \right.
\end{equation}
where $\epsilon(q) = 10^{-q/10}$ is the error rate implied by the phred-scaled quality $q$.

\section{Dynamic Programming}

Define the matrices $M$, $I$ and $D$ by $M_{ij} = $ the total likelihood of \textit{all} paths from the beginning of the read to position $i$ that end in a match state, and likewise for $D$ and $I$.  Then the recursions
\begin{align}
M_{ij} =& P(r_i | h_j, q_i) \left( M_{i-1,j-1} T_{MM} + I_{i-1,j-1} T_{IM} + D_{i-1,j-1} T_{DM} \right) \label{match} \\
I_{ij} =& M_{i-1,j} T_{MI} + I_{i-1,j} T_{II} \label{ins} \\
D_{ij} =& M_{i,j-1} T_{MD} + D_{i, j - 1} T_{DD} \label{del}
\end{align}
define the entire pair HMM algorithm:

\begin{algorithm}
\begin{algorithmic}[1]
\State Initialize $M_{0,j} = I_{0,j} = 0$ and $D_{0,j} = 2^{1020}/|\mc{H}|$ for $1 \le j \le |\mc{H}|$.
\For{$1 \le i \le | \mc{R} |$}
	\For{$1 \le j \le | \mc{H} |$}
		\State Calculate $M_{ij}$ via Equation \ref{match}.
		\State Calculate $I_{ij}$ via Equation \ref{ins}.
		\State Calculate $D_{ij}$ via Equation \ref{del}.
	\EndFor
\EndFor
\State Total likelihood $P(\mc{R} | \mc{H})$ is $\sum_j \left(M_{\mc{R},j} + I_{\mc{R},j} \right)$.
\end{algorithmic}
\caption{Pair HMM algorithm}
\label{pairHMM}
\end{algorithm}

That the $i = 0$ rows of $M$ and $I$ are initialized to zero corresponds to starting at an imaginary position one base before the read start in a deletion state\footnote{Since $T_{DI} = 0$ this means that an alignment may not start with an insertion, though it may begin with one.  This limitation is unnecessary and could be fixed by initializing the $i = 0$ row of $I$ to a non-zero value as well.}.  The factor of $1/|\mc{H}|$ corresponds to a flat prior on which $j$ an alignment starts at.  This is important for a local alignment because we don't want to penalize reads that start in the middle of the haplotype.  The curious factor of $2^{1020}$ is a huge number to prevent underflow -- all multiplications are by numbers less than 1, so we needn't worry about overflow -- which is a much more efficient approach than performing the computation in log space.  The omission of $D$ from the returned value recognizes the fact that a terminal deletion is meaningless.

Finally, we note a shortcut that the GATK exploits: when two consecutive haplotypes of the same length\footnote{The condition on the same length could easily be relaxed simply by accounting for the constant $1/|\mc{H}|$ initialization.}  agree up to the $k$th position, the first $k$ columns of $M$, $D$ and $I$ are recycled and the inner loop is over $k < j \le \mc{H}$.

\end{document}


\documentclass[12]{scrartcl}
\usepackage{amssymb,amsmath,gensymb,dsfont,calc,multicol,fullpage}
\makeatletter
\newcommand\Aboxed[1]{
   \@Aboxed#1\ENDDNE}
\def\@Aboxed#1&#2\ENDDNE{%
   &
   \settowidth\@tempdima{$\displaystyle#1{}$}
   \setlength\@tempdima{\@tempdima+\fboxsep+\fboxrule}
   \kern-\@tempdima
   \boxed{#1#2}
}
\makeatother

\begin{document}

\title{Homework 24, Section 4.6: 3, 5, 14, 16, 18}
\author{Alex Gordon}
\date{\today}
\maketitle
\section*{Homework}
\subsection*{3.}
$round(x)=\lfloor x+0.5 \rfloor$

\subsection*{5.}
478 digits.\
$1000(log_{10}(3)$
\subsection*{14.}
2000/3 = 666. Therefore, 666-333 =  333 positive odd multiples of 3
\subsection*{16.}
There are 933 - 467 = 466 odd multiples of 3 or 5.
\subsection*{18.}
99!, or 99 factorial.
\end{document}


\documentclass[11pt]{amsart}
\usepackage{geometry}                % See geometry.pdf to learn the layout options. There are lots.
\geometry{letterpaper}                   % ... or a4paper or a5paper or ...
%\geometry{landscape}                % Activate for for rotated page geometry
%\usepackage[parfill]{parskip}    % Activate to begin paragraphs with an empty line rather than an indent
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{epstopdf}
\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}

\title{Brief Article}
\author{The Author}
%\date{}                                           % Activate to display a given date or no date

\begin{document}
\maketitle
%\section{}
%\subsection{}

$P(football \mod \Theta_0)=2.40/6.0=0.40 $

\end{document}