%!TEX root = ../gronskiy_phd_thesis.tex
\chapter[Minimum Spanning Tree Algorithms: Regularization by Stopping]{Minimum Spanning Tree
  Algorithms: \\ Regularization by Stopping}
\label{ch:mst}

\hfill
\begin{minipage}[t]{.75\textwidth}
\textit{``Besser ein Spatz in der Hand, als eine Taube auf dem Dach.'' \\
  (germ. ``A bird in the hand is worth two in the bush.'')} \\
  \hrule
  \vspace{.2cm}
  \hfill
  \textsc{--- Proverbs}
\end{minipage}

\section{Introduction}

\subsection{Motivation and Examples}

\index{Minimum Spanning Tree}
\index{MST|see{Minimum Spanning Tree}}
Noise perturbed combinatorial optimization problems arise in various real-world
applications where problem instances are abstracted by weighted\footnote{We will
further utilize both the terms ``costs'' and ``weights'' in the same meaning.}
graphs with fluctuations in the weights. In this chapter, we
investigate the noisy Minimum Spanning Tree (MST) problems and their ability to
infer robust spanning trees. Examples of real-world applications of minimum
spanning trees come from various fields of human activity, such as
\emph{communication networks with delays} (optimizing message delivery times,
cf.~\citep{Bertsekas:1987}) or \emph{stock markets} (analyzing stock exchange
correlations, cf.~\citep{Sandoval:2012}), etc. All the applications require
trees with a high level of robustness to fluctuations since the quality of trees
is measured by expected costs. Wide real-world demand and applicability
stimulated extensive research on the robust minimum spanning trees
problem~\citep{Aron:2004, Kozina:1994, Sandoval:2012, Yaman:Karasan:Pinar:2001},
which addressed different aspects of the robust spanning tree setting, such as
development of algorithms, measuring algorithmic complexity, or comparing
robustness criteria.

In this chapter, we use an information-theoretic regularization approach
introduced in Chapter~\ref{ch:gen_appch} to analyse and validate MST algorithms
from the point of view of how well they can recognize (localize) the true
Minimum Spanning Tree under unknown noise in the graph instance. The validation
concept is developed for ``contractive'' algorithms that follow a step-by-step
strategy of shrinking the solution space until a single best solution is
identified.  This validation approach is based on the \emph{two-instance
scenario} \citep{Vapnik:1982} that requires statistical estimates to generalize
to test instances when the estimates have been inferred from a training
instance. For spanning trees, that means that they have to yield low costs on at
least two problem instances without being explicitly adapted to the test
instance.%%% JB: the reference to the IB method is obscure. We should
%%% reformulate it.
Such regularization remains in the spirit of the \emph{information bottleneck
method}~\citep{Tishby:1999} in the sense that it tries to optimize the amount of
information that the algorithm might transfer from an artificial channel (see
below, Section~\ref{sec:ASReg}).
\index{Infromation bottleneck method}

%Suppose we need to construct a delivery
%network with minimal travel time along a given transportation system,
%where travel times along each road section fluctuate due to random
%transport delays caused by possibly unforeseeable events. Another
%real-world application arises from stock financial markets,
%which exhibit strong correlations that can be
%modeled by a temporally evolving dependency graph. A minimalistic
%description of these correlations with a simple conditional
%independence structure requires to find a minimum spanning tree in
%such a noise contaminated graph (see~\citep{Sandoval:2012}). Both
%applications

\subsection{Contributions and Outline of the Chapter}
\label{sec:mst_contribs}
As main contributions of this chapter, we
\begin{itemize}
  \item define a notion of contractive algorithm and give an general
  extension of the ASC regularization approach (introduced earlier in
  Chapter~\ref{ch:gen_appch}) for the case of stepwise contractive algorithmic
  problems. We call it Algorithmic ASC;
  \item provide a specialization of such extension to the case of three major
  algorithms solving the Minimum Spanning Tree (MST) problem, including the
  necessary tools to avoid brute-force enumeration known as a computational
  bottleneck of ASC;
  \item carry out experiments which justify usage of Algorithmic ASC score as
  a ranking tool related to expected localization error: higher Algorithmic ASC
  score yields lower error.
\end{itemize}

The chapter is outlined as follows. First, a related work overview is given in
Section~\ref{sec:mst_related_work}. Then, in Section~\ref{sec:mst_related_work}
we revisit those ingredients of ASC approach relevant for this chapter. A
comprehensive introduction into the original approximation set-based approach is
then given in Section~\ref{sec:asc_original}. Later, the main contribution is
made in Section~\ref{sec:mst_algorithmic_gen}, where we make a generalization of
the ASC to algorithmic problems, and Section~\ref{sec:applying_asc_to_mst} where
we directly apply it to Minimum Spanning Tree (MST) problem and three most
important algorithms solving it. Experimental results follow in
Section~\ref{sec:mst_results}. Finally, we discuss our findings in
Section~\ref{sec:mst_conclusion}.

\section{Related Work Overview}
\label{sec:mst_related_work}

Literature entries on robust spanning trees vary by research purpose
(e.g.~algorithm complexity, robustness), adopted noise model
(e.g.~interval data), regularization strategy (regularizing by
pruning, graph preprocessing).  For example,
\citep{Yaman:Karasan:Pinar:2001} investigated graphs with edge weights
which are supposed to fall uniformly into a predefined interval. The
approach is based on regularization \emph{by preprocessing the graph},
which, in turn, involves eliminating selected edges and forcing other
edges to be included into the spanning tree.

\index{Pruning (tree)}
Regularizing an MST by \emph{pruning} as proposed
by~\citep{Sandoval:2012} identifies the presumably robust part of a
spanning tree. This goal is achieved by means of analyzing the
survival matrix constructed from adjacency matrices for training
minimum spanning trees.

Our approach is similar in that pruning (more precisely, optimal
stopping) is used as a regularization tool, but it rather aims at
\emph{analysis and comparison} of the algorithm's robustness by means
of such regularization. This approach also exhibits technical
differences from the mentioned work, relying only on \emph{two} data
instances, and it follows the spirit of learning theory with its goal
to analyse algorithms in a noise distribution independent way.

In Section~\ref{regularization_gen}, the general concepts of the
information-theoretic framework are given, as well as the
information-theoretic motivation. Then, in
Section~\ref{sec:mst_algorithmic_gen}, the application to stepwise algorithms
is discussed. The results and conclusion are summarized in
Sections~\ref{sec:mst_results} and~\ref{sec:mst_conclusion}, respectively.

\section{Approximation Set Coding Regularization}
\label{regularization_gen}

To make this chapter self-consistent, we first give a short overview of the approach
first introduced in Chapter~\ref{ch:gen_appch}, as well as of the
notation and terms used throughout this chapter.
\index{Regularization}

\subsection{Notation and Definitions}

Assume, there is a data instance $X$ given, for example
measurements in a data space $\mathcal{X}$, so that $X \in
\mathcal{X}$. In case of the MST problem, $\mathcal{X}$ is the set of
all possible combinations of the edge costs and $X$ is a particular realization
of such costs. Let $c$ denote a solution in a general set of solutions $\C$, so
that $c \in \C$. In case of the MST, $\C$ is the set of feasible spanning trees.

An optimization problem is defined by a cost (objective) function $R:
\C\times\mathcal{X}\rightarrow \mathbb{R}_{+}$ that assigns
each solution $c$ a real value $R(c,X)$. Furthermore,
$c^{\bot}(X) \in \arg\min_{c \in \C} R(c, X)$
denotes\footnote{In the context where it is clear what $X$ is, we
will omit the ``$(X)$'' in the notation: $c^\bot(X) \equiv
c^\bot$; $R(c,X) \equiv R(c)$; $\C_\gamma(X) \equiv
\C_\gamma$.} the minimum cost solution.

For the sake of referring to it in the rest of the chapter, we give here a
definition from Chapter~\ref{ch:gen_appch}:
\begin{definition}
\label{def:mst_ch_approximation_set}
For a given real number $\gamma \ge 0$, an approximation set is defined as follows:
\begin{equation}
  \mathcal{C}_\gamma (X, R) \coloneqq
  \{c \in \mathcal{C} \mid R(c, X) - R^\bot(X) \le \gamma\},
\end{equation}
\end{definition}

In fact, it is desirable to find a \emph{robust set} of solutions
rather than a single solution, since we cannot trust the global
minimizer $c^\bot$ to achieve low costs on test instance due to noise
in the measurements. In this light, the sets $\C_\gamma(X)$ play
the role of a ``trade-off agent'' in the process of finding the robust
solution to optimization problem. This trade-off is
%%% meant to be the one
balancing the solution sets between overfitting ($\gamma = 0$) and underfitting (large
$\gamma$).
\index{Global minimizer}
\index{Overfitting}
\index{Underfitting}

%%%%%%%%%%%%%%%%% JB corrections up to here %%%%%%%%%%%%%%%%%

\subsection{Robust Solving via ASC Regularization}
\label{subsec:asc_regularization_crit}
Regularization by the Approximation Sets applies to a situation, when two data
instances $X', X'' \in \mathcal{X}$ are available and when both are generated by
injecting two noise realizations into the true data instance $X^0 \in
\mathcal{X}$ (see Section~\ref{sec:data_generation_model}). It is required for
both cases that the noise follows the same noise distribution: $X', X'' \sim
PG(X^0)$. In the spirit of statistical learning theory, we are interested in
distribution independent results since the distribution $PG(X^0)$ might be
unknown.

A robust solution to the noisy optimization problem $(\mathcal{X}, \C, R)$ is
obtained by the two-step process described in
Algorithm~\ref{alg:robust_solving_via_similarity}.

\begin{algorithm}[hb!]
\caption{Robust Solving via ASC Regularization}
\label{alg:robust_solving_via_similarity}
  \KwData{\\
  \quad two instances of the $X', X''$, \\ \\
  \quad cost function $R(c, X)$}

  \KwResult{Solution $c \in \C$.}

  {
    Find the optimal regularization parameter $\gamma^*$ that
    maximizes the log-ratio
    % involving \emph{the first two data instances}:
    \begin{equation}\label{eq:mst_asc_ratio}
      \hat I_\gamma(X', X'') \coloneqq \log
      \Bigl(
        \frac{|\C| \; |{\C}_{\gamma}(X') \cap {\C}_{\gamma}(X'')|}%
          {|\mathcal{C}_\gamma(X')| \; |\mathcal{C}_\gamma(X'')|}
      \Bigr),
    \end{equation}
  }

  {Choose a solution uniformly at random from the intersection of
        optimal approximation sets: $c \in C_{\gamma^*}(X') \cap
        C_{\gamma^*}(X'')$.}
\end{algorithm}

\myremark Note the difference of~\eqref{eq:mst_asc_ratio} with previously
defined~\eqref{eq:asc_mutual_information_formula}: dropping the
expectation. This is perfectly legitimate in case when we have no access to the
problem generation distribution and have to replace the actual value by its
estimator.

\subsection{Information-Theoretic Basis for ASC Regularization}
\label{sec:ASReg}

Applying of ASC bases itself on the information-theoretic ground which uses
approximation sets $\C_{\gamma}(X)$ in a fictitious communication scenario.
This communication scenario involves sending and receiving a
\emph{transformation} $\tau\colon
\mathcal{X} \to \mathcal{X}$ that maps the data space onto
itself. We briefly recap the necessary information here: for a full
introduction, refer to Chapter~\ref{ch:gen_appch}
(Section~\ref{sec:communication_learning_stability}).

Communicating a true $\tau_{\mathrm{send}}$ is performed by sending ${\tau_{\mathrm{send}} \circ
X'}$ to the receiver, while the channel perturbs this ``message''
by substituting $X'$ with $X''$. Finally, the receiver obtains
$\tau_{\mathrm{send}} \circ X''$. The reader should note that $X'$ is known to
both sender and receiver.
\index{Sender}
\index{Receiver}
\index{Approximation Set Coding!Encoding and transmission}
\index{Approximation Set Coding!Decoding}

As the receiver
%%% JB: receives
accepts $\tau_{\mathrm{send}} \circ X''$, it has has to distinguish the
fluctuations in $X''$ relative to $X'$ from the applied
transformation $\tau_{\mathrm{send}}$. Error free communication is guaranteed if the
receiver can identify the correct transformation $\tau_{\mathrm{send}}$ without
being deceived by these fluctuations.  The straightforward decoding
rule consists in finding the ``nearby'' transformation $\hat \tau$,
which maximizes the overlap between the received approximation set
$\C_\gamma(\tau_{\mathrm{send}} \circ X'')$ and the ``nearby'' approximation set
$\C_\gamma(\hat\tau \circ X')$.

The role of the codebook vectors is played by all such transformations
which enable us to cover the complete solution space $\C$
with approximation sets $\C_\gamma(\tau \circ X')$. Obviously,
$\gamma$ controls the number of such codebook vectors since for large
$\gamma$ we can only select few such sets to cover $\C$;
otherwise we risk decoding errors. The concept is analogous to
classical coding theory where every codebook vector defines the
``center'' of an ``error-correcting'' sphere, and the elements of this
sphere are \emph{indistinguishable} from the data transmission point
of view.
\index{Indistinguishable solutions|see{Solutions}}
\index{Solutions!Indistinguishable}

As derived in Section~\ref{sec:communication_learning_stability}, an
asymptotically vanishing error probability is achievable for coding rates
bounded by
%%
\begin{align}
  I_\gamma(X', X'') &= \Expct \hat I_\gamma(X', X'') \notag \\
  &=
  \Expct \log\left( \frac{|\C| \, |{\C}_{\gamma}(X') \cap
      {\C}_{\gamma}(X'')|}{|{\C}_{\gamma}(X')|
      \, |{\C}_{\gamma}(X'')|}\right), \label{eq:mst_ch_mutual_inf}
\end{align}
The estimator $\hat I_\gamma(X', X'')$ in (\ref{eq:mst_ch_mutual_inf}) measures
the total information content of a message.

For a fixed $\gamma$, a large overlap means that the evaluation of the first
dataset generalizes to the second dataset, whereas a small or empty intersection
indicates lack of generalization. The fraction of approximation set
cardinalities in (\ref{eq:mst_ch_mutual_inf}) measures stability of the solutions under
noise fluctuations.

The value $\Expct \hat I_{\gamma}(X', X'')$ is an estimate for the \emph{mutual
information} and, in analogy to information theory
(cf.~\citep[Ch.~7]{Cover:2006}), the \emph{approximation capacity} is defined as
$C \coloneqq\max_{\gamma}\Expct [\hat I_{\gamma}(X', X'')]$ (cf.
Definition~\ref{def:asc_score}). The expectation $\Expct$ is taken with respect
to the random variables $X', X''$. If the distribution $PG(X^0)$ is unknown then
we have to derive learning theoretic large deviation bounds based on the
empirical quantity $\hat I_{\gamma}(X', X'')$ and appropriate complexity
penalties.

In summary, maximizing the ratio~\eqref{eq:mst_asc_ratio} allows us to select the
optimal resolution for the solution set, and elements of this approximation set
are considered indistinguishable given the present noise process.

\section{ASC Regularization for Stepwise Algorithms}
\label{sec:mst_algorithmic_gen}

\subsection{Application to Stepwise Algorithms: Main Idea}

Direct application of ASC regularization requires
optimizing~\eqref{eq:mst_asc_ratio} w.r.t.~$\gamma$, which, in turn, amounts
to compute the cardinalities
\begin{equation}
  |{\C}_{\gamma}(X') \cap {\C}_{\gamma}(X'')|,
  \quad |{\C}_{\gamma}(X')|,
  \quad |{\C}_{\gamma}(X'')|
\end{equation}
(cf. Definition~\ref{def:mst_ch_approximation_set}).  In general, computation or
at least estimation of these cardinalities requires to enumerate the elements of
$\C$ and to test if they belong to $\C_\gamma$. Such enumeration is
computationally hard, since $\C$ grows sometimes very fast (exponentially in
cases of some combinatorial problems).\footnote{This difficulty emerges as a
common bottleneck in many problem settings. We elaborated on it in
Section~\ref{sec:gen_appch_conclusion} of the previous chapter.}

For algorithms, these enumeration problems arise in a constraint form, i.e.,
\emph{the algorithm itself} might help to optimize this enumeration, eliminating
all those solutions that get ``out of consideration'' as the algorithm
progresses. In other words, some algorithms feature a step-by-step nature, and
they shrink the set of feasible solutions at each next step, ending up with the
optimal solution at the last step. Figure~\ref{fig:mst_contractive_algorithm}
illustrates this intuition. This contraction is very similar to the shrinkage of
approximation sets, as they shrink to the optimal solution as $\gamma$
decreases. The feasible sets induced by the algorithm often turn out to be
efficiently computable~-- and we will exploit this property of contractive
algorithms.

\subsection{Contractive Algorithms and Their Approximation Sets}
\index{Contractive algorithm}We define the algorithmic approximation set at computational step $t$ as the set
of solutions that are still considered as potential answers to be returned by
the algorithm $\algo$. This generalized notion of an approximation set for
algorithm $\algo$ measures the statistical behavior of a specific algorithm
during its execution.
\nomenclature[E, 05a]{$\algo$}{algorithm\nomnorefeq}%

\myremark Even if $\algo$ calculates the global minimum of the cost
function, the algorithm may not follow a gradient flow on that costs to
determine the global minimum.

\begin{figure}[ht!]
  \centering
  \includegraphics[width=\textwidth]{figures/ch_mst/contractive_alg}
  \\[.5cm]
  \caption{Illustration of a contractive algorithmic flow: the set of
    feasible (i.e. still possible) solutions contracts from step to step,
    shrinking to $c^\bot$ at the end of execution.}
  \label{fig:mst_contractive_algorithm}
\end{figure}

\begin{definition}
\label{def:contractive_algorithm}
  Assume that for a given data instance $X$ the stepwise execution of the
  algorithm $\algo$ evaluated on this instance can be expressed as a sequence of
  subsets of feasible solutions
  %%
  \begin{align}
    \algo(X) &= \langle A_0(X), \ldots, A_T(X)
    \rangle,
    \quad\text{where} \notag \\
  %%
    A_t(X) &\subseteq \C, \quad t = 0, \ldots, T, \qquad\quad
    \text{and} \notag\\
  %%
    A_0(X) &= \C \notag \\
    A_T(X) &= \{c^\bot\}.
  \end{align}
  \nomenclature[E, 05b]{$A_i(X)$}{algorithmic feasible set\mynomdef{def:contractive_algorithm}}%
  \nomenclature[E, 05b]{$T$}{total number of steps\nomnorefeq}%
  %%
  We call $\algo$ contractive, if $A_{t+1} \subseteq
  A_{t}$.
\end{definition}

\begin{definition}\label{def:algorithmic_approximation_set}
  For a contractive algorithm it is natural to define an algorithmic
  $t$-approximation set as follows: $\C^\algo_t(X) \coloneqq A_t(X)$.
  \nomenclature[E, 05d]{$\C^\algo_t(X)$}{algorithmic $t$-approximation set\mynomdef{def:algorithmic_approximation_set}}%
  \index{Algorithmic!Approximation set}
  \index{Approximation set!Algorithmic|see{Algorithmic}}
\end{definition}

An example of such algorithm, as well as comprehensive illustration, will be
given in Section~\ref{sec:applying_asc_to_mst} and
Figure~\ref{fig:ch_mst_illustration}.

Speaking informally, the algorithmic $t$-approximation set is an approximation
set as defined in Definition~\ref{def:mst_ch_approximation_set}, but taken
w.r.t. the flow of a particular algorithm at update step $t$. The role of the
$\gamma$ parameter is now (in contrast to
Definition~\ref{def:mst_ch_approximation_set}) played by a discrete step
variable $t$ spanning from $0$ to $T$ (note that, generally, $T$ is not a
constant). All the other notions remain the same when adopting the new
definition of the approximation sets $A_t(X)$. For example, the following
defines the algorithmic ASC score:

\begin{definition}
As an analogy to~\eqref{eq:mst_ch_mutual_inf}, we will call the quantity
%%
\begin{align}\label{eq:alg_eq:mst_asc_ratio}
  I_t^\algo = \Expct \hat I_t^{\algo}( X', X'' )
    &\coloneqq \Expct \log\biggl( \frac{|\C| \, |{\C}^\algo_{t}(X') \cap
      {\C}^\algo_{t}(X'')|}{|{\C}^\algo_{t}(X')|
      \, |{\C}^\algo_{t}(X'')|}\biggr) \notag \\
    &\equiv \Expct \log \biggl(\frac{
      |\C|\; |A_t(X') \cap A_t(X'')|
    }{
      |A_t(X')|\; |A_t(X'')|
    }\biggr),
\end{align}
\nomenclature[E, 05g]{$I_t^\algo$}{algorithmic ASC $t$-score}%
\nomenclature[E, 05ga]{$\hat I_t^\algo$}{empirical algorithmic ASC $t$-score}%
where $A_t(\cdot)$ are defined above, an algorithmic ASC $t$-score.
\index{Algorithmic!ASC score}
\index{Approximation Set Coding!Algorithmic|see{Algorithmic}}
\end{definition}
As in Chapter~\ref{ch:gen_appch}, we will distinguish between
expected and empirical values.

\subsection{Algorithmic ASC Score and Optimal Stopping}
\begin{figure}[hb!]
  \centering
  \includegraphics[width=\textwidth]{figures/ch_mst/contractive_alg_opt_stop}
  \\[.5cm]
  \caption{The main question addressed in this chapter: which of the steps to
    choose for the optimal stopping.}
  \label{fig:mst_contractive_algorithm_opt_stop}
\end{figure}
As an empirical extension of the notions introduced in
Chapter~\ref{ch:gen_appch}, we claim that the algorithmic channel capacity
\begin{equation}
  C^\algo \coloneqq \max_{t = [0, \ldots, T]} I^\algo_{t}
    = \max_{t = [0, \ldots, T]}\Expct [\hat I^\algo_{t}(X', X'')],
\end{equation}
\nomenclature[E, 05gb]{$C^\algo$}{algorithmic approximation capacity}%
also referred as~\emph{algorithmic approximation capacity} or \emph{information
content}, equals the maximum amount of information which could be transferred
through a fictitious channel described in Section~\ref{sec:ASReg} (and, in more
detail, earlier in Section~\ref{sec:communication_learning_stability}), and
constrained to the specific algorithm (i.e., algorithmic approximation sets are
allowed). From the learning perspective, the algorithmic approximation capacity
shows how well the algorithm can filter out the noise (by $t$-optimization),
remaining robust to underfitting.
\index{Algorithmic!Approximation capacity}
\index{Optimal stopping}

In a full analogy with the approach of Chapter~\ref{ch:gen_appch}
(cf.\eqref{eq:asc_best_gamma}), one can find the optimal stopping time $t^*$ by
maximizing the algorithmic ASC $t$-score:
\begin{equation}\label{eq:mst_asc_best_step}
  t^* \in \arg \max_{t = [0, \ldots, T]} \hat \Expct I^\algo_t(X', X'').
\end{equation}
\nomenclature[E, 05g]{$t^*$}{optimal stopping time}%
We are going to use this approach as an optimal stopping criterion for the
rest of the chapter (see Figure~\ref{fig:mst_contractive_algorithm_opt_stop}).

\myremark We will call this approach \textit{algorithmic ASC} opposed to the
original ASC approach developed in Chapter~\ref{ch:gen_appch}.
\index{Algorithmic!Approximation Set Coding}

\section{ASC Regularization for MST Algorithms}
\label{sec:applying_asc_to_mst}

\subsection{Major MST Algorithms}
In the rest of the chapter, we will consider the Minimum Spanning Tree (MST)
problem. Its formulation is as follows: given a weighted undirected graph $G =
(V, E)$, find a spanning tree of the minimum possible weight. \textit{Spanning
tree} is defined as a connected subgraph of $G$ without cycles, whose vertex set
equals $V$.
\index{Tree!Spanning}
\index{Spanning tree|see{Tree}}

There are several classical solutions to this problem, of which we will focus on
\textit{Prim's}, \textit{Kruskal's} and \textit{reverse-delete} algorithms. We
bring their definitions in a pseudo-code in
Algorithms~\ref{alg:prim_alg_pseudocode},~\ref{alg:kruskal_alg_pseudocode}
and~\ref{alg:rd_alg_pseudocode} and explain a verbal intuition behind them below.
\index{MST Algorithm!Prim's}
\index{MST Algorithm!Kruskal's}
\index{MST Algorithm!Reverse-delete}
\index{Prim's|see{MST Algorithm}}
\index{Kruskal's|see{MST Algorithm}}
\index{Reverse-delete|see{MST Algorithm}}

\begin{algorithm}[ht!]
\caption{Prim's Algorithm for finding MST}\label{alg:prim_alg_pseudocode}
\KwIn{undirected graph $G(V, E)$ with non-negative weights}
\KwOut{spanning tree with minimum total weight}
{initialize current tree $B = (V_B, E_B)$ by choosing first vertex:
  $B \leftarrow (\{v^*\}, \emptyset)$\;}

\While{$V_B \ne V$ (not all vertices are in tree)}{
  {find the minimal edge $e_\mathrm{min} = (v, v_\mathrm{min})$
    from $B$ to the rest $G \setminus B$\;}
  {add $e_\mathrm{min}$ to the tree $B$\;} }

\KwRet{current tree $B$}
\end{algorithm}

\begin{algorithm}[ht!]
\caption{Kruskal's Algorithm for finding MST}\label{alg:kruskal_alg_pseudocode}
\KwIn{undirected graph $G(V, E)$ with non-negative weights}
\KwOut{spanning tree with minimum total weight}
{initialize current tree $B = (V_B, E_B)$: $B \leftarrow (\emptyset, \emptyset)$ \;}

\While{$V_T \ne V$ (not all vertices are in tree)}{
  {find the minimal edge $e_\mathrm{min}$ such that: \\
    \quad a) $e_\mathrm{min} \not \in E_B$ and \\
    \quad b) $B \cup e_\mathrm{min}$ has no cycles\;}
  {add $e_\mathrm{min}$ to the tree $B$\;}
}

\KwRet{current tree $B$}
\nomenclature[E, 10a]{$G(V, E)$}{graph\nomnorefeq}%
\end{algorithm}

\begin{algorithm}[ht!]
\caption{Reverse-Delete Algorithm for finding MST}\label{alg:rd_alg_pseudocode}
\KwIn{undirected graph $G(V, E)$ with non-negative weights}
\KwOut{spanning tree with minimum total weight}
{initialize current graph $B = (V_B, E_B)$ with the input graph: $B \leftarrow G$
\;}

\While{$B$ is connected and yet not a tree}{
  {find the maximal edge $e_\mathrm{max}$ in $B$ such that: \\
    \quad deleting $e_\mathrm{max} $ does not disconnect $B$\;}
  \If{no such edge found}{break\;}
  \Else{remove $e_\mathrm{max}$ from $B$\;}
}

\KwRet{current tree $B$}
\end{algorithm}

\begin{figure}[th!]
  \centering
  \begin{subfigure}[b]{.48\textwidth}
      \includegraphics[width=\linewidth]{figures/ch_mst/mst_illustration_0}
      \caption{Step 2: candidate edges set is large, no restricted edges yet}
      \label{fig:ch_mst_illustration-0}
  \end{subfigure}
  \hfill
  \begin{subfigure}[b]{.48\textwidth}
      \includegraphics[width=\linewidth]{figures/ch_mst/mst_illustration_1}
      \caption{Step 3: candidate edges set is large gets smaller, restricted edges appear}
      \label{fig:ch_mst_illustration-1}
  \end{subfigure}
  \\[.5cm]
  \begin{subfigure}[b]{.48\textwidth}
      \includegraphics[width=\linewidth]{figures/ch_mst/mst_illustration_2}
      \caption{Step 4: candidate edges set is large gets even smaller, restricted edges build up}
      \label{fig:ch_mst_illustration-2}
  \end{subfigure}
  \hfill
  \begin{subfigure}[b]{.48\textwidth}
      \includegraphics[width=\linewidth]{figures/ch_mst/mst_illustration_3}
      \caption{Step 5: almost everything is restricted}
      \label{fig:ch_mst_illustration-3}
  \end{subfigure}
  \\[.5cm]
  \caption{Prim's MST Algorithm: freedom reduces as information
    grows. Edge weights are not shown (figure from the introduction recreated
    here for convenience).}
  \label{fig:ch_mst_illustration}
\end{figure}

\emph{Prim's algorithm} (``growing tree'' strategy) starts with a tree $B=(V_B,
E_B)$ on an empty set of edges $E_P$ and a starting vertex $v^{*}$. The
algorithm enlarges $E_B$, adding one edge $e_t$ at step $t$ (the first step
adds an edge incident to $v^{*}$), so that $B$ remains to be a tree, until $B$
becomes a spanning tree of $G$. It takes $T = n-1$ steps.

\emph{Kruskal's algorithm} (``connecting trees in a forest'' strategy) of
finding MSTs starts with an empty set of edges $E_B$. The algorithm adds
a minimal possible $e_t$ at the step $t$, not allowing cycles in $E_B$,
but yet not requiring $B$ to be connected at all times, until $B$ becomes a spanning tree
of $G$. It takes $T = n-1$ steps.

\emph{Reverse-Delete} (``reducing graph'' strategy) algorithm starts
with a graph on the full set of edges $B = G$ and shrinks it, removing one maximal edge
$e_t$ per step and keeping the graph
$B$ connected, until $B$ becomes a spanning tree.
For a complete graph, it takes $T = n(n-1)/2 - n + 1$ steps.

\myremark All the three algorithms can be easily proven to reach the global
minimizer solution $c^\bot(X) \in \arg \min_c R(c, X)$.

\subsection{Counting Approximation Sets for MST}

\paragraph{MST algorithms are contractive} From the description of three
algorithms in the previous section one can see, that they all comply with the
Definition~\ref{def:contractive_algorithm} of a contractive algorithm, due to
the following line of reasoning. Since each of them yields a set (yet
\textit{not} an approximation set) of \textit{candidate edges} which still are
able to make it into the final tree:
\begin{itemize}
  \item in case of Prim's and Kruskal's: candidates are edges which are
    either already included or still can be included into $B$;
  \item in case of Reverse-Delete: candidates are edges which still remain (i.e.
    are no yet removed) in $B$.
\end{itemize}
To complete the reasoning, one should notice that an algorithmic approximation
set is exactly the set of spanning trees built on such candidate edges.

This concept is illustrated in Figure~\ref{fig:ch_mst_illustration}, where
several steps ($2$ to $5$) of the Prim's algorithm are shown. One can easily see
that as the algorithm flows, some edges are excluded from consideration (due to
no-cycle condition; shown in red) and hence less and less spanning trees remain
possible.

\paragraph{Computing cardinalities for MST} How can we calculate the
cardinalities of algorithmic approximation sets involved in the evaluation of
term~\eqref{eq:alg_eq:mst_asc_ratio}? Counting number of spanning trees for a
complete graph can be performed analytically via Cayley's
formula~\citep{Aigner2010}:
\newtheorem*{cayley_thm}{Theorem (Cayley's Formula for Spanning Trees)}
\begin{cayley_thm}
  The total number of labeled trees on $n$ vertices is equal to $n^{n-2}$.
  \index{Labeled tree|see{Tree}}
  \index{Tree!Labeled}
\end{cayley_thm}

\index{Cayley's formula}
However, Cayley's formula works only for complete graphs, while in our case, one
has to deal with a general case of counting trees on a non-complete subgraph
(candidate edges). In this work, we utilize the Matrix-Tree
Theorem~\citep[cf.][]{Harris:2008}. For a connected graph $G = (V,E)$, it
involves computing the adjacency matrix $M^G_{\mathrm{adj}}$, the degree matrix
\begin{equation}
  M^G_{\mathrm{deg}} = \mathop{\mathrm{diag}}(\deg{v_1}, \ldots,
\deg{v_n}),
\end{equation}
and uses a notion of a cofactor:
\index{Degree matrix}
\index{Adjacency matrix}
\nomenclature[E, 07a]{$M^G_{\mathrm{adj}}$}{adjacency matrix}%
\nomenclature[E, 07b]{$M^G_{\mathrm{deg}}$}{degree matrix}%
\nomenclature[E, 07f]{$\mathrm{deg}(v)$}{degree of vertex}%

\begin{definition}
  Given an $n \times n$ matrix M, the $i,j$ cofactor of $M$ is defined to be
  $(-1)^{i+j} \mathrm{det}(M_{i, j})$, where $M_{i, j}$
  is the submatrix of $M$, where $i$-th row and $j$-th column are removed.
  \nomenclature[A, 00j]{$\mathrm{det}(\cdot)$}{determinant}%
  \index{Cofactor}
  \index{Matrix cofactor!see{Cofactor}}
\end{definition}

\newtheorem*{matrix_tree_thm}{Theorem (Kirchhoff's Matrix-Tree Theorem)}
\begin{matrix_tree_thm}
  The total number of labeled trees on a graph $G = (V,E)$ with adjacency matrix
  $M^G_{\mathrm{adj}}$, degree matrix $M^G_{\mathrm{deg}}$ is equal to (any) cofactor
  of the matrix $L = M^G_{\mathrm{deg}}- M^G_{\mathrm{adj}}$.
  \index{Kirchhoff's matrix-tree theorem}
\end{matrix_tree_thm}

%%The algorithm of finding the stepwise algorithmic
%%rate~\eqref{eq:mst_ch_mutual_inf} is thus the one described as
%%Algorithm~\ref{alg_mutual_inf}.
%%\begin{algorithm}
%%\caption{\textsc AlgApproxSetRatio}
%%\label{alg_mutual_inf}
%%\begin{algorithmic}
%%  \STATE $I\gets [\;]$ \COMMENT{the ratio~\eqref{eq:alg_eq:mst_asc_ratio} array}
%%  \STATE $P_1, P_2 \gets [\;]$ \COMMENT{candidate edges left at the
%%    current step (both for $X', X''$)} \FORALL{t = 1 \ldots
%%    T} \STATE $P_1 \gets P_1\setminus\{e_t(X')\}$ \STATE $P_2
%%  \gets P_2\setminus\{e_t(X'')\}$ \STATE\COMMENT{Now compute the
%%    cardinalities of aqpproximation sets} \STATE $N_1\gets$
%%  ComputeMatrixTreeCount($P_1$) \STATE $N_2\gets$
%%  ComputeMatrixTreeCount($P_2$) \STATE $N_{12}\gets$
%%  ComputeMatrixTreeCount($P_1 \cap P_2$) \STATE $I[t]\gets$
%%  ComputeRatio($N_1$, $N_2$, $N_{12}$)
%%\ENDFOR
%%\end{algorithmic}
%%\end{algorithm}

In our cases, applying the Matrix-Tree theorem to count cardinalities of algorithmic
approximation sets  ${\C}^\algo_{t}(X')$ and
${\C}^\algo_{t}(X'')$ is straightforward. Computing an intersection of two
algorithmic approximation sets ${\C}^\algo_{t}(X') \cap
{\C}^\algo_{t}(X'')$ is simple as well.

As the last step, it we find the optimal stopping step $t^* \in
\arg\max_t
\hat I^{\algo}_t( X', X'' )$ which maximizes the
ratio. The optimal stopping time $t^*$ then defines a
pruning operation on the solution space to make $\algo$ robust.

\subsection{Uniform Sampling an Optimally Stopped Spanning Tree}

Once the optimal stopping time $t^*$ is defined, we can sample from
and intersection of two optimally-stopped algorithmic approximation sets
${\C}^\algo_{t}(X') \cap {\C}^\algo_{t}(X'')$ using the Pr\"ufer encoding of
labeled trees. This purely algorithmic task, however, goes beyond the scope
of this work and we thus leave it out.
\index{Pr\"ufer codes}

\section{Experimental Results}
\label{sec:mst_results}
We will experimentally check that algorithmic approximation capacity is a consistent
measure of the information that can be extracted by an algorithm from
noisy data by means of optimal stopping.

\begin{figure}[!t]
\centering
\includegraphics[width=.9\textwidth]{figures/ch_mst/gaus_inf_log}
\caption{Gaussian noise model: information content}
\label{fig:gaus_inf}
\end{figure}

\subsection{Experiment Setting: Gaussian Noise Model}
To investigate the general information-theoretic behavior of MST
algorithms in practice, we generate weighted complete graphs in a
hierarchical way. First, we generate a ``ground truth'' graph with
$n = 50$ vertices and with edges that are attributed by Gaussian weights,
sampled i.i.d. from a Gaussian distribution $\mathcal{N}(\mu_0 = 100,
\sigma_0^2 = 100)$.
Second, perturbed versions of this ground truth graph are then obtained by
adding Gaussian noise $\mathcal{N}(\mu = 0, \sigma^2)$ to the edge
weights for a given noise range $\sigma \in [0,  8\sigma_0]$.

In the experiment with approximation set-regularized algorithms we repeated the
experiment $400$ times to ensure the statistical significance of the
results. For some plots a semi-logarithmic scale was used for a better
visualization of small differences. Confidence intervals were also constructed
and plotted.

\subsection{Algorithmic Approximation Capacity Ranking of Algorithms}
\index{Ranking}
We plot the algorithmic information content $\max_t \Expct \hat
I_t^{\algo}(X', X'') $~(cf.~\eqref{eq:alg_eq:mst_asc_ratio}) for three algorithms (Figure~\ref{fig:gaus_inf}).
%%% JB: no new paragraph
At low noise levels (particularly $\sigma = 0$) all the three
algorithms exhibit the same information content, which is equal to
$\log_2{n^{n-2}} = 48 \log_2{50} \approx 270,9$ bits of information:
at zero noise, all the three algorithms choose the true MST out of
$|\C|$ possible spanning trees. For a complete graph,
Cayley's tree formula calculates the number of possible solutions as
$|\C| = n^{n-2}$.
%%% JB: In a noise-free setting, extracting the right tree corresponds
%%% to $\log_2{n^{n-2}} = 48 \log_2{50} \approx 270,9$ bits of
%%% information.

The plot in Figure~\ref{fig:gaus_inf} provides a clear ranking of the three algorithms
w.r.t. their information content dependent on the noise level. A qualitative explanation
which accounts for such ranking and
clarifies the idea behind evaluating information content, is the
following:
%
%\begin{figure}[!t]
%\centering
%\includegraphics[scale=0.3]{figures/ch_mst/gaus_inf_step_log}
%\caption{Gaussian noise model: information content per step}
%\label{fig:gaus_inf_step_log}
%\end{figure}

\emph{Prim's algorithm} considers for addition only the edges which
are already connected to the tree built so far.  Among the not yet
considered edges there might be low cost edges which are more
efficient to be added in the beginning of the run rather than in the
end, thus making the information extraction inefficient from the point
of view of the algorithm dynamics.

\emph{Kruskal's algorithm} explores all the possible edges as
candidates for addition at each step, thus being less inclined to add
inefficient edges first and using the algorithm dynamics in a
efficient way. This gain is reflected by its increased informativeness
relative to Prim's algorithm.

\emph{Reverse-Delete algorithm} is more informative than Kruskal's,
since it efficiently discards all those edges which should not be
included in any approximate spanning tree. It pursues a strategy of
delayed decision making, that proved to be favourable also in other
situations of decision making under uncertainty.

\begin{figure}[!t]
\centering
\includegraphics[width=0.9\textwidth]{figures/ch_mst/gaus_loc_err_mod}
\caption{Gaussian noise model: localization error}
\label{fig:gaus_loc_err}
\end{figure}

The above insights are proven by the plot showing the stepwise
dynamics of logarithm of the cardinalities $|\C^\algo_t(X')|$, $|\C^\algo_t(X'')|$
(Figure~\ref{fig:gaus_as_card}). It visualizes the
fact, that as $t$ progresses, Prim's algorithm contracts for the
solution faster than Kruskal's, and both contract faster that the
Reverse-Delete one, which, in turn, forces earlier stopping and thus
leads to the worse performance. In fact, we can formulate an informal statement:
\begin{statement}
  Assume that at the step $t$ the algorithm exhibits the edge set $B$ defined
  above in Section~\ref{sec:applying_asc_to_mst}. Then the reduction of the
  amount of feasible spanning trees with edges in $B$ obtained by adding
  $e_{t+1}$ which is adjacent to $B$  (Prim) is higher than the reduction
  obtained by adding non-adjacent $e_{t+1}$ (Kruskal in general), and both are
  higher than the reduction obtained by deleting $e_{t+1}$ from $B$
  (Reverse-Delete).
\end{statement}

\index{Stepwise dynamics}
The plot in the Figure~\ref{fig:gaus_ratio_dyn} explains the discussed ranking
from the stepwise dynamics prospective. The algorithm, which reaches maximum of
mutual information earlier, is less informative in overall, and vice versa. This
behavior reflects a natural trade-off between early decision and informativeness of the
solution.

\subsection{Localization Error Ranking of Algorithms}

\index{Ranking}
The three given algorithms ``explore'' the solution set with different
dynamic behavior, extracting different amount of the information
about the true solution. This behavior is related to the localization error.

For the algorithmically ASC-regularized
(Section~\ref{subsec:asc_regularization_crit}) solution $\hat c$
we plot (Figure~\ref{fig:gaus_loc_err}) the localization error, which is
computed as $E(\hat c) = 1 - |c^* \cap \hat c| / |c^*|$, where straight brackets
denote the cardinality of edges.

\begin{figure}[!t]
\centering
\includegraphics[width=.9\textwidth]{figures/ch_mst/gaus_as_card}
\caption{Gaussian noise model: stepwise approximation set log-cardinalities ($\sigma = 48$)}
\label{fig:gaus_as_card}
\end{figure}

\index{Localization error}
It can be seen from the figures, that the ranking of the three
algorithms according to their localization capability is in connection
with the information content ranking~--- the more informative
algorithm yields less error in localizing the true solution.

\begin{figure}[!t]
\centering
\includegraphics[width=.9\textwidth]{figures/ch_mst/gaus_ratio_dyn}
\caption{Gaussian noise model: stepwise algorithmic information defined in~\eqref{eq:alg_eq:mst_asc_ratio} ($\sigma = 48$)}
\label{fig:gaus_ratio_dyn}
\end{figure}
%
%\begin{figure}[!t]
%\centering
%\includegraphics[scale=0.3]{figures/ch_mst/gaus_as_intersection_card}
%\caption{Gaussian noise model: stepwise approximation set intersection log-cardinalities ($\sigma = 48$)}
%\label{fig:gaus_as_intersect_card}
%\end{figure}

\subsection{Algorithmic ASC vs. Original ASC}
\label{sec:aasc_vs_asc}

As the last experiment, we showed that the original ASC regularization (the one
via $\gamma$-parameter) works still better than algorithmic ASC and even beats
the Joint Minimizer (first introduced as a benchmark solution in
Chapter~\ref{ch:gen_appch}), which minimizes the average cost $R(c, X')+R(c, X'')$.
\index{Joint cost minimizer}

Due to the computation limitations of the original ASC~--- it yields enumerating
the whole set of spanning trees ($n^{n-2}$) for each step $t$ --- we could only
run the experiments for graphs with few vertices ($n = 6$).
Figure~\ref{fig:aasc_vs_original} shows that the original ASC remains
competitive with the Joint Minimizer solution, while algorithmic version shows
weak performance. This is not surprising and comes at a certain cost for which
we discuss in conclusion.

\begin{figure}[!t]
\centering
\includegraphics[width=.9\textwidth]{figures/ch_mst/gen_6_all}
\caption{Error plotted on the solutions obtained via original ASC
  (Chapter~\ref{ch:gen_appch} and algorithmic ASC (this chapter), as well as benchmarked on
  Joint Minimizer (Section~\ref{sec:aasc_vs_asc}).}
\label{fig:aasc_vs_original}
\end{figure}

\section{Discussion and Conclusion}
\label{sec:mst_conclusion}

\subsubsection{On the ASC-induced ranking of the algorithms}
The framework of approximation set coding is generalized from
the domain of models to the domain of algorithms in the
chapter. This framework enables us to apply an information-theoretic
regularization and quality assessment principle to algorithms,
and, in particular, to a minimum spanning tree problem with noisy
graphs as input.

We interpreted the results of the quantitative analysis of the
algorithms in a qualitative way, binding the strategy of the algorithm
to its information content, showing consistency and agreement with the
localization capability of the algorithm.

The ranking of the quantities plotted in
% Figure~\ref{fig:gaus_inf},~\ref{fig:gaus_loc_err},~
% \ref{fig:gaus_as_card},~ \ref{fig:gaus_ratio_dyn}
Figure~\ref{fig:gaus_inf}--\ref{fig:gaus_ratio_dyn}
support the following conjecture.
\begin{conj}
  The information content of contractive algorithms applied to the
  same problem establishes a ranking among them. This ranking is
  consistent with the average localization error of these algorithms.
\end{conj}

A possible way of investigating this relation is a rigorous analysis of the
approximation set dynamics using the mentioned Matrix-Tree theorem.
Although we utilized the simplest model of Gaussian noise,
independent on separate edges, there exists evidence, which
allows us to expect the same consistent results for a \emph{structured}
noise setting, when the edges of the graph are impacted by the noise
in a complex way, involving the statistical dependence of the noise
ingredients and other degrees of the noise model complexity.

\subsubsection{Algorithmic ASC vs. original ASC}

Should one use algorithmic approximation approach of this chapter or the
original $\gamma$-approximation approach form Chapter~\ref{ch:gen_appch}? In
this section, we address a question on whether the optimal stopping rule derived
by algorithmic ASC can compete with the original ASC (we initiated this discussion
in Section~\ref{sec:aasc_vs_asc}).

In fact, these two approaches serve quite different purposes and feature
different highlights. The original $\gamma$-parametrized ASC regularization
works in conjunction with the optimization goal $R(c, X)$, while its algorithmic
version makes use of algorithm-specific \textit{flow}. Although the algorithm
finds the same global optimizer as a bare $R(c, X)$-minimization procedure, the
structure of approximation sets if very different. Below, we list the main
points of difference:
\begin{itemize}
\item The original ASC relates to the continuous parameter, while algorithmic ASC
works with a discrete steps. This yields different power of resolution at which
we coarse-grain the solution set $\C$. For original ASC, this resolution power
is higher (it can allow much smaller step between approximation set sizes), while
for algorithmic ASC it fully depends on the algorithm.

\item Original ASC is hard to apply computationally, since it boils down to
computing and enumerating approximation sets. For algorithmic ASC, it is easier
to derive a problem-specific procedure which makes it \textit{orders of
magnitude} faster. For example, in the MST case, we utilized the Matrix-Tree
theorem and computed the cardinalities analytically at each step!
\end{itemize}

Hence, the trade-off between the power of resolution and the computational
power exists and should be addressed in each special case.

\subsubsection{General notes on the ASC approach to algorithms}

From the statistical physics perspective, a contractive algorithm can be
considered as a process, where the temperature decreases step by step, and thus
``freezes'' the solution space, until finding the optimal solution. The
survey~\citep[Sec.~3.2]{Merhav:2010} discusses the connection between such a
coding framework and statistical physics. This connection \citep{JB:ISIT:2010}
inspires us to explore the dynamics of the algorithm from a generalization point
of view and to relate it to the information theory~\citep{Merhav:2010}.

We will consider the statistical physics viewpoint on the ASC regularization in the next
chapter.


% COSC 3P98 Project
% gl-billiards
% Dan Lapp
% Taras Mychaskiw

\section{3D Engine}
	\subsection{Loading Models}
	To use 3d models that are more complex than simple primitive shapes, it is important to import custom models. gl-billiards employs 3D Studio Max
    to create and texture all of the models in the game. The main advantage of this approach is that all vertex coordinates and texture mapping
    coordinates can be handled by 3D Studio Max and loaded into OpenGL. Programming a pool table directly in OpenGL would be virtually impossible.
    \\
	gl-billiards uses the \textit{.3ds} file format to read models into OpenGL. This is a binary file format that is fairly simple to parse,
    well documented and natively supported by 3D Studio Max. The data is organized into a hierarchy of chunks. An imported model is
    abstracted into it's own class \textit{Model}. Each object in the scene, such as \textit{Table} or \textit{Cue} have a \textit{Model}.
    An example of the model initialization is shown below,
	\begin{eqnarray}
        Model{\enskip}model = Model(std::string{\enskip}modelPath,{\enskip}std::string{\enskip}texturePath) \nonumber
	\end{eqnarray}
    The texture specified must be a \textit{.bmp} file for the texture parser to read. When the model is initialized, the 3ds parser skips
    through the file chunks and stores three things. First are the actual \textit{(x, y, z)} coordinates of the vertices, then the polygon list,
    which is the order to draw the vertices. Last is the texture mapping coordinates.
    \\
    The bitmap parser reads in a \textit{.bmp} file used as a texture for the model. The texture must have a width and a height that is a power of $2$,
    and must be maximum $2048$ pixels in any dimension. The textures are then handed to OpenGL and a texture id is saved in the model.
	%\end{Loading Models}

	\subsection{Calculating Normals}
	After the model and texture is loaded, normals have to be calculated to be used with game lighting. gl-billiards calculates normals itself. This way
    we could get the vertex normals without having to parse the smoothing groups in the \textit{.3ds} file. Smoothing groups are a different representation
    of smoothed vertex normals used by 3D Studio Max, and are not directly used with OpenGL's $glNormal()$. Since every polygon in this file
    format is a triangle, calculating normals was fairly simple.
    The pseudocode for this is shown in Algorithm \ref{algo:calcNormals}.
	\begin{center}
	\begin{pseudocode}[framebox]{calculateNormals}{vertexList, polygonList}
	\label{algo:calcNormals}
	\LOCAL{numPolygons}[vertexList.size]    \\
	\FOREACH polygon \in polygonList \DO
	\BEGIN
	\vec v_1 \gets polygon.\vec P_1 - polygon.\vec P_2  \\
	\vec v_2 \gets polygon.\vec P_1  - polygon.\vec P_3 \\
	\vec n \gets \vec v_1 \times \vec v_2 \\\\
		\FOREACH coord \in polygon \DO
		\BEGIN
            normalList[coord] \gets normalList[coord] + \vec n    \\
			numPolygons[coord] \gets numPolygons[coord] + 1
		\END
	\END
	\\  \\
	\FOREACH i \in normalList \DO
        normalList[i] \gets normalList[i] \div numPolygons[i]
    \end{pseudocode}
    \end{center}

    First, for each triangular polygon $(P_1, P_2, P_3)$ polygonList, calculate two vectors $\vec v_1, \vec v_2$ to use for the cross
    product. The cross product returns a vector $\vec n$ perpendicular to the polygon face. \textit{numPolygons} keeps track of which surfaces are next
    to each vertex. Then in the final \textit{for} loop the polygon normals are averaged to give each vertex normal a smoothed effect.
	%\end{Calculating Normals}

	\subsection{Optimizations}
    To optimize 3d rendering, OpenGL display lists are implemented for \textit{Model}. Large floating point arrays used to store vertex coordinates
    require a lot of memory to manage these arrays. Using display lists significantly reduces the overhead associated with storing vertex arrays.
    From the results of running benchmarks on gl-billiards, the user interface used $\sim43\%$ of the program's computation time, and
    physics used $\sim27\%$. Therefore speeding up the model rendering provided a noticable speed increase of at least $5\times$.
	%\end{Optimizations}

%\end{3D Engine}


%
%  This file is included by Registration.tex
%
%
%

\index{itk::Image\-To\-Image\-Metric}

In OTB, \doxygen{itk}{ImageToImageMetric} objects quantitatively measure how well
the transformed moving image fits the fixed image by comparing the gray-scale
intensity of the images. These metrics are very flexible and can work with any
transform or interpolation method and do not require reduction of the
gray-scale images to sparse extracted information such as edges.

The metric component is perhaps the most critical element of the registration
framework. The selection of which metric to use is highly dependent on the
registration problem to be solved. For example, some metrics have a large
capture range while others require initialization close to the optimal
position.  In addition, some metrics are only suitable for comparing images
obtained from the same type of sensor, while others can handle
multi-sensor comparisons.
Unfortunately, there are no clear-cut rules as to how to choose a metric.

\index{itk::Image\-To\-Image\-Metric!GetValue()}
\index{itk::Image\-To\-Image\-Metric!GetDerivatives()}
\index{itk::Image\-To\-Image\-Metric!GetValueAndDerivatives()}

The basic inputs to a metric are: the fixed and moving images, a transform and
an interpolator. The method \code{GetValue()} can be used to evaluate the
quantitative criterion at the transform parameters specified in the argument.
Typically, the metric samples points within a defined region of the fixed
image.  For each point, the corresponding moving image position is computed
using the transform with the specified parameters, then the interpolator is
used to compute the moving image intensity at the mapped position. %% Details on
%% this mapping are illustrated in Figures \ref{fig:ImageOverlapIterator} and
%% \ref{fig:ImageOverlapInterpolator}.

The metrics also support region based evaluation. The \code{SetFixedImageMask()} and
\code{SetMovingImageMask()} methods may be used to restrict evaluation of the metric
within a specified region. The masks may be of any type derived from \doxygen{itk}{SpatialObject}.

Besides the measure value, gradient-based optimization schemes also require
derivatives of the measure with respect to each transform parameter. The
methods \code{GetDerivatives()} and \code{GetValueAndDerivatives()} can be
used to obtain the gradient information.

The following is the list of metrics currently available in OTB:
\begin{itemize}
\item Mean squares\\ \doxygen{itk}{MeanSquaresImageToImageMetric}
\item Normalized correlation \\ \doxygen{itk}{NormalizedCorrelationImageToImageMetric}
\item Mean reciprocal squared difference \\ \doxygen{itk}{MeanReciprocalSquareDifferenceImageToImageMetric}
\item Mutual information by Viola and Wells \\ \doxygen{itk}{MutualInformationImageToImageMetric}
\item Mutual information by Mattes \\ \doxygen{itk}{MattesMutualInformationImageToImageMetric}
\item Kullback Liebler distance metric by Kullback and Liebler \\ \doxygen{itk}{KullbackLeiblerCompareHistogramImageToImageMetric}
\item Normalized mutual information \\ \doxygen{itk}{NormalizedMutualInformationHistogramImageToImageMetric}
\item Mean squares histogram \\ \doxygen{itk}{MeanSquaresHistogramImageToImageMetric}
\item Correlation coefficient histogram \\ \doxygen{itk}{CorrelationCoefficientHistogramImageToImageMetric}
\item Cardinality Match metric \\ \doxygen{itk}{MatchCardinalityImageToImageMetric}
\item Kappa Statistics metric\\ \doxygen{itk}{KappaStatisticImageToImageMetric}
\item Gradient Difference metric \\ \doxygen{itk}{GradientDifferenceImageToImageMetric}
\end{itemize}

In the following sections, we describe each metric type in detail.
For ease of notation, we will refer to the fixed image $f(\bf{X})$
and transformed moving image $(m \circ T(\bf{X}))$ as images $A$ and $B$.

\subsection{Mean Squares Metric}
\label{sec:MeanSquaresMetric}
\index{itk::Mean\-Squares\-Image\-To\-Image\-Metric}

The \doxygen{itk}{MeanSquaresImageToImageMetric} computes the mean squared
pixel-wise difference in intensity between image $A$ and $B$ over a user
defined region:

\begin{equation}
MS(A,B) = \frac{1}{N} \sum_{i=1}^N \left( A_i - B_i \right)^2
\end{equation}
\begin{center}
$A_i$ is the i-th pixel of Image A\\
$B_i$ is the i-th pixel of Image B\\
$N$ is the number of pixels considered
\end{center}

The optimal value of the metric is zero. Poor matches between images $A$ and
$B$ result in large values of the metric. This metric is simple to compute and
has a relatively large capture radius.

This metric relies on the assumption that intensity representing the same
homologous point must be the same in both images. Hence, its use is restricted
to images of the same modality. Additionally, any linear changes in the
intensity result in a poor match value.

\subsubsection{Exploring a Metric}
\label{sec:ExploringAMetric}

Getting familiar with the characteristics of the Metric as a cost function is
fundamental in order to find the best way of setting up an optimization process
that will use this metric for solving a registration problem.

%% The following
%% example illustrates a typical mechanism for studying the characteristics of a
%% Metric. Although the example is using the Mean Squares metric, the same
%% methodology can be applied to any of the other metrics available in the
%% toolkit.

%% \ifitkFullVersion
%% \input{MeanSquaresImageMetric1.tex}
%% \fi

\subsection{Normalized Correlation Metric}
\label{sec:NormalizedCorrelationMetric}
\index{itk::Normalized\-Correlation\-Image\-To\-Image\-Metric}

The \doxygen{itk}{NormalizedCorrelationImageToImageMetric} computes pixel-wise
cross-correlation and normalizes it by the square root of the autocorrelation
of the images:

\begin{equation}
NC(A,B) = -1 \times \frac{ \sum_{i=1}^N \left( A_i \cdot B_i \right) }
        { \sqrt { \sum_{i=1}^N A_i^2  \cdot \sum_{i=1}^N B_i^2 } }
\end{equation}
\begin{center}
$A_i$ is the i-th pixel of Image A\\
$B_i$ is the i-th pixel of Image B\\
$N$ is the number of pixels considered
\end{center}

Note the $-1$ factor in the metric computation. This factor is used to make the
metric be optimal when its minimum is reached.  The optimal value of the metric
is then minus one. Misalignment between the images results in small measure
values.  The use of this metric is limited to images obtained using the same
imaging modality.  The metric is insensitive to multiplicative factors
-- illumination changes -- between
the two images.  This metric produces a cost function with sharp peaks and well
defined minima.  On the other hand, it has a relatively small capture radius.

\subsection{Mean Reciprocal Square Differences}
\label{sec:MeanReciprocalSquareDifferenceMetric}
\index{itk::Mean\-Reciprocal\-Square\-Difference\-Image\-To\-Image\-Metric}

The \doxygen{itk}{MeanReciprocalSquareDifferenceImageToImageMetric} computes
pixel-wise differences and adds them after passing them through a bell-shaped
function $\frac{1}{1+x^2}$:

\begin{equation}
PI(A,B) =  \sum_{i=1}^N \frac{ 1 }{ 1 + \frac{ \left( A_i - B_i \right) ^ 2}{ \lambda^2 }  }
\end{equation}
\begin{center}
$A_i$ is the i-th pixel of Image A \\
$B_i$ is the i-th pixel of Image B \\
$N$ is the number of pixels considered \\
$\lambda$ controls the capture radius
\end{center}

The optimal value is $N$ and poor matches results in small measure values.
The characteristics of this metric have been studied by Penney and Holden
\cite{Holden1999}\cite{Penney1998}.

This image metric has the advantage of producing poor values when few pixels
are considered.  This makes it consistent when its computation is subject to
the size of the overlap region between the images. The capture radius of the
metric can be regulated with the parameter $\lambda$.  The profile of this
metric is very peaky. The sharp peaks of the metric help to measure spatial
misalignment with high precision. Note that the notion of capture radius is
used here in terms of the intensity domain, not the spatial domain. In that
regard, $\lambda$ should be given in intensity units and be associated with
the differences in intensity that will make drop the metric by $50\%$.

The metric is limited to images of the same image modality.  The
fact that its derivative is large at the central peak is a problem for some
optimizers that rely on the derivative to decrease as the extrema are
reached.  This metric is also sensitive to linear changes in intensity.

\subsection{Mutual Information Metric}
\label{sec:MutualInformationMetric}

The \doxygen{itk}{MutualInformationImageToImageMetric} computes the mutual
information between image $A$ and image $B$.  Mutual information (MI)
measures how much information one random variable (image intensity in one
image) tells about another random variable (image intensity in the other
image). The major advantage of using MI is that the actual form of the
dependency does not have to be specified.  Therefore, complex mapping between
two images can be modeled.  This flexibility makes MI well suited as a
criterion of multi-modality registration~\cite{Pluim2003}.

Mutual information is defined in terms of entropy. Let
\begin{equation}
H(A) = - \int p_A(a) \log p_A(a)\, da
\end{equation}
be the entropy of random variable $A$, $H(B)$ the entropy of
random variable $B$ and
\begin{equation}
H(A,B) = \int p_{AB}(a,b) \log p_{AB}(a,b)\,da\,db
\end{equation}
be the joint entropy of $A$ and $B$. If $A$ and $B$ are independent, then
\begin{equation}
p_{AB}(a,b) = p_A(a) p_B(b)
\end{equation}
and
\begin{equation}
H(A,B) = H(A) + H(B).
\end{equation}
However, if there is any dependency, then
\begin{equation}
H(A,B)<H(A)+H(B).
\end{equation}
The difference is called Mutual Information : \( I(A,B) \)
\begin{equation}
I(A,B)=H(A)+H(B)-H(A,B)
\end{equation}

\subsubsection{Parzen Windowing}

\itkpiccaption[Parzen Windowing in Mutual Information]{
In Parzen windowing, a continuous density function is constructed by
superimposing kernel functions (Gaussian function in this case) centered on the
intensity samples obtained from the image.\label{fig:ParzenWindowing}}
\parpic(0.5\textwidth,5.5cm)[r]{\includegraphics[width=0.48\textwidth]{ParzenWindowing13.eps}}

In a typical registration problem, direct access to the marginal
and joint probability densities is not available and hence the
densities must be estimated from the image data. Parzen windows
(also known as kernel density estimators) can be used for this purpose.
In this scheme, the densities are constructed by taking intensity
samples $S$ from the image and super-positioning kernel functions
$K(\cdot)$ centered on the elements of $S$ as illustrated in
Figure \ref{fig:ParzenWindowing}:

A variety of functions can be used as the smoothing kernel with the
requirement that they are smooth, symmetric, have zero mean and
integrate to one. For example, boxcar, Gaussian and B-spline functions are
suitable candidates.  A smoothing parameter is used to scale the kernel
function.  The larger the smoothing parameter, the wider the kernel function
used and hence the smoother the density estimate. If the parameter is too
large, features such as modes in the density will get smoothed out.  On the
other hand, if the smoothing parameter is too small, the resulting density
may be too noisy. The estimation is given by the following equation.

\begin{equation}
p(a) \approx P^{*}(a) = \frac{1}{N} \sum_{s_j \in S} K\left(a - s_j\right)
\end{equation}

Choosing the optimal smoothing parameter is a difficult research problem and
beyond the scope of this software guide.  Typically, the optimal value of the
smoothing parameter will depend on the data and the number of samples used.

\subsubsection{Viola and Wells Implementation}

OTB, through ITK, has multiple implementations of the mutual information
metric. One of the most commonly used is
\doxygen{itk}{MutualInformationImageToImageMetric} and follows the method specified
by Viola and Wells in \cite{Viola1997}.

\index{itk::Mutual\-Information\-Image\-To\-Image\-Metric}

In this implementation, two separate intensity samples $S$ and $R$ are drawn
from the image: the first to compute the density, and the second to approximate
the entropy as a sample mean:
\begin{equation}
H(A) = \frac{1}{N} \sum_{r_j \in R} \log P^{*}(r_j).
\end{equation}
Gaussian density is used as a smoothing kernel, where the standard deviation
$\sigma$ acts as the smoothing parameter.

\index{itk::Mutual\-Information\-Image\-To\-Image\-Metric!SetNumberOfSpatialSamples()}

The number of spatial samples used for computation is defined using
the \code{SetNumberOfSpatialSamples()} method. Typical values range from 50 to 100.
Note that computation involves an $N \times N$ loop and hence, the computation
burden becomes very expensive when a large number of samples is used.

\index{itk::Mutual\-Information\-Image\-To\-Image\-Metric!SetFixedImageStandardDeviation()}
\index{itk::Mutual\-Information\-Image\-To\-Image\-Metric!SetMovingImageStandardDeviation()}
The quality of the density estimates depends on the choice of the standard
deviation of the Gaussian kernel. The optimal choice will depend on the
content of the images.  In our experience with the toolkit, we have found
that a standard deviation of 0.4 works well for images that have been
normalized to have a mean of zero and standard deviation of 1.0. The standard
deviation of the fixed image and moving image kernel can be set separately
using methods
\code{SetFixedImageStandardDeviation()} and \code{SetMovingImageStandardDeviation()}.

\subsubsection{Mattes et al. Implementation}
Another form of mutual information metric available in ITK follows the method
specified by Mattes et al. in \cite{Mattes2001} and is implemented by the
\doxygen{itk}{MattesMutualInformationImageToImageMetric} class.

\index{itk::Mattes\-Mutual\-Information\-Image\-To\-Image\-Metric}
In this implementation, only one set of intensity samples is drawn from the
image.  Using this set, the marginal and joint probability density function
(PDF) is evaluated at discrete positions or bins uniformly spread within the
dynamic range of the images. Entropy values are then computed by summing over
the bins.

\index{itk::Mattes\-Mutual\-Information\-Image\-To\-Image\-Metric!SetNumberOfSpatialSamples()}
\index{itk::Mattes\-Mutual\-Information\-Image\-To\-Image\-Metric!SetNumberOfHistogramBins()}

The number of spatial samples used is set using method
\code{SetNumberOfSpatialSamples()}. The number of bins used to compute
the entropy values is set via \code{SetNumberOfHistogramBins()}.

Since the fixed image PDF does not contribute to the metric derivatives, it
does not need to be smooth. Hence, a zero order (boxcar) B-spline kernel is
used for computing the PDF. On the other hand, to ensure smoothness, a third
order B-spline kernel is used to compute the moving image intensity PDF. The
advantage of using a B-spline kernel over a Gaussian kernel is that the
B-spline kernel has a finite support region. This is computationally
attractive, as each intensity sample only affects a small number of bins and
hence does not require a $N \times N$ loop to compute the metric value.

During the PDF calculations, the image intensity values are linearly scaled
to have a minimum of zero and maximum of one. This rescaling means that a
fixed B-spline kernel bandwidth of one can be used to handle image data with
arbitrary magnitude and dynamic range.

\subsection{Kullback-Leibler distance metric}
The \doxygen{itk}{KullbackLeiblerCompareHistogramImageToImageMetric} is yet another information based metric.
Kullback-Leibler distance measures the relative entropy between two
discrete probability distributions. The distributions are obtained from the
histograms of the two input images, $A$ and $B$.

The Kullback-Liebler distance between two histograms is given by
\begin{equation}
KL(A,B) =  \sum_i^N p_A(i) \times \log \frac{ p_A(i) }{p_B(i) }
\end{equation}

The distance is always non-negative and is zero only if the two distributions
are the same. Note that the distance is not symmetric. In other
words, $KL(A,B) \neq KL(B,A)$. Nevertheless, if the distributions are not too dissimilar,
the difference between $KL(A,B)$ and $KL(B,A)$ is small.

The implementation in ITK is based on \cite{Chung2002}.

\subsection{Normalized Mutual Information Metric}
Given two images, $A$ and $B$, the normalized mutual information may be computed as
\begin{equation}
NMI(A,B) = 1 + \frac{I(A,B)}{H(A,B)} = \frac{H(A) + H(B)}{H(A,B)}
\end{equation}
where the entropy of the images, $H(A)$, $H(B)$, the mutual
information, $I(A,B)$ and the joint entropy $H(A,B)$ are computed as mentioned
in \ref{sec:MutualInformationMetric}. Details of the implementation may be found in
the \cite{Hajnal2001}.

\subsection{Mean Squares Histogram}
\index{itk::Mean\-Squares\-Histogram\-Image\-To\-Image\-Metric}

The \doxygen{itk}{MeanSquaresHistogramImageToImageMetric} is an alternative
implementation of the Mean Squares Metric. In this implementation the joint
histogram of the fixed and the mapped moving image is built first. The user
selects the number of bins to use in this joint histogram. Once the joint
histogram is computed, the bins are visited with an iterator. Given that each
bin is associated to a pair of intensities of the form: \{fixed intensity,
moving intensity\}, along with the number of pixels pairs in the images that
fell in this bin, it is then possible to compute the sum of square distances
between the intensities of both images at the quantization levels defined by
the joint histogram bins.

This metric can be represented with
Equation~\ref{eqn:MeanSquaresHistogramImageToImageMetric}

\begin{equation}
\label{eqn:MeanSquaresHistogramImageToImageMetric}
MSH = \sum_f \sum_m { H(f,m) { \left( f - m \right) } ^ 2 }
\end{equation}

where $H(f,m)$ is the count on the joint histogram bin identified with fixed image
intensity $f$ and moving image intensity $m$.

\subsection{Correlation Coefficient Histogram}
\index{itk::Correlation\-Coefficient\-Histogram\-Image\-To\-Image\-Metric}

The \doxygen{itk}{CorrelationCoefficientHistogramImageToImageMetric} computes the
cross correlation coefficient between the intensities in the fixed image and
the intensities on the mapped moving image. This metric is intended to be used
in images of the same modality where the relationship between the intensities
of the fixed image and the intensities on the moving images is given by a
linear equation.

The correlation coefficient is computed from the Joint histogram as

\begin{equation}
\label{eqn:CorrelationCoefficientHistogramImageToImageMetric}
CC = \frac{ \sum_f \sum_m { \
            H(f,m) \left( f \cdot m - \
            \overline{f} \cdot \overline{m} \right)  } }{ \
            \sum_f { H(f) \left( (f - \overline{f})^2 \right) } \cdot \
            \sum_m { H(m) \left( (m - \overline{m})^2 \right) } }
\end{equation}

Where $H(f,m)$ is the joint histogram count for the bin identified with the
fixed image intensity $f$ and the moving image intensity $m$. The values
$\overline{f}$ and $\overline{m}$ are the mean values of the fixed and moving
images respectively.  $H(f)$ and $H(m)$ are the histogram counts of the fixed
and moving images respectively. The optimal value of the correlation
coefficient is $1$, which would indicate a perfect straight line in the
histogram.

\subsection{Cardinality Match Metric}
\index{itk::Match\-Cardinality\-Image\-To\-Image\-Metric}
The \doxygen{itk}{MatchCardinalityImageToImageMetric} computes cardinality of the
set of pixels that match exactly between the moving and fixed images. In other
words, it computes the number of pixel matches and mismatches between the two
images. The match is designed for label maps. All pixel mismatches are
considered equal whether they are between label 1 and label 2 or between label
1 and label 500. In other words, the magnitude of an individual label mismatch
is not relevant, or the occurrence of a label mismatch is important.

The spatial correspondence between the fixed and moving images is established using
a \doxygen{itk}{Transform} using the \code{SetTransform()} method and an interpolator
using \code{SetInterpolator()}. Given that we are matching pixels with labels,
it is advisable to use Nearest Neighbor interpolation.

\subsection{Kappa Statistics Metric}
\index{itk::Kappa\-Statistic\-Image\-To\-Image\-Metric}
The \doxygen{itk}{KappaStatisticImageToImageMetric} computes spatial intersection of
two binary images. The metric here is designed for matching pixels in two images
with the same exact value, which may be set using \code{SetForegroundValue()}.
Given two images $A$ and $B$, the $\kappa$ coefficient is computed as

\begin{equation}
\kappa = \frac{|A| \cap |B|}{|A| + |B|}
\end{equation}

where $|A|$ is the number of foreground pixels in image $A$.  This computes the
fraction of area in the two images that is common to both the images. In the
computation of the metric, only foreground pixels are considered.

\subsection{Gradient Difference Metric}
\index{it::Gradient\-Difference\-Image\-To\-Image\-Metric}

This \doxygen{itk}{GradientDifferenceImageToImageMetric} metric evaluates the
difference in the derivatives of the moving and fixed images. The derivatives
are passed through a function $\frac{1}{1+x}$ and then they are added. The
purpose of this metric is to focus the registration on the edges of structures
in the images.  In this way the borders exert larger influence on the result
of the registration than do the inside of the homogeneous regions on the image.


\documentclass[12pt]{article}

% packages
\usepackage{setspace}
\usepackage{hyperref}
\usepackage{array}
\usepackage[margin=0.75in]{geometry}
\usepackage{amsmath,bm}
\usepackage{amssymb}
\usepackage{bbold}
\usepackage{physics}
\usepackage{xcolor}
\usepackage{indentfirst}
\usepackage{enumerate}
\usepackage{mathtools}
\usepackage{fancyhdr}

\pagestyle{fancy}
\fancyhf{}
\rhead{Creative Destruction Lab}
\lhead{Introductions to Projects}
\rfoot{Page \thepage}

\allowdisplaybreaks

\title{Project 3: Calculating Franck-Condon Factors}

\begin{document}

\maketitle

\thispagestyle{empty}

\subsection*{Motivation}

Spectroscopy is the study of how light and matter (atoms and molecules) interact.
Light can be absorbed by matter (\textit{absorption}) or matter can emit light (\textit{emission}).
It turns out that spectroscopy is scientists' main tool for discovering properties of molecules (e.g. their molecular structure, bond strengths, etc.) and for chemical identification. Picture for instance an astronomer taking spectroscopic measurements from a gas cloud located millions of light years away from the Earth. To determine what chemicals this gas cloud comprises of, they certainly cannot travel there and take a physical sample of the gas cloud. The astronomer is tasked with utilizing spectroscopic theory (i.e. models wherein numerical studies can be carried out efficiently) to explain what they see.

Countless other applications like drug discovery, magnetic-resonance imaging (MRI) and climate science rely heavily on spectroscopic methods and theory. This week, you'll familiarize yourself with calculating Franck Condon Factors (FCFs), which are useful in studying {\it vibronic} transitions in molecules. You'll also get to compare your calculations to real experiments.

\subsection*{Gaussian Boson Sampling}
As you have learned in Week 2 in the presentation by Xanadu, Boson Sampling is a powerful tool which demonstrates that a non-universal quantum computer can display exponential speedup over classical computers, especially in the quantum optics or photonics field\cite{huh2015boson,aaronson2011computational, harrow2017quantum, quesadaFranckCondonFactorsCounting2019}. For our purpose, Gaussian Boson Sampling (GBS), which uses Gaussian input states, can be used to sample a distribution where the probabilities are proportional to the Franck-Condon Factors, which can then be used to construct vibrational spectra.

Franck-Condon Factors involve the overlap of the ``starting" and ``ending" wavefunctions of a transition and they are proportional to the amplitude of the vibronic spectrum. The method to calculate the Franck-Condon Factor between state $|m\rangle$ and $|n\rangle$ requires the use of the Doktorov operator, $U_{Dok}$, which allows the final state to be represented in terms of the initial state. The overlap can then be calculated as

\begin{equation}
FCF=|\langle m|U_{Dok}|n\rangle|^2    ~.
\end{equation}

This Doktorov operator can be written in terms of displacement, $D_\alpha$, squeezing,$\Sigma_r$ and interferometer operators $U_1$, $U_2$ as\cite{killoran2019strawberry, bromley2020applications}:

\begin{equation}
U_{Dok}=D_\alpha U_2 \Sigma_r U_1    ~.
\end{equation}

A GBS device can be programmed to perform these operations for a particular molecule, therefore can be used to calculate the FCFs and, in turn, the vibronic spectrum of a molecule. You will have an opportunity to simulate this ``experiment'' in Task 3.
\newpage

\subsection*{Your Tasks}

\subsubsection*{Task \#1}

In this task, you will calculate Franck-Condon Factors for H$_2-$H$_2^+$ using the harmonic oscillator approximation and compare to a real experiment! In a real experiment, we can excite many different vibronic transitions. We would therefore be able to calculate many FCFs that describe different vibronic transitions. For example, it could be that we see excitations corresponding to an H$_2$ molecule in its $n = 0$ vibrational state to H$_2^+$ in its $n = 2$ vibrational state (green line in Fig.~\ref{fig:visualize_vibronic}) and its $n = 4$ vibrational state. In this task, we will look at only transitions that have a high FCF (i.e. the corresponding transition is very intense).

\begin{figure}
    \begin{center}
        \includegraphics[width=0.5\linewidth]{../figures/potential_energy_curve.pdf}
    \end{center}
    \caption{A visualization of vibronic transitions. The blue and red curves represent H$_2$ and H$_2^+$, respectively.}
        \label{fig:visualize_vibronic}
\end{figure}

You are provided with a \texttt{python} notebook called \texttt{Task1.ipynb} which calculates all of the information you require. The \texttt{FCF\_helper.py} file contains all of the calculations done under the hood. \footnote{Calculating FCFs comes down to calculating the overlap between the wavefunctions before and after the vibronic transition. The overlap calculation is an integral which we chose to evaluate numerically. We emphasize that molecular parameters like the reduced mass of H$_2$ and fundamental frequencies of H$_2$ and H$_2^+$ are hard-coded in \texttt{FCF\_helper.py}} Currently, the \texttt{spectrum\_analysis} function in \texttt{FCF\_helper.py} only outputs the following.
\begin{itemize}
    \item \texttt{n\_0} and \texttt{n\_p}: The vibrational state numbers of H$_2$ and H$_2^+$, respectively, involved in the vibronic transition
    \item \texttt{FCF}: The corresponding FCF associated to the vibronic transition
\end{itemize}
Here's what we'd like you to do:
\begin{enumerate}
    \item Open up \texttt{FCF\_helper.py} and navigate to the \texttt{spectrum\_analysis} function. Modify the \texttt{data} variable so that it also includes the spectral intensity (\texttt{Ep - E0}).
    \item Plot the corresponding FCF versus spectral intensity (i.e. plot \texttt{FCF} versus \texttt{Ep - E0}) and show the results for \texttt{n\_0}=0 and \texttt{n\_p}=10.
\end{enumerate}

Congratulations, you have now successfully predicted the Franck-Condon Factors of H$_2-$H$_2^+$! Figure \ref{fig:h2_spectrum} shows the photoionization spectrum for H$_2$. Does it look like the real data in Fig.~\ref{fig:h2_spectrum}?

\begin{figure}
    \begin{center}
        \includegraphics[width=\linewidth]{../figures/H2-expspectrum.pdf}
    \end{center}
    \caption{
    Experimental photoionization spectrum of H$_2$-H$_2^+$ from Ref.~\cite{berkowitz1973comparison} with the vibrational level of H$_2$=0.
    }
    \label{fig:h2_spectrum}
\end{figure}

\subsubsection*{Task \#2}

You are provided with a \texttt{C++} code \texttt{FC.cxx} created by P.-N. Roy \cite{yang1995structure}, which calculates the photoionization spectrum for any molecule up to triple excitations and goes beyond the harmonic oscillator approximation. The theory is based on the paper by Ref~\cite{doktorov1977dynamical}. The molecule you will be investigating is $V_3$.
This code takes as input a file which requires the results of diagonalizing the mass-weighted hessian/force-constant matrix (2nd derivative of the Hamiltonian with respect to position). This input file is provided for you (V3).

Browse the following references (\cite{yang1995structure,doktorov1977dynamical,quesadaFranckCondonFactorsCounting2019}) to understand how the code works, as you will need a basic understanding of this for the next task, but it's not necessary to fully understand it.  Compile and run the code \texttt{./FC\_quick V3}. This code outputs the spectrum \texttt{V3.spec.out}. Plot it in your favourite plotting program.

\subsubsection*{Task \#3}

In this task, you will be simulating a Gaussian Boson Sampling (GBS) ``experiment." This code generates samples for computing a vibronic spectrum. Each sample that is generated starts from the vacuum state and the following gates are performed\cite{killoran2019strawberry, bromley2020applications}.

\noindent This code takes as input a file which requires the following information:
\begin{enumerate}
\item Two-mode squeezing on all  $2N$  modes with parameters \texttt{t}
\item Interferometer \texttt{U1} on the first $ N$ modes
\item Squeezing on the first $N$ modes with parameters \texttt{r}
\item Interferometer \texttt{U2} on the first  $N$ modes
\item Displacement on the first  $N$ modes with parameters \texttt{alpha}
\end{enumerate}

The energy of the resultant state is calculated and contributes to the spectrum. After running this simulation over many samples and plotting the energies against their frequency, those states (energies) with a high Franck-Condon Factor will have a larger peak on the spectrum and those with a low Franck-Condon Factor will have a smaller peak. It is important to note that it's not the exact values of the peak heights that we care about in this case (since it's based on a probability distribution), it's the relative heights of the peaks of the spectrum.

\hspace{20mm}

You are provided with a \texttt{python} notebook called \texttt{Task3.ipynb} which calculates all of the information you require. You will be required to use the \texttt{Strawberry Fields} library\cite{killoran2019strawberry, bromley2020applications}. Click  \href{https://strawberryfields.readthedocs.io/en/stable/_static/install.html}{\underline{\textbf{here}}} for the install instructions. However, to be able to use this code, you require an input file, which you will have to create. To do this, we will leverage the fact that we have another code that has done this work already, \texttt{FC.cxx}, which you became familiar with in Task 2. Each piece of information that you need to output has been clearly marked in the code and your task is to write that information to a file, which will then be used as your input file to \texttt{Sample\_Vibronic.py}. Once you have that input file, you should be able to produce the spectrum for $V_3$. Compare this spectrum to the previous method. What happens if you decrease the number of samples to 10? 100? 1000? At what number of samples do you feel the spectrum is converged?

\section*{Challenges}

\begin{enumerate}
    \item An alternative and analogous method to calculating these Franck-Condon Factors using matrix elements is to use a loop hafnian approach. This loop hafnian approach uses Gauss Boson Sampling which would allow these Factors to be calculated using a quantum circuit. Use the result of Task 3 to provide data to a skeleton code provided that uses loop hafnians to calculate the Franck-Condon Factors.
    \item Explain briefly the similarities and differences between these three methods.
\end{enumerate}

\section*{Possible Business Outcomes}

\begin{enumerate}
    \item Explain to a layperson what theoretical chemistry/physics is, in the general context of Franck-Condon Factors.
    \item What is the importance of theoretical chemistry/physics from an economic point of view.
    \item Explain to a layperson what a quantum circuit is and its relationship to theoretical chemistry/physics.
    \item These codes use General Public License and Apache 2.0 licenses. Discuss the similarities and differences of these with respect to intellectual property rights of the coder. What are the advantages and disadvantages of codes licensed for the public domain and those that are licensed for private use.
\end{enumerate}

\newpage

\bibliography{refs}
\bibliographystyle{unsrt}

\end{document}


\documentclass[12pt,letterpaper]{article}
\usepackage{geometry} % see geometry.pdf on how to lay out the page. There's lots.
%\geometry{a4paper} % or letter or a5paper or ... etc
% \geometry{landscape} % rotated page geometry

%% LaTeX - Article customise

%%% PACKAGES
\usepackage{booktabs} % for much better looking tables
\usepackage{array} % for better arrays (eg matrices) in maths
\usepackage{paralist} % very flexible & customisable lists (eg. enumerate/itemize, etc.)
\usepackage{verbatim} % adds environment for commenting out blocks of text & for better verbatim
\usepackage{subfigure} % make it possible to include more than one captioned figure/table in a single float
% These packages are all incorporated in the memoir class to one degree or another...

%%% PAGE DIMENSIONS
\usepackage{geometry} % to change the page dimensions
%\geometry{margins=2cm} % for example, change the margins to 2 inches all round
%\geometry{landscape} % set up the page for landscape
% read geometry.pdf for detailed page layout information

%%% HEADERS & FOOTERS
\usepackage{fancyhdr} % This should be set AFTER setting up the page geometry
\pagestyle{fancy} % options: empty , plain , fancy
\renewcommand{\headrulewidth}{0pt} % customise the layout...
\lhead{}\chead{}\rhead{}
\lfoot{}\cfoot{\thepage}\rfoot{}

%%%% SECTION TITLE APPEARANCE
%\usepackage{sectsty}
%\allsectionsfont{\sffamily\mdseries\upshape} % (See the fntguide.pdf for font help)
%% (This matches ConTeXt defaults)

%% LaTeX Preamble - Common packages

\usepackage[utf8]{inputenc} % Any characters can be typed directly from the keyboard, eg éçñ
\usepackage{textcomp} % provide lots of new symbols
\usepackage{graphicx}  % Add graphics capabilities
%\usepackage{epstopdf} % to include .eps graphics files with pdfLaTeX
\usepackage{flafter}  % Don't place floats before their definition
%\usepackage{topcapt}   % Define \topcation for placing captions above tables (not in gwTeX)
\usepackage{natbib} % use author/date bibliographic citations

\usepackage{amsmath,amssymb}  % Better maths support & more symbols
\usepackage{bm}  % Define \bm{} to use bold math fonts

\usepackage[pdftex,bookmarks,colorlinks,breaklinks]{hyperref}  % PDF hyperlinks, with coloured links
%\definecolor{dullmagenta}{rgb}{0.4,0,0.4}   % #660066
%\definecolor{darkblue}{rgb}{0,0,0.4}
%\hypersetup{linkcolor=red,citecolor=blue,filecolor=dullmagenta,urlcolor=darkblue} % coloured links
%%\hypersetup{linkcolor=black,citecolor=black,filecolor=black,urlcolor=black} % black links, for printed output

\usepackage{memhfixc}  % remove conflict between the memoir class & hyperref
% \usepackage[activate]{pdfcprot}  % Turn on margin kerning (not in gwTeX)
\usepackage{pdfsync}  % enable tex source and pdf output syncronicity

% Defining Itemized List Levels
\renewcommand{\labelitemi}{$-$}
%\renewcommand{\labelitemii}{$\cdot$}
%\renewcommand{\labelitemiii}{$\diamond$}
%\renewcommand{\labelitemiv}{$\ast$}

% Better margins
\usepackage{fullpage}

% Matlab code includer
% load package with ``framed'' and ``numbered'' option.
\usepackage[framed,numbered,autolinebreaks,useliterate]{mcode}

%%% BEGIN DOCUMENT
\begin{document}

% Maybe \noindent

\begin{center}
\huge
NTRT Scaling Analysis work\\
\normalsize
%\vspace{0.1in}
Andrew P. (Drew) Sabelhaus\\
apsabelhaus@berkeley.edu
\end{center}

\vspace{-2em}

\section{Nonlinear system example}

In this example system, there is a rod on a hinge (hinge at (x,y) = (0,0) ) with length $l$.
There is an angular (rotational) spring, and an angular damper, acting at the origin also.
Gravity is present, and we assume it acts fully at the center of mass for the rod.

With the state variables for this system being

\[
x = [\theta, \dot \theta]^\top
\]

The rod makes a triangle with the x-axis and the vertical, and the spring makes a triangle with the y-axis and a horizontal axis drawn from the top of the rod.

This top triangle has dimensions

\[
(l_{horiz-top}, l_{vert-top}) = (l cos \theta, h_s - l sin\theta)
\]

The length of the spring (the hypotenuse of this top triangle) is then

\[
x_s = \sqrt( l^2 cos^2\theta + (h_s - l sin \theta)^2)
\]

For a given $\theta$, the force from the spring onto the rod can then be decomposed into the following, assuming that $x_{s0}$ is the rest length of the spring, and we let $\theta_2$ be the acute angle in the upper triangle close to the rod end:

\[
F_{spring} = k x_s = \sqrt( l^2 cos^2\theta + (h_s - l sin \theta)^2)
\]
\[
F_{s_x} = F_{spring} cos \theta_2 = F_{spring} \frac{ l cos\theta }{ x_s } = (k)(l)cos\theta
\]
\[
F_{s_y} = F_{spring} sin \theta_2 = F_{spring} \frac{ h_s - l sin\theta}{ x_s} = (k) (h_s - l sin\theta)
\]

So, then, by Newton's 3rd law, our rigid body dynamics are:
\[
\sum T = I \ddot \theta, (mg)(\frac{m}{2} sin\theta) - F_{s_x} l sin\theta - F_{s_y} l cos\theta = I \ddot \theta
\]

For a solid rod,
\[
I = \frac{m l^3}{3}
\]

\end{document}


\documentclass[letterpaper]{article}

\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{September 17, 2014}
\maketitle
\section*{assignment}
Section 1.4: \# 17, 19, 20, 23, 24, 27.
\renewcommand{\labelenumi}{2.\arabic{enumi}}
\renewcommand{\labelenumii}{\arabic{enumii}.}
\renewcommand{\labelenumiii}{(\alph{enumiii})}
\begin{enumerate}
\setcounter{enumi}{2}
\item
  \begin{enumerate}
  \setcounter{enumii}{4}
  \item

  \item
  (1),(12),(13),(14),(23),(24),(34),(123),(124),(134),(234),(132),(243),(142),(143),(1234)
  \end{enumerate}
\end{enumerate}

\section*{2.3.5 theorem}
sketch of a proof
\begin{align*}
  \tau&=\left(
  \begin{aligned}
    1&\quad2&\quad3&\quad\dots& n\\
    \sigma(1)&\quad\sigma(2)&\quad\sigma(3)&\quad\dots&\sigma(n)
  \end{aligned}
  \right)
\end{align*}
\section*{example}
\begin{align*}
  \sigma&=\left(
  \begin{aligned}
    &1&&\quad2&&\quad3&&\quad4&&\quad5&&\quad6&&7\\
    &7&&\quad2&&\quad6&&\quad3&&\quad1&&\quad5&&4\\
  \end{aligned}
  \right)=(1,7,4,3,6,5)
\end{align*}
definition: let $\sigma\in S$ the least positive integer $m$ such that $\sigma^m=(1)$ is called the order of $\sigma$
\subsubsection*{example}
$\sigma=(123)(45)\to(123)^3(45)^2=\sigma^6$
\subsubsection*{observation for 2.3.8 proof}
$(a1a2a3\dots ak)^i=(1)$ and $(b1b2b3\dots bk)^i=1$ because they are disjoint and applying the a permutions don't change the b elements and vice versa
\end{document}


\subsection{Optimising convex functions}


%
% CMPT 310: Artificial Intelligence - A Course Overview
% Section: Problem Solving
%
% Author: Jeffrey Leung
%

\section{Problem Solving}
	\label{sec:problem-solving}
\begin{easylist}

& Environment: Observable, deterministic, discrete
& \textbf{State}: Possible arrangement of entities and agents in the environment
	&& Transition: Change from an initial state to another state
		&&& Successor function: Mapping from the current state and action to a new state
			&&&& Notation: $(\textrm{state}, \textrm{action}) \rightarrow \textrm{state}$
		&&& Goal state: Desired end state
		&&& Goal test: Condition which determines whether or not the current state is the goal state
			&&&& Notation: $(\textrm{state}) \rightarrow \textrm{true/false}$
		&&& Path cost: Numeric cost to take a path
	 	&&& Includes the possibility of no change
	&& E.g. For a two-square grid of a possibly dirty carpet with a vacuum cleaner, there are 4 possibilities for the dirty squares and 2 possibilities for the vacuum cleaner

\end{easylist}
\clearpage