\section{Additional visualizations}

\begin{figure}[h]
\begin{subfigure}[t]{0.3\textwidth}
    \centering
    \includegraphics[width=\textwidth]{img/loracs/mnist/tsne/mnist2-tsne-normal.png}
    \caption{Normal prior}
\end{subfigure}
\begin{subfigure}[t]{0.3\textwidth}
    \centering
    \includegraphics[width=\textwidth]{img/loracs/mnist/tsne/mnist2-tsne-noprior.png}
    \caption{No prior}
\end{subfigure}
\begin{subfigure}[t]{0.3\textwidth}
    \centering
    \includegraphics[width=\textwidth]{img/loracs/mnist/tsne/mnist2-tsne-vamp.png}
    \caption{Vam\p(500) prior}
\end{subfigure}
~
\begin{subfigure}[t]{0.3\textwidth}
    \centering
    \includegraphics[width=\textwidth]{img/loracs/mnist/tsne/mnist2-tsne-maf.png}
    \caption{MAF prior}
\end{subfigure}
\begin{subfigure}[t]{0.3\textwidth}
    \centering
    \includegraphics[width=\textwidth]{img/loracs/mnist/tsne/mnist2-tsne-tmc.png}
    \caption{\acronym(200) prior}
\end{subfigure}
\caption{TSNE visualizations of the latent space of the MNIST test set with various prior distributions, color-coded according to class.}
\label{fig:tsne}
\end{figure}

\begin{figure}[h]
    \centering
    \includegraphics[width=0.5\textwidth]{img/loracs/mnist/tsne/mnist2-tsne-tmc-tree.png}
    \caption{A TSNE visualization of the latent space for the TMC(200) model with inducing points and one sample from $\q(\tree; \inducingpoints)$ plotted. Internal nodes are visualized by computing their expected posterior values, and branches are plotted in 2-d space.}
    \label{fig:mnist-tsne-tree}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics[width=0.5\textwidth]{img/loracs/mnist/mnist-vamp-inputs.png}
\caption{MNIST VampPrior learned pseudo-inputs.}
\label{fig:mnist-vamp-inducing}
\end{figure}
\begin{figure}[h]
\centering
\includegraphics[width=0.5\textwidth]{img/loracs/mnist/mnist-vamp-outputs.png}
\caption{MNIST VampPrior reconstructed pseudo-inputs obtained by
deterministically encoding and decoding each pseudo-input.}
\label{fig:mnist-vamp-inducing-outputs}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics[width=0.4\textwidth]{img/loracs/omniglot/inducing-points.png}
\caption{Omniglot learned inducing points.}
\label{fig:omniglot-inducing-points}
\end{figure}

\begin{figure}[h]
\centering
\includegraphics[width=0.8\textwidth]{img/loracs/celeba/inducing-points.png}
\caption{CelebA learned inducing points.}
\label{fig:celeba-inducing-points}
\end{figure}

\section{Empirical results}

\begin{sidewaystable}
\scriptsize
\centering
\caption{MNIST few-shot classification results.}
\setlength{\tabcolsep}{2pt}
\input{results/loracs/cf-mnist.tex}
\label{tab:semisupervised-mnist}
\end{sidewaystable}

\begin{sidewaystable}
\scriptsize
\setlength{\tabcolsep}{2pt}
\centering
\caption{Omniglot few-shot classification results.}
\input{results/loracs/cf-omniglot.tex}
\label{tab:semisupervised-omniglot}
\end{sidewaystable}

\begin{table}
\centering
\caption{MNIST few-shot classification with labeled inducing points.}
\scriptsize
\setlength{\tabcolsep}{2pt}
\begin{tabular}{r|cc}
\toprule
     \# of inducing points &  200 & 500\\
\midrule
        & 0.9428 & 0.9474\\
\bottomrule
\end{tabular}
\label{tab:inducing-point-labels}
\end{table}

\begin{figure}
\centering
\begin{subfigure}[h]{0.4\textwidth}
    \centering
    \includegraphics[width=\textwidth]{img/loracs/mnist/ir-mnist-joint.png}
    \caption{MNIST}
\end{subfigure}
\begin{subfigure}[h]{0.4\textwidth}
    \centering
    \includegraphics[width=\textwidth]{img/loracs/omniglot/ir-omniglot-joint.png}
    \caption{Omniglot}
\end{subfigure}
\caption{Averaged precision-recall curves over test datasets.}
\label{fig:prec-rec}
\end{figure}

\section{Algorithm details}
\label{sec:algorithm-details}

\subsection{Stick breaking process}
\label{sec:stick-breaking}
Consider inserting a node $\numdata + 1$ into the tree in between vertices $u$ and $v$
such that $t_v > t_u$,
creating branch $e_{\numdata + 1}$.
The inserted node has time $t_{\numdata + 1}$ with probability according to the stick breaking process, i.e.
\begin{equation}
\textstyle r(t_{\numdata+1}\given e_{\numdata+1}, V, E) =
\mathrm{Beta}\left(\frac{t_v - t_{\numdata + 1}}{1 - t_{\numdata + 1}}; a, b\right)\mathrm{Beta}\left(\frac{t_{\numdata + 1} - t_u}{1 - t_u}; a, b\right).
\end{equation}

\subsection{Belief propagation in TMCs}

The TMC is at the core of the \acronym\;
prior.
Recall that the TMC is
a prior over phylogenies $\tree$,
and after attaching a Gaussian random walk (GRW),
we obtain a distribution over $N$ vectors in $\mathbb{R}^d$,
corresponding to the leaves, $r(\latentdataset \given \tree)$.
However, the GRW samples
latent vectors at internal nodes $z_{V_{\text{int}}}$.
Rather than explicitly representing these values,
in this work we marginalize them out, i.e.
\begin{equation}
    r(\latentdataset \given \tree) = \int r(\latentdataset \given z_{V_{\text{int}}}, \tree)\p(z_{V_{\text{int}}} \given \tree) dz_{V_{\text{int}}}
\end{equation}
This marginalization process can be done efficiently, because our graphical model
is tree-shaped and all nodes have Gaussian likelihoods. Belief propagation is a message-passing
framework for marginalization and we utilize message-passing
for several TMC inference queries. The main queries we are interested in are:
\begin{enumerate}
    \item $r(\latentdataset, \tree)$ - for the purposes of MCMC, we are interested in computing the joint likelihood of a set of observed
    leaf values and a phylogeny.
    \item $r(\z_\n \given z_{\backslash n}, \tree)$ - this query computes the posterior density over one leaf given all the others;
    we use this distribution when computing the posterior predictive density of a TMC.
    \item $\nabla_{z_{\backslash n}} r(\z_\n \given z_{\backslash n}, \tree)$ - this query is the gradient of the predictive density
    of a single leaf with respect to the values at all other leaves. This query is used
    when computing gradients of the ELBO w.r.t $\inducingpoints$ in the \acronym\;prior.
\end{enumerate}

\textbf{Message passing} Message
passing treats the tree as an undirected graph.
We first pick start node $v_{\text{start}}$
and request messages from each of $v_{\text{start}}$'s neighbors.

Message passing is thereafter defined recursively.
When a node $v$ has requested messages from a source node $s$,
it thereafter requests messages from all its neighbors but $s$.
The base case for this recursion is a leaf node $v_\n$,
which returns a message with the following contents:
\begin{equation}
    \nu_\n = \bm{0};\quad \mu_\n = \z_\n;\quad \log \z_\n = 0;\quad \nabla_{\nu_\n}(\nu) = \bm{1};\quad \nabla_{\nu_\n}(\mu) = \bm{0};\quad \nabla_{\mu_\n}(\mu) = \bm{1}
\end{equation}
where bold numbers $\bm{0}\triangleq (0,\ldots,0)^\top$ and $\bm{1}\triangleq (1,\ldots,1)^\top$ denote vectors obtained by repeating a scalar $d$ times.

In the recursive case, consider being at a node $i$ and receiving a set of messages from its neighbors $M$.
\begin{equation}
\begin{split}
    \nu_i = \frac{1}{\sum_{m \in M}\frac{1}{\nu_m + e_{im}}} ;\quad \mu_i = v_i \sum_{m \in M} \frac{\mu_m}{\nu_m + e_{im}}\\
\end{split}
\end{equation}
where $e_{im}$ is the length of the edge between nodes $i$ and $m$.
These messages are identical to those used in \citet{Boyles2012}.

Additionally, our messages include gradients w.r.t. \emph{every} leaf node
downstream of the message.
We update each of these gradients when computing the new message
and pass them along to the source node.
Gradients with respect to one of these nodes $j$ are calculated as
\begin{equation}
\begin{split}
    \nabla_{\nu_j}(\nu) &= \nabla_{\nu_j} \nu_i\\
    \nabla_{\nu_j}(\mu) &= \nabla_{\nu_j} \mu_i\\
    \nabla_{\mu_j}(\mu) &= \nabla_{\mu_j} \mu_i\\
\end{split}
\end{equation}
The most complicated message is the $\log Z_i$ message, which depends
on the number of incoming messages. $v_{\text{start}}$ gets
three incoming messages,
all other nodes get only two. Consider
two messages from nodes $v_k$ and $v_l$:
\begin{equation}
\begin{split}
    \Sigma_i &\triangleq (\nu_k + e_{ik} + \nu_l + e_{il})I \\
    \log Z_i &= -\frac{1}{2}\|\mu_k - \mu_l\|^2_{\Sigma_i} - \frac{1}{2}\left(\log|\Sigma_i| _ d\log2\pi\right)
\end{split}
\end{equation}
For three messages from nodes
$v_k$, $v_l$, and $v_m$:
\begin{equation}
\begin{split}
    \Sigma_i &\triangleq \left((\nu_k + e_{ik})(\nu_l + e_{il}) + (\nu_l + e_{il})(\nu_m + e_{im}) + (\nu_m + e_{im})(\nu_k + e_{ik})\right)I \\
    \log Z_i &= -\frac{1}{2}\left((\nu_m + e_{im})\|\mu_k - \mu_l\|^2_{\Sigma_i} + (\nu_k + e_{ik})\|\mu_l - \mu_m\|^2_{\Sigma_i} + (\nu_l + e_{il})\|\mu_m - \mu_k\|^2_{\Sigma_i}\right)  \\
             &- \frac{1}{2}\log|\Sigma_i| - \log2\pi
\end{split}
\end{equation}

With these messages, we can answer all the aforementioned inference queries.
\begin{enumerate}
    \item We can begin message passing at any internal node and compute: $\log r(\latentdataset, \tree) = \sum_{v \in V} \log Z_v$
    \item We start message passing at $v_\n$. $r(\z_\n \given z_{\backslash n}, \tree)$ is a Gaussian with mean $\mu_\n$ and variance $\nu_\n$.
    \item $\nabla_{z_{\backslash n}} r(\z_\n \given z_{\backslash n}, \tree)$ is $\nabla_{z_{\backslash n}} \N(\z_\n \given \mu_\n, \nu_\n I)$,
    which in turn utilizes gradients sent via message passing.
\end{enumerate}

\textbf{Implementation} We chose
to implement the TMC and message
passing in Cython because we found raw Python to be
too slow due to function call and type-checking
overhead. Furthermore, we used diagonal rather than
scalar variances in the message passing implementation
to later support diagonal variances handed
from the variational posterior over $\z_\n$.

\subsection{Variational inference for the \acronym\;prior}
\label{sec:inference-details}

The \acronym\; prior involves first sampling a tree
from the posterior distribution over TMCs
with $\inducingpoints$ as leaves.
We then sample a branch and time
for each data $\z_\n$ according
to the posterior predictive
distribution described in \autoref{sec:bnhc}.
We then sample a $\z_\n$ from
the distribution induced by the GRW likelihood model.
Finally, we pass the sampled $\z_\n$
through the decoder.
\begin{equation}
    \begin{split}
        \tree &\sim \p(\tree ; \inducingpoints) \\
        e_\n, t_\n &\sim \p(e_\n, t_\n | \tree) \\
        \z_\n | e_\n, t_\n, \tree &\sim \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints) \triangleq r(s_{M + 1} = \z_\n | e_\n, t_\n, \tree) \\
        \x_\n | \z_\n &\sim \p_\genparam(\x_\n | \z_\n)
    \end{split}
\end{equation}

Consider sampling the optimal $q^*(\tree; \inducingpoints)$.

\begin{equation}
\begin{split}
    q^*(\tree; \inducingpoints)
    &\propto \exp{\E_\q\left[\log \p(\tree, \latentdataset, \dataset)\right]} \\
    &\propto \exp{\log \p(\tree ; \inducingpoints) + \sum_\n \E_\q\left[\p(\z_\n | e_\n, t_\n, \tree)\right]} \\
    &\propto \exp{\log \mathrm{TMC}_\n(\tree; a, b) + \sum_{m = 1}^M \log r(\inducingpoint_m | \inducingpoint_{1:m - 1}, \tree) \\&+ \sum_\n \E_\q\left[\log \p(\z_\n | e_\n, t_\n, \tree)\right]} \\
\end{split}
\end{equation}
We  set $\q(\tree; \inducingpoints) = r(\tree \given \inducingpoints)$.
We use additional variational factors
$\q(e_\n)$,
$q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)$,
and
$q_\recparam(\z_\n | \x_\n)$.
$q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)$ is a
recognition network that outputs
the attach time for a particular branch.
Since the $\q(\tree; \inducingpoints)$ and $\p(\tree; \inducingpoints)$ terms
cancel out, we obtain the following ELBO.
\begin{equation}
    \begin{split}
    \L[q] &\triangleq \E_\q \left[\log \frac{\prod_\n \p(e_\n, t_\n | \tree) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right] \\
    \end{split}
\end{equation}

\textbf{Inference procedure}
In general, $\q(\tree; \inducingpoints)$ can
be sampled using vanilla SPR Metropolis-Hastings,
so samples from this distribution are readily available.

For each data in the minibatch $\x_\n$, we pass it
through the encoder to obtain $\q(\z_\n | \x_\n)$.
We then compute
\begin{equation}
    q^*(e_\n) = \exp{\E_\q\left[\log \p(e_\n | t_\n, \z_\n, \tree; \inducingpoints)\right]}
\end{equation}
This quantity is computed by looping
over every branch $b$ of
a sample from $\q(\tree)$,
storing incoming messages at each node,
passing the $\mu$ and $\nu$
and a sample from $\q(\z_\n | \x_\n)$
into $q_\xi(t_\n | e_\n, \inducingpoints, \z_\n)$,
outputting a logistic-normal distribution
over times for that branch. We sample that
logistic normal to obtain a time $t$
to go with branch $b$. We can then compute
the log-likelihood of $\z_\n$ if it were
to attach to $b$ and $t$, using TMC
inference query \#2.
This log-likelihood is added to the TMC prior log-probability
of the branch being selected to obtain
a joint probability $\E_\q\left[\log \p(e_\n)\p(t_\n)\p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\right]$ over the branch.
After doing this for every branch, we normalize
the joint likelihoods to obtain
the optimal categorical distribution over every branch
for $\z_\n$, $q^*(e_\n)$. We then sample this
distribution to obtain an attach location and time
$e_\n, t_\n$ for each data in the minibatch.

The next stage is to compute gradients w.r.t. to the
learnable parameters of the model ($\genparam$, $\inducingpoints$, $\recparam$, and $\xi$).
In the process of calculating $q^*(e_\n)$,
we have obtained samples from its corresponding
$q_\xi(t_\n | e_\n, \z_\n, \tree; \inducingpoints)$ and $\q(\z_\n | \x_\n)$.
We plug these into the ELBO and can compute
gradients via automatic differentiation w.r.t. $\recparam$,
$\genparam$, and $\xi$. Computing gradients w.r.t.
$\inducingpoints$ is more tricky. We first examine the ELBO.

\begin{equation}
    \L[\q] = \E_\q \left[\log \frac{\prod_\n \p(e_\n|\tree) \p(t_\n) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)\q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right]
\end{equation}

Consider the gradient of the ELBO with respect to $\inducingpoints$.

\small
\begin{equation}
    \begin{split}
    \nabla_{\inducingpoints}\L[q] &=
    \nabla_{\inducingpoints}\E_\q \left[\log \frac{\prod_\n \p(e_\n|\tree) \p(t_\n) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right] \\
    &= \nabla_{\inducingpoints}\sum_{\tree} \q(\tree; \inducingpoints)\E_\q \left[\log \frac{\prod_\n \p(e_\n|\tree) \p(t_\n) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right]\\
    &= \sum_\tree \q(\tree; \inducingpoints)\nabla_{\inducingpoints}\E_\q\left[\log \frac{\prod_\n \p(e_\n|\tree) \p(t_\n) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right]\\
    &+ \sum_\tree \left(\nabla_{\inducingpoints} \q(\tree; \inducingpoints)\right)\E_\q \left[\log \frac{\prod_\n \p(e_\n|\tree) \p(t_\n) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right]\\
    &= \sum_\tree \q(\tree; \inducingpoints)\nabla_{\inducingpoints}\E_\q\left[\log \frac{\prod_\n \p(e_\n|\tree) \p(t_\n) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right]\\
    &+ \sum_\tree \left(\q(\tree;\inducingpoints)\nabla_{\inducingpoints} \log \q(\tree; \inducingpoints)\right)\E_\q \left[\log \frac{\prod_\n \p(e_\n|\tree) \p(t_\n) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right]\\
    &= \E_{\q(\tree)}\left[\nabla_{\inducingpoints}\E_\q\left[\log \frac{\prod_\n \p(e_\n|\tree) \p(t_\n) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right]\right.\\
    &+ \left.\nabla_{\inducingpoints} \log \q(\tree; \inducingpoints)\E_\q \left[\log \frac{\prod_\n \p(e_\n|\tree) \p(t_\n) \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\p_\genparam(\x_\n | \z_\n)}{\prod_\n \q(e_\n)q_{\xi}(t_\n | e_\n, \z_\n; \inducingpoints)q_\recparam(\z_\n | \x_\n)}\right]\right]\\
    &= \E_{q}\left[
        \nabla_{\inducingpoints}\left(-\log \q(e_\n) - \log \q(t_\n \given \z_\n, e_\n, \tree; \inducingpoints) + \log \p(\z_\n \given e_\n, t_\n, \tree; \inducingpoints)\right)
    \right] \\
    &+ \E_\q\left[\nabla_{\inducingpoints}\left(\log \q(\tree) + \log \q(e_\n)\right)\log\frac{\p(e_\n | \tree)}{\q(e_\n)}\frac{\p(\z_\n \given \z_\n, e_\n, t_\n, \tree; \inducingpoints)}{\q(t_\n \given \z_\n, e_\n, \tree; \inducingpoints)}\right]
    \end{split}
\end{equation}
\normalsize

In the last step, we expand out expectation over $e_\n$ and
then pass the derivative through like we did for $\tree$.
The gradients w.r.t. $\q(e_\n)$ are zero, since $q^*(e_\n)$ is
a partial optimum of the ELBO and we are left with:.
\begin{equation}
    \begin{split}
        \nabla_{\inducingpoints}\L[q] &= \E_\q\left[\nabla_{\inducingpoints}\log \p(\z_\n | e_\n, t_\n, \tree; \inducingpoints)\right] - \E_\q\left[\log \q(t_\n \given \z_\n, e_\n, \tree; \inducingpoints)\right] \\
        &+ \E_\q\left[\nabla_{\inducingpoints} \log \q(\tree; \inducingpoints)\log\frac{\p(e_\n | \tree)}{\q(e_\n)}\frac{\p(\z_\n \given \z_\n, e_\n, t_\n, \tree; \inducingpoints)}{\q(t_\n \given \z_\n, e_\n, \tree; \inducingpoints)}\right]
    \end{split}
\end{equation}
The first term of the gradient is the expected gradient
of the posterior predictive density w.r.t $\inducingpoints$.
This can be calculated by using TMC inference query \#3
using samples from $\q(e_\n)$ and $\q(t_\n \given \z_\n, e_\n, \tree; \inducingpoints)$. The second term also uses the same gradients, by means
of the chain rule to differentiate through the time-amortization network.
The third term of this gradient is a score function gradient,
which we decide to not use due to the high-variance nature of score function gradients. We found that we were able to obtain strong results even with biased gradients.

\section{Details of experiments}
\label{sec:implementation-details}

We implemented the \acronym\;prior in Tensorflow
and Cython.
For MNIST and Omniglot, our architectures are in \autoref{tab:mnist-arch} and CelebA is in \autoref{tab:celeba-arch}.

\begin{table}
\centering
\caption{Network architectures for MNIST and Omniglot}
\begin{subfigure}[h]{0.4\textwidth}
\caption{Encoder}
\begin{tabular}{lll}
\toprule
Layer type & Shape \\
\midrule
Conv + ReLU & [3, 3, 64], stride 2 \\
Conv + ReLU & [3, 3, 32], stride 1 \\
Conv + ReLU & [3, 3, 16], stride 2 \\
FC + ReLU & 512 \\
Gaussian & 40 \\
\bottomrule
\end{tabular}
\end{subfigure}
\begin{subfigure}[h]{0.4\textwidth}
\caption{Decoder}
\begin{tabular}{lll}
\toprule
Layer type & Shape \\
\midrule
FC + ReLU & 3136 \\
Deconv + ReLU & [3, 3, 32], stride 2 \\
Deconv + ReLU & [3, 3, 32], stride 1 \\
Deconv + ReLU & [3, 3, 1], stride 2 \\
Bernoulli & \\
\bottomrule
\end{tabular}
\end{subfigure}
\label{tab:mnist-arch}
\end{table}

\begin{table}
\centering
\caption{Network architectures for CelebA}
\begin{subfigure}[h]{0.4\textwidth}
\caption{Encoder}
\begin{tabular}{lll}
\toprule
Layer type & Shape \\
\midrule
Conv + ReLU & [3, 3, 64], stride 2 \\
Conv + ReLU & [3, 3, 32], stride 1 \\
Conv + ReLU & [3, 3, 16], stride 2 \\
FC + ReLU & 512 \\
Gaussian & 40 \\
\bottomrule
\end{tabular}
\end{subfigure}
\begin{subfigure}[h]{0.4\textwidth}
\caption{Decoder}
\begin{tabular}{lll}
\toprule
Layer type & Shape \\
\midrule
FC + ReLU & 4096 \\
Deconv + ReLU & [3, 3, 32], stride 2 \\
Deconv + ReLU & [3, 3, 32], stride 1 \\
Deconv + ReLU & [3, 3, 3], stride 2 \\
Bernoulli & \\
\bottomrule
\end{tabular}
\end{subfigure}
\label{tab:celeba-arch}
\end{table}

In general, we trained the model interleaving
one gradient step with 100 sampling steps for $\q(\tree; \inducingpoints)$.
We also found that experimenting with values of $a$ and $b$
in the TMC prior did not impact results significantly.
We initialized the networks with weights
from a VAE trained for 100 epochs and inducing points
were initialized using k-means. All parameters
were trained using Adam \citep{Kingma2014adam} with a $10^{-3}$ learning rate
for an 100 epochs with learning rate decay to $10^{-5}$ for the last 20 epochs.
Finally, we initialized trees with all node times close to 0,
to emulate a VAE prior.

\subsection{Baseline details}
All baselines were trained with the default architecture.
They were trained for 400 epochs,
with KL warmup ($\beta$ started at $10^{-2}$, and ramped up to $\beta = 1$
linearly over 50 epochs). They were trained using Adam with a
learning rate of $10^{-3}$, with a learning rate of $10^{-5}$ for
the last 80 epochs.

DVAE\# was trained using the default implementation from
\url{https://github.com/QuadrantAI/dvae}, which is hierarchical
VAE consisting of two Bernoulli latent variables, 200-dimensional each.
Each is learned via a feed-forward neural network 4-layers deep.
The default DVAE\# implementation also uses statically binarized MNIST
where we use dynamically binarized.


After a successful installation, you are ready to use {\IRSTLM}.

\noindent
In this Section, a basic 4-step procedure is given to estimate a LM and to compute its perplexity on a text.
Many changes to this procedure can be done in order to optimize effectiveness and efficiency according to your needs.

\noindent
Please refer to Section~\ref{sec:commands} to learn more about each IRSTLM commands, and
to Section~\ref{sec:functions} to get hints about IRSTLM functionalities.

\IMPORTANT{All programs assume that the environment variable {\bf IRSTLM} is correctly set to {\tt /path/to/install/doc}, and that that environment variable {\bf PATH} includes the command directory {\tt /path/to/install/bin}. see above}

\noindent
Data used in the following usage examples can be found in an archive you can download from the official website of {\IRSTLM}.
Most of them are very little, so the reported figures are not reliable.

\subsection{Preparation of Training Data}
In order to estimate a Language Model, you first need to prepare your training corpus. The corpus just consists of a text.
We assume that the text is already preprocessed according to the user needs; this means that lowercasing, uppercasing, tokenization, and any other text transformation has to be performed beforehand with other tools.

\noindent
You can only decide whether you are interested that {\IRSTLM} is aware of sentence boundaries, i.e. where a sentence starts and ends. Otherwise, it considers the corpus as a continuous stream of text, and does not identify sentence splits.

\noindent
The following script adds start and end symbols ({\tt <s>} and {\tt </s>}, respectively) to all lines in your training corpus.
\begin{verbatim}
$> cat train.txt | add-start-end.sh > train.txt.se
\end{verbatim}

\noindent
{\IRSTLM} does not compute probabilities for cross-sentence $n$-grams, i.e. $n$-grams including the pair {\tt </s>  <s>}.

\IMPORTANT{{\IRSTLM} assumes that each line corresponds to a sentence, regardless the presence of punctuation inside or at the end of the line.}
\IMPORTANT{Start and end symbols ({\tt <s>} and {\tt </s>}) should be considered reserved symbols, and used only as sentence boundaries.}

\subsection{Computation of $n$-gram statistics}
\noindent
You can now collect $n$-gram statistics for your training data (3-gram in this example) by running the command:

\begin{verbatim}
$> ngt -i=train.txt.se -n=3 -o=train.www -b=yes
\end{verbatim}

\noindent
The $n$-grams counts are saved in the binary file "train.www".

\subsection{Estimation of the LM}
\noindent
You can now estimate a $n$-gram LM (3-gram LM in this example) smoothed according to the Linear Witten Bell method by running the command:

\begin{verbatim}
$> tlm -tr=train.www -n=3 -lm=LinearWittenBell -obin=train.blm
\end{verbatim}
\noindent
The estimated LM is saved in the binary file "train.blm".

\subsection{Computation of the Perplexity}
\noindent
With the estimated LM, you can now compute the perplexity of any text contained in "test.txt" by running the commands below.

\noindent
To be compliant with the training data actually used to estimate the LM, start and end symbols are added to the text as well.

\begin{verbatim}
$> cat test.txt | add-start-end.sh > test.txt.se
$> compile-lm  train.blm  --eval=test.txt.se
\end{verbatim}

\noindent
which produces the output:
\begin{verbatim}
%% Nw=1009 PP=8547.90 PPwp=6870.51 Nbo=983 Noov=119 OOV=11.79%
\end{verbatim}

\noindent
The output shows the number of  words ({\tt Nw}), the LM perplexity ({\tt PP}), the portion of PP due to the out-of-vocabulary words ({\tt PPwp}), the amount of backoff calls({\tt Nbo}) required for computing PP, the amount of out-of-vocabulary words ({\tt Noov}), and the out-of-vocabulary rate ({\tt OOV}).


\chapter{Implementation}
In this chapter, we describe POXVine's implementation in detail.

\section{MinSwitchMapper}
The POXVine System can use any host mapper heuristic to map the virtual hosts. For \emph{modularization}, each host mapper takes input the physical Topology and virtual Topology object and stores the mapping in text files for other modules of the system to use. This also allows us to use different host mappers simultaneously for different tenants. \\

MinSwitchMapper minimises the diameter of the network graph connecting the hosts of the virtual topology as seen in Chapter 5. Algorithm 1 is used to find the virtual-to-physical mapping which minimises the diameter. The intuition behind the algorithm is that the switches connected to hosts initially carry the capacity of the host. Every round, we \emph{percolate} the capacity to its neighbours, so, the first round we get a switch with capacity exceeding the required capacity, we can terminate the algorithm. A host list obtained in a \emph{later round} will have greater distance among hosts as we percolate to farther hosts each round, increasing the distance, and thus the diameter of the network graph formed by the virtual hosts. \\

Once the algorithm provides a list of physical hosts with sufficient capacity, we try to map the largest virtual host on the largest physical host, and if we can, repeat the step again till all the virtual hosts are mapped. If such a mapping is not possible, then we reiterate Algorithm 1 to find a different host list to map the virtual hosts.

\IncMargin{1em}
\begin{algorithm}
  \SetKwData{swl}{swlist1}\SetKwData{round}{round}\SetKwData{Up}{up}
  \SetKwData{swll}{swlist2}
  \SetKwFunction{len}{len}\SetKwFunction{append}{append}
  \SetKwFunction{capacity}{capacity}\SetKwFunction{mapHosts}{mapHosts}
  \SetKwFunction{neighbours}{neighbours}
  \SetKwInOut{Input}{input}\SetKwInOut{Output}{output}
  \Input{Physical Topology $P$, Virtual Host List $vhosts$}
  \Output{Virtual-to-physical mappings of each host in $vhost$}
  \emph{Heuristic: Minimise the constructed network graph.}
  \BlankLine
  \emph{Initialization:}\\
  	\swl $\leftarrow$  [] \\
  	\swll $\leftarrow$ [] \\
  	\round $\leftarrow$ 0 \\
  \For{host in P.phosts}{
  	host$\rightarrow$switch$\rightarrow$hostlist = [host] \\
  	\swl.\append(host$\rightarrow$switch) \\
}
	\emph{Start the Percolation Rounds.}\\
  \While{MappingNotFound} {
  		\While{ \len(\swl) $\neq$ 0} {
  				sw = \swl.pop() \\
  				\If{ \capacity( sw$\rightarrow$hostlist ) $\ge$ \capacity(vhosts)} {
  					MappingNotFound = False \\
  					\mapHosts ( sw$\rightarrow$hostlist )
  				}
  				nlist = \neighbours(sw) \\
  				\For{n in nlist} {
  					\emph{Add all the hosts in sw$\rightarrow$hostlist to n$\rightarrow$hostlist} without duplicates.\\
  					If any change in n$\rightarrow$hostlist, add n to \swll
  				}
  		}
  		\swl $\leftarrow$ \swll \\
  		\swll $\leftarrow$ [] \\
  		\round $\leftarrow$ \round + 1
	}
  \caption{Minimum Network Diameter Host Mapping Algorithm}\label{algo_disjdecomp}
\end{algorithm}\DecMargin{1em}

\section{NetworkMapping}
The NetworkMapping class uses the \emph{virtual-to-physical} mappings provided by the host mapper, and calculates all the network routes on the physical topology required for the virtual hosts to talk to each other. Consider the topology in Figure 4.3. The NetworkMapping object will first find the route from $v1$ to $v2$ in the virtual topology, that is $v1 \rightarrow vs1 \rightarrow vs2 \rightarrow v2$.  \\

The Physical Topology object has a method \verb|getCompleteRoute()| which translates the virtual route to the corresponding physical topology route by finding the physical route for each hop in the virtual topology. Thus, from the virtual route, we obtain the following physical route : $v1 \rightarrow s1 \rightarrow s2 \rightarrow s3  \rightarrow vs1 \rightarrow s3 \rightarrow s2  \rightarrow s1 \rightarrow vs2 \rightarrow s1  \rightarrow s2 \rightarrow s3 \rightarrow v2$. \\
\begin{center}
	Pseudo Code for Network Route Generation
\end{center}
\begin{lstlisting}
for each pair of virtual hosts{h1, h2}) :
  virtualRoute = virtualTopology.getRoute(h1, h2)
  physicalRoute =
    physicalTopology.getCompleteRoute(virtualRoute)
  physicalRoute.setRouteTags()
  add physicalRoute to networkPaths
end for
\end{lstlisting}
Note that we need to add rules from $v1$ to $v2$ and the other way as well, $v2$ to $v1$. After generating all the routes for a virtual topology, this is passed to the NetworkMapper module, which translates them into OpenFlow rules to install at the respective switches.

\section{NetworkMapper}
\subsection{Route Tagging}
\begin{figure}
	\noindent
	\makebox[\textwidth]{\includegraphics[width=17cm]{Figures/routetag.png}}%
	\caption{Setting Up Route Tags}
\end{figure}
The NetworkMapper is a POX application. Using the NetworkMapping object, it obtains all the routes it needs to install OpenFlow rules for at the mininet switches. As discussed in the previous chapter, we need \emph{Route Tags} to distinguish between different segments in the network route. The Route class has a function \verb|SetRouteTags()| which differentiates the segments and marks those switches in the route where there is a need to match and increment the route tags (start and end of segments). We illustrate an example in Figure 5.1. \\
The technique of finding these route tags is, in a network, the start and end of a segment will always be a \emph{virtual entity}. Thus, the switch preceeding and succeeding the virtual entity(will be the same in our case) will need to check and increment the Route Tag. This is done by the function \verb|SetRouteTags()|.

\subsection{Proactive Rule Installation}
When a packet reaches a switch which does not have any rule for the packet, a \emph{PacketIn} message is sent to the POX Controller which can install the rule for the corresponding packet at the switch. However, this \emph{reactive} rule installation will have increased latencies for the first flows. POXVine, after receiving the first packet, \emph{proactively} installs all the network rules across all the switches (which will take some time), but any new flow will not be suffer from increased latency due to addition of OpenFlow rules by the controller. However, one problem with \emph{proactive} rule installation is that we need to add rules for communication for all hosts to all other hosts for all tenants, which can impose resource constraints on the switch flow table sizes, thus hindering scalability. With \emph{reactive} rule installation, you add only rules for hosts which are communicating with one other.

\subsection{Discovery}
The POX distribution comes with many useful applications. One of them is \emph{Discovery} module, which helps detect the links in the topology. Using this, the NetworkMapper can create the \emph{port map} for each switch(the mapping of switch port and neighbour). This is then used by to install the required rules (therefore, POXVine should not start the installation of rules before all the link discoveries have been received by the controller.) Another use of the Discovery module is that in case of link failures, the NetworkMapper must set up alternate routes to maintain connectivity.

\subsection{Notes}
\begin{itemize}
	\item ARP packets are tricky to handle (in general, broadcast is in such a system). The virtual hosts do not need a ARP protocol for communication, however, to adhere to legacy protocols, ARP packets are flooded across the network.
	\item To check IP headers at the switch, we need the packet to match this.
	\begin{center}
						\verb|msg.match.dl_type = ethernet.IP_TYPE|
	\end{center}
	\item The \emph{VLAN ID} header field is used to store the tenant ID and RouteTag. The 12=bit VLAN ID has tenant ID as the most significant 6 bits and route tag as the least significant 6 bits.
	\item To send a packet back the input port (in case of loops), the controller needs to add a pecial rule and the outport should be \emph{OFFP\_IN\_PORT}.
\end{itemize}


% !TeX root = ../../main.tex
% !TEX spellcheck = en_GB

\section{Design}
\label{ch:Design}
On \cref{fig:DetailedDesign_prepro} is a detailed description of the preprocessing block.
\begin{figure}
	\centering
	\includegraphics[width=1\linewidth]{gfx/Design/DesignDetailed_Preprocesing.pdf}
	\caption{Detailed description of preprocessing in \systemName.}
	\label{fig:DetailedDesign_prepro}
\end{figure}

Input is a time signal, and the output is a discrete signal. Firstly the time signal will be sampled with \SI{48000}{\hertz}. After the signal is sampled it will enter the decimation part, which consist of a lowpass filter with a cutoff at \SI{1200}{\hertz} and stop at \SI{2400}{\hertz}. The downsampling will be a factor $M=10$. The last phase of the proprocessing is too gather the data in a buffer. The buffer size will be 4069, hence the frequency resolution will be $f_{resolution} = \frac{fs_{new}}{size}= \frac{\SI{4800}{\hertz}}{4096}= \SI{1.17}{\hertz}$. An problem with this method is, the time delay that will be created. The closer the size of the buffer gets to $4800$ the closer the delay will become \SI{1}{\second}. An advantage is that the memory usage will be significantly lower then if there was sampled 48000 samples.\newline The output from the preprocessing block will continue too the signal processing block, here a detailed block can be seen on \cref{fig:DetailedDesign_sigpro}.
\begin{figure}
	\centering
	\includegraphics[width=1\linewidth]{gfx/Design/DesignDetailed_Signalprocessing.pdf}
	\caption{Detailed description of signal processing in \systemName.}
	\label{fig:DetailedDesign_sigpro}
\end{figure}

Input to the signal processing block is the discrete signal which was the output from the preprocessing block. The output from signal processing is, the new changed discrete signal where the frequency has been changed. The input signal will go too the FFT block, here $fs=\SI{4800}{\hertz}$ and $n=4096$. The FFT block will have as output the transformed signal called xfft. xfft will be passed onto the block FindMax which will find the max frequency and the corresponding bin nr. FindMax will then pass on this information to the PitchShift Algorithm which will use this information on the discrete input signal. As output will the new discrete signal with a new frequency. The challenge is the end of each processed block has to match the next block of data. Therefore a future extension would be to make each block overlap so they match eachother. The output from signal processing will be passed on to the postprocessing block which can be seen on \cref{fig:DetailedDesign_postpro}.

\begin{figure}
	\centering
	\includegraphics[width=1\linewidth]{gfx/Design/DesignDetailed_Postprocessing.pdf}
	\caption{Detailed description of postprocessing in \systemName.}
	\label{fig:DetailedDesign_postpro}
\end{figure}

The main function of the post processing block is to upsample the signal so it can be played send out by the DAC at a $fs=\SI{48000}{\hertz}$. The interpolation is an integer of 10, followed up by a lowpass filter. The DAC will send out the final signal which shoulde have a new frequency now compared to the input signal. \fxnote{Move this to an appendix}

\FloatBarrier


% !TeX root = ../solution.tex

\hypertarget{he22.teaser}{%
\chapter{Teaser Challenge}\label{hv22.teaser}}

\begin{figure}
	\includegraphics[width=100mm]{00teaser/banner.jpg}
\end{figure}

\begin{center}
CraCC this, you HAXXOR!

\verb+a4a9fefcfefeb7b8fff8bfa9bee1a8fca2ffedb1+
\end{center}

\section{Solution}\label{he22.01-solution}

HAXXOR $\rightarrow$ XOR encoding

\noindent CraCC hints at 0xCC as the key to XOR with

\begin{minted}{python}
msg = 'a4a9fefcfefeb7b8fff8bfa9bee1a8fca2ffedb1'
res = ''
for i in range(len(msg)//2):
    s = int(msg[2*i:2*i+2],16)
    c = s ^ 0xcc
    res += chr(c)
    print(f'{s} -> {c} %s'%chr(c))
print(res)
\end{minted}

\noindent \verb+he2022{t34ser-d0n3!}+.


\section{Recommendations}
\subsection{Ground-track}
This simulation has not been able to show the ground-track the satellite goes through. As a future improvement this simulation should be able to show the trace that a satellite has on the ground.
\subsection{Earth's rotation}
This simulation has not been able to show the rotation of the earth as the satellite moves around the earth. However, this does not reduce the accuracy of the program in determining the satellites position along it orbit rather it decreases the ability to trace the ground track of the satellite. As a future improvement this simulation should be able to show the earth moving about its axis.

\subsection{Pertubations}
This simulation has not been able to show the effects of perturbations mentioned earlier on the satellite's position along its orbit. This can be done by deriving equations that govern perturbations on the satellite. As a future improvement this simulation should be able to show the effects of perturbations on the satellites motion.

\subsection{Realtime location}
As a future improvement this simulation should be able to locate the subject satellite in realtime. It was not possible for this case because of the recommendations mentioned above. Ideally, one should be able to determine the ground track to successfully for one to locate the satellite correctly.

\subsection{Conclusions}
The objectives of this project were fully met as we were able to simulate the satellite's motion of 1KUNS-PF in 3 dimensions. The accuracy of up to 12 decimal places was satisfactory.

Future improvements can be made to make the program more accurate and include many bodies to the simulation.


\subsection{\acrshort{nrpa}, a story of code}%
\label{sub:nrpa_a_story_of_code}

As seen in the playout (see section~\ref{sub:nrpa_playout}) and the adapt (see section~\ref{sub:nrpa_adapt}), the \gls{nrpa} uses a function named \textit{code}.
This function is essential because it changes what the algorithm learns entirely.
This function takes an action (and can also take a state) and returns a number corresponding to it.
It can be viewed as an hash function for moves.
It helps to store the weight inside the policy that can be seen as a list.
The size of this list will be the number of code possible but that is just one way to see it or implement it.

When we say that it changes the knowledge acquired by the \gls{nrpa} it is undertone.
Let's consider the \gls{lmp} as defined in section~{sub:mdp} and examine two way of coding the actions.

\[
    code(a)=
    \begin{cases}
        1 & \text{if } a = left\\
        0 & \text{otherwise }
    \end{cases}
\]

In this first case, the code is produces from the action itself.
Going to the left is 1 and going to the right is 0.
This means that there is only 2 codes possible; therefore, their are two weights to adjust.
The policy will be binary and only learn to go left or right.
This is very adapted to the problem since the question does not change according to the number of turn already played.

\[
    code(a, s=(r, d)) = d*2 +
    \begin{cases}
        1 & \text{if } a = left\\
        0 & \text{otherwise }
    \end{cases}
\]

In this second case, there are \(2 \times N\) codes and the same number of weight to adjust.
This means that, at each level of deepness, the algorithm will wonder if it is better to go left or right.
If he knows that, at the root level, it is always better to go left, he does not apply it on other levels.
This is a real problem in the modelization of the problem since it does not capture well what the algorithm have to learn.
That said, even with this version, it probably find the best solution but in more time.

The codes are an essential part of the \gls{nrpa} since they modelize the knowledge the algorithm is acquiring.
Bad codes will result in bad learning and bad learning results in bad results.


\section{DistilBERT}
\label{chp:approach:sec:distilbert}

Parameter-heavy deep learning models like \ac{BERT} tend to raise scaling issues the more parameters they include \parencite{Schwartz:2019}.
To tackle this concern, \textcite{Sanh:2019} introduced \ac{DistilBERT}.

% Paragraph architecture and learning
The architecture of \ac{DistilBERT} is similar to that of \ac{BERT}.
However, \ac{DistilBERT} uses only half as many layers.
To obtain a much smaller model the authors use \textit{knowledge distillation} as proposed by \textcites{Bucilua:2006}{Hinton:2015}.
When applying this technique a smaller student model is trained to replicate the behavior of the larger model, the teacher.
The teacher can also be a set of different models.
With this technique the authors successfully trained \ac{DistilBERT} to keep 97\% of \ac{BERT}'s performance.
Compared to \ac{BERT}, \ac{DistilBERT} consists of 40\% less parameters and trains 60\% faster. \parencite{Sanh:2019}

Since \ac{DistilBERT} and \ac{BERT} share the overall same architecture, \ac{DistilBERT} also exposes a [CLS] token which again can be seen as an aggregation of the input sequence.
Similar to our \ac{BERT}-based model, we use this token as input for a fully connected feed forward \ac{NN} with 1,538 trainable parameters.


\chapter{Roll the dice!}
World of Artograch - in spite of being intended more to serve as the base of CRPG adaptations, rather than a tabletop RPG meant to be played among friends - is, at the end of the day, still a tabletop RPG, as such, it is only expected that it will rely on dice-rolls to determine things. Events will be generally detailed in percentages rather than dice rolls by default, but game masters are expected to adapt this to dice rolls: for example, if we state that character has 50\% of succeeding at a task, the game master must ask the character's controller to roll the dice. If it's a 20-sided dice, they must roll 10 or greater, if it's a 6-sided dice, they must roll 3 or greater, and so on. House rules. Nevertheless, for the sake of convenience, sometimes, dice measurements will be also given.


% !TEX root=/home/tavant/these/manuscript/src/manuscript.tex

\section{Electron-induced secondary electron emission}
\label{sec-seemodel}
When an incident electron reaches the wall material, several scenarii are possible, as described in \citet{villemant2018}
\begin{enumerate}
  \item Elastic reflection\string: the electron encounters only elastic collision with the material, hence its energy is constant. However, its reflection is not necessary specular.
  \item Inelastic reflection\string: the electron looses some of its energy to the material before returning to the plasma.
  \item Secondary electron emission\string: the energy of the primary electron is enough to extract one or more electrons from the material.
  \item No emission, the electron is absorbed by the wall.
\end{enumerate}

The probability \proba{}  that one event happens instead of another depends predominantly on the particle energy, and weakly on its  impact angle.
Concerning the mean flux of electron incident and emitted, we use the mean emission rate, or yield, \rate
\begin{equation*} \label{eq-ratedifinition}
  \rate = \frac{\Gamma_{e, \rm secondary}}{\Gamma_{e, \rm primary}}
\end{equation*}
which can be developed using the distribution function to
\begin{equation}
  \label{eq-ratedifinition_evdf}
  \rate = \frac{\iiint_{\Omega} v_x \proba(\vect{v_e}) f(\vect{v_e}) d^3v}{\iiint_{\Omega} v_x  f(\vect{v_e}) d^3v}
\end{equation}
with $\Omega$ the ensemble of $\vect{v_e}$ directed towards the wall\string: $\vect{v_e} \cdot \vect{n} > 0$, with $\vect{n} $ the unit vector normal to and towards the wall.

\subsection{Models of emission } \label{subsec-seemodels}
Several models can be used to describe the electron emission.

\paragraph{Monte Carlo models} are the most realistic.
 They are based on the computation of the trajectory of the electrons through the material, during which the electron can encounter several interactions with the material.
 Each interaction can modify the electron direction, energy, and generate new electrons.
 Several models have been proposed, as \citet{furman2002,pierron2017}.
 These models allow a precise characterization of the processes, but depend on a large number of parameters difficult to obtain due to the lack of experimental data.

\paragraph{Analytical models} provide a simplified description of the rate of emission.
Their complexity depends on the accuracy desired.
The most largely used are the models of \citet{vaughan1989,barral2003a,sydorenko2006b}.

In this work, we are interested only in representing qualitatively the electron emission.
Moreover, \citet{croes2017} showed that changing the model used does not affect significantly the results.
Hence, we will use the model of \citet{barral2003a} for its simplicity.

\subsection{Barral electron emission model}
\label{sec-modelused}

The emission model used follows a linear-saturated law for the probability of emission with three parameters.
It describes the total emission corresponding to the sum of the elastic and inelastic backscattering and the secondary electron emission.
\begin{equation} \label{eq-proba_barral}
  \proba(\ek) =
  \begin{cases}
    \proba_0 + (1 - \proba_0) \frac{\ek}{\crover}   &\text{ if } \ek <  \ek_{\max} \\
    \probamax &\text{ if } \ek \geq \ek_{\max}
  \end{cases}
\end{equation}
where $\ek$ is the kinetic energy of the incoming electron, $\proba_0$ is the asymptotic probability of emission at null energy, $\crover$ is the crossover energy above which the probability of emission is higher than one, \probamax{} is the maximum probability and $\ek_{\max}= \frac{\probamax - \proba_0}{1 - \proba_0} \crover $ is the minimum energy for which $\rate = \probamax$.
\Cref{eq-proba_barral} is illustrated in \Cref{fig-modelbarral}.

\begin{figure}[hbt]
  \centering
  \includegraphics[width=\defaultwidth]{barral}
  \caption{Linear-saturated emission model from \citet{barral2003a}.}
  \label{fig-modelbarral}
\end{figure}

 We suppose that all the electrons are isotropically emitted following a Maxwellian flux distribution function of temperature $\Tsee$.
 The parameters $\proba_0$,  $\probamax$ and $\crover$ can be obtained from experiments.
 \Cref{tab-seeparames} shows the crossover energy and the  probability of emission at energy null for different materials.
 The value of $\proba_0$ is always close to $0.5$, but $\crover$ can vary from 18 to 305 V.
 Hence, in the following parametric studies, $\proba_0$ will be kept at 0.5 while we vary $\crover$ from low values, corresponding to highly emissive materials, to high values, representing less emissive materials.

 \begin{table}[hbt]
   \ra{1.3}
   \centering
   \caption{Emission parameters for different materials, from \citet{barral2003a}.}
   \label{tab-seeparames}
   \begin{tabular}{@{}lll@{}} \toprule
   Material & $\crover$ (V)& $\proba_0$ \\ \midrule
   BN-SiO$_2$ & 53 & 0.45 \\
   Al$_2$O$_3$ & 18  & 0.57 \\
   SiC     &  43  &0.69  \\
   Graphite & 305  & 0.40 \\
   \bottomrule
   \end{tabular}
 \end{table}

 Since the model used describes both the reflected and true secondary electron emission, we will use the general name \emph{electron emission}  to refer to the flux of electrons emitted by the walls towards the plasma.
 In contrast, the electrons reaching the plasma are referred to as the \emph{primary electrons}.


\chapter{De-Entanglement}

\section{How to accomplish de-entanglement?}

The most popular approach to obtain isolation of factors of variation in neural models is by employing stochastic random variables. This approach provides flexibility to jointly train the latent representations as well as the downstream  network. It has been observed that the latent representations resemble disentangled representations under certain conditions \citep{isolating_sources_betavae, understanding_disentanglement_betavae,structured_disentangled_representations,hyperprior_disentanglement}. Note that although obtaining such degenerate representations is considered typical, it is not the only manifestation: it also manifests as continuous representations\citep{ravanelli2018interpretablesyncnet} and other abstract phenomena(e.g. grounding). I argue that explicitly controlling what and how much gets de-entangled \citep{understanding_disentanglement_betavae} is better than implicit disentanglement as is followed today\citep{locatello2018challenging}. I identify four ways to computationally control de-entanglement in encoder decoder models

\begin{itemize}
    \item (1) By employing suitable priors about task or data distribution
    \item (2) By incorporating additional adversarial or multi task objectives within the model
    \item (3) By utilizing a different divergence objective
    \item (4) By employing an alternative formulation of probability density estimation
\end{itemize}

I will  expand on each of these in the following chapters, providing one task from language technologies as a case study.

I posit that designing learning paradigms such that we explicitly control de-entanglement of relevant factors of variation while marginalizing the nuisance factors of variation leads to massive improvements. Such an approach, I claim, leads to further advantages in the context of both generative processes: in terms of generation of novel content and discriminative processes: in terms of robustness of such models to noise and attacks. Let us consider a typical deep learning architecture such as AlexNet\citep{alexnet}. It is characterized by a series of convolutional layers (feature extraction module) followed by a pooling layer and a SoftMax layer(classification module). Note that while I mention AlexNet as an example, this abstraction can be extended to most sequence to sequence architectures with encoder as feature extraction module and decoder as the classification module\citep{tutorial_dataaugmentation} across modalities and tasks. It can be shown that the pooling layer acts as information bottleneck\citep{tishby2000information} module in such architectures. I point out\citep{variational_attention_rsk} that in case of conventional Seq2Seq architectures deployed today, attention plays the role of information bottleneck module regulating the amount of information being utilized by the decoder. In \citep{variational_attention_rsk,vyas2019learning} I show that this module controls optimization in encoder decoder models leading to (1) Disentanglement of Causal Factors of variation in the data distribution (2) Marginalization of nuisance factors of variation from the input distribution. In case of models that employ stochasticity, two more effects can be observed : (a) Posterior collapse or Degeneration due to powerful decoders and (b) Loss of output fidelity due to finite capacity decoders. In current architectures, marginalization and disentanglement are realized implicitly and often lead to (a) and (b) when deployed in practise.

The most popular approach to obtain isolation of factors of variation in neural models is by employing stochastic random variables. This approach provides flexibility to jointly train the latent representations as well as the downstream  network. It has been observed that the latent representations resemble disentangled representations under certain conditions \citep{isolating_sources_betavae, understanding_disentanglement_betavae,structured_disentangled_representations,hyperprior_disentanglement}. Note that although obtaining such degenerate representations is considered typical, it is not the only manifestation: it also manifests as continuous representations\citep{ravanelli2018interpretablesyncnet} and other abstract phenomena(e.g. grounding). I argue that explicitly controlling what and how much gets de-entangled \citep{understanding_disentanglement_betavae} is better than implicit disentanglement as is followed today\citep{locatello2018challenging}. I identify four ways to computationally control de-entanglement in encoder decoder models

\begin{itemize}
    \item (1) By employing suitable priors about task or data distribution
    \item (2) By incorporating additional adversarial or multi task objectives within the model
    \item (3) By utilizing a different divergence objective
    \item (4) By employing an alternative formulation of probability density estimation
\end{itemize}

\subsection{Case for Controlled De-Entanglement}

I believe that complete disentanglement of input data into its independent causal factors of variation is not fully useful. A more attractive option is to control what and how much gets de-entangled in a a task dependent manner. It has to be noted that given a particular downstream task, some causal factors of variation might not be relevant, in which case modeling them would be unnecessary. Let us consider a data distribution X which consists of class examples $\{ x_1, x_2...,x_n \}$, where each $x_i$ is described by attribute-set $(a,b,c)$. The prior distribution of X can be represented by a parameteric function g such that g maximizes the likelihood of X over the set of its attributes:

\begin{equation}
P_{\omega}(X) = g_{\omega}(a,b,c)
\end{equation}

Note that the attribute-set can either contain individual entities or the relationships between them or both.  To illustrate this, let us consider a toy-example where we build a binary classifier to predict if a given integer triplet is a Pythagorean triplet. Pythagorean triplets are a triplet of numbers that follow Pythagoras Theorem such as $\{3,4,5\}$ and $\{5,12,13 \}$. In this task, the attribute-set consists of the relationship between the first two-elements of the triplet. If the model is able to discover this attribute, it can generalize for any given numbers. %Another example is to build a generative model for predicting the next number in the Fibonacci series. The attribute-set consists of the relation between the last two elements on the input list.
However, if we have a more complicated task like building a classifier for MNIST digits, then the attribute-set has multiple first and second order relations like brush strokes, shape of the digits etc. The success of modelling $P_{\omega}(X)$, and ultimately the success on the downstream task, relies on how well can the model isolate these individual attributes from the observed data $X_{t}$. This isolation ability becomes even more important in case we want to regenerate the digits using a generative model. Mathematically, let us consider the posterior probability of a training instance $x_1$ expressed as

\begin{equation}
P_{\theta}(x_1) = f_{\theta}(x_1)
\end{equation}

where \textit{f} denotes arbitrary function and $\theta$ denotes the parametric family used to model the distribution $X$. It can be seen that compositionality over an unseen training instant $x_{new}$ would be possible if \textit{f} is related to \textit{g}. In other words, \textit{f} needs to intuitively have some information about the latent causal factors of variation that generated $X$ in the first place. In such scenarios, the test instance can be appropriately expressed as

\begin{equation}
P_{\theta}(x_{new}) = h(a,k(b,c))
\end{equation}

where h and k can be a novel combination of functions that embed these attributes in the manifold of original distribution of $X$. Not tracking the relevant factors of variation typically leads to model memorizing only the surface level associations leading to mode collapse and lack of diversity in the generated outputs. On the other hand, explicitly caring about the factors of variation can be seen as a way of incorporating inductive bias into the model and has the potential to avoid such pitfalls.

\iffalse
\subsection{Inductive Bias}
Inductive bias typically refers to the set of constraints applied on the model that help restrict the hypothesis space apriori based on the background knowledge about the distribution of data. Having such priors helps the model reach the solution faster. For instance, consider the prior of linearity imposed on a curve fittign task between two points in cartesian co ordinates. Having such prior extremely simplifies the task since there is only one line through two points while there are infinitely many curves. Not having any sort of priors about the data distribution might also lead to uninterpretible and black box results even if the solution is infact correct. While this might be okay in the beginning, to make the technology closer to acceptance, we need models that are interpretible atleast conditionally. Not having any sort of inductive biases might lead to model memorizing or just learning surface level associations. This leads to the model becoming vulnerable to adversarial attacks. Voice impersonation attacks for example can be very serious. Additionally, having such priors in place also acts as a probe to let us diagnose the performance of the model when it fails. This we believe is a more principled way towards building and debugging reliable systems as opposed to post hoc fixes based on the errors made by such models. Community today is two paced: either based on end to end neural approaches where the aim is to discover everything through data or based on hand crafted features. We believe that a far more interesting approach is to take a leaf out of research from epigenetics that shows biology builds both by nature as well as nurture and not absolute. Specifically, we believe going forward it is important to design techniques that have quirks from the nature - prior knowledge incorporated into the nurture - end to end approaches. In this paper, we perform a case study on a manifestation of speech generation, voice conversion, and present approaches that enable us to incorporate inductive biases into the model.
\fi

\begin{center}
\begin{table}
\caption{Implicit Realization of information bottleneck in popular deep learning mechanisms}
\begin{tabular}{  | c | c | c| }
\hline
\textbf{Architecture} & \textbf{Manifestation of Information Bottleneck} & \textbf{Type of Bottleneck} \\
\hline
AlexNet & Pooling  & Spatial \\
 \hline
Attention & Activation & Temporal\\
\hline
VAE & Priors & Spatio temporal\\
\hline
Neural Module Networks & Softmax over Modules & Spatial \\
\hline
LISA & Linguistic Priors & Temporal\\
\hline
BERT & Masking  & Spatio temporal\\
\hline
XLNet & Permutation & Spatio temporal \\
 \hline
\end{tabular}

\end{table}
\label{Message}
\end{center}

\subsection{Implicit De-Entanglement in Seq2Seq Models: Deterministic Attention vs Stochastic Attention}

I will illustrate this sub section with a typical generative model of speech: Text to Speech. Consider that we are interested in building a code mixed version: a model that can accomodate two languages in a single utterance. Let us also consider a speech corpus $X$ consisting of languages $\{ l_1,...,l_n \}$, where each $l_i$ might comprise of multiple speakers.  Let \textit{$y_1$,...,$y_n$} denote acoustic frames in the target sequence \textit{y}  while \textit{$x_1$,...,$x_n$} denote the encoded text sequence \textit{x} from one of the languages. A typical attention based encoder decoder network such as Tacotron\citep{tacotron_original} factorizes the joint probability of acoustic frames as product of conditional probabilities. Mathematically, this can be shown as below:

\begin{equation}
P_\theta(y|x) = \Pi_{ t=1}^{t=n} P(y_t | x_1...x_m,s_t)
\end{equation}
where $s_t$ is a decoder state summarizing $y_1$,...$y_{t-1}$. Parameters $\theta$ of the model are set by maximizing either the log likelihood of training examples or the divergence between predicted and true target distributions. At each time step t in these models, an attention variable $a_t$ is used to denote which encoded  state of \textbf{$x_1...x_m$} aligns with \textit{$y_t$}. The most common form of attention used is soft attention, a convex combination from encoded representation of input text. It has to be noted that soft attention in such scenarios is essentially a latent deterministic variable that computes an expectation over the alignment between input and output sequences. Empirically, soft attention provides surprisingly good alignment often  correlating with human intuitions. Having said that, to synthesize speech from different languages at test time, the generative process needs to disentangle appropriate individual language attributes from observed data $X_{obs}$ and also compose them to form a coherent utterance in the voice of desired speaker. However, presence of deterministic alignment method limits the ability of models to generalize to such scenario.

%Incorporating latent alignment into Seq2Seq models by injecting stochastic latent variables allows to efficiently marginalize the likelihood of data distribution.

On the other hand, variational attention\citep{latentalignment_variationalattention} provides a mechanism to factorize this alignment and mediate the generative process of $y$ through a stochastic variable $z$. In addition, both soft and hard attention mechanisms can be shown as special cases of ELBO\citep{latentalignment_variationalattention}. Therefore, incorporating latent stochastic variables allows us to directly optimize ELBO. In this context, model parameters are set by maximizing the log marginal likelihood of the training samples. But direct maximization of this marginal in the presence of latent variable  is often difficult due to expectation involved. To address this, a recognition network $q$ is employed to approximate the posterior probability using reparameterization. It is interesting to note that the encoder in a deterministic Seq2Seq network functions as the recognition network in latent stochastic variable models and is incentivized to search over variational distributions to improve ELBO. Intuitively, the lower bound is tight when the inferred variational distribution is closer to the true posterior of the data. This has a sense of grounding in our understanding of the task as well. Perhaps there are a set of universal phonemes, around 120, which should be enable us to speak in any language subject to the phonotactic constraints of the language. Having such prior information greatly reduces the model size as opposed to naively using a combination of all phones from all the languages to build a polyglot model.

\subsection{Analysis of role of priors in Latent Stochastic Models}
\label{analysis}

The choice of priors plays a significant role in optimization within latent stochastic models. In this subsection, we present an analysis to show that priors control the disentanglement of causal factors of variation in such models. Let us consider the ELBO being optimized in a VAE:

\begin{equation}
E_{q_{\phi}(z|x,c)}[log p_{\theta} (x|c,z)] -  |D_{KL}(q_{\phi} (z|x,c) || p_{\theta} (z|c))|
\end{equation}

where the first term is the reconstruction error while the second is the divergence between approximate and true posteriors. Here are the four phenomenon that are manifested due to choices of priors:

(1) \textit{Disentanglement or Factorization of causal factors of variation} \\
The KL divergence forces the posterior distribution output by encoder to follow an appropriate prior about the data generation process. Typically, prior space is assumed to be continuous distribution and a unit Gaussian. The global optimum value for the divergence in such cases is 0 and is reached only when both the distributions exactly match each other. Since the prior information about the data generation process typically involves some causal factors of variation of the data, this naturally is assumed to translate to a constraint on the encoder to track such factors. Thus, such models have potential to disentangle or factorize the causal factors of variation in the distribution.

(2) \textit{Marginalization of Nuisance Factors of Variation} \\
It has to be noted that during training optimization is performed in expectation over minibatches. Therefore, the expectation of KL divergence can be rewritten as related to the amount of mutual information between the latent representation and the data distribution \citep{pixelgan_autoencoder}. As this divergence decreases, the amount of information the encoder can place in the latent space also decreases. As a result, encoder is forced to discard some nuisance factors that may not have contributed to the generation of data. Thus, KL divergence also forces the model to marginalize the nuisance variables.

%Thus, the KL divergence forces the encoder network to disentangle the causal factors - akin to the basis vectors - of variation in the data and ignore the nuisance factors that may have contributed to the generation of data.

(3) \textit{Posterior Collapse due to simple priors} \\
Consider the scenario where the prior is too simplistic, such as the aforementioned unit normal distribution. In such cases, the model is incentivized to force the posterior distribution to closely follow the Gaussian distribution \citep{lossy_vae}. Typically the decoders in variational models are implemented using universal approximators such as RNNs. In the context of a TTS systems, decoder segment of the acoustic model along with the neural vocoder act as the decoders. Since such decoders are very powerful, they are able to learn or ignore the priors about data distribution themselves and hence marginalize out the latent representation input from the encoder. In other words, the prediction of next sample is based solely on the marginal distribution at the current timestep which can be implemented by learning a dictionary per time step. Therefore, the encoder is no longer forced to track the causal factors of variation in the data. This is referred to as posterior collapse or mode collapse.

(4) \textit{Loss of output fidelity due to complex priors} \\
A reasonable and intuitive solution to posterior collapse is making the prior space more complex thereby pressurizing the posterior distribution to track the prior space more closely. For instance, \citep{beta_analysis} attempt to accomplish this by adding a hyperparameter $\beta$ to promote disentanglement and gradually increasing channel capacity, something that increases loss. However, it has to be noted that simply making the prior distribution arbitrarily complex also perhaps leads to unreasonable constraints on the decoder. For instance, in scenarios that have categorical distribution as their output (tasks such as language modeling, machine translation, image captioning among others) it is unintuitive to assume that the true prior that generates latent distribution is a Gaussian when the likelihood is based on discrete sequential data in such tasks. Having such strong priors directly affects the reconstruction ability in these models.

Therefore, priors in latent stochastic models play a significant role in the optimization and facilitate disentanglement of causal factors of variation on the one hand, as well as help the ability of the model to reconstruct the data distribution on the other. Having this knowledge enables us to engineer various components to tune the model behavior as per our requirements.


\section{sub3 with spaces}\label{sec:sub3 with spaces}

\begin{equation}
  f(x) = 42
  \label{eq:inputted sub with spaces is working}
\end{equation}


\subsubsection*{Filter Functions\hspace*{\fill}\hyperlink{SignalProcessing}{(up)}\hypertarget{filterFunctions}{}} \addcontentsline{toc}{subsubsection}{Filter Functions}
Finite Impulse Response (FIR) functions defined in C VSIPL are included in \pyjv.
\begin{table}[H]
\caption{Filter Functions}
\label{tab:filterFunctions}
\begin{center}
\begin{tabular}{|l|l|}
\multicolumn{2}{c}{\hyperlink{firFunc}{\rmfamily \bfseries Finite Impulse Response Filter Class}} \\ \hline
fir\_create & Create FIR Object\\
fir\_destroy & Free FIR Object\\
firflt & Filter Data\\
fir\_getattr & Get attributes of FIR Object\\
fir\_reset & Reset FIR Object to just created state\\ \hline
\end{tabular}
\end{center}
%\label{default}
\end{table}%


\documentclass[a4paper,12pt]{amsart}
\usepackage[utf8]{inputenc}\usepackage[T1]{fontenc}

\title[Multi-Agent Deep Deterministic Policy Gradients]{The Tennis environment solved using a Deep Deterministic Policy Gradient algorithm}
\author{Julian Wergieluk}\address{}\email{julian.wergieluk@risklab.com}
\input{preamble}
\input{commands}

\usepackage[url=false,backend=biber]{biblatex}
\addbibresource{rl.bib}
\addbibresource{ddpg.bib}

\newcommand{\stateSpace}{\mathbb S}
\newcommand{\stateSpaceAlg}{\mathcal S}
\newcommand{\coffinSpace}{\Delta}
\newcommand{\actionSpace}{\mathbb A}
\newcommand{\actionSpaceAlg}{\mathcal A}
\newcommand{\stateValueFunc}{V}
\newcommand{\actionValueFunc}{Q}
\newcommand{\advantageFunc}{A}
\newcommand{\policy}{\pi}
\newcommand{\policyLik}{f}
\newcommand{\discountFactor}{\gamma}
\newcommand{\prob}{\mathbb P}
\newcommand{\rewardFunc}{\phi}
\newcommand{\trajectory}{\tau}
\newcommand{\trajectorySpace}{\mathbb T}
\newcommand{\startStateDist}{\rho_0}

\begin{document}

\maketitle

\begin{abstract}
This short note provides a concise description of the model architecture and
learning algorithms of the agent developed in this project. We also report learning
performance of the agent and provide a list of possible future model improvements.
\end{abstract}

\section{Description of the learning algorithm}

For this problem, we use a standard Deep Deterministic Policy Gradient
algorithm (DDPG) and parameterize it in a way to make it suitable for multi-agent
environments.

Deep deterministic policy gradient algorithm is an actor-critic type method
with a deterministic actor policy $\pi$ mapping states from $\stateSpace$ to
actions in $\actionSpace$. During the training, the action-value
function $Q$ is optimized using a standard DQN algorithm as described in
\cite{mnih2015humanlevel} employing a replay buffer and a separate target
network.

The policy network is trained using a policy gradient derived from the action-value
function approximation. Specifically, the parameter vector $\theta$ is given as
$\theta = (\theta^{Q}, \theta^{\pi})$, where $\theta^{Q}$ determines $Q$ and
$\theta^{\pi}$ determines $\pi$. Let $J(\theta)$ be the cumulative expected
reward. Then the policy gradient can be approximated by
\begin{align*}
    \nabla_{\theta^{\pi}} J & \approx
    \E \left[ \nabla_{a} Q(S_t, \pi(S_t)) \nabla_{\theta^\pi} \pi(S_t) \right].
\end{align*}

To improve the algorithm stability we use soft target updates for the
parameters of both actor and critic networks, described in
\cite{mnih2015humanlevel}, and batch normalization.

\section{Training analysis}

We train a DDPG agent for 1900 episodes consisting of at most 1000 time steps
each. The cumulative reward averaged over 100 episodes reaches the level of
0.5017 after completing the episode 494.

\begin{figure}[tb]
    \centering
    \includegraphics[width=\textwidth]{{scores}.pdf}
    \caption{We train a DDPG agent for 1900 episodes. The ``reward'' curve gives
        the maximum of individual agents rewards at the end of each episode. The
        ``average reward'' curve is the rolling mean of the ``reward'' curve
        calculated over 100 episodes. (For episodes $1,\cdots, 99$ we use an
        expanding window.)}
    \label{fig:scores}
\end{figure}

\section{Algorithm parametrization}

\paragraph{The Actor network architecture.} The actor is a feed-forward neural
network mapping the state of the environment to a deterministic action. The
network consists of three layers sandwiched with batch normalization. The final
layer uses the tanh nonlinearity to produce an action vector in the cube $[-1, 1]^{2}$.

\verbatiminput{actor-net.txt}

\paragraph{The Critic network architecture.} The critic is a feed-forward neural
network approximating the state-action value function $Q$ mapping state-action
pairs to the expected cumulative reward. The critic network consists of two subnetworks:
a one-layer network combined with batch normalization that maps the state to a high-dimensional
state representation, and a two-layer network that maps the aforementioned representation
of the state concatenated with the action vector to the approximation of the state-action
value $Q(s,a)$.

\verbatiminput{critic-net.txt}

The list of hyperparameters used by the agent is listed in Table \ref{tab:hyperparameters}.

\begin{table}
%\centering
\caption{List of hyperparameters and their values}
\begin{tabular}{|l|l|l|l|}
    \hline
Hyperparameter & Variable name & Value \\
    \hline \hline
    Replay buffer size & \texttt{BUFFER\_SIZE} &  2e5 \\ \hline
    Batch size & \texttt{BATCH\_SIZE} &  128  \\  \hline
    Discount factor ($\gamma$) & \texttt{GAMMA} &  0.99 \\ \hline
    Tau & \texttt{TAU} & 1e-3 \\ \hline
    Actor learning rate & \texttt{LR\_ACTOR} & 0.001 \\ \hline
    Critic learning rate & \texttt{LR\_CRITIC} & 0.001 \\ \hline
    Initial value of the & \texttt{EPSILON} & 1.0 \\
    OU process scaling factor $\varepsilon$ && \\ \hline
    OU mean reversion level & & 0.0 \\ \hline
    OU mean reversion speed & & 0.15 \\ \hline
    OU volatility (sigma) & & 0.2 \\ \hline
    Learning frequency & \texttt{} &  every 5 time steps \\ \hline
\end{tabular}
\label{tab:hyperparameters}
\end{table}

\section{Ideas for future work}

\begin{itemize}
    \item Use of a real multi-agent reinforcement learning algorithm explicitly taking the
        non-stationarity of the environment into the account.
    \item Actor and critic could use a common network for state processing.
\end{itemize}

\nocite{lillicrap2015continuous}
\printbibliography

\end{document}

% vim: spelllang=en_us:spell:


\section{Ridge regression}


\subsection*{Array Support}

The following array processing functions are supported:

\begin{itemize}
\item \lstinline{array_length(x)}: \textit{primitive}, returns
  the current length of array \lstinline{x}, which is 1 plus the
highest index \lstinline{i} that has been used so far in an array assignment on \lstinline{x}.
\item \lstinline{is_array(x)}: \textit{primitive}, returns
  returns \texttt{true} if \texttt{x} is an array, and \texttt{false} if it is not.
\end{itemize}


\section{Implementation Optimization} \label{sec:Implementation}

This section describes optimization in our implementations of VGLCS
algorithms, both in sequential and parallel environments.  Note that
some of these techniques address hardware characteristics, e.g.,
memory cache behavior, and are not confined to asymptotically analysis
only.

\subsection{The Strategy of Disjoint Set Implementation}

We now describe the optimization in the implementations of disjoint
sets, whose applications include VGLCS.

%% There two techniques to efficiently merge two disjoint sets -- {\em
%%   path compression}, and {\em rank strategy}.

\subsubsection{Cache Performance}

Memory cache is essential to efficient implementation of disjoint
sets.  Patwary, Blair, and Manne~\cite{Patwary2010ExperimentsOU}
conducted experiments on disjoint-set and showed that different
implementations have different impact on different levels of caches
misses.  In practice, the cache miss is strongly related to how we go
from a child to its ancestors through pointer chasing during path
compression.  Usually, an algorithm with a lower time complexity,
e.g., will have more ``long jumps'' than an algorithm with a higher
time complexity.  Here the long jump means the pointer chasing will go
from a memory address to a far away memory address.  Please refer to
Figure~\ref{fig:long-short-jump-disjoint} for an illustration.

\begin{figure}[!thb]
  \centering \subfigure[An algorithm with a lower time complexity] {
    \includegraphics[width=0.42\linewidth]{\GraphicPath/fig-rem-long-jump.pdf}
    \label{fig:long-short-jump-disjoint-long}
  } \subfigure[An algorithm with a higher time complexity] {
    \includegraphics[width=0.42\linewidth]{\GraphicPath/fig-rem-short-jump.pdf}
    \label{fig:long-short-jump-disjoint-short}
  }
  \caption{The parent jump in disjoint set}
  \label{fig:long-short-jump-disjoint}
\end{figure}

The Rem's algorithm~\cite{dijkstra1976a} achieves better cache
performance by a {\em merge-by-index} technique.  Traditional disjoint
set merging techniques are {\em merge-by-rank} and {\em
  merge-by-size}~\cite{Tarjan1975EfficiencyOA}, which determine the
root of new tree by the ranks and the sizes of the two trees
respectively.  Despite that they do have asymptotically better time
complexity, their performance in practice is not so impressive due to
the previously mentioned cache issue.  In contrast, Rem's algorithm
assigns an {\em index} to each node, and merges two disjoint sets
according the index of the roots of the two trees being merged.  That
is, the root of the new merged tree will be the root with the
``larger'' index.

In our experiments, we still use merge-by-rank to merge disjoint set
trees.  However, when we have two {\em equally ranked} disjoint set
trees to merge, we will apply the {\em merge-by-index} technique to
break the tie, so as to improve cache performance.
Figure~\ref{fig:fig-perf-cache-miss-rem} compares the cache miss rate
of merge-by-index tie-breaking and the usual random tie-breaking
approach in both merge-by-rank and merge-by weight, and under
different application contexts.  The experiments show that the
merge-by-index tie-breaking technique in average reduces cache miss
rate by 3\% of the usual random tie-breaking approach.

\begin{figure}
  \centering
  \includegraphics[width=0.95\linewidth]{\GraphicPath/fig-perf-cache-miss-rem.pdf}
  \caption{The cache miss rate of the {\em merge-by-index
      tie-breaking} for different applications and merge techniques in
    disjoint set data structure on an E5-2620 server.  The statistics
    are measured by the {\tt perf} analyzing tool in Linux.}
  \label{fig:fig-perf-cache-miss-rem}
\end{figure}

\subsubsection{Application on VGLCS}

% VGLCS

It is possible to further optimize the implementation of Peng's
sequential algorithm (Algorithm~\ref{alg:serial-VGLCS}) by a {\em lazy
  insertion} technique.  Recall that in
Algorithm~\ref{alg:serial-VGLCS}, when the $i$-th character from the
first input string does not match the $j$-th character of the second
input string, we need to place a zero into the dynamic programming
table at position $V[i][j]$.  In practice, this mismatch will happen
{\em very frequently}, and will cause frequent insertions of zeros
into the disjoint sets.  Each insertion then links the newly inserted
singleton node of zero to the node of zero inserted one iteration ago.

We implemented an optimization that resolves this repeated insertion,
and linking, of zeros.  Our implementation will scan through a series of
zeros, and locate the next non-zero $V$ element (denoted as $v$) in the
same column, and insert them as a batch.  Since all values in $V$ are
non-negative, we can link all zeros we have seen {\em directly} to $v$.
From the experiments, we observe that this technique causes less pointer
chasing and updates and does improve performance.

Note that we do {\em not} apply the lazy insertion optimization in our
{\em parallel} implementation.  For a multi-core platform, the
efficiency of thread synchronization is essential to the performance.
Since the threads of a parallel VGLCS algorithm needs to synchronize at
every column of the dynamic programming table in order to process a row,
the lazy insertion, which can ignore this dependency in a sequential
environment, is not beneficial, and is not adopted in our parallel
implementation.

\subsection{Parallel Range Query in VGLCS}

We further improve the performance of range query by maintaining two
extra tables in the blocked sparse table approach described in
Section~\ref{sec:blocked-sparse-table}.  There are now three tables -- a
sparse table on the block maximum $T_S$, a {\em prefix maximum table}
$P$, and a {\em suffix maximum table} $S$.  As described in
Section~\ref{sec:blocked-sparse-table}, the sparse table $T_S$ can
easily answer range query of consecutive blocks on the input $A$.  The
prefix maximum table $P$ maintains the maximum of the prefix {\em
within} a block, and similarly the suffix maximum table $S$ maintains
the maximum for prefix.  Please refer to
Figure~\ref{fig:compressed-sp-opt} for an illustration.

It is easy to see that any range query can be answered by {\em two}
queries into sparse table $T_S$, {\em one} query into prefix maximum
table $P$, and {\em one} into suffix maximum table $S$.  For example, in
Figure~\ref{fig:compressed-sp-opt}, the query from index 2 to 18 can be
answered by two queries into the sparse table $T_S$ -- one from block 1
to block 2, and one from 2 to 3, a query for the suffix of length 2 into
the first block of $S$, and a query for the prefix of length 3 into the
last block of $P$.  Please refer to Figure~\ref{fig:compressed-sp-opt}
for an illustration.

%% The in-block query is a very small probability event due to small $s =
%% \frac{\log n}{4}$.

%% We can use prefix and suffix maximum array to reduce the number of
%% lookup operation on LCA table.

\begin{figure}[!thb]
  \centering \subfigure[The prefix/suffix maximum tables for blocks] {
    \includegraphics[width=0.55\linewidth]{\GraphicPath/fig-compressed-sp-prefix-suffix.pdf}
    \label{fig:compressed-sp-opt-all}
  } \subfigure[A Sparse Table for $T_S$] {
    \includegraphics[width=0.35\linewidth]{\GraphicPath/fig-sparse-table-opt.pdf}
    \label{fig:compressed-sp-opt-ts}
  } \caption{Block maximum $T_S$, prefix maximum $P$, and suffix maximum $S$.}
  \label{fig:compressed-sp-opt}
\end{figure}

We argue that the {\em order} to access these maximum tables is
important.  Our implementation accesses the block maximum $T_S$ {\em
  first}, then the prefix maximum $P$ {\em second}, then the suffix
maximum $S$ {\em last}.  The reasoning for this order is as follows.
Since we need to access two elements in the sparse table $T_S$ in {\em
  the same level}, it is very likely that they will be in the same
cache line, so access the first will bring in the other automatically
by the hardware caching mechanism.  For example, in
Figure~\ref{fig:compressed-sp-opt-ts}, we will access both
$T_{S}[1][2]$ and $T_{S}[1][3]$ in order to find the maximum from
block 1 to 3.

Our implementation improves performance by ``peeking'' into two
neighboring elements in the $T_{S}$ table.  To be more specific, we
peek the element {\em before} the first element, and the one {\em
  after} the second element, which we just accessed from $T_S$.

For example, in Figure~\ref{fig:compressed-sp-opt-ts}, we will peek
into $T_{S}[1][1]$ and $T_{S}[1][4]$, which are very likely also
present in cache because they are in the same level of the sparse
table as the previously accessed elements $T_{S}[1][2]$ and
$T_{S}[1][3]$.  The idea is that if the maximum from the block
maximum, i.e., the maximum of $T_{S}[1][2]$ and $T_{S}[1][3]$, is
already {\em larger} than the overall block maximum where the prefix
belongs, i.e., in $T_{S}[1][4]$, then we do {\em not} need to check
the $P$ table.  Similarly, if the maximum from the block maximum {\em
  and} the prefix maximum is larger than the block maximum where the
suffix belongs, i.e., in $T_{S}[1][1]$, then we do not need to check
the $S$ table either.  This peeking is efficient since the data being
peeked are most likely in the same cache line, and can save
unnecessary access to $P$ and $S$ tables.

Algorithm~\ref{alg:rmq-access-order-2e} shows the access order for the
range maximum query to reduce the cache miss.  Note that we check the
prefix maximum $P$ {\em before} the suffix maximum $S$ since the value
of the dynamic programming table in the same row is {\em increasing},
so it is more likely that a prefix maximum, which has a larger column
index, can save a check into the suffix table, which has a smaller row
index.

Our experiments show that despite that these maximum arrays require
additional $O(n)$ memory, the peeking technique improves overall
performance by up to 8\% when $n = 10^4$, and it improves the overall
performance of query operations by up to 35\% when $n = 10^7$.  Please
refer to Figure~\ref{fig:fig-perf-cache-miss-peek} for an
illustration.

\begin{figure}
  \centering
  \includegraphics[width=0.95\linewidth]{\GraphicPath/fig-perf-cache-miss-peek.pdf}
  \caption{The cache miss rates of the ``peeking'' for different
    applications and environments in blocked sparse table on an
    E5-2620 server.  The cache miss rate is measured by the {\tt perf}
    analyzing tool in Linux.}
  \label{fig:fig-perf-cache-miss-peek}
\end{figure}

% One can consider the
% first level of the compressed sparse table $T_S[0]$ as a small cache,
% which may prevent access to the prefix/suffix maximum array, as
% described earlier.

\input{\AlgoPath/alg-rmq-access-order-2e}


    \filetitle{group}{Retrieve VAR object from panel VAR for specified group of data}{VAR/group}

	\paragraph{Syntax}\label{syntax}

\begin{verbatim}
V = group(V,Grp)
\end{verbatim}

\paragraph{Input arguments}\label{input-arguments}

\begin{itemize}
\item
  \texttt{V} {[} VAR {]} - Panel VAR object estimated on multiple groups
  of data.
\item
  \texttt{Grp} {[} char {]} - Requested group name; must be one of the
  names specified when the panel VAR object was constructed using the
  function \href{VAR/VAR}{\texttt{VAR}}.
\end{itemize}

\paragraph{Output arguments}\label{output-arguments}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{V} {[} VAR {]} - VAR object for the \texttt{K}-th group of
  data.
\end{itemize}

\paragraph{Description}\label{description}

\paragraph{Example}\label{example}

Create and estimate a panel VAR for three variables, \texttt{x},
\texttt{y}, \texttt{z}, and three countries, \texttt{US}, \texttt{EU},
\texttt{JA}. Then, retrieve a plain VAR for an individual country.

\begin{verbatim}
v = VAR({'x','y','z'},{'US','EU','JA'});
v = estimate(v,d,range,'fixedEffect=',true);
vi_us = group(v,'US');
\end{verbatim}