\documentclass[dvisvgm,hypertex,aspectratio=169]{beamer}
\usefonttheme{serif}

%\usepackage[utf8]{inputenc}
%\usepackage[T1]{fontenc}

%\usepackage[draft]{animate}
\usepackage[final]{animate}
\usepackage{ifthen}

%\usepackage{pythontex} % <--
\usepackage{graphicx}

\usepackage{tikz}
\usepackage{pgfplots}
\usepackage{pgfplotstable}
\pgfplotsset{compat=1.16}
\usetikzlibrary{calc}
\usetikzlibrary{decorations.pathmorphing,patterns}
\usepackage{amsmath}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Define footer
\usepackage{ccicons}

\makeatletter
\setbeamertemplate{footline}
{
  \leavevmode%
  \hbox{%
  %\begin{beamercolorbox}[wd=.333333\paperwidth,ht=2.25ex,dp=1ex,center]{title in head/foot}%
    %\usebeamerfont{title in head/foot}\insertsubsection
  %\end{beamercolorbox}%
  %\begin{beamercolorbox}[wd=.333333\paperwidth,ht=2.25ex,dp=1ex,right]{date in head/foot}%
  %  \usebeamerfont{date in head/foot}\insertshortdate{}\hspace*{2em}
  %  \insertframenumber{} / \inserttotalframenumber\hspace*{2ex}
  %\end{beamercolorbox}}%
  %\vskip0pt%
  \begin{beamercolorbox}[wd=.92\paperwidth,ht=2.25ex,dp=1ex,right]{author in head/foot}%
    \usebeamerfont{author in head/foot}\insertauthor
  \end{beamercolorbox}%
  \begin{beamercolorbox}[wd=.08\paperwidth,ht=2.25ex,dp=1ex,right]{date in head/foot}%
    \ccbysa
  \end{beamercolorbox}}%
  \vskip0pt%
}
\makeatother
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\author{\href{mailto:kjartan@tec.mx}{kjartan@tec.mx}}

\begin{document}

\section{Create animation}

  \def\tend{30}
  \def\nframes{24}
  \def\nsamples{200}
  \def\wexpstart{-2}
  \def\wexpend{1}
  \def\ttau{1}

\begin{frame}[label=A]{First-order Bode plot}
      \begin{center}
        \begin{tikzpicture}[scale=0.7, transform shape,]
          \tiny
          \node[draw, minimum width=14mm, minimum height=12mm] (sys) {$G(s)=\frac{1}{s\tau + 1}$};
          \node[coordinate, left of=sys, node distance=46mm] (input) {};
          \node[coordinate, right of=sys, node distance=46mm] (output) {};
          \draw[->] (input) -- node[ above] { $\textcolor{blue!80!green}{u(t)=\sin(\omega_1 t)}$} (sys);
          \draw[->] (sys) -- node[near end, above] { $\textcolor{green!80!black}{y(t)=|G(i\omega_1)|\sin\big(\omega_1 t + \arg G(i\omega_1)\big)}$} (output);
        \end{tikzpicture}
      \end{center}
      \begin{center}
        \begin{animateinline}[controls, loop, palindrome]{4}
          \multiframe{\nframes}{n=0+1}{

            \pgfmathsetmacro{\wexp}{\wexpstart + (\wexpend-\wexpstart)/\nframes*\n}
            \pgfmathsetmacro{\ww}{pow(10, \wexp)}
            \pgfmathsetmacro{\wstart}{pow(10, \wexpstart)}
            \pgfmathsetmacro{\wend}{pow(10, \wexpend)}
            \pgfmathsetmacro{\gain}{1/sqrt(pow(\ttau*\ww,2) + 1)}
            \pgfmathsetmacro{\phshift}{-atan2(\ttau*\ww,1)}

            \begin{tikzpicture}[scale=0.5, transform shape]

            \begin{loglogaxis} [
            width=7cm,
            height=5cm,
            ylabel=$|G|$,
            %xticklabels=\empty,
            xtick={0.01, 0.1, 1, 10},
            xticklabels={$\frac{0.01}{\tau}$, $\frac{0.1}{\tau}$, $\frac{1}{\tau}$, $\frac{10}{\tau}$},
            grid=both,
            minor y tick num=9,
            % extra y ticks={.5}, % how to convert to fixed point tick label ?
            % extra y tick style={log identify minor tick positions=true},
            every major grid/.style={red, opacity=0.5},
            ymin=0.01, ymax=2,
            xmin = 0.01, xmax=10,
            ]
            \addplot+[thick, orange!80!black, no marks, domain=\wstart:\wend, samples=100]
                  {1/sqrt(pow((\ttau*\x),2) + 1)};
            \draw[black!90, ] (axis cs: \ww, 0.01) -- (axis cs: \ww, 10);
            \node[magenta!80!black, circle, draw, inner sep=2pt, thick,] at (axis cs: \ww, \gain) {};
          \end{loglogaxis}
          \begin{semilogxaxis} [
            xlabel=$\omega$,
            ylabel=$\arg G$,
            xshift = 8cm,
            width=7cm,
            height=5cm,
            grid=both,
            ytick={0, -90},
            ymin=-90, ymax = 0,
            xtick={0.01, 0.1, 1, 10},
            xticklabels={$\frac{0.01}{\tau}$, $\frac{0.1}{\tau}$, $\frac{1}{\tau}$, $\frac{10}{\tau}$},
            xmin = 0.01, xmax = 10,
            minor y tick num=2,
            every major grid/.style={red, opacity=0.5},
            %legend entries={Bessel filter, Delay of one},
            %legend pos={south west},
            ]
            \addplot+[thick, orange!80!black, no marks, domain=\wstart:\wend, samples=100]
               {-atan2(\x*\ttau,1)};
            \draw[black!90,] (axis cs: \ww, -90) -- (axis cs: \ww, 10);
            \node[magenta!80!black, circle, draw, inner sep=2pt, thick,] at (axis cs: \ww, \phshift) {};
          \end{semilogxaxis}
          \begin{axis} [
            width = 10cm,
            height= 5cm,
            yshift = -5cm,
            xshift = 5cm,
            xlabel = {$t$},
            ymin=-1.2,
            ymax = 1.2,
            ]
            \addplot+[thick, blue!80!green, no marks, domain=0:\tend, samples=\nsamples, smooth] {sin(\ww*180/3.14*x)};
            \addplot+[thick, green!80!black, no marks, domain=0:\tend, samples=\nsamples, smooth] {\gain*sin(\ww*180/3.14*x + \phshift)};

          \end{axis}
          \begin{scope}[yshift=-4cm, xshift=-4.5cm]
            \pgfmathsetmacro{\www}{min(\ww, 8)};
            \clip (-2, -1) rectangle (2, 10);
            \draw[->] (-2, 0) -- (0.3, 0) node[below] {Re};
            \draw[->] (0, -1) -- (0, 8) node[left] {Im};
            \node[red] at (-1/\ttau, 0) {\large $\times$};
            \node[magenta!80!black, circle, draw, inner sep=2pt, thick,] at (0, \www) {};
            \draw[<->, black!60] (-1/\ttau, 0) to (0, \www);
            \node at (0.9, \www) {$\frac{\www}{\tau}$};
            \node[]  at (-1/\ttau, -0.5){$-\frac{1}{\tau}$};
            \node at (-1, 6) {$s$-plane};
          \end{scope}
          \begin{scope}[yshift=-3cm, xshift=0cm, ]
            \pgfmathsetmacro{\xx}{2*\gain*cos(\phshift)};
            \pgfmathsetmacro{\yy}{2*\gain*sin(\phshift)};
            \draw[->] (-2, 0) -- (3, 0) node[below] {Re};
            \draw[->] (0, -2) -- (0, 1) node[left] {Im};
            \node[magenta!80!black, circle, draw, inner sep=2pt, thick,] at (\xx, \yy) {};
            \node at (1.5, 1) {$G(i\omega)$-plane};
            \draw (2,0) -- (2, 0.2) node[above] {1};
            \draw[orange!80!black, domain=0:100, smooth, samples at={0, 0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.4, 0.64, 0.8, 1.0, 1.28, 2, 2.56, 3, 4,  5.12, 6, 8, 10.24, 20}, variable=\t] plot ({2*(1/sqrt(1 + pow(\ttau*\t, 2)))*cos(-atan2(\ttau*\t, 1))}, {2*(1/sqrt(1 + pow(\ttau*\t, 2)))*sin(-atan2(\ttau*\t, 1))});
          \end{scope}
        \end{tikzpicture}
      }
    \end{animateinline}
  \end{center}
\end{frame}

\end{document}


\hypertarget{classtuple__element}{}\section{tuple\+\_\+element Class Reference}
\label{classtuple__element}\index{tuple\+\_\+element@{tuple\+\_\+element}}

\hyperlink{classtuple__element}{tuple\+\_\+element} specialization for \hyperlink{classnumpp_1_1matrix_1_1dense}{numpp\+::matrix\+::dense}

{\ttfamily \#include $<$utils.\+hpp$>$}

\subsection{Detailed Description}
\hyperlink{classtuple__element}{tuple\+\_\+element} specialization for \hyperlink{classnumpp_1_1matrix_1_1dense}{numpp\+::matrix\+::dense}

\begin{DoxyTemplParams}{Template Parameters}
{\em Index} & Index inside the vector container, specifying concrete element \\
\hline
{\em T} & arithmetic type contained in matrix class \\
\hline
{\em Rows} & number of matrix rows \\
\hline
{\em Columns} & number of matrix columns\\
\hline
\end{DoxyTemplParams}
Check C++ standard library for information about this structure.

The documentation for this class was generated from the following file\+:\begin{DoxyCompactItemize}
\item
structures/matrices/dense/utils.\+hpp\end{DoxyCompactItemize}


This chapter details the proposed framework and implementation. I start by giving an overview of the proposed framework, and then five crucial components in the framework are carefully explained: Deep Denoising Autoencoders on audio-visual modalities, Fisher Vector Encoding, Document Embedding on textual modality, Tree-based Feature Selection and Multi-Task Learning.

\section{Overview}

Figure \ref{fig:pipeline} illustrates the framework of my proposed method for automatic Bipolar Disorder BD recognition. For acoustic modality, I extract 39-dimensional Mel-Frequency Cepstrum Coefficients (MFCCs)\footnotemark as the Low-Level Descriptors (LLDs) with OpenSMILE and for visual modality, I extract the 132-dimensional facial landmarks, 6-dimensional head pose, 6-dimensional eye gaze, and 35-dimensional Facial Action Units (FAUs) as the LLDs with OpenFace. To discover the correlation across audio-visual modalities and produce robust representations, I propose a Deep Denoising Autoencoder (DDAE) that learns a shared and joint representation on different number of modalities. More specifically, according to the number of modalities on which DDAEs are built, I define uni-DDAE, bi-DDAE, and multi-DDAE. Before feeding multimodal features into DDAE, features must be aligned on frame-level first to ensure they are extracted from the same time interval. I then compute the dynamic changes of the latent representations in DDAE as each representation is regarded as a specific movement of the subject. After computing the velocity ($1^{st}$-order derivative) and the acceleration ($2^{nd}$-order derivative) of the latent representations, the three features are concatenated as frame-level descriptors. Because the video clips vary in length, I encode the frame-level descriptors with a Fisher Vector (FV), a fixed-length descriptor on session-level, by fitting them into a Gaussian Mixture Model (GMM). To reduce redundancy and select the most discriminative feature set, a tree-based model is used for feature selection by evaluating the feature importance.

\begin{figure}[ht]
    \centering
    \includegraphics[width=13.5cm]{images/design/general_pipeline.png}
    \caption{Pipeline of the proposed framework that includes different learning architectures on different modalities}
    \label{fig:pipeline}
\end{figure}

On the other hand, I obtain the transcripts of video interviews and apply a document embedding model (doc2vec) to learn fixed-length representations of the transcripts on session-level. To boost the performance of the doc2vec model, the model is pre-trained on an additional Turkish corpus.

With both fixed-length FVs on audio-visual modalities and fixed-length document embeddings on textual modality, a Multi-Task Deep Neural Network (MT-DNN) is built to handle the overfitting due to the limited size of the BD corpus. The MT-DNN is learned with a weighted loss from the ternary classification task of Mania Level and the regression task of Young Mania Rating Scale (YMRS). Furthermore, to address the imbalance between three classes, I duplicate the training instances of the minority class and apply Unweighted Average Recall (UAR) as my metric for evaluation. Finally, with the best-performing MT-DNN, the final decision on each video interview is inferred: depression, hypo-mania, or mania.

\footnotetext{The 39-dimensional MFCC features are obtained by appending additional 13 delta and 13 acceleration coefficients to the conventional 13-dimensional MFCC features.}

\section{Deep Denoising Autoencoders}
\label{sec:DDAE}

Autoencoders (AEs) are an unsupervised learning algorithm that learns a latent-space representation of given data with hidden layers constrained while reconstructing the data from the representation. The representations learnt from AEs generally have a lower dimensionality and have been proven effective as high-level features in the following classification tasks, which in many cases are competitive or even superior to the hand-engineered representations \cite{ng2011}. Autoencoders are composed of two parts: a) encoder, which compresses the input into a latent representation with the function $h=f(x)$, and b) decoder, which reconstructs the input from the latent representation with the function $r=g(h)$, as shown in Figure \ref{fig:unimodal_ae}. Therefore, autoencoders can be described by the function $g(f(x))=r$, and the reconstruction error $\parallel x-r \parallel ^ 2$ is minimized in the training processing.

\begin{figure}[ht]
    \centering
    \includegraphics[height=6.5cm]{images/design/autoencoder.png}
    \caption{Schematic for Unimodal Deep Denoising Autoencoders}
    \label{fig:unimodal_ae}
\end{figure}

Autoencoders were first introduced as an approach to initialize the weights of neural networks \cite{ballard1987}, and because of the \textit{curse of dimensionality} \cite{bellman1966}, the low-dimensional hidden layers enable AEs to be commonly employed in feature engineering and representation learning \cite{charte2018}. Several variations of AEs have been developed with different constraints on the hidden layers, such as sparse AEs \cite{ng2011}, denoising AEs \cite{vincent2008}, and variational AEs \cite{kingma2013}.

Denoising AEs, proposed by \cite{vincent2008}, stochastically corrupts part of the original input $x$ and reconstructs the original input $x$ with the corrupted version $\tilde{x}$. The noise can be of different forms, such as additive isotropic Gaussian noise and masking noise \cite{vincent2010}. The Gaussian noise $\tilde{x}$ is obtained with $\tilde{x}|x \sim \mathcal{x, \sigma^2,I}$ and the masking noise is calculated by forcing a fraction $v$ of the elements of input $x$ to 0. Although the Gaussian noise is reported to be the natural choice for real-valued inputs \cite{vincent2010}, I only consider the masking noise in the experiment as the Gaussian noise could introduce a small change in facial expressions, which could be interpreted as a misleading descriptor for the BD symptoms. The objective of denoising AEs accordingly becomes minimizing the reconstruction error $\parallel \tilde{x}-r \parallel ^ 2$. The denoising AEs are reported to produce representations that are robust to small irrelevant changes in the input.

With the development of multimedia data processing, a more recent learning architecture is introduced for feature fusion: the multi-modal AEs \cite{mangai2010}. Feature fusion aims to learn a shared representation cross modalities without redundant or irrelevant information, which is one of the biggest challenges in multi-modal data processing \cite{mangai2010}. In the work of \cite{hong2015}, authors aimed to perform multimodal fusion of 2D images and 3D human poses to obtain high-level representations.
Inspired by \cite{vincent2008}, \cite{mangai2010}, and \cite{dibekliouglu2017}, I propose a series of 3-layer Deep Denoising Autoencoders (DDAEs) (i.e., DDAEs with 3 hidden layers) with different number of modalities to investigate the multimodal fusion in the BD detection task.

\subsection{Unimodal Deep Denoising Autoencoders}

I first define the unimodal DDAEs on acoustic and visual features respectively as shown in Figure \ref{fig:unimodal_ae}, where features of one modality are fed into DDAEs to learn a compact-size representation after adding masking noise. This architecture is considered as a baseline for the following architectures as it does not discover the correlation across modalities.

In many implementations of AEs, the binary cross-entropy (BCE) is set as the loss function, which measures the amount of information is preserved in the reconstruction compared to the original input \cite{de2005}. BCE is calculated with Equation \ref{eq:binarycrossentropy}, in which $x_k$ represents one node in the input layer and $r_k$ represents the corresponding node in the output layer. The input data must be therefore normalized to the range $[0,1]$ with the sigmoid activation function in the output layer. BCE could apply to binary-value image pixel intensities, such as MNIST dataset, because the range of original intensities lies in $[0, 255]$, but other features, like MFCC, rarely share the same range across the dataset and normalization could wrongly corrupt the correlation in features, thus hurting the extraction of emotion-related information. In my framework, instead, I define the loss function as the mean squared error (MSE) (Equation \ref{eq:mse}) with the linear activation function in the output layer and no normalization. The DDAEs are trained on minimizing the distance between reconstructed input and original input, and the reconstruction could be easily interpreted and visualized in the evaluation.

\begin{equation}
    J(x,r) = - \sum_{k=1}^d x_k \log(r_k) + (1-x_k) \log(1-r_k)
    \label{eq:binarycrossentropy}
\end{equation}

\begin{equation}
    J(x,r) = \frac{1}{N} \sum_{k=1}^d (x_k - r_k)^2
    \label{eq:mse}
\end{equation}

\begin{figure}[ht]
    \centering
    \includegraphics[height=7.5cm]{images/design/bimodal_ae.png}
    \caption{Schematic for Bimodal Deep Denoising Autoencoders (Different colours indicate different modalities)}
    \label{fig:bimodal_ae}
\end{figure}

\subsection{Bimodal Deep Denoising Autoencoders}

I continue to define the 3-layer bimodal DDAEs on acoustic and visual features altogether as displayed in Figure \ref{fig:bimodal_ae}. The acoustic features are Low-Level Descriptors (LLDs), either MFCC or eGeMAPS features, and the visual features in this architecture are set as facial landmarks whose dynamics have been proven useful in depression detection \cite{dibekliouglu2017}. As suggested in the work of \cite{ngiam2011}, two separate encoders are merged into one shared hidden layer after one hidden layer, which outputs the first-order representations. From the shared hidden layer, two modalities are then reconstructed via their decoders with the sum of MSE on both reconstructed inputs as the loss function.
Acoustic and visual modalities need to be aligned beforehand to ensure they are within the same window \cite{ngiam2011}, and I concatenate 3 contiguous acoustic features as each input that has approximately the same duration as 1 visual feature. Because audio-visual modalities have different data formats and ranges, they are normalized and whitened separately.

\begin{figure}[ht]
    \centering
    \includegraphics[height=7.5cm]{images/design/multimodal_ae.png}
    \caption{Schematic for Multimodal Deep Denoising Autoencoders (Different colours indicate different modalities)}
    \label{fig:multimodal_ae}
\end{figure}

\subsection{Multimodal Deep Denoising Autoencoders}

Given the importance of head pose, eye gaze, and action units in emotion recognition \cite{adams2015, ekman2013}, the 3-layer multimodal DDAEs are defined on a total of five modalities, namely facial landmarks, MFCC / eGeMAPS, head pose, eye gaze, and action units (shown in Figure \ref{fig:multimodal_ae}). Following the same design principle as the bimodal DDAEs, five modalities are merged into the shared representation layer with their ``mid-level" representation output by their first hidden layer, and masking noise is added to modalities individually. The reconstruction loss of each modalities is assigned with equal weights in the joint reconstruction loss function.

\subsection{Implementation Details}

The list of investigated hyperparameters for all DDAEs is given in Table \ref{tab:param_DDAE}, in which the hidden ratio is defined as the ratio between two consecutive hidden layers, and for instance, with hidden ratio 0.5, the dimensions of all hidden layers would be $\{0.5d, 0.25d, 0.5d\}$ where $d$ represents the input dimensionality. In addition, I evaluate the denoising effect in DDAEs by setting different masking noise levels.

\begin{table}[ht]
    \centering
    \caption{Hyperparameter settings to investigate for DDAEs}
    \begin{tabular}{l|l}
        \Xhline{2\arrayrulewidth}
        Hyperparameter & Values \\
        \hline
        hidden ratio & \{0.4, 0.5\} \\
        noise level & \{0.1, 0.2, 0.4\} \\
        batch size & 1024 \\
        learning rate & 0.01 \\
        epochs & 100 \\
        \Xhline{2\arrayrulewidth}
    \end{tabular}
    \label{tab:param_DDAE}
\end{table}

The dimensions of all input modalities are listed in Table \ref{tab:dim_modality}. Because the shared representation is trained from only one acoustic features, either MFCC features or eGeMAPS features are input into bimodal DDAEs (Table \ref{fig:bimodal_ae}) and multimodal DDAEs (Table \ref{fig:multimodal_ae}). The sum of the input dimensions would thus be either 300 or 252, and the shared representation has four different dimensions with different hidden ratio settings (either 0.4 or 0.5), as shown in the last two row of Table \ref{tab:dim_modality}. The dimension of facial landmarks is obtained by $x$ and $y$ coordinates of 68 landmarks, while to acoustic features, dimensions of MFCC and eGeMAPS are multiplied with 3 because of the alignment of audio-visual modalities.

The training of DDAEs can be considered as the unsupervised feature learning, in which more data benefit the encoding of DDAEs \cite{ngiam2011}. All DDAEs are therefore trained with all available labelled (training set and development set) and unlabelled (test set) audio-visual data.

\begin{table}[ht]
    \centering
    \caption{Dimension of all five modalities. The two dimensions of representations in the last two rows are based on the different hidden ratios, defined in \ref{tab:param_DDAE}}
    \begin{tabular}{l|l}
        \Xhline{2\arrayrulewidth}
        Modality & Dimension \\
        \hline
        Facial Landmarks & 136 ($68 \times 2$) \\
        MFCC features & 117 ($39 \times 3$) \\
        eGeMAPS features & 69 ($23 \times 3$) \\
        Head Pose & 6 \\
        Eye Gaze & 6 \\
        Action Units & 35 \\
        \hline
        Sum of input (MFCC) & 300 \\
        Sum of input (eGeMAPS) & 252 \\
        \hline
        Representation (MFCC) & 48 (0.4) / 75 (0.5) \\
        Representation (eGeMAPS) & 40 (0.4) / 63 (0.5) \\
        \Xhline{2\arrayrulewidth}
    \end{tabular}
    \label{tab:dim_modality}
\end{table}

\subsection{Computing Dynamics of Latent Representations}

After training, each DDAE (unimodal, bimodal, or multimodal) learns a presentation for the per-frame extracted features. These representations, however, only encode the static element of the input, such as locations of facial landmarks or loudness of audio signal, but without any temporary information. Following the ideas proposed in \cite{dibekliouglu2017}, I extend these representations with the dynamics. Considering DDAE-based representations as a matrix $H \in \mathcal{R}^{n \times d}$, in which $n$ denotes the number of frames and $d$ the final dimension of representations. Each column in $H_i(i \in \{1,2...d\})$ corresponds to one node in the representation layer, or for example, to one static point in unimodal DDAEs. I then compute the first-order dynamics, velocity $V$, of $H$ by the $1^{st}$ derivative $V_i = \frac{d H_i}{dt}$, measuring the velocity of the change between per-frame representations. I continue to calculate the second-order dynamics, acceleration $A$, of $H$ by the $2^{nd}$ derivative $A_i = \frac{d^2 H_i}{d^2 t}$, measuring the acceleration of the change. To align $H$, $V$, and $A$, I discard the first two frames in each video sessions, and I concatenate $H$, $V$, and $A$ for the final representations on frame-level.

\section{Fisher Vector Encoding}
\label{sec:fisher}

Feature aggregation is an approach via which low-level descriptors (LLDs) can be summarised to produce fixed-length high-level descriptors on variable-length audio-visual recordings, which encodes more global information. Many approaches within feature aggregation exist, such as the Bag-of-Words (BoW) \cite{csurka2004} and the Fisher Vector (FV) \cite{perronnin2010}. In my framework, to encode the frame-level representations learnt from DDAEs into a fixed-length vector on session-level, I investigate and implement the improved Fisher Vector (FV) \cite{perronnin2010, sanchez2013}. The BoW representations are briefly explained first as they are one of the baseline features used in Chapter \ref{ch:evaluation}.

\subsection{Bag-of-Words Representation}

Bag-of-Words originates from natural language processing and as a semi-supervised representation learning, it represents the distribution of LLDs based on a dictionary or codebook learned from them \cite{csurka2004, peng2016}. Generally, BoW is composed of five steps: a) feature extraction, b) feature-preprocessing, c) codebook generation, d) feature encoding, and e) pooling and normalization. According to the modality of representations, BoW can be categorised into Bag-of-Audio-Words (BoAW) and Bag-of-Visual-Words (BoVW). The feature extraction is completed as described in Chapter \ref{ch:background} and these extracted LLDs are usually high dimensional and strong correlated. Principal Component Analysis (PCA), a statistical procedure, is therefore used in BoW to preprocess the LLDs to low-dimensional and de-correlated features \cite{peng2016}. For codebook generation, two approaches are often considered, (i) partitioning the feature space into regions, each represented by its centre, such as $k$-means \cite{bishop2006}, and (ii) using a generative model to capture the probability distribution of features, such as Gaussian Mixture Model (GMM) \cite{bishop2006}. With parameters indicating either cluster centres or parameters for GMM, the LLDs extracted from one video are encoded into a fixed-length vector via voting-based or reconstruction-based encoding method \cite{peng2016}. In the last step, a pooling operation is used to obtain a global per-video representation and normalization enables the representations to be invariant to the variable number of LLDs extracted from different videos \cite{peng2016}.

\subsection{Fisher Vector}

Fisher Vector (FV) extends the bag-of-words (BoW) representations to learn the distribution of LLDs with their mean and variance, and it is commonly used as a global image descriptors in image classification \cite{perronnin2010, krapac2011}. More recently, FV has become popular for a variety of applications in social signal processing, such as depression estimation \cite{jain2014, dhall2015} and emotion recognition \cite{kaya2015}, because it combines advantages of both the generative and discriminative approaches \cite{sanchez2013} in machine learning. In the workflow of FV encoding, a generative model, typically Gaussian Mixture Model (GMM), is firstly built on LLDs and the Fisher kernel is then computed from this generative model. FVs are quantified using first and second order statistics of the gradient of the sample log-likelihood with respect to GMMs' parameters \cite{sanchez2013}.

Formally, let $X = \{x_t; t=1...T\}$ be an element in the time-series representations with the number of frames, $T$. Let $\Theta = \{\mu_k, \Sigma_k, \pi_k; k=1...K\}$ be the parameter of a Gaussian Mixture Model (GMM) fitting the distribution of the representations where $\mu_i$ and $\Sigma_i$ are respectively the mixture mean vector and covariance matrix and priors of GMM. GMM estimates a probability distribution of multiple multivariate Gaussian distributions and it is trained while maximizing the likelihood $p(X|\Theta)$:

\begin{equation}
    \Theta^{*} = \arg \max_{\Theta} p(X|\Theta) = \arg \max_{\Theta} \prod_{i=1}^{N} p(x_i|\Theta)
\end{equation}

GMM associates each vector $x_t$ to a mode $k$ in the mixture with a strength given by the posterior probability:

\begin{equation}
    q_{tk} = \frac{\exp [-1/2 (x_t-\mu_k)^{T} \Sigma_k^{-1}(x_t-\mu_k)]}{\sum_{i=1}^{K} \exp [-1/2 (x_t-\mu_i)^{T} \Sigma_k^{-1}(x_t-\mu_i)]}
\end{equation}

For each mode $k$, the mean and covariance deviation vectors are defined as

\begin{align}
    u_{jk} &= \frac{1}{N\sqrt{\pi_k}} \sum_{t=1}^T q_{tk}\frac{x_{jt}-\mu_{jk}}{\sigma_{jk}} \\
    v_{jk} &= \frac{1}{N\sqrt{2\pi_k}} \sum_{t=1}^T q_{tk}[(\frac{x_{jt}-\mu_{jk}}{\sigma_{jk}})^2 - 1]
\end{align}

where $j=1,2...D$ spans the vector dimensions. The FV of the time-series representations is the stacking of the vector $u_k$ and $v_k$ for each of the K modes in the Gaussian mixtures:

\begin{equation}
    \Phi(X) = [...u_k...v_k...]^T
\end{equation}

\subsection{Improved Fisher Vectors}

In addition to the traditional FV, the improved FV was introduced by \cite{perronnin2010} for better classification performance with the following ideas :
\begin{itemize}
    \item Power normalization. As the number of Gaussians increases, FV becomes sparser and the distribution of features becomes more ``peaky" around zero. The kernel is therefore replaced with the Laplacian kernel, which is more robust on sparse vectors, by applying the function $|z| \sign z$ to each dimension of the vector $\Phi(X)$.
    \item $l_2$ normalization. Before using the representations in a linear model, the vector $\Phi(X)$ is further by the $l_2$ normalization to discard the video-independent information, or descriptors which are likely to occur in any video. FV thus focus on video-specific features.
\end{itemize}

I compute the Fisher Vectors using GMMs with empirically 16 or 32 Gaussian distributions \cite{syed2018, dibekliouglu2017} to estimate the distribution on DDAE-based $d$-dimensional representations. The resulting feature vectors are $96 \times d_{latent}$ dimensional ($16 \times 2 \times 3 \times d_{latent}$) or $192 \times d_{latent}$ dimensional ($32 \times 2 \times 3 \times d_{latent}$), where $d_{latent}$ denotes the dimension of latent representation learnt from DDAEs.

\section{Tree-based Feature Selection}

Feature selection is a key step to reduce redundancy and improve the accuracy of classifiers by selecting the most informative features \cite{hira2015}. The reduced dimensional features should preserve as much information as the original high-dimensional features do. Considering the high dimension of the obtained Fisher Vectors with respect to the limited size of the dataset, feature selection is deemed necessary in my framework. Many approaches have been proposed to select the discriminative features, such as Min-Redundancy Max-Relevance (mRMR) algorithm \cite{peng2005}, Analysis of Variance (ANOVA) \cite{moran1918}, and correlation-based feature selection (CFS) \cite{pudil1994}, I apply tree-based feature selection in my framework by evaluating and ranking the importance of each feature. More specifically, a Random Forest (RF) classifier is used to compute the information gain $(S,A)$ for a feature $A$ relative to a dataset $S$:

\begin{equation}
    Gain(S,A) = Entropy(S) - \sum_{v \in values(A)} \frac{|S_v|}{|S|} Entropy(S_v)
\end{equation}

where $values(A)$ is the set of all possible values for feature $A$, and $S_v$ is the subset of $S$ for which feature $A$ has value $v$. As RF is robust to redundant features and insensitive to the irrelevant information, the resulting feature importance leads to a reliable and discriminative subset of features. The split criterion is preset as entropy, and the number of trees as 800, determined by the baseline system describe in Chapter \ref{ch:evaluation}. I select the top 100 features in the Fisher Vector ranked with the importance, and in other words, the dimension for features in audio-visual modality is 100.

\section{Document Embedding}

Since strong correlations have been found between interview contents and depression symptoms \cite{morales2016, pampouchidou2016}, analyzing emotion-related textual modality has been emerged as a new approach in depression detection. More specifically, the use of negatively-valenced words and pronouns has been stressed in \cite{morales2016}. To incorporate textural modality into my framework, I first transcribe the recordings from BD corpus into plain texts using Google Speech-To-Text API \footnote{\url{https://cloud.google.com/speech-to-text/}}, and then make use of Paragraph Vector (PV) \cite{mikolov2014} to embed to session-level transcripts into textual features of the same size.

The PV model, or doc2vec, is an unsupervised learning algorithm to learn the distributed representations of a variable-length piece of texts \cite{mikolov2014}, and it is usually considered as an extension of word2vec, which aims to learn the word representations \cite{mikolov2013}.

\subsection{word2vec}

The two architectures within word2vec, Continuous Bag-Of-Words (CBOW) and Skip-Gram (SG), are variants of neural networks with one hidden layer \cite{mikolov2013}. While CBOW architecture takes the context words (multiple word vectors) as input and predicts the target word (one word vector), SG architecture infers the context words given the target word, as shown in Figure \ref{fig:word2vec}.

Formally, in word2vec-CBOW architecture, the objective is to maximize the log probability of the target word given context words with Equation \ref{eq:cbow}.

\begin{equation}
    \frac{1}{T} \sum_{t=c}^{T-c} \log p(w_t | w_{t-c} ... w_{t+c})
    \label{eq:cbow}
\end{equation}

where $c$ indicates the size of context windows (which equals to 2 in Figure \ref{fig:word2vec}) and $T$ represents the length of the word sequence. On the other hand, in word2vec-SG architecture, the optimization function is based on the averaged log probability as Equation \ref{eq:sg}, where $c$ and $T$ share the same notations as Equation \ref{eq:cbow}.

\begin{equation}
    \frac{1}{T} \sum_{t=i}^{T} \sum_{-c\leq j \leq c} \log p(w_{t+j}|w_t)
    \label{eq:sg}
\end{equation}

A large $c$ means a broader context window and thus a higher probability to capture the semantics of the words, but it could also lead to more expensive computation \cite{mikolov2013}. Furthermore, since the high dimensionality of input vectors can easily cause difficulties in the computation of condition probability $p(w_O|w_I)$ obtained by the softmax function, shown in Equation \ref{eq:softmax}, where $w_O$ is denoted as the ``output vector", $w_I$ as the ``input vector", and $W$ as the weight matrix that is updated via backpropagation with loss function $E = -\log p(w_O|w_I)$. Mikolov \textit{et al.} \cite{mikolov2013efficient} proposed three optimization techniques to improve the quality of output vectors and also the training speed: hierarchical softmax, negative sampling, and subsampling \cite{mikolov2013, mikolov2013efficient}.

\begin{equation}
    p(w_O|w_I) = \frac{\exp({v^{'}_{w_O}}^{T} v_{w_I})}{\sum_{w=1}^{W} \exp({v^{'}_w}^{T} v_{w_{I}})}
    \label{eq:softmax}
\end{equation}

As an efficient approach of computer softmax, the hierarchical softmax applies a binary tree to represent $W$ words in the output layer \cite{mikolov2013}, and the evaluation of $W$ nodes in the output weight matrix is therefore reduced to $\log_2 W$ nodes. An alternative to the hierarchical softmax is negative sampling: instead of updating the entire weight matrix, only a limited number of ``negative words" are selected to update the weights \cite{mikolov2013efficient}. The ideas underlying subsampling is straightforward: the most frequent words in a large corpus generally provides less semantic information, such as ``a", ``the" or ``some" \cite{mikolov2013efficient}. Subsampling of such words can decrease the number of training samples and also help the neural networks to assign higher weights to rare words, which reflects more emotion-related information.

\begin{figure}[ht]
    \centering
    \begin{minipage}{0.47\textwidth}
        \centering
        \includegraphics[height=4.3cm]{images/design/word2vec_cbow.png} \\
        (a) word2vec-CBOW
    \end{minipage}
    \begin{minipage}{0.48\textwidth}
        \centering
        \includegraphics[height=4.3cm]{images/design/word2vec_sg.png} \\
        (b) word2vec-SG
    \end{minipage}
    \caption{Schematic for two Word Vector (word2vec) architectures}
    \label{fig:word2vec}
\end{figure}

\subsection{doc2vec}

To extend the embeddings to a higher level, the Paragraph Vector (doc2vec) is proposed to learn the representation for a variable-length of texts: sentence, paragraph and document. There are also two architectures within doc2vec, Paragraph Vector with Distributed Memory (PV-DM), corresponding to word2vec-CBOW, and Paragraph Vector with Distributed Bag-Of-Words (PV-DBOW), corresponding to word2vec-SG. As displayed in Figure \ref{fig:doc2vec}, both architectures introduce an additional document token that could be considered as the topic of each document (or in my framework, the mania level of transcript). The input in PV-DM is the concatenation of document vectors and word vectors, both of which are learned in the training process, but only the document vector is used for the inferring process. On the contrary, the PV-DBOW ignores the context words and predicts the words randomly sampled from the inferred document. It is obvious that PV-DBOW requires fewer data storage as it only saves softmax weights and PV-DM also needs to save the word vectors \cite{mikolov2014}.

\begin{figure}[ht]
    \centering
    \begin{minipage}{0.55\textwidth}
        \centering
        \includegraphics[height=5.4cm]{images/design/doc2vec_dm-m.png} \\
        (a) doc2vec-DM
    \end{minipage}
    \begin{minipage}{0.43\textwidth}
        \centering
        \includegraphics[height=5.4cm]{images/design/doc2vec_dbow.png} \\
        (b) doc2vec-DBOW
    \end{minipage}
    \caption{Schematic for two Paragraph Vector (doc2vec) architectures}
    \label{fig:doc2vec}
\end{figure}

\subsection{Implementation Details}
Some researchers have reported that PV-DBOW outperforms PV-DM in the sentiment analysis, contradictory results in the work of \cite{mikolov2014}. In addition, Yang \textit{et al.} only considers PV-DM architectures in their depression detection framework, and therefore, in my experiment, I investigate both PV-DM and PV-DBOW in the BD detection with different hyperparameter settings, as shown in Table \ref{tab:param_doc2vec}.

\begin{table}[ht]
    \centering
    \caption{Hyperparameter settings to investigate for document embedding}
    \begin{tabular}{l|l}
        \Xhline{2\arrayrulewidth}
        Hyperparameter & Values \\
        \hline
        model & \{PV-DM, PV-DBOW\} \\
        vector size & \{25, 50, 100\} \\
        window size & \{5, 10\} \\
        negative words & \{5, 10\} \\
        hierarchical softmax & \{0, 1\} \\
        \Xhline{2\arrayrulewidth}
    \end{tabular}
    \label{tab:param_doc2vec}
\end{table}

The translation of transcripts to English prior to the document embedding has been considered as it might help to understand the transcripts and to evaluate the embeddings. Nonetheless, I apply the doc2vec model directly on the Turkish transcripts to ensure the completeness of semantic content, which could be compromised in the translation, and after learning process, the document embeddings are evaluated with translation for better understanding.

Due to the limited size of the BD corpus, I pre-train the doc2vec models on an external, large-scale Turkish corpus,``trwiki", to improve the performance of my models \cite{lau2016}. The ``trwiki" corpus is based on Wikimedia\footnote{\url{https://dumps.wikimedia.org/trwiki/}} dump service, which contains various kinds of texts in Turkish, such as articles, templates, media/file descriptions, and primary meta-pages. After training separate doc2vec models following the hyperparameter settings in Table \ref{tab:param_doc2vec}, I first evaluate their performance with a simple Random Forest (RF) classifier, the same classifier used in the baseline, and select the best-performing doc2vec model to infer the document vector for each transcript. These document vectors are used in the final fusion stage to predict the mania level of each session. In addition, visualizing the similar groupings of transcripts via Embedding Projector\footnote{\url{http://projector.tensorflow.org}} and the similar words stored in PV-DM models are presented to qualitatively examine the embedding space inferred by the doc2vec models.

\section{Multi-Task Learning}

In the context of Deep Learning, the generalization of the model could be improved if representations being shared across related tasks \cite{ruder2017}, and this approach is called Multi-Task Learning (MTL). There are two typical structures in MTL, hard parameter sharing and soft parameter sharing. Hard parameter sharing is applied by sharing the hidden layers between all tasks, with unchanged task-specific output layers (as shown in Figure \ref{fig:mtl}). Soft parameter sharing, on the other hand, separate all tasks with their own model and parameters (as shown in Figure \ref{fig:mtl}) with an objective to minimize the distance between parameters of different models.

\begin{figure}[ht]
    \centering
    \begin{minipage}[c]{0.42\textwidth}
    \centering
    \includegraphics[height=4.2cm]{images/design/multitask_hard.png} \\
    (a) Hard parameter sharing
    \end{minipage}
    \begin{minipage}[c]{0.55\textwidth}
    \centering
    \includegraphics[height=4.2cm]{images/design/multitask_soft.png} \\
    (b) Soft parameter sharing
    \end{minipage}
    \caption{Schematic for two architectures in Multi-Task Learning frameworks}
    \label{fig:mtl}
\end{figure}

Hard parameter sharing architecture greatly reduces the risk of overfitting \cite{baxter1997} and more specifically, the risk of overfitting the shared parameters is an order N smaller than the risk of overfitting the task-specific parameters, where N which represents the number of tasks. I therefore adjust my deep neural network (DNN) classifier to learn on both classification task (mania levels) and regression task (YMRS scores). The joint loss function is defined as the weighted sum of cross entropy loss for classification and the Euclidean loss for regression, as shown in Equation \ref{eq:multi_loss}.

\begin{equation}
\begin{split}
 \mathcal{L} & = \mathcal{W}_c \mathcal{L}_c + \mathcal{W}_r \mathcal{L}_r \\
             & = \mathcal{W}_c (-\sum_{c=1}^N y_c \log(p_c)) + \mathcal{W}_r (\frac{1}{M} \sum_{i=1}^M \parallel y_r - p_r \parallel ^ 2)
\end{split}
\label{eq:multi_loss}
\end{equation}

where $\mathcal{W}_c$ and $\mathcal{W}_r$ represent the weight for two losses, $y_c$ and $y_r$ are values for two tasks while $p_c$ and $p_r$ are the predicted values, and $N$ is the number of classes and $M$ is the number of samples in the training set.


Input is converted into a sequence of tokens during the lexical analysis.
Tokens end if the next character is a whitespace character or if the next
character cannot be added to it.

\subsection{Whitespace characters}

The characters
` ' (ASCII 32),
`\textbackslash f' (ASCII 12),
`\textbackslash v' (ASCII 11) and
`\textbackslash t' (ASCII 9) are whitespace charakters.

Whitespace characters delimite tokens but are otherwise ignored.

\subsection{Tokens recoginzed}

Recognized token kinds are one of: \\[0.3cm]

\begin{tabular}{lp{9cm}}
\lexkeyword{UNKNOWN}
    & Illegal character (i.e. not a whitespace and can not be part of any
    token) was found and consumed.\\[0.2cm]
\lexkeyword{EOI} & End of input \\[0.2cm]
\nonterminal{float-literal}
    & floating point literal matching the regular expression\\
    & \verb#[0-9]+(([.][0-9]*)([eE][+-]?[0-9]+)?)?#\\
    & or \\
    & \verb#[0-9]*(([.][0-9]+)([eE][+-]?[0-9]+)?)?#\\
    & Note: The first regular expression requires at least one leading digit, the
    second at least one trailing digit. Hence, \verb#0.# and \verb#.0# are
    legal, but \verb#.# is not \\[0.2cm]
\lexkeyword{+} & Plus operator\\[0.2cm]
\lexkeyword{-} & Minus operator or unary minus\\[0.2cm]
\lexkeyword{*} & Multiplication operator\\[0.2cm]
\lexkeyword{/} & Division oprtator \\[0.2cm]
\lexkeyword{\textasciicircum}  & Power operator \\[0.2cm]
\lexkeyword{(}  & Left parenthesis \\[0.2cm]
\lexkeyword{)}  & Right parenthesis\\
\end{tabular}


\documentclass[10pt]{article}
\usepackage{a4wide}
\usepackage{listings}
\usepackage{listings}
\newcommand{\be}{\begin{equation}}
\newcommand{\ee}{\end{equation}}
\newcommand{\OP}[1]{{\bf\widehat{#1}}}

\begin{document}

\section*{Project 3, Variational Monte Carlo studies of helium and beryllium}

The final aim of this project is to develop a variational Monte Carlo program which can be used to obtain ground state properties of atoms like He, Be, O, Ne, Si etc as well as diatomic molecules.
for important molecules

The final project builds on project 1 and 2 and adds the optimization procedure for the variational parameter $\beta$ using for example Newton's methods, the steepest descent method or the conjugate gradient method.
Furthermore, the single-particle wave functions used in the computation
of the Slater determinant will be replaced by Gaussian type orbitals (GTO) using already optimized parameters.

Thus, in addition to these ingredients, the final project contains all the elements you have included inprojects 1 and 2. This means that you can include all results you have obtained in projects 1 and 2 in the wrap-up of the final report. It is the final report which counts for the final mark.

You will thus see that this report contains much of the same text as included in projects 1 and 2.

{\bf The deadline for project 3 is June 15, at noon}.  See below for delivery format.

\section*{Part 1: Variational Monte Carlo calculations of the helium atom}

We will start with the simplest possible system beyond hydrogen, namely the helium atom.
We label $r_1$ the distance from electron 1 to the nucleus and similarly
$r_2$ the distance between electron 2 and the nucleus.
The contribution to the potential energy from the interactions between the
electrons and the nucleus is
\be
   -\frac{2}{r_1}-\frac{2}{r_2},
\ee
and if we add the electron-electron repulsion with
$r_{12}=|{\bf r}_1-{\bf r}_2|$, the total potential energy
$V(r_1, r_2)$ is
\be
 V(r_1, r_2)=-\frac{2}{r_1}-\frac{2}{r_2}+
               \frac{1}{r_{12}},
\ee
yielding the total Hamiltonian
\be
   \OP{H}=-\frac{\nabla_1^2}{2}-\frac{\nabla_2^2}{2}
          -\frac{2}{r_1}-\frac{2}{r_2}+
               \frac{1}{r_{12}},
\ee
and Schr\"odinger's equation reads
\be
   \OP{H}\psi=E\psi.
\ee
All equations are in so-called atomic units. The distances
$r_i$ and $r_{12}$ are dimensionless. To have energies in electronvolt
you need to multiply all results with
$2\times E_0$,
where $E_0=13.6$ eV.
The experimental binding energy for helium in atomic units a.u. is $E_{\mathrm{He}}=-2.9037$ a.u..

\begin{enumerate}

\item[1a)] We want to perform  a Variational Monte Carlo calculation of the ground state of the helium atom.
In our first attempt we will use a brute force Metropolis sampling with a trial wave function which has the following form
\begin{equation}
   \psi_{T}({\bf r_1},{\bf r_2}, {\bf r_{12}}) =
   \exp{\left(-\alpha(r_1+r_2)\right)}
   \exp{\left(\frac{r_{12}}{2(1+\beta r_{12})}\right)},
\label{eq:trial}
\end{equation}
with $\alpha$ and $\beta$ as variational parameters.

Your task is to perform a Variational Monte Carlo calculation
using the Metropolis algorithm to compute the integral
\begin{equation}
   \langle E \rangle =
   \frac{\int d{\bf r_1}d{\bf r_2}\psi^{\ast}_T({\bf r_1},{\bf r_2}, {\bf r_{12}})\OP{H}({\bf r_1},{\bf r_2}, {\bf r_{12}})\psi_T({\bf r_1},{\bf r_2}, {\bf r_{12}})}
        {\int d{\bf r_1}d{\bf r_2}\psi^{\ast}_T({\bf r_1},{\bf r_2}, {\bf r_{12}})\psi_T({\bf r_1},{\bf r_2}, {\bf r_{12}})}.
\end{equation}
You should parallelize your program.   Find the  energy minimum and compute also the mean distance
$r_{12}$ between the two electrons for the optimal set of the variational parameters.
A code for doing a VMC calculation for the helium atom can be
found on the webpage of the course, see under programs.

Your Monte Carlo moves are determined by
\begin{equation}
   {\bf R}' = {\bf R} +\delta \times r,
\end{equation}
where $r$ is a random number from the uniform distribution and $\delta$
a chosen step length.
In solving this exercise you need to devise an algorithm which finds
an optimal value of $\delta$ for the variational parameters $\alpha$ and $\beta$,
resulting in roughly $50\%$ accepted moves.

Give a physical  interpretation of the best value of $\alpha$.
Make a plot of the variance as a function of the number of Monte Carlo
cycles.
\item[1b)]
Find closed form expressions for the local energy (see below) for the above
trial wave function and explain shortly how this
trial function satisfies
the cusp condition when $r_1\rightarrow 0$ or
$r_2\rightarrow 0$ or  $r_{12}\rightarrow 0$. Show that
closed-form expression for the trial wave function is
\[
E_{L2} = E_{L1}+\frac{1}{2(1+\beta r_{12})^2}\left\{\frac{\alpha(r_1+r_2)}{r_{12}}(1-\frac{\mathbf{r}_1\mathbf{r}_2}{r_1r_2})-\frac{1}{2(1+\beta r_{12})^2}-\frac{2}{r_{12}}+\frac{2\beta}{1+\beta r_{12}}\right\},
\]
where
\[
E_{L1} = \left(\alpha-Z\right)\left(\frac{1}{r_1}+\frac{1}{r_2}\right)+\frac{1}{r_{12}}-\alpha^2.
\]

Compare the results of with and without the closed-form expressions (in terms of CPU time).
\item[1c)] Introduce now importance sampling and study the dependence of the results as a function of the time step $\delta t$.
Compare the results with those obtained under 1a) and comment eventual differences.
In performing the Monte Carlo analysis you should use blocking as a technique  to make the statistical analysis of the numerical data.
The code has to run in parallel.
\item[1d)]  With the optimal parameters for the ground state wave function, compute the onebody density. Discuss your results and compare the results with those obtained with a pure hydrogenic wave functions. Run a Monte Carlo calculations without the Jastrow factor as well
and compute the same quantities. How important are the correlations induced by the Jastrow factor?

\item[1e)]  Repeat step 1c) by varying the energy using the
conjugate gradient method or similar methods to obtain the best possible parameter
$\beta$.  Replace now the hydrogen-like single-particle wave functions with the 3-21G basis defined
at the EMSL website \url{https://bse.pnl.gov/bse/portal}.

\end{enumerate}

\section*{Part 2: Variational Monte Carlo calculations of the Beryllium and  Neon atoms}
The previous exercise has prepared you for extending your calculational machinery  to other systems.
Here we will focus on the neon and beryllium atoms.
It is convenient to make modules or classes of trial wave functions, both many-body wave functions
and single-particle wave functions  and the quantum numbers  involved, such as spin, orbital momentum and principal
quantum numbers.

The new item you need to pay attention to is the calculation of the Slater Determinant. This is an additional complication
to your VMC calculations.
If we stick to hydrogen-like wave functions,
the trial wave function for Beryllium can be written as
\begin{equation}
   \psi_{T}({\bf r_1},{\bf r_2}, {\bf r_3}, {\bf r_4}) =
   Det\left(\phi_{1}({\bf r_1}),\phi_{2}({\bf r_2}),
   \phi_{3}({\bf r_3}),\phi_{4}({\bf r_4})\right)
   \prod_{i<j}^{4}\exp{\left(\frac{r_{ij}}{2(1+\beta r_{ij})}\right)},
\end{equation}
where $Det$ is a Slater determinant and the single-particle wave functions
are the hydrogen wave functions for the $1s$ and $2s$ orbitals. Their form
within the variational ansatz are given by
\begin{equation}
\phi_{1s}({\bf r_i}) = e^{-\alpha r_i},
\end{equation}
and
\begin{equation}
\phi_{2s}({\bf r_i}) = \left(1-\alpha r_i/2\right)e^{-\alpha r_i/2}.
\end{equation}
For neon, the trial wave function can take the form
\begin{equation}
   \psi_{T}({\bf r_1},{\bf r_2}, \dots,{\bf r_{10}}) =
   Det\left(\phi_{1}({\bf r_1}),\phi_{2}({\bf r_2}),
   \dots,\phi_{10}({\bf r_{10}})\right)
   \prod_{i<j}^{10}\exp{\left(\frac{r_{ij}}{2(1+\beta r_{ij})}\right)},
\end{equation}
In this case you need to include the $2p$ wave function as well.
It is given as
\begin{equation}
\phi_{2p}({\bf r_i}) = \alpha {\bf r_i}e^{-\alpha r_i/2}.
\end{equation}
Observe that $r_i = \sqrt{r_{i_x}^2+r_{i_y}^2+r_{i_z}^2}$.

You can approximate the Slater determinant for the ground state of the Beryllium atom
by writing it out as
\begin{equation}
   \psi_{T}({\bf r_1},{\bf r_2}, {\bf r_3}, {\bf r_4}) \propto
\left(\phi_{1s}({\bf r_1})\phi_{2s}({\bf r_2})-\phi_{1s}({\bf r_2})\phi_{2s}({\bf r_1})\right)
\left(\phi_{1s}({\bf r_3})\phi_{2s}({\bf r_4})-\phi_{1s}({\bf r_4})\phi_{2s}({\bf r_3})\right).
\end{equation}
Here you can see a simple code example which implements the above expression
\begin{lstlisting}
 for (i = 0; i < number_particles; i++) {
    argument[i] = 0.0;
    r_single_particle = 0;
    for (j = 0; j < dimension; j++) {
      r_single_particle  += r[i][j]*r[i][j];
    }
    argument[i] = sqrt(r_single_particle);
  }
// Slater determinant, no factors as they vanish in Metropolis ratio
wf  = (psi1s(argument[0])*psi2s(argument[1])
       -psi1s(argument[1])*psi2s(argument[0]))*
      (psi1s(argument[2])*psi2s(argument[3])
       -psi1s(argument[3])*psi2s(argument[2]));
\end{lstlisting}
For beryllium we can easily implement the explicit evaluation of the Slater determinant.  The above will serve as a useful check
for your function which computes the Slater determinat.
The derivatives of the single-particle wave functions can be computed analytically and you should consider
using the closed form expression for the local energy (not mandatory, you can use numerical derivatives as well although a closed form expressions speeds up your code).

For the correlation part
\[
\Psi_C=\prod_{i< j}g(r_{ij})= \exp{\left\{\sum_{i<j}\frac{ar_{ij}}{1+\beta r_{ij}}\right\}},
\]
we need to take into account whether electrons have equal or opposite spins since we have to obey the
electron-electron cusp condition as well.  For Beryllium, as an example,  you can fix electrons 1 and 2 to have spin up while
electrons 3 and 4 have spin down.
When the electrons have  equal spins
\[
a= 1/4,
\]
while for opposite spins (as for the ground state of  helium)
\[
a= 1/2.
\]

\begin{enumerate}
\item[(2a)]   Write a function which sets up the Slater determinant for beryllium and neon and can be generalized to
handle larger systems as well.
Compute the ground state energies of neon and beryllium as you did for the helium atom
in 1d).
The calculations should include  parallelization, blocking, importance sampling and energy minimization using the conjugate gradient approach or related approaches like the steepest descent method or Newton's method.  You should also replace the hydrogen-like single-particle wave functions with the 3-21G basis set defined
at the EMSL website \url{https://bse.pnl.gov/bse/portal}. If you get time, you should also try the 6-311G basis set from the same website.

\item[2b)]  With the optimal parameters for the ground state wave function, compute again the onebody density. Discuss your results and compare the results with those obtained with a pure hydrogenic wave functions. Run a Monte Carlo calculations without the Jastrow factor as well
and compute the same quantities. How important are the correlations induced by the Jastrow factor?

\end{enumerate}

\section*{Brief summary on how ot write a report}

Here follows a brief recipe and recommendation on how to write a report for each
project.
\begin{itemize}
\item Give a short description of the nature of the problem and the eventual
numerical methods you have used.
\item Describe the algorithm you have used and/or developed. Here you may find it convenient
to use pseudocoding. In many cases you can describe the algorithm
in the program itself.

\item Include the source code of your program. Comment your program properly.
\item If possible, try to find analytic solutions, or known limits
in order to test your program when developing the code.
\item Include your results either in figure form or in a table. Remember to
       label your results. All tables and figures should have relevant captions
       and labels on the axes.
\item Try to evaluate the reliabilty and numerical stability/precision
of your results. If possible, include a qualitative and/or quantitative
discussion of the numerical stability, eventual loss of precision etc.

\item Try to give an interpretation of you results in your answers to
the problems.
\item Critique: if possible include your comments and reflections about the
exercise, whether you felt you learnt something, ideas for improvements and
other thoughts you've made when solving the exercise.
We wish to keep this course at the interactive level and your comments can help
us improve it. We do appreciate your comments.
\item Try to establish a practice where you log your work at the
computerlab. You may find such a logbook very handy at later stages
in your work, especially when you don't properly remember
what a previous test version
of your program did. Here you could also record
the time spent on solving the exercise, various algorithms you may have tested
or other topics which you feel worthy of mentioning.
\end{itemize}

\section*{Format for electronic delivery of report and programs}
%
The preferred format for the report is a PDF file. You can also
use DOC or postscript formats.
As programming language we prefer that you choose between C++, Fortran2008 or Python.
Finally,
we recommend that you work together. Optimal working groups consist of
2-3 students, but more people can collaborate. You can then hand in a common report.

\section*{Literature}
\begin{enumerate}
\item B.~L.~Hammond, W.~A.~Lester and P.~J.~Reynolds, Monte Carlo methods
in Ab Inition Quantum Chemistry, World Scientific, Singapore, 1994, chapters
2-5 and appendix B.

\item B.H.~Bransden and C.J.~Joachain, Physics of Atoms and molecules,
Longman, 1986. Chapters 6, 7 and 9.
\item S.A.~Alexander and R.L.~Coldwell,
Int.~Journal of Quantum Chemistry, {\bf 63} (1997) 1001.  This article is available
at the webpage of the course as the file jastrow.pdf under the project 1 link.
\item C.J.~Umrigar, K.G.~Wilson and J.W.~Wilkins, Phys.~Rev.~Lett.~{\bf 60}
(1988) 1719.

\item Moskowitz and Kalos, Int.~Journal of Quantum Chemistry {\bf XX}, 1107 (1981).
Results for He and H$_2$.
\item Filippi, Singh and Umrigar, J.~Chemical Physics {\bf 105}, 123 (1996).   Useful results on
Be$_2$ to which you can benchmark your results against.

\end{enumerate}

\end{document}

\section*{How to write the report}
What should the report contain and how can I structure it? A typical structure follows here.
\begin{itemize}
\item An abstract with the main findings.
\item  An introduction where you explain the aims and rationale for the physics case and what you have done. At the end of the introduction you should give a brief summary of the structure of the report
\item Theoretical models and technicalities. This sections ends often being the methods section.
\item Results and discussion
\item Conclusions and perspectives
\item Appendix with extra material
\item Bibliography
\end{itemize}
Keep always a good log of what you do.

\subsection*{What should I focus on? Introduction.}
You don't need to answer all questions in a chronological order. When you write the introduction you could focus on the following aspects
\begin{itemize}
\item Motivate the reader, the first part of the introduction gives always a motivation and tries to give the overarching ideas
\item What I have done
\item The structure of the report, how it is organized etc
\end{itemize}
\subsection*{What should I focus on? Methods sections.}
\begin{itemize}
\item Describe the methods and algorithms
\item You need to explain how you implemented the methods and also say something about the structure of your algorithm and present some parts of your code
\item You should plug in some calculations to demonstrate your code, such as selected runs used to validate and verify your results. The latter is extremely important!! A reader needs to understand that your code reproduces selected benchmarks and reproduces previous results, either numerical and/or well-known closed form expressions.
\end{itemize}

\subsection*{What should I focus on? Results sections.}
\begin{itemize}
\item Present your results
\item Give a critical discussion of your work and place it in the correct context.
\item Relate your work to other calculations/studies
\item An eventual reader should be able to reproduce your calculations if she/he wants to do so. All input variables should be properly explained.
\item Make sure that figures and tables contain enough information in their captions, axis labels etc so that an eventual reader can gain a first impression of your work by studying figures and tables only.
\end{itemize}

\subsection*{What should I focus on? Conclusions sections.}
\begin{itemize}
\item State your main findings and interpretations
\item Try as far as possible to present perspectives for future work
\item Try to discuss the pros and cons of the methods and possible improvements
\end{itemize}

\subsection*{What should I focus on? Additional material, appendices.}
\begin{itemize}
\item Additional calculations used to validate the codes
\item Selected calculations, these can be listed with few comments
\item Listing of the code if you feel this is necessary
\item You can consider moving parts of the material from the methods section to the appendix. You can also place additional material on your webpage.
\end{itemize}
\subsection*{What should I focus on? References.}
\begin{itemize}
\item Give always references to material you base your work on, either scientific articles/reports or books.
\item Refer to articles as: name(s) of author(s), journal, volume (boldfaced), page and year in parenthesis.
\item Refer to books as: name(s) of author(s), title of book, publisher, place and year, eventual page numbers
\end{itemize}

\section*{Format for electronic delivery of report and programs}
%
Your are free to choose your format for handing in. The simplest way is that you send us your github link that contains the report in your chosen format(pdf, ps, docx, ipython notebook etc) and the programs.
As programming language you have to choose either C++ or Fortran or Python. We recommend C++ or Fortran.
Finally,
we recommend that you work together. Optimal working groups consist of
2-3 students, but more people can collaborate. You can then hand in a common report.

\section*{Literature}
\begin{enumerate}
\item B.~L.~Hammond, W.~A.~Lester and P.~J.~Reynolds, Monte Carlo methods
in Ab Inition Quantum Chemistry, World Scientific, Singapore, 1994, chapters
2-5 and appendix B.

\item B.H.~Bransden and C.J.~Joachain, Physics of Atoms and molecules,
Longman, 1986. Chapters 6, 7 and 9.
\item S.A.~Alexander and R.L.~Coldwell,
Int.~Journal of Quantum Chemistry, {\bf 63} (1997) 1001.  This article is available
at the webpage of the course as the file jastrow.pdf under the project 1 link.
\item C.J.~Umrigar, K.G.~Wilson and J.W.~Wilkins, Phys.~Rev.~Lett.~{\bf 60}
(1988) 1719.

\end{enumerate}

\section*{Unit tests, how and why?}
Unit Testing is the practice of testing the smallest testable parts, called units, of an application individually and independently to determine if they behave exactly as expected. Unit tests (short code fragments) are usually written such that they can be preformed at any time during the development to continually verify the behavior of the code. In this way, possible bugs will be identified early in the development cycle, making the debugging at later stage much easier. There are many benefits associated with Unit Testing, such as
\begin{itemize}
\item It increases confidence in changing and maintaining code. Big changes can be made to the code quickly, since the tests will ensure that everything still is working properly.
\item Since the code needs to be modular to make Unit Testing possible, the code will be easier to reuse. This improves the code design.
\item Debugging is easier, since when a test fails, only the latest changes need to be debugged.
\item Different parts of a project can be tested without the need to wait for the other parts to be available.
\item A unit test can serve as a documentation on the functionality of a unit of the code.
\end{itemize}
Here follows a simple example, see the website of the course for more information on how to install unit test libraries.
\begin{verbatim}
#include <unittest++/UnitTest++.h>

class MyMultiplyClass{
public:
   double multiply(double x, double y) {
      return x * y;
   }
};
TEST(MyMath) {
     MyMultiplyClass my; CHECK_EQUAL(56, my.multiply(7,8));
}
int main()
{
return UnitTest::RunAllTests();
}
\end{verbatim}
For Fortran users, the link at \url{http://sourceforge.net/projects/fortranxunit/} contains a similar software for unit testing.
\end{document}


\chapter{Constraining the equation of state with astrophysics}
%\chapter{Astrophysics around neutrons stars}
\chapterimage[width=15cm]{wordcloud/chap3b.png}

\subsection{Accretion disks}

Gravitational potential energy release
\be
\Delta E_{\mathrm{acc}} = \frac{G M m}{R} \sim 10^{20}  \left( \frac{10\km}{R} \right) \left( \frac{M}{\Msun} \right) \unitspace\erg\unitspace\g^{-1}
\ee

Roche Lobe \cite{PRP02} \cite{LL15}

LMXB \cite{TH06}

Hard and soft state \cite{HvdK89}
Alternates between these two states \cite{MDF14} \cite{DGK07}

\subsection{Between the disk and the star: boundary layers}

\be
\Omega(R) \approx \Omega_{\mathrm{K}}(R) = \left( \frac{G M}{R^3} \right)^{1/2}
\ee

Layer of thickness $b$ equals $\Omega(R + b) \approx \Omega_{\mathrm{K}}(R + b)$ that must slow down to $\Omega_{*}$.

Energy difference
\be
\dot{E} = \frac{1}{2} \Mdot R^2 (\Omega_{\mathrm{K}}^2 - \Omega_{*}^2) =
\frac{1}{2} \Mdot \frac{GM}{R} \left[ 1 - \left(\frac{\Omega_*}{\Omega_{\mathrm{K}}} \right)^2 \right]
\ee

Viscous torque $G_{\mathrm{T}} = \Mdot R^2 (\Omega_{\mathrm{K}} - \Omega_*)$
Hence,
\be
\dot{E} = \frac{1}{2} \frac{G M \Mdot}{R} \left( 1 - \frac{\Omega_*}{\Omega_{\mathrm{K}}} \right)^2
\ee

\section{X-ray bursts}
\subsection{Unstable thermonuclear burning on top of neutron stars}

\subsection{Constraining the size of bursting source}

%--------------------------------------------------
\section{Accretion}
%infalling matter leads to X-rays \cite{Lewin93}

\subsection{Source of energy}
In the heart of this whole problem is an astrophysical process called accretion.

Gravitational potential energy release
\be
\Delta E_{\mathrm{acc}} = \frac{G M m}{R} \sim 10^{20}  \left( \frac{10\km}{R} \right) \left( \frac{M}{\Msun} \right) \unitspace\erg\unitspace\g^{-1}
\ee

Eddington luminosity
\be
L_{\mathrm{Edd}} = \frac{ 4 \pi G M \mproton c }{\sigma_{\mathrm{T}} } \approx \Ten{1.3}{38} \left( \frac{M}{\Msun} \right) \ergs
\ee

Accretion luminosity
\be
L_{\mathrm{acc}} = \frac{G M \Mdot}{R}
\ee

\subsection{Binary systems}

\begin{figure}[t]
\centering
\includegraphics[width=7.5cm]{figs/astro/roche.pdf}
\includegraphics[width=7.5cm]{figs/astro/roche.pdf}
\caption{\label{fig:roche}
Roche potential for binary systems.
}
\end{figure}

\subsubsection{Roche lobes and mass transfer}
Roche Lobe \cite{PRP02} \cite{LL15}

A flow of gas between two stars can be described by the Euler equation.
It gives the time evolution of the velocity $\vec{v}$ of the gas that has a pressure of $P$ and density $\rho$.
In a reference frame rotating together with the binary system with angular velocity $\omega$ the Euler equation takes the form
\be
\frac{ \partial \vec{v} }{\partial t} + (\vec{v} \cdot \nabla)\vec{v} = -\nabla \Phi_{\mathrm{R}} - 2 \vec{ \omega } \times \vec{v} - \frac{1}{\rho} \nabla P,
\ee
where the angular velocity of the binary is then
\be
\vec{ \omega } = \left( \frac{ G M }{a^3} \right)^{1/2} \vec{e},
\ee
as given with the unit vector $\vec{e}$, normal to the orbital plane.
Here $M$ is the total mass of the system, i.e. $M = M_1 + M_2$, where $M_1$ and $M_2$ are the individual masses of the two stars in the system, respectively, and $a$ is their orbital separation.

The effects originating from the gravitation and from the centrifugal forces are encapsulated in the so-called Roche potential, given as a function of radial vector $\vec{r}$ as
\be
\Phi_{\mathrm{R}}(\vec{r}) = -\frac{G M_1}{|\vec{r} - \vec{r_1}|} -\frac{G M_2}{|\vec{r} - \vec{r_2}|} - \frac{1}{2} ( \vec{ \omega } \times \vec{v} )^2,
\ee
where the location of the stars is given with $\vec{r_1}$ and $\vec{r_2}$.

By studying the shape of the potential, we see that in between the stars, in the so-called $L_1$ point there exists a location where the individual gravitational pull from the stars is balanced.
This leads to a kinda of a nozzle in the system from which the material can leak from the less massive star to the more massive object.
Such a leaking, or a Roche lobe overflow, will then occur if the companion star's radius exceeds the size of its Roche lobe.
Typically such a thing can happen when the star evolves and expands at the end of its life cycle.

LMXB \cite{TH06}

\subsection{Accretion disks}
Hard and soft state \cite{HvdK89}
Alternates between these two states \cite{MDF14} \cite{DGK07}

\section{Accretion to a neutron star}

\subsection{Boundary layers}

\be
\Omega(R) \approx \Omega_{\mathrm{K}}(R) = \left( \frac{G M}{R^3} \right)^{1/2}
\ee

Layer of thickness $b$ equals $\Omega(R + b) \approx \Omega_{\mathrm{K}}(R + b)$ that must slow down to $\Omega_{*}$.

Energy difference
\be
\dot{E} = \frac{1}{2} \Mdot R^2 (\Omega_{\mathrm{K}}^2 - \Omega_{*}^2) =
\frac{1}{2} \Mdot \frac{GM}{R} \left[ 1 - \left(\frac{\Omega_*}{\Omega_{\mathrm{K}}} \right)^2 \right]
\ee

Viscous torque $G_{\mathrm{T}} = \Mdot R^2 (\Omega_{\mathrm{K}} - \Omega_*)$
Hence,
\be
\dot{E} = \frac{1}{2} \frac{G M \Mdot}{R} \left( 1 - \frac{\Omega_*}{\Omega_{\mathrm{K}}} \right)^2
\ee

\subsection{X-ray bursts}
Thermonuclear runaway.

\subsection{Constraining the size of the star}
Cooling tail method.

% --------------------------------------------------
\newpage
\section{Appendix A: Real astrophysics}

%\begin{itemize}
%    \item Accretion
%    \item Disks
%    \item Soft and hard states
%    \item Boundary layers
%    \item Ignition?
%    \item X-ray bursts
%    \item Cooling tail method
%\end{itemize}
%
%Possible layout:
%
%v1
%\begin{enumerate}
%    \item Accretion
%    \begin{itemize}
%        \item Disks
%        \item Soft and hard states
%        \item Boundary layers
%    \end{itemize}
%    \item X-ray bursts
%    \begin{itemize}
%        \item Unstable thermonuclear burning
%        \item Cooling tail method
%    \end{itemize}
%\end{enumerate}
%
%v2
%\begin{enumerate}
%    \item Accretion disks
%    \begin{itemize}
%        \item Accretion
%        \item ?Roche lobe overflow
%        \item Soft and hard state
%        \item Boundary layers
%    \end{itemize}
%    \item X-ray bursts
%    \begin{itemize}
%        \item Unstable thermonuclear burning
%        \item Cooling tail method
%    \end{itemize}
%\end{enumerate}

\sect{Energetics}
Let us try and estimate the energetics of different phenomena of what we can observe from neutron stars.\mnote{Energy output}
Few possible stable sources of energy exists: thermal, gravitational, rotational, and magnetic.
In addition, unstable fusion processes can also power some observable phenomena.


\section{Interpretation of QM}
\subsection{Supporting definitions}

To provide our interpretation of quantum mechanics we need to develop
a number of supporting definitions. As the reader familiar with
process algebraic systems can readily verify, these definitions make
\emph{essential} use of the reflective operations and as such identify
this calculus as uniquely suited to this particular task.

Among these operations we find a notion of \emph{multiplication} of
names that interacts well with a notion of \emph{tensor product} of
processes. Even more intriguingly, we find a notion of a \emph{dual}
to a process in the form of maps from processes to names. While
notions of composite names have been investigated in the process
algebraic literature, it is the fact that names reflect process
structure that enables the collection of duals to enjoy an algebraic
structure dual to the collection of processes (i.e. there are
operations available to duals that reflect the operations on
processes). Moreover, it is this structure that enables an effective
definition of inner product.

\subsubsection{Multiplication}
\begin{mathpar}
  \quotep{Q} \cdot \quotep{R} := \quotep{Q|R}
  \and \\
  \quotep{Q} \cdot P := P\{ \quotep{Q|R} / \quotep{R} : \quotep{R} \in \freenames{P} \}
\end{mathpar}

\paragraph{Discussion}
The first equation needs little explanation; the second says that each
free name of the process is replaced with the multiplication of that
name by the scalar. Multiplication of a scalar (name) by a state
(process) results in a process all the names of which have been `moved
over' by parallel composition with the process the scalar
quotes. There is a subtlety that the bound names have to be
manipulated so that multiplied names aren't accidentally
captured. There are many ways to achieve this.

\begin{remark}\label{rem:multiplication_identities}
  The reader is invited to verify that for all $x,y,z \in \QProc$ and $P \in \Proc$
  \begin{mathpar}
    x \cdot \quotep{0} \equiv x
    \and
    x \cdot y \equiv y \cdot x
    \and
    x \cdot (y \cdot z) \equiv (x \cdot y) \cdot z
    \and \\
    \quotep{0} \cdot P \equiv P
    \and \\
    x \cdot (y \cdot P) \equiv (x \cdot y) \cdot P
    \and \\
    x \cdot (P|Q) \equiv (x \cdot P) | (x \cdot Q)
    \and \\
  \end{mathpar}
\end{remark}

\subsubsection{Tensor product}

We define a tensor product on processes by structural induction.

\paragraph{Tensor of sums} First note that all summations, including
$\pzero$ and sequence, can be written $\Sigma_{i} x_{i}.A_{i} +
\Sigma_{j} x_{j}.C_{j}$, where we have grouped input-guarded processes
together and output-guarded processes together.

Thus, we can define the tensor product of two summations, $N_{1}\otimes N_{2}$, where

\begin{mathpar}
  N_{1} := \Sigma_{i} x_{i}.A_{i} + \Sigma_{j} x_{j}.C_{j}
  \and
  N_{2} := \Sigma_{i'} y_{i'}.B_{i'} + \Sigma_{j'} y_{j'}.D_{j'}
\end{mathpar}

as follows.

\begin{mathpar}
  \Sigma_{i} x_{i}.A_{i} + \Sigma_{j} x_{j}.C_{j} \otimes \Sigma_{i'}
  y_{i'}.B_{i'} + \Sigma_{j'} y_{j'}.D_{j'}
  \and \\
  := \; \Sigma_{i} \Sigma_{i'} \quotep{\stackrel{\vee}{x_{i}}| \stackrel{\vee}{y_{i'}}}.(A_{i}\otimes B_{i'}) \; | \; \Sigma_{i'} \Sigma_{i} \quotep{\stackrel{\vee}{y_{i'}}|\stackrel{\vee}{x_{i}}}.(B_{i'}\otimes A_{i})
  \and
  \;\; | \;\; \Sigma_{j} \Sigma_{j'} \quotep{\stackrel{\vee}{x_{j}}|\stackrel{\vee}{y_{j'}}}.(A_{j}\otimes B_{j'}) \; | \; \Sigma_{j'} \Sigma_{j} \quotep{\stackrel{\vee}{y_{j'}}|\stackrel{\vee}{x_{j}}}.(B_{j'}\otimes A_{j})
\end{mathpar}

\begin{remark}
  Do we need to $x^{L}$ and $y^{R}$ for this construction as well?
\end{remark}

\paragraph{Tensor of parallel compositions} Next, we distribute tensor
over par.

\begin{mathpar}
  P_{1}|P_{2} \otimes Q_{1}|Q_{2} := (P_{1} \otimes Q_{1}) | (P_{1}
  \otimes Q_{2}) | (P_{2} \otimes Q_{1}) | (P_{2} \otimes Q_{2})
\end{mathpar}

\paragraph{Tensor with dropped names} We treat tensor of a
process with a dropped name as parallel composition.

\begin{mathpar}
  P \otimes \dropn{x} := P | \dropn{x}
\end{mathpar}

\paragraph{Tensor of agents}

Finally, we need to define tensor on agents. Note that the definition
of tensor on summations only tensors inputs with inputs and outputs
with outputs. Thus, we only have to define the operation on
``homogeneous'' pairings.

\begin{mathpar}
  (\vec{x})P \otimes (\vec{y})Q
  \and \\
  := (x_{0}^{L}|y_{0}^{R},\ldots,x_{0}^{L}|y_{n}^{R},\ldots,x_{m}^{L}|y_{0}^{R},\ldots,x_{m}^{L}|y_{n}^R)(P\{ \vec{x}^{L}/\vec{x}\} \otimes Q \{ \vec{y}^{R}/\vec{y}\})
  \and \\
  \clift{\vec{P}} \otimes \clift{\vec{Q}}
  \and \\
  := \clift{P_{0}\otimes Q_{0},\ldots,P_{0}\otimes Q_{n},\ldots,P_{m}\otimes Q_{0},\ldots,P_{m}\otimes Q_{n}}
\end{mathpar}

\begin{remark}
  Observe that arities of tensored abstractions matches arities of
  tensored concretions if the original arities matched. Note also that
  the length of the arities corresponds to the increase in dimension
  we see in ordinary vector space tensor product.
\end{remark}

\begin{remark}
  Operationally, this definition distributes the tensor down to
  components ``linked'' by summation. Tensor over summation is
  intriguing in that it mixes names. Moreover, as a consequence of the
  way it mixes names we have the identities for all $x \in \QProc$ and
  $P,Q \in \Proc$

  \begin{mathpar}
    (x \cdot P) \otimes Q \equiv x \cdot (P \otimes Q) \equiv P \otimes (x \cdot Q)
    \and \\
    P \otimes \pzero \equiv P
  \end{mathpar}

  that the reader is invited to verify.
\end{remark}

\subsubsection{Annihilation}
\begin{mathpar}
  P^{\perp} := \{ Q : \forall R. P|Q \red^{*} R \Rightarrow R \red^{*} \pzero \}
  \and \\
  \annihilate{P} := \Sigma_{Q \in P^{\perp}} \quotep{Q}?(y).(\dropn{y}|Q) | \Sigma_{Q \in P^{\perp}} \quotep{Q}\clift{\Box}
\end{mathpar}

\paragraph{Discussion} The reader will note that $P^{\perp}$ is a
\emph{set} of processes, while $\annihilate{P}$ is a
\emph{context}. We call the set $P^{\perp}$ the \emph{annihilators} of
$P$. The parallel composition of a process in the annihilators of $P$
with $P$ will result in a process, the state space of which has all
paths eventually leading to $\pzero$. Execution may endure loops; but
under reasonable conditions of fairness (naturally guaranteed under
most notions of bisimulation) such a composite process cannot get
stuck in such a loop and will, eventually pop out and terminate.

The context $\annihilate{P}$ is ready and willing to ``take the
$P$ out of'' the process to which it is applied. It will effectively
transmit the code of the process to which it is applied to one of the
annihilators and run the process against it.

\begin{remark}
  Note that ${\annihilate{P}}^*$ is the abstraction corresponding to
  context $\annihilate{P}$. We will set $\dualize{P} := {\annihilate{P}}^*$.
\end{remark}

\subsubsection{Evaluation}
We fix $M$ a domain of fully abstract interpretation with an equality
coincident with bisimulation. We take $\meaningof{\cdot} : \Proc \to
M$ to be the map interpreting processes and $\nmeaningof{\cdot} : \M
\to Proc$ to be the map running the other way. Then we define

\begin{mathpar}
  \int P := \nmeaningof{\meaningof{P}}
\end{mathpar}

\paragraph{Discussion}
There are many fully abstract interpretations of Milner's
$\pi$-calculus. Any of them can be used as a basis for interpreting
the reflective calculus here. Equipped with such a domain it is
largely a matter of grinding through to check that the Yoneda
construction for the normalization-by-evaluation program can be
extended to this setting.

\begin{remark}
  The reader is invited to verify that $\int (\annihilate{P}[P]) = 0$,
  and equivalently $(\nu\; x)\int \dualize{P}\langle x \rangle |
  x\clift{P} = 0$.
\end{remark}

\subsection{Quantum mechanics}

\subsubsection{What is the quantum mechanical notion of continuation?}\label{sec:quantum_continuation}

Imagine the following experimental set-up. Alice, our intrepid quantum
investigator, prepares a state by performing some operation on some
initial state. Then she performs some measurement to obtain an
observation. Using the information of the observation, she selects a
new initial state, operation and measurement and repeats the steps
above. She iterates this procedure until she obtains some desired
observation. What is the expression of this procedure in the language
of quantum mechanics?

\begin{figure}[htp]\label{fig:iterated_experiment}
%  \fbox{
    \begin{lstlisting}[mathescape]
      $\mathcal{E} ::=$
      let S = $U \state{L}$ in (* prepare state*)
      let m = $\innerprod{M}{S}^2$ in (* take measurement *)
      match m with (* use m to decide next experiment *)
      v$_0$ -> $\mathcal{E}$
      | $\ldots$
      | v$_N$ -> $\mathcal{E}$
      | v$_{Exit}$ -> m (* return observation *)
    \end{lstlisting}
%    }
  \caption{Iterated experiment schematic}
\end{figure}

Figure \ref{fig:iterated_experiment} gives a schematic description of
such an iterated experimental procedure. The question is how do we
write down this iterated procedure without stepping outside the
language of quantum mechanics? Note that accounts of famous composite
quantum experiments, like the Stern-Gerlach experiment leave the
language of quantum mechanics to describe the iterated experiment.

We ask this question for many reasons, but one of them is to help set
up the exegesis of our interpretation. In our framework
\emph{everything} is a computation, both the quantum operations and
processes (classical or quantum) that invoke those operations. There
is no need to step out of the conceptual (and more pragmatically, the
computational) framework to describe these kinds of experiments. More
to the point, the framework we are proposing is -- like the hybrid
functional language employed in the schema -- \emph{compositional}:
experiments, computations are built out of experiments and
computations. This is of enormous pragmatic value if we are to build
and reason about systems of significant scale.

\begin{remark}
  It is also worth noting in this connection that this schema is the
  core of a wide range of recursive functions. Further, this
  connection to calculations of fixpoints makes it a close neighbor of
  search techniques like natural selection and the scientific
  method. This is a theme to which we will return, for quantum
  information seems very \emph{unlife-like} in it's uncloneable,
  undeleteable nature.
\end{remark}

Returning the matter of the computational interpretation, our
interpretation will take the form of a map, written
$\meaningof{-}(-)$, from expressions in Dirac notation to expressions
in our target reflective calculus. The map takes an \emph{ancillary}
argument, a channel along which to communicate results to subsequent
computations. This is how we communicate, for example, the results of
taking a measurement to a subsequent step in an experiment.

\subsubsection{Interpretation}

Table \ref{tbl:core_qm_op_defns} gives the core operational
correspondences. It is meant as an intuitive guide.

\begin{table}[htp]\label{tbl:core_qm_op_defns}
  \center{
    \fbox{
      \begin{tabular}{c|c}
        quantum mechanics & process calculus \\
        \hline
        scalar & $x := \quotep{P}$ \\
        state vector & $\state{P} := P$ \\
        dual & $\state{P}^{*} := \event{\annihilate{P}} := \quotep{\annihilate{P}}[-]$ \\
        matrix & $ \Sigma_{\alpha} \state{P_{\alpha}}x_{\alpha}\event{Q_{\alpha}}$ \\
        vector addition & $\state{P} + \state{Q} := \state{P | Q}$ \\
        tensor product & $\state{P} \otimes \state{Q} := \state{P \otimes Q}$ \\
        inner product & $\innerprod{P}{Q} := \quotep{\int \annihilate{P}[Q]}$ \\
      \end{tabular}
    }
  }
  \caption{QM - operational definitions}
\end{table}

\paragraph{Discussion}
The process algebraic view of a state is called, ironically, a
process, and that is what we map vectors to in our interpretation. It
has long been noted in the process algebraic community that names play
a role somewhat similar to scalars in a vector space. What is unique
about the reflective calculus, and makes it suitable for an
interpretation of this form is that with the structure of names
reflecting the structure of processes we can both make this similarity
in a precision instrument; and find a notion of \emph{dual} to a
state.

If we posit names as scalars, then in perfect analogy with vector
spaces a dual is a map from processes to names. We actually have two
candidates for this interpretation: nominal contexts, $\quotep{M}$,
and their corresponding abstraction, $\quotep{M}^{*}$. The goal of
supporting a notion of continuation selects the latter of the two for
our interpretation.

Taking these as the basis of the interpretation together with the
algebraic identities required by the Dirac notation more or less fixes
the definitions of the rest of the operations. Among the interesting
particularities, the definition of inner product finds near perfect
mirroring of the Feyman interpretation.

\begin{mathpar}
  \inferrule* [lab=states] {} {\meaningof{\state{P}}(c) = c?(l,r).r\clift{P}}
  \and
  \inferrule* [lab=events] {} {\meaningof{\event{P}}(c) = (x)c?(l,r).l\clift{\dualize{P}\langle x \rangle} }
  \and
  \inferrule* [lab=vector addition] {} {\meaningof{\state{P} + \state{Q}}(c) = \meaningof{\state{P | Q}}(c)}
  \and
  \inferrule* [lab=tensor product] {} {\meaningof{\state{P} \otimes \state{Q}}(c) = \meaningof{\state{P\otimes Q}}(c)}
  \and
  \inferrule* [lab=inner product] {} {\meaningof{\innerprod{P}{Q}}(c) = (\nu\; x)c\clift{\int \dualize{P}\langle x \rangle | x\clift{Q}}}
  \and
  \inferrule* [lab=matrix] {} {\meaningof{\fprmatrix{P}{x}{Q}}(c) = \\\\ (u)(\nu \; lr) c!(l,r).(l?(e). (\nu\; y)\meaningof{\innerprod{\dropn{e}}{Q}}(x) | x?(z).x?(a).c\clift{(\dualize{P}\sigma(z,a))\langle u \rangle}|y!(x) \\\\
    + r?(e). (\nu\; x)\meaningof{\innerprod{P}{\dropn{e}}}(x) | x?(z).x?(a).c\clift{Q\sigma(z,a)}|x!(x_{\alpha}))}
  \and
  \inferrule* [lab=matrix application] {} {\meaningof{(\fprmatrix{P}{x}{Q})(\state{S})}(c) = (\nu\; c'u)\meaningof{\fprmatrix{P}{x}{Q}}(c')\langle u \rangle | \meaningof{\state{S}}(c') | c'?(a).c!(a)
  \\\\
  \meaningof{(\fprmatrix{P}{x}{Q})(\event{S})}(c) = (\nu\; c'u)\meaningof{\fprmatrix{P}{x}{Q}}(c')\langle u \rangle | \meaningof{\event{S}}(c') | c'?(a).c!(a)}
\end{mathpar}

where

\begin{mathpar}
  P\sigma(z,a) := P\{ z\cdot a\cdot r/r : r \in \freenames{P} \}
\end{mathpar}

\begin{remark}
  The reader is invited to verify that
  \begin{mathpar}
    \meaningof{\innerprod{P}{Q}}(c)
    \and \\
    \wbbisim
    \and \\
    %\and
    (\nu\; x)(\nu c'lr)\meaningof{\event{P}}(c')\langle x \rangle \;|\; \meaningof{\state{Q}}(c')
    %\and
    |\; c'!(l,r).l?(p).r?(q).c\clift{\int \dropn{p} | x!(q)}
  \end{mathpar}
  This provides a (more) compositional definition of inner product. It
  also illustrates an important point of the computational
  interpretation. We have a notion of equivalence providing a crucial
  proof method: bisimulation.
\end{remark}

\begin{remark}
  Assuming $\int (\annihilate{P}[P]) = 0$, the reader is
  invited to verify that $(\fprmatrix{P}{x}{P})(\state{P}) = x \cdot \state{P}$.
\end{remark}

% \begin{remark}
%   The reader is invited to verify that $\innerprod{P}{Q}$ could
%   equally well have been written $\quotep{\int \stackrel{\vee}{x}}$
%   where $x = \event{\annihilate{P}}(Q)$.

%   One of the motivations for this remark is that there is another way
%   to factor these operations. We could package up evaluation in the dual:

%   \begin{mathpar}
%     \state{P}^{*} := \event{\int \annihilate{P}} := \quotep{\int \annihilate{P}}[-]
%   \end{mathpar}

%   and then have inner product defined by

%   \begin{mathpar}
%     \innerprod{M}{Q} := \event{M}(Q)
%   \end{mathpar}

%   where we use $M$ to label the dual to emphasize that it is a context.

%   Hopefully, experience with the calculations will provide guidance on
%   the best factoring.
% \end{remark}

\begin{remark}\label{rem:abstract_scalars}
  Assuming $\int (\annihilate{P}[P]) = 0$, the reader is
  invited to verify that $\forall P,Q. (\prmatrix{0}{Q})(\state{0}) =
  \state{0}$ and dually $(\prmatrix{P}{0})(\event{0}) = \event{0}$.
\end{remark}

\subsubsection{Interpreting continuations}

As promised, we can combine these interpretations with standard
semantics for conditionals and continuations to provide an
interpretation of the iterated experiment.

\begin{lstlisting}[mathescape]
  $\ldb$ let S = $U \state{L}$ in
  let m = $\innerprod{M}{S}^2$ in
  match m with
  v$_0$ -> $\mathcal{E}$
  | $\ldots$
  | v$_N$ -> $\mathcal{E}$
  | v$_{Exit}$ -> m $\rdb(c)$
  $=$
  $(\nu\; c')\meaningof{\innerprod{M}{U\state{L}}}(c')$
  $| c'?(m).m!(m) | \Sigma_{i=0}^{N}v_{i}?(m).\meaningof{\mathcal{E}} + v_{Exit}?(m).c!(m)$
\end{lstlisting}

\paragraph{A quick tally}
Already the interpretation is beginning to show signs of
promise. First of all, it is no more notationally cumbersome than the
notation used in QM calculations. Beyond syntax, we have a new proof
principle in hand and the ability to reason about more complex
experimental situations than is directly calculable in ordinary
quantum mechanics.

\subsubsection{Adjointness}

We need to give a definition of $(\cdot)^{\dagger}$ for matrices. The
obvious candidate definition is
\begin{mathpar}
\meaningof{(\fprmatrix{P}{x}{Q})^{\dagger}}(c)
= (\nu\; u)\meaningof{\fprmatrix{\dualize{Q}\langle u \rangle}{\overline{x}}{\dualize{P}\langle u \rangle}}(c)
\end{mathpar}

% But, $(Q_{\alpha}^{\underline{\perp}})^{*}$ requires a name along
% which to communicate the process to achieve the context application.

\begin{remark}
  i'm a little worried that i don't (yet) have proper support for
  complex conjugacy. But, the observation above may give us a
  clue. According to Abramsky, it must be the case that the scalars
  are iso to the homset of the identity for the tensor -- which the
  observation above (\ref{rem:abstract_scalars}) characterizes. For
  now, we will simply bookmark the notion with $\overline{x}$.
\end{remark}

\subsubsection{Basis for a basis}
If processes label states and ``addition'' of states (a.k.a. vector
addition) is interpreted as parallel composition, what corresponds to
notions of linear independence and basis? Here, we recall that Yoshida
has developed a set of \emph{combinators} for an asynchronous verison
of Milner's $\pi$-calculus \cite{DBLP:conf/concur/Yoshida98}. These
are a finite set of processes such any process can be expressed as
parallel composition of these combinators together with liberal uses
of the new operator and replication. We can simply give a translation
of these into the present calculus and have reasonable expectation
that the property carries over. That is, that the resultant set allows
to express all processes via parallel composition. Note, however, that
there is no new operator or replication in this calculus. As a result,
we expect that the corresponding set is actually infinite. That is, we
expect that the space is actually infinite dimensional.

\begin{remark}
  The reader familiar with the lambda calculus may reasonably object:
  certainly, the collection $S$, $K$ and $I$ is a finite set of
  combinators \cite{Barendregt84}. Shouldn't we expect to see a finite
  set of combinators for an effectively equivalent system? i am very
  sympathetic to this critique and feel it warrants full attention. On
  the other hand, i also have in mind the following analogy. The
  natural numbers, as a monoid under addition, has exactly $1$
  generator, while the natural numbers, as a monoid under
  multiplication, has countably many generators (the primes). We
  observe that the application of the lambda calculus is much less
  resource sensitive than the parallel composition of the
  $\pi$-calculus. Could it be the case that we have an analogy of the
  form

  \begin{mathpar}
    m + n : MN :: m*n : M|N
  \end{mathpar}

  giving a similar blow up in the set of ``primes''?  This is such a
  wonderful thought that, even if it's not true, i think it's worth
  writing down.
\end{remark}


\subsection{Propositional logic}

\subsection{Zero-order logic}

Equality, inequality


\subsubsection{\stid{3.05} Enabling Time Integrators for Exascale Through SUNDIALS}

\paragraph{Overview}

This project is enhancing the SUNDIALS library of numerical software packages for integrating differential systems in time using state-of-the-art adaptive time step technologies for use on exascale systems.  Through software infrastructure developments, this project is enabling the efficient and robust SUNDIALS time integrator packages to easily interoperate with external linear and nonlinear solver packages developed for exascale computing.  In addition, this project is providing a many-vector capability so that SUNDIALS time integrators can more easily operate on data divided over heterogeneous architectures.  Lastly, this project is supporting the deployment and use of SUNDIALS packages within ECP applications, mainly through incorporation into the discretization-based Co-Design Centers, AMReX and CEED.

Efficient time integrators are essential for ECP because they are at the core of every time-dependent simulation application.  However, many applications do not use state-of-the-art methods, and if they do, they often do not yet use them fully on their systems.  For example, at the start of the ECP the astrophysics code, Nyx, used an adaptive integration package for solving individual reactions.  However, by applying a time integration package to a larger reaction system, the code is able to vectorize more of the calculations and get an accurate solution much faster.  By allowing for solvers tuned to exascale systems and vectors that are heterogeneous, SUNDIALS will be more applicable for use in multiphysics systems running on exascale platforms.

\paragraph{Key  Challenges}

Current implementations of efficient time integrators face challenges on many fronts.  First, integrators typically have treated the full physical problem with a single step size or have relied on low order operator splitting methods to couple physical processes at different time scales. While research is moving forward within the time integration community on methods for multirate systems, the software infrastructure needs to be in place to accommodate these schemes once they are developed.   Second, typical integrators operate on problem data in the form of vectors.  These operations suffer from low arithmetic intensity, and their efficiency is often memory bandwidth limited.  Lastly, implicit integrators, which are required in many exascale systems, require efficient linear and nonlinear solvers to be highly effective.  In addition, by applying integrator-dependent controls on these solvers, their efficiency can be significantly increased.  Applying these controls, however, often requires that information about the integrator and its progress be passed to the solver, and software must be designed to effectively pass that information while ensuring adequate encapsulation to provide ease of maintenance and software extension.

\paragraph{Solution Strategy}

This project includes a number of implementation activities that will prepare the SUNDIALS suite of time integrators for exascale systems. The main activity is a redesign of all linear solver interfaces and encapsulation of the nonlinear solvers within the time integrators in SUNDIALS.  The new linear solver interfaces will make it much easier to interface external solver packages while maintaining the efficiency of SUNDIALS integrators. Encapsulating the nonlinear solvers, will reduce redundant code and allow the time integrators to better leverage common code thus lowering the code maintenance burden with SUNDIALS.  In addition, the integrators will be able to take advantage of outside nonlinear solvers.

This project is also introducing a set of optional fused vector kernels into SUNDIALS.  The goal of these kernels is to execute multiple vector operations at once thereby reducing the number of kernel launches in GPU environments and also reducing the number of communications required for reduction operations.  These new kernels will be added to all supplied SUNDIALS vectors and will be invoked through optional interfaces.

Lastly, this project is developing a many-vector capability for SUNDIALS.  Due to the tight data encapsulation within SUNDIALS, users are able to supply any vector they would like underneath the integrators.  This project will supply the infrastructure needed to make it easy to place a vector of vectors underneath the integrators.  This vector of vectors is essential for later implementation of time integrators that will advance various parts of the system with different time step sizes.  This many-vector capability will also ease the use of different programming environments as differing vectors can be instantiated on different parts of a hybrid machine.

\paragraph{Recent Progress}

In September of 2017, SUNDIALS 3.0.0 including new linear solver and matrix interfaces, was released \cite{SUNDIALSweb}.  Figure \ref{fig:sunorg1} shows the new organization of SUNDIALS where separate linear solver interfaces are provided for direct and iterative linear solver methods.  These interfaces are shared across all SUNDIALS integrators.  Individual integrators have the freedom to supply specific information from the integrator that controls the linear solver.  In addition, a single interface to each external solver, such as SuperLU\_MT or KLU, is shared across the suite, as opposed to the prior situation where each integrator included its own interface to each solver.  In addition, a SUNMATRIX class was developed that can be instantiated with a dense, banded, or sparse matrix.  Again, this class is shared by all integrators.

\begin{figure}[htb]
	\centering
	\includegraphics[width=6in]{projects/2.3.3-MathLibs/2.3.3.05-SUNDIALS/sunorg1.pdf}
	\caption{\label{fig:sunorg1}New structure of SUNDIALS showing options for the new SUNLINEARSOLVER and SUNMATRIX classes.}
\end{figure}

\paragraph{Next Steps}

During the remainder of FY18, this project team will:
\begin{enumerate}
\item Complete a release of SUNDIALS with the new fused vector routines implemented within each supplied vector and optionally used from the integrator packages.
\item Complete a release of SUNDIALS with all nonlinear solvers encapsulated and with implementations of the new nonlinear solver interfaces for both the Newton and fixed point solvers used in the integrator packages.
\end{enumerate}