 \begin{table}[!ht]
  \begin{tabular}{|l|l|}
    \hline
    author  & O.Bonnefon, V. Acary\\
    \hline
    date    & Sept, 07, 2007 \\
    last update        & Feb, 2011 \\
                       & April, 2014 \\
    \hline
    version &  \\
    \hline
  \end{tabular}
\end{table}

This section is devoted to the implementation and the study  of the algorithm. The interval of integration is $[0,T]$, $T>0$, and a grid $t_{0}=0$, $t_{k+1}=t_{k}+h$, $k \geq 0$, $t_{N}=T$ is constructed. The approximation of a function $f(\cdot)$ on $[0,T]$ is denoted as $f^{N}(\cdot)$, and is a piecewise constant function, constant on the intervals $[t_{k},t_{k+1})$. We denote $f^{N}(t_{k})$ as $f_{k}$. The time-step is $h>0$.

\section{Various first order dynamical systems with input/output relations}

\paragraph{FirstOrderR. Fully nonlinear case}
Let us introduce the following system,
\begin{equation}
\begin{array}{l}
M \dot{x}(t) = f(x(t),t) + r(t)  \\[2mm]
y(t) = h(t,x(t),\lambda (t)) \\[2mm]
r(t) = g(t,x(t),\lambda (t) ) \\[2mm]
\end{array}
\label{first-DS}
\end{equation}
where $\lambda(t) \in \RR^m$  and $y(t) \in \RR^m$ are  complementary variables related through a multi-valued mapping.   According to the class of systems, we are studying, the function $f$ and $g$ are defined by a fully nonlinear framework or by affine functions. We have decided to present the time-discretization in its full generality and specialize the algorithms for each cases in Section~\ref{Sec:Spec}. This fully nonlinear case is not  implemented in Siconos yet. This fully general case is not yet implemented in Siconos.

This case is implemented in Siconos with the relation {\tt FirstOrderR} using the subtype {NonLinearR}

\paragraph{FirstOrderType1R}
Let us introduce a new notation,
\begin{equation}
\begin{array}{l}
M \dot{x}(t) = f(x(t),t) + r(t)  \\[2mm]
y(t) = h(t,x(t)) \\[2mm]
r(t) = g(t,\lambda (t) ) \\[2mm]
\end{array}
\label{first-DS1}
\end{equation}
This case is implemented in Siconos with the relation {\tt FirstOrderType1R}.

\paragraph{FirstOrderType2R}
Let us introduce a new notation,
\begin{equation}
\begin{array}{l}
M \dot{x}(t) = f(x(t),t) + r(t)  \\[2mm]
y(t) = h(t,x(t),\lambda (t)) \\[2mm]
r(t) = g(t,\lambda (t) ) \\[2mm]
\end{array}
\label{first-DS2}
\end{equation}
This case is implemented in Siconos with the relation {\tt FirstOrderType2R}.

\paragraph{Linear case }Let us introduce a new notation,
\begin{equation}
\begin{array}{l}
M \dot{x}(t) = Ax(t) + r(t)  +b(t)\\[2mm]
y(t) = h(x(t),\lambda (t),z) = Cx + Fz + D \lambda  \\[2mm]
r(t) = g(t,\lambda (t) ) = B \lambda \\[2mm]
\end{array}
\label{first-DS3}
\end{equation}

\section{Time--discretizations}

\subsection{Standard $\theta-\gamma$ scheme.}
Let us now proceed with the time discretization of (\ref{first-DS3}) by a fully implicit scheme :
\begin{equation}
  \begin{array}{l}
    \label{eq:toto1}
     M x_{k+1} = M x_{k} +h\theta f(x_{k+1},t_{k+1})+h(1-\theta) f(x_k,t_k) + h \gamma r(t_{k+1})
     + h(1-\gamma)r(t_k)  \\[2mm]
     y_{k+1} =  h(t_{k+1},x_{k+1},\lambda_{k+1}) \\[2mm]
     r_{k+1} =  g(t_{k+1},x_{k+1},\lambda_{k+1})\\[2mm]
     \mbox{NsLaw} ( y_{k+1} , \lambda_{k+1})
  \end{array}
\end{equation}
where $\theta = [0,1]$ and $\gamma \in [0,1]$. As in \cite{acary2008}, we call the problem \eqref{eq:toto1} the ``one--step nonsmooth problem''.

In the Siconos/Kernel module, the use of $\gamma$  is activated in the class {\tt EulerMoreauOSI} by the boolean {\tt \_useGamma}.

 This time-discretization is slightly more general than a standard implicit Euler scheme. The main discrepancy lies in the choice of a $\theta$-method to integrate the nonlinear term. For $\theta=0$, we retrieve the explicit integration of the smooth and  single valued term $f$. Moreover for $\gamma =0$, the term $g$ is explicitly evaluated. The flexibility in the choice of $\theta$ and $\gamma$ allows the user to improve and control the accuracy, the stability and the numerical damping of the proposed method. For instance, if the smooth dynamics given by $f$ is stiff, or if we have to use big step sizes for practical reasons, the choice of $\theta > 1/2$ offers better stability with the respect to $h$.

\subsection{Full $\theta-\gamma$ scheme}

Another possible time--discretization is as follows.
\begin{equation}
  \begin{array}{l}
    \label{eq:toto1-ter}
    M x_{k+1} = M x_{k} + h\theta f(x_{k+1},t_{k+1})+h(1-\theta) f(x_k,t_k) + h r(t_{k+\gamma}) \\[2mm]
    y_{k+\gamma} = h(t_{k+\gamma},x_{k+\gamma},\lambda _{k+\gamma}) \\[2mm]
    r_{k+\gamma} = g(t_{k+\gamma},x_{k+\gamma},\lambda _{k+\gamma})\\[2mm]
    \mbox{NsLaw} ( y_{k+\gamma} , \lambda_{k+\gamma})
  \end{array}
\end{equation}
We call the scheme~(\ref{eq:toto1-ter}) the full $\theta-\gamma$ scheme since it uses also the evaluation at $t_{k+\gamma}$ for the relation.

In the Siconos/Kernel module, the time--stepping scheme is activated in the class {\tt EulerMoreauOSI} by the boolean {\tt \_useGammaForRelation}.

Another possibility for the time discretization in the nonlinear case would be
\begin{equation}
  \begin{array}{l}
    \label{eq:toto1-quat}
    M x_{k+1} = M x_{k} +h f(x_{k+\theta},t_{k+\theta}) + h r(t_{k+\gamma}) \\[2mm]
    y_{k+\gamma} =  h(t_{k+\gamma},x_{k+\gamma},\lambda _{k+\gamma}) \\[2mm]
    r_{k+\gamma} = g(t_{k+\gamma},x_{k+\gamma},\lambda _{k+\gamma})\\[2mm]
    \mbox{NsLaw} ( y_{k+\gamma} , \lambda_{k+\gamma})
  \end{array}
\end{equation}
This scheme has not been yet implemented in Siconos/Kernel.

\clearpage
\section{Newton's linearization of~(\ref{eq:toto1})}

\input{MCP_linearized.tex}
%\input{MCP_linearized_old.tex}

\input{MCP_linearized_2.tex}
\input{MCP_linearized_linear.tex}

\section{Newton's linearization of~ (\ref{eq:toto1-ter}) }

\input{MCP-FullThetaGamma.tex}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "DevNotes"
%%% End:


\subsection{Spacetime interval}

\subsection{Proper time}


%!TEX root = ../paper.tex
\section{Evaluation and Results}
\label{sec:results}

To evaluate our two separate steps, we need a suitable data set with certain characteristics.
Firstly we want to measure the performance of our clustering into similar author groups.
Based on how this performance changes, when adding or removing features, we can also evaluate the value gain of features.
After we have the results from the clustering, we then use this mapping of blog posts on clusters to evaluate the prediction of new blog posts.
For example the cluster a blog post belongs to in the clustering result, is also the one we want to identify the blog post as, if it was not part of the clustering and classified afterwards.

\input{sections/04a_data_sources}

\subsection{Evaluation of Clustering}
\label{sec:evaluation_clustering}
To evaluate our \textit{k-means} algorithm we evaluated the feasibility and usefulness of our features (see Table~\ref{tab:featureTable}) by splitting them into 12 semantically related groups and then ran our \textit{k-means} clustering algorithm on each of the $2^{12} = 4,096$ possible permutations of active features using our small test data set (see Section~\ref{sec:data_sources}).
We set the number of desired clusters to six, allowing for the possibility of each author getting their own cluster.
We then used this scenario as a gold standard and calculated precision, recall, and F-measure wrt. an author as shown here:
\begin{equation}
	Precision = \frac{TP}{TP + FP} \\
	\label{eq:precision}
\end{equation}
\begin{equation}
	Recall = \frac{TP}{TP + FN} \\
	\label{eq:recall}
\end{equation}
\begin{equation}
	F-measure = \frac{2 \times Recall \times Precision}{Recall + Precision}
	\label{eq:fMeasure}
\end{equation}
For each author we find the cluster, which contains the highest number of blog posts from that author, called dominant cluster.
We assume that this is the best matching cluster for the author, so all blog posts from that author should belong to that cluster.
Based on this assumption, if author A has the dominant cluster C, we then define $TP$, $FP$ and $FN$ in the following way:
\begin{itemize}
	\item[$TP$:] Number of blog posts of author A, which are assigned to cluster C. \\
	\item[$FP$:] Number of blog posts from other authors, which are assigned to cluster C. \\
	\item[$FN$:] Number of blog posts of author A, which are not assigned to cluster C.
\end{itemize}

From these $4,096$ individual test runs we selected some meaningful data points to base our conclusions on.
One of these meaningful feature combinations was the one that gave the best overall F-measure of $61.62\%$.
This feature combination did not contain the \textit{prefix/suffix} and the \textit{part-of-speech tag} features.
This was surprising for us, because~\cite{madigan2005author} listed these as some of the features that performed best for them and because our own tests showed that having only these features active on their own provided us with better results than having only one of the other features turned on.
Looking further into the data, we found that turning off the \textit{prefix/suffix} or the \textit{part-of-speech tag} feature while all other features remained active provided us with better results than having all features active at a time (see Table~\ref{tab:feature_evaluation_1}).

\begin{table}[ht!]
	\begin{center}
    \begin{tabular}{l|r|r}
	Feature name		& F-measure without & Difference to all features \\ \hline \hline
	All features active	& 0.4152833313 & $\pm$0.0 \\ \hline \hline
	Emoticon			& 0.3757936508 & +0.0438422688 \\ \hline
	Blank line			& 0.4074074074 & +0.0078759239 \\ \hline
	Post length			& 0.4196359196 & +0.0043525883 \\ \hline
	Word frequency		& 0.4130983959 & +0.0021849354 \\ \hline
	Upper case			& 0.4144100548 & +0.0008732765 \\ \hline
	Capital letter		& 0.4148865024 & +0.0003968289 \\ \hline
	Function word		& 0.4150935674 & +0.0001897639 \\ \hline
	Character frequency	& 0.4151904736 & +0.0000928577 \\ \hline
	Word length			& 0.4152373925 & +0.0000459388 \\ \hline
	Sentence length		& 0.4152833313 & $\pm$0.0 	   \\ \hline
	Part-of-speech tag				& 0.4196359196 & -0.0043525883 \\ \hline
	Prefix/suffix		& 0.4712048408 & -0.0559215095 \\
    \end{tabular}
    \end{center}
	\caption{The F-measure having all features but one active and the difference to the F-measure having all features active. Having the \textit{part-of-speech tag} feature or the \textit{prefix/suffix} feature active, decreases the overall F-measure.}
	\label{tab:feature_evaluation_1}
\end{table}

We concluded that the \textit{prefix/suffix} and \textit{part-of-speech tag} features provided merely decent (but not great) results on their own.
Due to their high number of individual features, we can use them alone quite well to distinguish blog posts, however the overall quality is not as good as other single features combined.
Features with only one individual value might provide a clearer separation but only in a single dimension.
This means that they are not good for creating a larger number of clusters, which is what would have been required for a good performance in our test with only one active feature.
However, due to the clear separation they provide, adding more small features (and thus dimensions) drastically improves the test results.
If we then add in the high dimensional but only mediocre features, such as \textit{prefix/suffix} and \textit{part-of-speech tag} as well, the test results become worse, because the positive impact of the low dimensional features is lessened by the sheer amount of dimensions added to the feature vector.
Thus, their good performance is being overshadowed by the high dimensional features' mediocrity.

Ultimately, we decided not to use the \textit{prefix/suffix} and \textit{part-of-speech tag} features, since they both lowered performance significantly.
Dropping them both meant that the number of features we had in a vector was now only a fraction of the previous amount, which gave our small but well-performing features much more impact.

The resulting F-measure for our \textit{k-means} algorithm when having all but the \textit{prefix/suffix} and \textit{part-of-speech tag} feature enabled is $61.64\%$.
When looking at the individual blog posts' classification, we noticed that the blog posts of two authors would mostly be sorted into the same cluster due to their feature vectors being quite similar.
We concluded that these two authors actually had a similar writing style and that this meant that the F-measure could not become much better due to our gold standard assuming one cluster per author.

\subsection{Evaluation of Classification}
\label{sec:evaluation_classification}

For evaluating our classification methods we used ten-fold cross validation~\cite{kohavi1995study}.
We divide our test data set into ten sets, each set containing nine blog posts.
Then we train our classification algorithms on nine of these ten sets and use the last set for testing.
This process is repeated ten times, so each one of these sets was a test set once.
Each time the precision is calculated and at the end the overall average precision is determined.

Training the \textit{support vector machine} means that a new model based on the training data set is created.
This model is then used for determining a cluster for each blog post in the test set.
The \textit{k-nearest neighbor} algorithm uses the training data set as known data points (see Section~\ref{sec:k_nearest_neighbor}).

After finding the optimal parameters for both methods, we achieved $97.77\%$ precision for \textit{k-nearest neighbor} and $98.89\%$ precision for our \textit{support vector machine} in the ten-fold cross validation.

Thus, our result of $\sim60\%$ F-measure for clustering appears to be a lot better than those from related work in this field.
This is mainly due to the different problem domain though:
We merely try to find similar writing style groups, while most of the related work tried to identify the exact author to a document.
Since this problem is harder than ours, simply because the solution space is a lot larger, results that appear worse at first glance (such as the $20\%$ accuracy of Narayanan et al.~\cite{narayanan2012feasibility}) are to be expected.
On the other hand, Koppel et al.~\cite{koppel2003automatically}, who try to correctly identify an author's gender by their writing style, achieve a precision of over $80\%$ since this problem is smaller than even our six assumed writing styles.
The fact that our results lie somewhere between these two other approaches to (respectively) more and less difficult problems is thus exactly what was to be expected.


% File SDSS2020_SampleExtendedAbstract.tex
\documentclass[10pt]{article}
\usepackage{sdss2020} % Uses Times Roman font (either newtx or times package)
\usepackage{url}
\usepackage{latexsym}
\usepackage{amsmath, amsthm, amsfonts}
\usepackage{algorithm, algorithmic}
\usepackage{graphicx}

\title{Sparse Bayesian Predictive Modelling of Tumor Response
	from Radiomic Data}

\author{
  Shirin Golchi\\
  Department of Biostatistics\\ McGill University \\
  Purvis Hall, 1020 Pine Ave W\\
  Montreal QC H3A 1A2 \\
  {\tt shirin.golchi@mcgill.ca} \\\And
  Reza Forghani \\
  Department of Diagnostic Radiology \\ McGill University \\
  1650 Cedar Avenue \\
  Montreal QC H3G 1A4 \\
  {\tt reza.forghani@mcgill.ca} \\\And
  Sahir Bhatnagar \\
  Department of Biostatistics\\ Department of Diagnostic Radiology \\ McGill University \\
  Purvis Hall, 1020 Pine Ave W\\
  Montreal QC H3A 1A2 \\
  {\tt sahir.bhatnagar@mcgill.ca}
}

\date{}

\begin{document}
\maketitle
\begin{abstract}
We propose a Bayesian hierarchical model for the analysis of radiomic data for characterization of head and neck squamous cell carcinoma (HNSCC). The proposed model facilitates radiomic feature selection, dealing with missing values in key predictors as well as prediction in a unified framework.

\end{abstract}

{\bf Keywords:} Radiomics, Multilevel Modeling, Horseshoe prior, Missing Data

\section{Introduction}

An important objective in oncology is the creation of a standardized set of criteria to predict and monitor tumor response to treatment and for outcome prognosis based on objectively measured biomarkers. In addition to the traditional role of imaging for staging and post treatment follow-up of HNSCC, there is increasing interest in the use of quantitative image extracted or radiomic features for characterization of HNSCC. Image analysis algorithms extract mathematically defined features of the tumor's appearance giving rise to high-dimensional matrix covariates.

Many challenges arise from the structure of radiomic data. Namely, an efficient and reliable variable selection technique is required to select a reasonable number of radiomic features to be used in prediction of key outcomes such as lymph node metastasis. Variable selection and prediction should reflect the heterogenity among tumor sites, however, site-stratified inference can result in low statistical power. In addition, there is a considerable amount of missing data among important predictors such as the presence/absence of human papilloma virus (HPV).

We propose a Bayesian hierarchical model that can address radiomic feature selection and prediction in a unified framework while dealing with complexities such as missing values in predictors. The hierarchical nature of the model enables information borrowing across tumor sites while allowing site-specific variable selection and parameter estimation. Integrating variable selection and missing data handling together with inference, results in predictions with adequate representation of uncertainty associated with each of these procedures. We present the results of the analysis as Bayesian feature selection outcomes across sites and the accuracy for predictions of lymph node metastasis.

\section{Methods}

Below we describe a Bayesian hierarchical model that takes advantage of regularized horseshoe priors \cite{PiiVeh2017} to perform site-specific radiomic feature selection while borrowing information across sites.  Let $y_{1n}$ and $y_{2n}$ denote the binary outcomes lymph node metastasis and HPV for patient $n = 1, \ldots, N$, respectively. While HPV may be predicted by a number of covariates such drinking and smoking habits, it is an important predictor for lymph node metastasis. Therefore, we define the model as follows,
\begin{equation}
\label{eqn:model}
y_{1n} \sim \text{Bernoulli}(\pi_{1n}),\hskip 10pt y_{2n} \sim \text{Bernoulli}(\pi_{2n})
\end{equation}
where,
\begin{equation*}
\text{logit}(\mathbf{\pi}_{1n}) = \phi \pi_{2n} + \mathbf{z}_n \boldsymbol{\eta}_1+ \mathbf{x}_n\boldsymbol{\beta}_{1s_n}, \hskip 20pt n = 1, \ldots,N
\end{equation*}
where $\pi_{2n}$ is the risk of HPV for patient $n$ that is in turn modelled as,
\begin{equation*}
\text{logit}(\pi_{2n}) = \mathbf{z}_n \boldsymbol{\eta}_2+ \mathbf{x}_n\boldsymbol{\beta}_{2s_n}, \hskip 20pt n = 1, \ldots,N
\end{equation*}
where $s_n = 1,\ldots, S$ are the tumor site, $ \mathbf{z}_n$ are the set of covariates (drinking, smoking and T-stage group)  and  $\mathbf{x}_n$ is the $F\times 1$ vector of radiomic features for patient $n$.

Allowing for the feature selection to be performed separately across the three sites introduces $F\times S$ coefficients. The notation $\boldsymbol{\beta}_{s_n}$ is used to represent the site-specific set of the radiomic feature coefficients matrix.

Following \cite{PiiVeh2017} the coefficients, $\beta_j$, $j = 1, \ldots, J$, of the radiomic features are assigned the following prior distribution,
\begin{align*}
&\beta_{j,s_n} \sim \mathcal{N}(0, \tau^2_{s_n}\tilde{\lambda}_j^2)\\
& \tilde{\lambda}_j^2 = \frac{c^2\lambda_j^2}{c^2 + c^2\lambda_j^2}
\end{align*}
where
\begin{align*}
&\lambda_j \sim \mathcal{C}^+(0,1),\\
& c^2\sim \mathcal{IG}(\frac{\nu}{2},\frac{\nu}{2}s^2),\\
& \tau_{s_n} \sim \mathcal{C}^+(0, \tau_0).
\end{align*}
where $\nu = 20$, $s^2 = 4$ and $\tau_0=0.001$. These values are chosen according to recommendations in \cite{PiiVeh2017}. Note the subscript $s_n$ for parameter $\tau$ that represents the site specific variance for $\beta_{j,s_n}$.

As mentioned earlier, a considerable portion of the patients have a missing HPV outcome. The above model is dealing with this issue by augmenting the HPV data with the unknown values of HPV and estimating the missing values together with the rest of the model parameters.

The inference relies on the posterior distribution of parameters of interest, i.e., $(\phi, \boldsymbol{\eta}_1, \boldsymbol{\eta}_2, \boldsymbol{\beta}_{1s_n}, \boldsymbol{\beta}_{2s_n})$. Sampling from the posterior is performed using Stan.

\section{Data/Results}

The data comprise 603 contrast enhanced pre-treatment neck CT scans evaluated from patients diagnosed with HNSCC, with tumors arising in three sites: 241 from the larynx or hypopharynx (LHP), 162 oral cavity (OC), and 200 oropharynx (OP), further stratified based on HPV status to avoid its confounding effects. HPV status was missing in 55\% of the sample, with 175 missing from the LHC, 3 from the OPC and 155 from the OSCC. First order texture features with additional filtrations were extracted from each tumor using TexRAD software (TexRAD; University of Sussex, Falmer, England) and used in conjunction with patient age, smoking status, drinking status, and tumor T-stage to construct models for predicting lymph node metastasis, the presence of lymphovascular invasion (LVI) and perineural invasion (PNI).
%***Can you please add a little more about the missing portions, etc?***
We apply the proposed model to the described data set. Figure~\ref{fig1} shows the point estimates and 95\% credible intervals of the radiomic features by tumor site. Only a handful of radiomic features are selected by the shrinkage prior to predict the lymph node metastasis and selected features vary by tumor site.

\begin{figure}[htbp]
	\begin{center}
		\centerline{\includegraphics[width=\columnwidth]{mv_NS_feat.png}}
		\caption{Site-specific point estimates and 95\% credible intervals for the radiomic feature coefficients in the predictive model for lymph node metastasis}
		\label{fig1}
	\end{center}
\end{figure}

Prediction accuracy of the proposed model is illustrated in Figure~\ref{fig2} in form of 100 draws from the posterior distribution of the Receiver Operating Curves (ROC). The Area Under the Curve (AUC) varies from 0.81 to 0.84.

\begin{figure}[htbp]
	\begin{center}
		\centerline{\includegraphics[width=0.9\columnwidth]{ROC.png}}
		\caption{Posterior draws from the Receiver Operating Curve for prediction of lymph node metastasis; The Area Under the Curve varies from 0.81 to 0.84.}
		\label{fig2}
	\end{center}
\end{figure}

\section{Discussion/Conclusions}

We have proposed a Bayesian hierarchical predictive model to analyse complex and high dimensional radiomic data. The novelty of the proposed model is in dealing with multiple challenging issues within a unified framework.  Namely, variable selection, missing data imputation and prediction are performed simultaneously within the proposed model. The main advantage of integrating variable (feature) selection and missing data handling together with estimation and prediction over multi-step procedures, is adequate representation of uncertainty in the final results. We further compared our apporach to popular machine learning techniques often used in the radiomics literature. For example, the random forest method applied to this data achieved an AUC of 0.76, but there was no way to obtain a site specific feature importance measure and HPV status was ignored in the training due to missingness.
%*** Any comments regarding the performance in compare to existing methods? ***

\bibliographystyle{sdss2020} % Please do not change the bibliography style
\bibliography{SampleReferencesForExtendedAbstract}

\end{document}


\documentclass[12pt,letterpaper]{article}
\usepackage{fullpage}
\usepackage[top=2cm, bottom=4.5cm, left=2.5cm, right=2.5cm]{geometry}
\usepackage{amsmath,amsthm,amsfonts,amssymb,amscd}
\usepackage{hyperref}
% \usepackage{xcolor}
\usepackage[dvipsnames]{xcolor}
\usepackage{fancyhdr}
\usepackage{mathrsfs}
\usepackage{amsmath}
\usepackage{dsfont}

\usepackage{mathtools}
\DeclarePairedDelimiter{\ceil}{\lceil}{\rceil}
\DeclarePairedDelimiter{\set}{\left\{}{\right\}}

\usepackage{fontspec}

\setromanfont{PTSerif}[
    Path=./fonts/,
    Extension = .ttf,
    UprightFont = *-Regular,
    BoldFont = *-Bold,
    ItalicFont = *-Italic,
    BoldItalicFont = *-BoldItalic,
]

\hypersetup{%
  colorlinks=true,
  linkcolor=blue,
  linkbordercolor={0 0 1}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Define Colors
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\definecolor{light-gray}{gray}{0.85}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newcommand\hwnumber{16}
\newcommand\student{Ivan Zhytkevych}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\pagestyle{fancyplain}
\headheight 35pt
\lhead{\textit{\student}}
\chead{\textbf{\Large PT Homework \hwnumber}}
\rhead{\textit{\today}}
\lfoot{}
\cfoot{}
\rfoot{\small\thepage}
\headsep 1.5em

% \usepackage{import}
% \usepackage{pdfpages}
% \usepackage{transparent}
% \usepackage{xcolor}

% \newcommand{\incfig}[2][1]{%
%     \def\svgwidth{#1\columnwidth}
%     \import{./figures/}{#2.pdf_tex}
% }
\usepackage{import}
\usepackage{xifthen}
\usepackage{pdfpages}
\usepackage{transparent}

\newcommand{\incfig}[1]{%
    \def\svgwidth{\columnwidth}
    \import{./figures/}{#1.pdf_tex}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}
% \tableofcontents
% \newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Problem 15.18}

\[ \varphi_{S_n}(t) = \varphi_{\xi_1 + \xi_2 + \dots + \xi_n}(t) = \varphi_{\xi_1}(t) \dots \varphi_{\xi_n}(t) \]
\[ \varphi_{\xi_i}(t) = \sum_{k=1}^{\infty} e^{itx_k} \cdot P(\xi = x_k) =
\frac{1}{2} e^{-it} + \frac{1}{2} \cdot e^{it} =  \cos(t) \]
\[ \varphi_{S_n}(t) = \cos^n (t) \]

\begin{figure}[ht]
    \centering
    \incfig{graph}
    \caption{Graph}
    \label{fig:graph}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Problem 11.21}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Problem 11.22}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Problem 11.23}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Problem 11.24}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Problem 11.25}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\end{document}


\documentclass{article}
\usepackage{graphicx}
\usepackage[utf8]{inputenc}
\usepackage{amsmath, amssymb, latexsym}

\usepackage{pgfplots}
\usepackage{algorithm}
\usepackage[noend]{algpseudocode}
\usepackage{tikz}
\usepackage{nicefrac}
\usepackage{placeins}
\pgfplotsset{every axis legend/.append style={
at={(0,0)},
anchor=north east}}
\usetikzlibrary{shapes,positioning,intersections,quotes}
\usetikzlibrary{arrows.meta,
                bending,
                intersections,
                quotes,
                shapes.geometric}

\definecolor{darkgreen}{rgb}{0.0, 0.6, 0.0}
\definecolor{darkred}{rgb}{0.7, 0.0, 0.0}
\makeatletter
\def\BState{\State\hskip-\ALG@thistlm}
\makeatother
\title{Week 17}
\begin{document}
\pagenumbering{gobble}
\maketitle
\newpage
\pagenumbering{arabic}

\section*{Learning with large datasets}
Taking a low bias algorithm and training it on a large amount of data is one of the greatest approaches to get high performance.

\begin{itemize}
  \item Assume we have a data collection with m = 100,000, 000.
  \item On such a large system, how can we train a logistic regression model?

        $$\theta_j := \theta_j - \alpha \frac{1}{m} \sum_{i=1}^m(h_{\theta}(x^{(i)})- y^{(i)})x_j^{(i)}$$

  \item As a result, for each gradient descent step, you must add up more than 100,000,000 terms.
  \item The first step is to inquire whether we may train on 1000 examples rather than 100 000 000.
  \item Pick a small sample at random. Can you build a system that performs well?
\end{itemize}

~\\
\includegraphics[width=0.5\textwidth]{resources/learning_curve}

\begin{itemize}
  \item If the gap is large, it is a high variance problem. More examples should lead to better results.
  \item If the gap is tiny, it is a high bias problem. More examples may not be beneficial.
\end{itemize}

\section*{Stochastic Gradient Descent}

Hypothesis:
$$h_{\theta}(x) = \sum_{n}^{j=0}\theta_jx_j$$

~\\
Cost function:
$$\quad J_{train}(\theta) = \frac{1}{2m} \sum_{m}^{i=1}(h_{\theta}(x^{(i)}) - y^{(i)})^2$$

~\\
We get bowl shape surface plots if we plot our two parameters against the cost function:

\includegraphics[width=0.5\textwidth]{resources/surface_cost_function}

~\\
How does gradient descent work?

\begin{algorithm}
  \caption{Gradient Descent}\label{euclid}
  \begin{algorithmic}[1]
    \Large
    \While {not converged}
    \State $\theta_j := \theta_j - \alpha \frac{\partial}{\partial \theta_j} J(\theta_0, ..., \theta_n)$
    \State $(for\ j = 0, ..., n)$
    \EndWhile
  \end{algorithmic}
\end{algorithm}

~\\
Although we have just referred to it as gradient descent so far, this type of gradient descent is known as batch gradient descent. This simply implies that we examine all of the examples at the same time. Batch gradient descent is unsuitable for large datasets.

~\\
When discussing stochastic gradient descent, we will use linear regression as an algorithmic example, however the concepts apply to other algorithms as well, such as logistic regression and neural networks.

~\\
Define our cost function in a slightly different way, as follows:
$$\quad J_{train}(\theta) = \frac{1}{m} \sum_{m}^{i=1}cost(\theta, (x^{(i)}, y^{(i)}))$$
$$cost(\theta, (x^{(i)}, y^{(i)}))=\frac{1}{2}(h_{\theta}(x^{(i)}) - y^{(i)})^2$$

\FloatBarrier
\begin{algorithm}
  \caption{Stochastic Gradient Descent}\label{euclid}
  \begin{algorithmic}[1]
    \Large
    \State Randomly shuffle training examples
    \For{\texttt{$i:=1,...,m$}}
    \State \texttt{$\theta_j := \theta_j - \alpha (h_{\theta}(x^{(i)}) - y^{(i)})x_j^{(i)}$}
    \State \texttt{$(for\ j = 0, ..., n)$}
    \EndFor
  \end{algorithmic}
\end{algorithm}
\FloatBarrier

The random shuffling at the start ensures that the data is in a random sequence, preventing bias in the movement.

~\\
Although stochastic gradient descent is similar to batch gradient descent, instead of waiting for the gradient terms to be summed across all m examples, we pick only one example and make work on enhancing the parameters right away.

~\\
This means that we change the parameters on EVERY step through the data, rather than at the end of each loop over all of the data.

~\\
\includegraphics[width=0.6\textwidth]{resources/stochastic}

~\\
For our stochastic gradient descent, we might plot the cost function vs the number of iterations.
We should be able to see if convergence is occurring by looking at the graphs.
A smoother curve may be obtained by averaging across many (e.g. 1000 and 5000) instances.

\includegraphics[width=0.45\textwidth]{resources/stochastic_convergence}

\section*{Online learning}

\begin{itemize}
  \item Allows us to model problems in which there is a continuous stream of data from which an algorithm should learn.
  \item In the same way that stochastic gradient descent is used, slow updates are performed.
  \item To learn from traffic, web companies employ several sorts of online learning algorithms.
\end{itemize}

\subsection*{Example - product search}
\begin{itemize}
  \item Assume you own a cellphone-selling website.
  \item You have a user interface where the user may enter in a query such as "Android phone 1080p camera."
  \item We want to provide the user ten phones per query, with the phones ranked from most appealing to the user.
  \item We generate a feature vector (x) for each phone based on a specific user query.
  \item We want to determine the likelihood of a user picking a phone.
  \item $y = 1$ if a user clicks on a link.
  \item $y = 0$ otherwise.
  \item We learn $p(y=1|x;\theta)$ - this is the problem of learning the predicted click through rate (CTR).
  \item If you can estimate the CTR for any phone, we can utilize it to show the phones with the highest likelihood first.
\end{itemize}

\section*{Map reduce and data parallelism}
Some problems are simply too large for a single CPU to handle.

~\\
\includegraphics[width=0.6\textwidth]{resources/map_reduce}

Parallelization can come from:
\begin{itemize}
  \item Multiple machines.
  \item Multiple CPUs.
  \item Multiple cores in each CPU.
\end{itemize}

~\\
Certain numerical linear algebra libraries can automatically parallelize your calculations over several cores, depending on the implementation details.
So, if this is the case and you have a decent vectorization implementation, you don't have to worry about local parallelization because the local libraries will handle optimization for you.

~\\
Hadoop is an example of Map Reduce implementation.

\end{document}


% paper # 3887, pw CF8FECE1
% Note: the original submission was rev. 530...
% submitted again at 539 before review..
% Template for SLT-2006 paper; to be used with:
%          spconf.sty  - ICASSP/ICIP LaTeX style file, and
%          IEEEbib.bst - IEEE bibliography style file.
% --------------------------------------------------------------------------
\documentclass{article}
\usepackage{spconf,amsmath,amssymb,bm,epsfig}
\usepackage{color}
\usepackage{multirow}

\definecolor{MidGray}{rgb}{0.3, 0.3, 0.3}
\definecolor{Pink}{rgb}{1.0, 0.4, 0.4}
\definecolor{Red}{rgb}{0.9, 0.1, 0.1}

% Example definitions.
% --------------------

%%%%%%
% Begin Dan's math definitions.
%%%%%%
\def\Re{{\mathbb{R}}}
\def\x{{\mathbf x}}
\def\a{{\mathbf a}}
\def\U{{\mathbf U}}
\def\HCLG{{\mathit{HCLG}}}
\def\bDelta{{\mathbf \Delta}}
\def\V{{\mathbf V}}
\def\u{{\mathbf u}}
\def\s{{\mathbf s}}
\def\t{{\mathbf t}}
\def\G{{\mathbf G}}
\def\F{{\mathbf F}}
\def\r{{\mathbf r}}
\def\semicolonsep{\,;\,}
\def\tr{{\operatorname{tr}\,}}
\def\z{{\mathbf z}}
\def\y{{\mathbf y}}
\def\Y{{\mathbf Y}}
\def\Z{{\mathbf Z}}
\def\gammatilde{{\gamma}}  %% This was \tilde{\gamma} in the tutorial, mapping to \gamma2 to preserve the distinction in the latex source in case later I want to resurrect that.
\def\R{{\mathbf R}}
\def\D{{\mathbf D}}
\def\half{ {\textstyle {\frac{1}{2}}} }
\def\L{{\mathbf L}}
\def\P{{\mathbf P}}
\def\Q{{\mathbf Q}}
\def\B{{\mathbf B}}
\def\S{{\mathbf S}}
\def\T{{\mathbf T}}
\def\K{{\mathbf K}}
\def\H{{\mathbf H}}
\def\g{{\mathbf g}}
\def\I{{\mathbf I}}
\def\btheta{{\mathbf\theta}}
\def\calN{{\cal N}}
\def\calQ{{\cal Q}}
\def\diag{{\mathrm{diag}}}
\def\pre{{\mathrm{pre}}}
\def\inv{{\mathrm{inv}}}
\def\calT{{\cal T}}
\def\bmu{{\bm \mu}} % mathbf does not work for lowercase greek letters!  But mathbf looks better than bm in most other cases so I only use bm here.
\def\bSigma{{\mathbf \Sigma}}
\def\M{{\mathbf M}}
\def\N{{\mathbf N}}
\def\bv{{\mathbf v}}
\def\w{{\mathbf w}}
\def\W{{\mathbf W}}
\def\A{{\mathbf A}}
\def\o{{\mathbf o}}
\def\b{{\mathbf b}}
\def\p{{\mathbf p}}
%%\newcommand\refeqn[1]{(\ref{#1})}

%%%%%%
% End Dan's math definitions.
%%%%%%

% Title.
% ------
\title{Generating exact lattices in the WFST framework}

\makeatletter
\def\name#1{\gdef\@name{#1\\}}
\makeatother

\name{ \em Daniel Povey$^1$, Mirko Hannemann$^{1,2}$,  \\
 \em {Gilles Boulianne}$^3$, {Luk\'{a}\v{s} Burget}$^{2,4}$, {Arnab Ghoshal}$^5$, {Milo\v{s} Janda}$^2$, {Martin Karafi\'{a}t}$^2$, {Stefan Kombrink}$^2$, \\
  \em {Petr Motl\'{i}\v{c}ek}$^6$, {Yanmin Qian}$^7$, {Korbinian Riedhammer}$^9$, {Karel Vesel\'{y}}$^2$, {Ngoc Thang Vu}$^8$
    \thanks{Thanks to Honza \v{C}ernock\'{y}, Renata Kohlov\'{a}, and Tom\'{a}\v{s} Ka\v{s}p\'{a}rek for their help relating to the Kaldi'11 workshop at BUT, and to Sanjeev Khudanpur for his help in preparing the paper.  Researchers at BUT were partly supported by Technology Agency of the Czech Republic grant No. TA01011328, Czech Ministry of Education project No. MSM0021630528, and Grant Agency of the Czech Republic project No. 102/08/0707.  Arnab Ghoshal was supported by EC FP7 grant 213850 (SCALE), and by EPSRC grant EP/I031022/1 (NST).}}

%%% TODO: fix thanks.
%\thanks{Arnab Ghoshal was supported by the European Community's Seventh Framework
% Programme under grant agreement no. 213850 (SCALE); BUT researchers were partially
%supported by Czech MPO project No. FR-TI1/034.}}

\address{{ $^1$ Microsoft Research, Redmond, WA, {\normalsize \tt dpovey@microsoft.com} }\\
         { $^2$ Brno University of Technology, Czech Republic, {\normalsize \tt ihannema@fit.vutbr.cz}} \\
         { $^3$ CRIM, Montreal, Canada \ \  $^4$ SRI International, Menlo Park, CA, USA} \\
         { $^5$ University of Edinburgh, U.K. \ \  $^6$  IDIAP, Martigny, Switzerland } \\
         { $^7$ Tsinghua University, Beijing, China \ \ $^8$ Karlsruhe Institute of Technology, Germany} \\
         { $^9$ Pattern Recognition Lab, University of Erlangen-Nuremberg, Germany}}

\begin{document}

\ninept
%
\maketitle
%
%\pagestyle{plain} % was set to empty in spconf.sty... this makes page number appear.

\begin{abstract}
We describe a lattice generation method that is exact, i.e. it satisfies all the
natural properties we would want from a lattice of alternative transcriptions of
an utterance.  This method does not introduce substantial overhead above one-best decoding.  Our method is
most directly applicable when using WFST decoders where the WFST is
``fully expanded'', i.e. where the arcs correspond to HMM transitions.  It outputs
lattices that include HMM-state-level alignments as well as word labels.
The general idea is to create a state-level lattice during decoding, and to do
a special form of determinization that retains only the best-scoring path
for each word sequence.
This special determinization algorithm is a solution to the following problem:
Given a WFST A, compute a WFST B that, for each input-symbol-sequence of A,
contains just the lowest-cost path through A.
\end{abstract}

\begin{keywords}
  Speech Recognition, Lattice Generation
\end{keywords}

\vspace*{-0.075in}
\section{Introduction}
\vspace*{-0.05in}

In Section~\ref{sec:wfst} we give a Weighted Finite State Transducer
(WFST) interpretation of the speech-recognition decoding problem, in order
to introduce notation for the rest of the paper.  In Section~\ref{sec:lattices}
we define the lattice generation problem, and in Section~\ref{sec:previous}
we review previous work.
In Section~\ref{sec:overview} we give an overview of our method,
and in Section~\ref{sec:details} we summarize some aspects of a determinization
algorithm that we use in our method.  In Section~\ref{sec:exp} we give
experimental results, and in Section~\ref{sec:conc} we conclude.

\vspace*{-0.075in}
\section{WFSTs and the decoding problem}
\vspace*{-0.05in}
\label{sec:wfst}

The graph creation process we use in our toolkit, Kaldi~\cite{kaldi_paper},
is very close to the standard recipe described in~\cite{wfst},
where the Weighted Finite State Transducer (WFST) decoding graph is
\begin{equation}
  \HCLG = \min(\det(H \circ C \circ L \circ G)),
\end{equation}
where $H$, $C$, $L$ and $G$ represent the HMM structure, phonetic
context-dependency, lexicon and grammar respectively,
and $\circ$ is WFST composition (note: view $\HCLG$ as a single symbol).
For concreteness we will speak of ``costs'' rather
than weights, where a cost is a floating point number that typically represents a negated
log-probability.  A WFST has a set of states with one distinguished
start state\footnote{This is the formulation that corresponds best with the toolkit we use.};
each state has a final-cost (or $\infty$ for non-final states);
and there is a set of arcs between the states, where each arc has an
input label, an output label, and a weight
(just think of this as a cost for now).
In $\HCLG$, the input labels are the identifiers of context-dependent
HMM states, and the output labels represent words.  For both the input and output
labels, the special symbol $\epsilon$ may appear, meaning ``no label is present.''

\begin{figure}
\begin{center}
\includegraphics[height=2.1in,angle=270]{figures/acceptor_utterance.eps}
\vspace*{-0.03in}
  \caption{Acceptor $U$ describing the acoustic scores of an utterance}
\vspace*{-0.15in}
\label{fig:acceptor}
\end{center}

\end{figure}

Imagine we want to ``decode'' an utterance of $T$ frames, i.e. we want to
find the most likely word sequence and its corresponding state-level alignment.  A WFST
interpretation of the decoding problem is as follows.  We construct an
acceptor, or WFSA, as in Fig.~\ref{fig:acceptor} (an acceptor is represented as a
WFST with identical input and output symbols).  It has $T{+}1$ states,
with an arc for each combination of (time, context-dependent HMM state).  The
costs on these arcs correspond to negated and scaled acoustic log-likelihoods.
Call this acceptor $U$.  Define
\begin{equation}
   S \equiv U \circ \HCLG,
\end{equation}
which we call the {\em search graph} of the utterance.  It has approximately $T{+}1$ times
more states than $\HCLG$ itself.  The decoding problem is equivalent to finding
the best path through $S$.  The input symbol sequence for this best path represents
the state-level alignment, and the output symbol sequence is the corresponding
sentence.  In practice we do not do a full search of $S$, but use beam pruning.
Let $B$ be the searched subset of $S$, containing a subset of the states and arcs
of $S$ obtained by some heuristic pruning procedure.
When we do Viterbi decoding with beam-pruning, we are finding the best path through $B$.
%
Since the beam pruning is a part of any practical search procedure and cannot
easily be avoided, we will define the desired outcome of lattice generation in terms
of the visited subset $B$ of $S$.

\vspace*{-0.075in}
\section{The lattice generation problem}
\vspace*{-0.05in}
\label{sec:lattices}

There is no generally accepted single definition of a lattice.  In~\cite{efficient_general}
and~\cite{sak2010fly}, it is defined as a labeled, weighted, directed acyclic graph
(i.e. a WFSA, with word labels).  In~\cite{ney_word_graph}, time information
is also included.  In the HTK lattice format~\cite{htkbook}, phone-level time alignments
are also supported (along with separate language model, acoustic and pronunciation-probability
scores), and in~\cite{saon2005anatomy}, HMM-state-level alignments are also produced.
In our work here we will be producing state-level alignments; in fact, the input-symbols
on our graph, which we call {\em transition-ids}, are slightly more fine-grained
than acoustic states and contain sufficient information to reconstruct the phone
sequence.

There is, as far as we know, no generally accepted problem statement
for lattice generation, but all the the authors we cited seem to
be concerned with the accuracy of the information in the lattice (e.g. that the
scores and alignments are correct) and the completeness of such information (e.g.
that no high-scoring word-sequences are missing).  The simplest
way to formalize these concerns is to express them in terms of a lattice
pruning beam $\alpha > 0$ (interpret this as a log likelihood difference).
\begin{itemize}
  \item The lattice should have a path for every word sequence within $\alpha$ of the best-scoring one.
  \item The scores and alignments in the lattice should be accurate.
  \item The lattice should not contain duplicate paths with the same word sequence.
\end{itemize}
We need to be a little more precise about what we mean by the scores
and alignments being ``accurate''.  Let the lattice be $L$.  The
way we would like to state this requirement is:
\begin{itemize}
  \item For every path in $L$, the score and alignment corresponds to
   the best-scoring path in $B$ for the corresponding word
   sequence\footnote{Or one of the best-scoring paths, in case of a tie.}.
\end{itemize}
The way we actually have to state the requirement in order to get an efficient procedure is:
\begin{itemize}
  \item For every word-sequence in $B$ within $\alpha$ of the best one, the score and alignment
    for the corresponding path in $L$ is accurate.
  \item All scores and alignments in $L$ correspond to actual paths through $B$ (but not always
   necessarily the best ones).
\end{itemize}
The issue is that we want to be able to prune $B$ before generating a lattice from it,
but doing so could cause paths not within $\alpha$ of the best one to be lost, so we have
to weaken the condition.  This is no great loss, since regardless
of pruning, any word-sequence not within $\alpha$ of the best one could be omitted altogether,
which is the same as being assigned a cost of $\infty$).
By ``word-sequence'' we mean a sequence of whatever symbols are on the
output of $\HCLG$.  In our experiments these output symbols represent words, but
silences do not appear as output symbols (they are
represented via alternative paths in $L$).

\vspace*{-0.075in}
\section{Previous lattice generation methods}
\vspace*{-0.05in}
\label{sec:previous}

Lattice generation algorithms tend to be closely linked to particular types of decoder,
but are often justified by the same kinds of ideas.
A common assumption underlying lattice generation methods is the {\em word-pair assumption}
of~\cite{ney_word_graph}.  This is the notion that the time boundary between a pair of words
is not affected by the identity of any earlier words.  In a decoder in which there is
a different copy of the lexical tree for each preceding word,
assuming the word-pair assumption holds, in order to generate an accurate lattice, it is
sufficient to store a single Viterbi back-pointer at the word level; the entire set of
such back-pointers contains enough information to generate the lattice.  Authors who have used
this type of lattice generation method~\cite{ney_word_graph,odell_thesis} have generally
not been able to evaluate how correct the word-pair assumption is in practice, but it seems
unlikely to cause problems.  Such methods are not applicable for WFST based decoders anyway.

The lattice generation method described in~\cite{efficient_general}
is applicable to decoders that use WFSTs~\cite{wfst}
expanded down to the $C$ level (i.e. $CLG$), so the input symbols represent
context-dependent phones.  In WFST based decoding networks, states normally do
not have a unique one-word history, but the authors of~\cite{efficient_general}
were able to satisfy a similar condition at the phone level.  Their method was
to store a single Viterbi back-pointer at the phone level; use this to
create a phone-level latice; prune the resulting
lattice; project it to leave only word labels; and then remove $\epsilon$
symbols and determinize.
Note that the form of pruning referred to here is not the same as beam pruning as
it takes account of both the forward and backward parts of the cost.
The paper also reported experiments with an accurate, ``reference'' method that did not require any
phone-pair assumption; these experiments showed that the main method they were describing
had almost the same lattice oracle error rate as the reference method.  However, the
experiments did not evaluate how much impact the assumption had on the accuracy of the scores,
and this information could be important in some applications.

The lattice generation algorithm that was described in~\cite{saon2005anatomy}
is applicable to WFSTs expanded down to the $H$ level (i.e. $\HCLG$),
so the input symbols represent context-dependent states.  It keeps both scores and
state-level alignment information.  In some sense this algorithm also relies
on the word-pair assumption, but since the copies of the lexical tree in the decoding
graph do not have unique word histories, the resulting algorithm has to be quite
different.  Viterbi back-pointers at the word level are used, but the algorithm keeps track
of not just a single back-pointer in each state, but the $N$ best back-pointers for the $N$ top-scoring distinct
word histories.  Therefore, this algorithm has more in common with the sentence N-best
algorithm than with the Viterbi algorithm.  By limiting $N$ to be quite small (e.g. $N{=}5$),
the algorithm was made efficient, but at the cost of losing word sequences
that would be within the lattice-generation beam.

\vspace*{-0.075in}
\section{Overview of our algorithm}
\vspace*{-0.05in}
\label{sec:overview}

\subsection{Version without alignments}
\vspace*{-0.05in}

In order to explain our algorithm in the easiest way, we will first explain how it would
be if we did not keep the alignment information, and were storing only a single cost
(i.e. the total acoustic plus language-model cost).  This is just for didactic purposes;
we have not implemented this simple version.
In this case, our algorithm would be quite similar
to~\cite{efficient_general}, except at the state level rather than the phone level.
We actually store forward rather than backward pointers: for each active state on each
frame, we create a forward link record for each active arc out of that state; this points
to the record for the destination state of the arc on the next frame (or on
the current frame, for $\epsilon$-input
arcs).   As in~\cite{efficient_general}, at the end of the utterance,
we prune the resulting graph to discard any paths that are not within the beam $\alpha$ of the best cost.
Let the pruned graph be $P$, i.e.
\begin{equation}
  P = \mathrm{prune}(B, \alpha),
\end{equation}
where $B$ is the un-pruned state-level lattice.
We project on the output labels (i.e. we keep only the word labels), then remove $\epsilon$
arcs and determinize.  In fact, we use a determinization algorithm that does $\epsilon$
removal itself.

As in~\cite{efficient_general}, to save memory, we actually do the pruning
periodically rather than waiting for the end of the file (we do it every 25 frames).
Our method is equivalent to their method of linking all currently active states to a ``dummy''
final state and then pruning in the normal way.  However, we implement it in such a way
that the pruning algorithm does not always have to go back to the beginning of the utterance.
For each still-active state, we store the cost difference between the best path including that
state, and the best overall path.  This quantity does not always change between different
iterations of calling the pruning algorithm, and when we detect that these quantities are
unchanged for a particular frame, the pruning algorithm can stop going backward in time.

After the determinization phase, we prune again using the beam $\alpha$.  This is needed because
the determinization process can introduce a lot of unlikely arcs.  In fact, for particular
utterances, the determinization process can cause the lattice to expand enough to
exhaust memory.  To deal with this, we currently just detect when determinization
has produced more than a pre-set maximum number of states, then we prune with a tighter
beam and try again.

This ``simple'' version of the algorithm produces an acyclic, deterministic
WFSA with words as labels.  This is sufficient for applications such as language-model
rescoring.

\vspace*{-0.075in}
\subsection{Keeping separate graph and acoustic costs}
\vspace*{-0.05in}

A fairly trivial extension of the algorithm described above is to store separately
the acoustic costs and the costs arising from $\HCLG$.  This enables us to do things
like generating output from the lattice with different acoustic scaling factors.
We refer to these two costs as the graph cost and the acoustic cost, since the cost
in $\HCLG$ is not just the language model cost but also contains components
arising from transition probabilities and pronunciation probabilities.  We implement
this by using a semiring that contains two real numbers, one for the graph and one
for the acoustic costs; it keeps track of the two costs separately, but its
$\oplus$ operation returns whichever pair has the lowest sum of costs (graph plus acoustic).

Formally, if each weight is a pair $(a,b)$, then $(a,b) \otimes (c,d) = (a{+}c, b{+}d)$,
and $(a,b) \oplus (c,d)$ is equal to $(a,b)$ if $a{+}b < c{+}d$ or if $a{+}b = c{+}d$ and $a{-}b < c{-}d$,
and otherwise is equal to $(c,d)$.  This is equivalent to the normal lexicographic semiring (see~\cite{roark2011lexicographic}) on the pair $((a{+}b),(a{-}b))$.

\vspace*{-0.075in}
\subsection{Keeping state-level alignments}
\vspace*{-0.05in}

It is useful for various purposes, e.g. discriminative training and certain kinds of
acoustic rescoring, to keep the state-level alignments in the lattices.  We will now
explain how we can make the alignments ``piggyback'' on top of the computation
defined above, by encoding them in a special semiring.

First, let us define $Q = \inv(P)$, i.e. $Q$ is the inverted, pruned state-level lattice,
where the input symbols are the words and the output symbols are the p.d.f. labels.
We want to process $Q$ in such a way that we keep only the best path through it
for each word sequence, and get the corresponding alignment.  This is possible
by defining an appropriate semiring and then doing normal determinization.  We shall
ignore the fact that we are keeping track of separate graph and acoustic costs,
to avoid complicating the present discussion.

We will define a semiring in which symbol sequences are encoded into the weights.
Let a weight be a pair $(c, s)$, where $c$ is a cost and $s$ is a sequence of symbols.
We define the $\otimes$ operation as $(c, s) \otimes (c', s') = (c+c', (s,s'))$, where
$(s,s')$ is a concatenation of $s$ and $s'$.  We define the $\oplus$ operation so that
it returns whichever pair has the smallest cost: that is, $(c,s) \oplus (c',s')$
equals $(c,s)$ if $c < c'$, and $(c',s')$ if $c > c'$.  If the costs are identical,
we cannot arbitrarily return the first pair because this would not satisfy the semiring
axioms.  In this case, we return the pair with the shorter string part, and if
the lengths are the same, whichever string appears first in dictionary order.

Let $E$ be an encoding of the inverted state-level lattice $Q$
as described above, with the same number of states and arcs; $E$ is an acceptor, with its symbols
equal to the input symbol (word) on the corresponding arc of $Q$, and
the weights on the arcs of $E$ containing both the weight and the output symbol (p.d.f.),
if any, on the corresponding arcs of $Q$.  Let $D = \mathrm{det}(\mathrm{rmeps}(E))$.
Determinization
will always succeed because $E$ is acyclic (as long as the original decoding
graph $\HCLG$ has no $\epsilon$-input cycles).  Because $D$ is deterministic
and $\epsilon$-free, it has only one path for each word sequence.
Determinization preserves equivalence,
and equivalence is defined in such a way that the $\oplus$-sum of the
weights of all the paths through $E$ with a particular word-sequence, must be the same
as the weight of the corresponding path through $D$ with that word-sequence.
It is clear from the definition of $\oplus$ that this path through
$D$ has the cost and alignment of the lowest-cost path through $E$ that has the
same word-sequence on it.

\vspace*{-0.075in}
\subsection{Summary of our algorithm}
\vspace*{-0.05in}

During decoding, we create a data-structure
corresponding to a full state-level lattice.  That is, for every arc of $\HCLG$, we traverse
on every frame, we create a separate arc in the state-level lattice.  These arcs
contain the acoustic and graph costs separately.  We prune the state-level graph using
a beam $\alpha$; we do this periodically (every 25 frames) but this is equivalent to
doing it just once at the end, as in~\cite{efficient_general}.  Let the
final pruned state-level lattice be $P$.
Let $Q = \inv(P)$, and let $E$ be an encoded version of $Q$ as described above (with the
state labels as part of the weights).  The final lattice is
\vspace*{-0.075in}
\begin{equation}
   L = \mathrm{prune}(\mathrm{det}(\mathrm{rmeps}(E)), \alpha) . \vspace*{-0.075in}
\end{equation}
The determinization and epsilon removal are done together by a single algorithm
that we will describe below.  $L$ is a deterministic, acyclic weighted acceptor with the
words as the labels, and the graph and acoustic costs and the alignments
encoded into the weights.  The costs and alignments are not ``synchronized''
with the words.

\vspace*{-0.075in}
\section{Details of our determinization algorithm}
\vspace*{-0.05in}
\label{sec:details}

We implemented $\epsilon$ removal and determinization as a single algorithm
because $\epsilon$-removal using the traditional approach would greatly
increase the size of the state-level lattice (this is mentioned
in~\cite{efficient_general}).  Our algorithm uses data-structures
specialized for the particular type of weight we are using.  The issue
is that the determinization process often has to append a single symbol to
a string of symbols, and the easiest way to
do this in ``generic'' code would involve copying the whole sequence each time.
Instead we use a data structure that enables this to be done in linear
time (it involves a hash table).

We will briefly describe another unique aspect of our
algorithm.  Determinization algorithms involve weighted subsets of states, e.g.:
\begin{equation}
   S = \{ (s_1, w_1), (s_2, w_2), \ldots \} .
\end{equation}
Let this weighted subset, as it appears in a conventional determinization
algorithm with epsilon removal, be the {\em canonical representation} of a state.
A typical determinization algorithm would maintain a map from this representation
to a state index.  We define a {\em minimial representation} of a state
to be like the canonical representation, but only keeping states that
are either final, or have non-$\epsilon$ arcs out of them.  We maintain
a map from the minimal representation to the state index.  We can
show that this algorithm is still correct (it will tend to give more
minimal output).   As an optimization for speed, we also define
the {\em initial representation} to be the same type of subset, but prior
to following through the $\epsilon$ arcs, i.e. it only contains the states
that we reached by following non-$\epsilon$ arcs from a previous determinized
state.  We maintain a separate map from the initial representation to
the output state index; think of this as a ``lookaside buffer'' that helps us
avoid the expense of following $\epsilon$ arcs.

%Although we have not written a proof of correctness of this algorithm, we
%have tested it thoroughly.  This is done by generating random WFSTS,
%applying this algorithm, and if it terminates, checking that the result
%is deterministic and is equivalent (in this semiring) to the original WFST
%represented as an acceptor in this semiring.

Since submitting this paper, we have become aware of~\cite{shafran2011efficient},
which solves the exact same problem for a different purpose.  They use a semiring
which is more complicated than ours (the string part of the semiring becomes a
structured object with parentheses).  They use this semiring instead of the one
 we describe here, because in our semiring the $\oplus$-sum of two weights
does not necessarily left-divide the weights, and this is a problem for
a typical determinization algorithm.  We bypass
this problem by defining a ``common divisor'' operation $\boxplus$ with the
right properties (it $\oplus$-adds the weight part and returns the longest
common prefix of the string part).  We use this instead of
$\oplus$ when finding divisors in the determinization algorithm.

%% The algorithm for $\epsilon$-removal and determinization is also optimized
%% for our particular case.  We will first explain what a generic $\epsilon$-removal
%% and determinization algorithm would look like.  We will explain this using
%% generic weights.  Let us use the notation
%% \begin{equation}
%%    S = \{ (s_1, w_1), (s_2, w_2), \ldots \}
%% \end{equation}
%% for a weighted subset of states in the original FST.  Each subset $S$ corresponds
%% to a state in the determinized FST, i.e. we maintain a map $m(S) \rightarrow s$
%% where $s$ is a state number in the output (in practice there are tolerances
%% involved in this map when the weights contain floating-point numbers).  We need
%% to normalize these subsets to remove any ``common part'' of the weights.  We
%% need to define an associative and commutative ``common divisor'' operation
%% on weights (call this $\boxplus$; it can be the same as $\oplus$ for many semirings),
%% that gives us the normalizer for a pair of weights.
%% We must be able to  left-divide each individual weight by the result of this operation, i.e.
%% if $c = a \boxplus b$, then we need to be able to solve $a = c a'$ and $b = c b'$ for $a'$ and $b'$.
%% Here, $a'$ and $b'$ become the ``normalized weights''.  Also $\otimes$-multiplication
%% must be left-distributive over this operation, which ensures the right kind of
%% ``uniqueness'' of the normalization operation.  In the particular case of interest
%% $w$ is a pair $(w', s)$ of a ``base-weight'' and a string $s$, and the
%% operation $(a,s) \boxplus (b,t)$ returns $(a\oplus b, u)$ where $u$ is the longest
%% common prefix of $s$ and $t$ and $a\oplus b$ essentially returns the minimum cost.

%% For the determinization algorithm, we define
%% $\mathrm{normalizer}(S)$ to be the $\boxplus$-sum of the weights in the subset $S$,
%% and $\mathrm{normalize}(S)$ to be $S$ with each weight left-divided by
%% $\mathrm{normalizer}(S)$.  We also need to define an operation that ``follows through''
%% $\epsilon$ transitions.  Call this $\mathrm{follow-eps}(S)$

%% The key characteristic of our method is that unlike all previously reported
%% methods that we are aware of (with the sole exception of the inefficient method used
%% as a baseline in~\cite{efficient_general}), our method is exact even if the word-pair
%% assumption is invalid.  In addition, we are able to generate a lattice with full
%% state-level alignment information, which would not be possible using the approaches
%% used in~\cite{efficient_general}.  Our algorithm also does not need to store $N$
%% back-pointers like the lattice-generation algorithm previously reported in~\cite{saon2005anatomy}
%% for $H$-level WFSTs (i.e. fully-expanded WFSTs).

\vspace*{-0.1in}
\section{Experimental results}
\vspace*{-0.075in}
\label{sec:exp}

We do not compare with any other algorithms,
as~\cite{ney_word_graph,odell_thesis,efficient_general} are designed for
different types of decoders than ours, and the lattices contain less information,
making comparisons hard to interpret; the algorithm of~\cite{saon2005anatomy} has
similar requirements and outputs as ours, but besides being inexact, it is bound
to be slower due to the need to store $N$ back-pointers, so we did not view it as
worthwhile to do the experiment.

We report experimental results on the Wall Street Journal database of
read speech.
Our system is a standard mixture-of-Gaussians system trained on the SI-284
training data; we test on the November 1992 evaluation data.
We generated lattices with the bigram language model
supplied with the WSJ database, and for rescoring experiments we
use the trigram language model.  The acoustic scale was $1/16$ for first-pass
decoding and $1/15$ for LM rescoring.
For simplicity, we used a decoder that
does not support a ``maximum active states'' option, so the only variables
to consider are the beam used in the Viterbi beam search, and the separate
beam $\alpha$ used for lattice generation.

\begin{figure}[t]
\centering
   \includegraphics[width=0.9\columnwidth]{figures/latbeam.eps}
 \vspace*{-0.1in}

   \caption{ { Lattice properties, varying lattice beam $\alpha$ (Viterbi pruning beam fixed at 15)} }
  \vspace{-0.15in}
  \label{fig:latbeam}
\end{figure}

Figure~\ref{fig:latbeam} shows how the lattice properties change as we vary
$\alpha$, with the Viterbi beam fixed at 15; Figure~\ref{fig:viterbibeam} varies
the Viterbi decoding beam, leaving $\alpha$ fixed at 7.  Lattice density is
defined as the average number of arcs crossing each frame.  We get all the
improvement from LM rescoring by increasing $\alpha$ to 4, and time taken
increases rapidly when $\alpha > 8$, so we recommend roughly $4 < \alpha < 8$ for
LM rescoring purposes.  We do not display the real-time factor of the
non-lattice-generating decoder on this data (2.26xRT) as it was actually slower
than the lattice generating decoder; this is possibly due to the overhead of
reference counting.  Out of vocabularly words (OOVs) provide a floor on the
lattice oracle error rate: of 333 test utterances, 87 contained at least one OOV
word, yet only 93 sentences (6 more) had oracle errors with $\alpha=10$.

\begin{figure}[t]
\centering
   \includegraphics[width=0.9\columnwidth]{figures/decodebeam.eps}
 \vspace*{-0.1in}
    \caption{ {Lattice properties, varying Viterbi pruning beam (lattice beam $\alpha$ fixed at 7)}}
  \label{fig:viterbibeam}
  \vspace{-0.15in}
\end{figure}

\vspace*{-0.1in}
\section{Conclusions}
\label{sec:conc}
\vspace*{-0.075in}

We have described a lattice generation method that is to our knowledge the first
efficient method that does not rely on the word-pair assumption of~\cite{ney_word_graph}.
It includes an ingenious way of obtaining HMM-state-level alignment information
via determinization in a specially designed semiring.

\vspace*{-0.04in}
\footnotesize {
\vspace*{-0.1in}
\bibliographystyle{IEEEbib}
\bibliography{refs}
}

\end{document}

steps/decode_tri3a_latgen.sh
acwt=0.0625
beam=[15.0-7.0]
max_active=15000
lat_beam=[10.0-1.0]
max_arcs=50000 (75000 for lat_beam 10.0)
model=exp/tri3a/final.mdl
graph=exp/graph_tri3a_bg_left/HCLG.fst

scp=data/eval_nov92.scp

333 utterances
87 utterances contain OOV
42 utterances contain annotated noise (actually some more, but listed as OOV utterances here)

5641 tokens (some scripts count 5700 with 59 noise annotations)
107 OOV tokens
1.88% OOV token rate

253204 frames (2532 seconds)

19976 words in decoding vocabulary and LM (not counting <s>,</s>,<unk>)

average lattice link density per frame:
have a wide range over files: for example beam=15.0, lat_beam=9.0:
ranging from 1.5 to 167

running on blades, using 64bit version with double float and lattice-simple-decoder
realtime factor running on pcburget, 22989 frames (decode3.sh)

decoding with bigram LM:
decoding WER:      11.51%
ins/del/sub:       118/57/474
utterances wrong:  227

1) experiment: changing lattice beam:
(lattices 443c040w and 444c040i have been reduced from 10.0 to 8.1 because of determinization memory problems)

decoding beam:     15.0         15.0       15.0       15.0       15.0         15.0        15.0        15.0        15.0         15.0        15.0
lattice beam:      10.0          9.0        8.0        7.0        6.0          5.0         4.0         3.0         2.0          1.0         0.0
avg. link density: 23.56        16.0       10.82       7.26       4.91         3.38        2.38        1.77        1.4          1.17        1.0
realtime factor:    2.317        2.114      2.006      1.941     1.900         1.855       1.835       1.797       1.769        1.737       ---

oracle WER:         2.62%        2.78%      2.85%      3.08%      3.35%        3.86%       4.52%       5.46%       6.98%        8.62%      11.51%
ins/del/sub:       22/0/126     25/0/132   29/0/132   31/1/142   38/5/146     51/11/156   57/15/183   64/20/224   76/32/286    89/40/357   118/57/474
oracle utt wrong:  93           96         96         101        105          116         129         147         168          191         227

rescoring WER(15):  9.59%        9.59%      9.59%      9.59%      9.56%        9.59%       9.61%       9.70%       9.80%       10.21%      11.51%
ins/del/sub:       117/36/388                                    116/36/387   117/35/389  115/35/392  111/38/398  108/39/406   113/42/421  118/57/474
rescore utt wrong: 205                                           204          204         204         202         205          212         227

2) experiment: changing decoding beam:

decoding beam:     16.0        15.0          14.0          13.0         12.0         11.0         10.0          9.0 (1 utt partial) 8.0 (2 partial)
lattice beam:       7.0         7.0           7.0           7.0          7.0          7.0          7.0          7.0                7.0
avg. link density:  7.4         7.26          7.04          6.74         6.33         5.71         4.9          4.08               3.09
realtime factor:    2.646       1.941         1.428         1.052        0.775        0.555        0.388        0.267              0.177
decoding WER:      11.51%       11.51%       11.54%        11.58%       11.75%       11.93%       12.32%        13.47%             15.41%
ins/del/sub:       118/57/474   118/57/474   118/57/476    119/56/478   124/57/482   131/58/484   140/58/497    164/56/540         197/61/611
utterances wrong:  227          227          227           227          228          228          229           236                246

oracle WER:        3.03%        3.08%        3.24%         3.37%        3.56%        4.02%        4.72%         6.45%              9.18%
ins/del/sub:       30/1/140     31/1/142     35/2/146      38/2/150     42/3/156     54/6/167     55/7/204      73/13/278          118/22/378
oracle utt wrong:  100          101          105           108          112          119          128           153                184

rescoring WER(15): 9.59%        9.59%        9.73%         9.80%        10.05%       10.26%       10.55%        11.82% (1 failed)  13.79% (2 failed)
ins/del/sub:       116/36/389   117/36/388   115/37/397    117/38/398   123/39/405   132/39/408   141/37/417    ---                ---
rescore utt wrong: 205          205          206           206          207          209          208           220                235

3) bigram vs. trigram decoding:

decoding lm:       bigram       trigram,pruned  bigram        trigram,pruned   bigram (just to see whether wider beams still help)
decoding beam:     15.0         15.0            13.0          13.0             16.0
lattice beam:       9.0          9.0             9.2           9.2             10.0 (already quite big for lattice-oracle - more than 6GB RAM used)
avg. link density: 16.0         14.98           14.86         13.58            24.84
rt factor(old):     3.17         3.26            1.75         1.77              4.54

decoding WER:      11.51%       10.57%          11.58%        10.80%           11.51%
ins/del/sub:       118/57/474   116/44/436      119/56/478    120/47/442       118/57/474
utterances wrong:  227          214             227           215              227

oracle WER:        2.78%        2.84%           3.05%         3.19%            2.61%
ins/del/sub:       25/0/132     30/0/130        30/2/140      33/1/146         21/0/126
oracle utt wrong:  96           96              103           107              92

rescoring WER(15):  9.59%       9.64%           9.80%         9.87%            9.59%
ins/del/sub:       117/36/388   115/34/395      117/38/398    119/38/400       116/36/389
rescore utt wrong: 205          205             206           209              205

4) lattice vs. n-best lists
decoding with trigram, pruned

decoding:          lattice      n-best         n-best/beam
decoding beam:     13.0         13.0           13.0
lattice beam:      9.2          --             9.35 [6.0-12.5] (few outliers below/above)
N best:            --           <=20           206.96 [2-1600] (according to beam range and utterance)
max tokens:        15000        150k,700k      150000-770000
avg. link density: 13.58        14?            231.05
realtime factor:   1.77         12             crap

decoding WER:      10.80%       12.78% (why?)  14.96% (probably too low max tokens)
ins/del/sub:       120/47/442
utterances wrong:  215

oracle WER:        3.19%
ins/del/sub:       33/1/146
oracle utt wrong:  107

rescoring WER(15):  9.87%
ins/del/sub:       119/38/400
rescore utt wrong: 209

2.259 is baseline RT factor w/ decoding beam of 15, with simple-decoder.

note: paper# is 3887
passwd  CF8FECE1


% Abstract

%\renewcommand{\abstractname}{Abstract} % Uncomment to change the name of the abstract

\pdfbookmark[1]{Abstract}{Abstract} % Bookmark name visible in a PDF viewer

\begingroup
\let\clearpage\relax
\let\cleardoublepage\relax
\let\cleardoublepage\relax

\chapter*{Abstract}
	Ranking functions used in information retrieval are primarily used in the search engines and they are often adopted for various language processing applications.
This project introduces some novel heuristics combined with probabilistic retrieval functions and are employed in the domain of approximate string similarity problem.
A lot of algorithms have been proposed in the literature to solve approximate string similarity problems, however none of them makes use of probabilistic retrieval functions.
We are the first to explore the intersection between these two areas and propose heuristic designs to resolve this problem.
First we propose chunking heuristic function, BREAK.
We show the variants BREAK-1, 2, OFF which splits up the terms with the sequential notion.
Then we propose BREAK-n which generalizes these variants and scales to larger datasets.
In order to relate these split ups, we propose a graphical error modelling heuristics MAKE over the BREAK variants.
Finally, we propose TAKE curve, a novel feature engineering probabilistic distribution which replaces the prevalent normalization heuristics.
Taking the advantage of flexibility over the choice of heuristics, we assess the variants on the cognate detection, mutant identification, and isolated spelling correction based problems.
In the extensive evaluation methods, we found that our designs perform better than prevalent heuristics and are robust against database characteristics.

\endgroup

\vfill


\documentclass{documentation}

\title{ExceptionAlge Package}

\author{James B. Wilson}
\address{Colorado State University}
\email{james.wilson@colostate.edu}

\author{Joshua Maglione}
\address{Universit\"at Bielefeld}
\email{jmaglione@math.uni-bielefeld.de}

\version{2.1}
\date{\today}
\copyrightyear{2016--2019}

\input{preamble.tex}

%-----------------------------------------------------------------------------
\begin{document}

\frontmatter

\dominitoc
\maketitle
\tableofcontents

\mainmatter

\chapter{Introduction}

\subsection*{Citing ExceptionAlge}

\section{Overview}

\section{Version}

\chapter{Exceptional Algebras}

Magma provides functionality with common exceptional tensors.  Many are used in the
construction of nonassociative algebras.  A few supporting functions for nonassociative algebras
are also provided.

\section{Generics for nonassociative algebras}

\subsection{Nonassociative algebras with involutions}~

\index{IsStarAlgebra}
\begin{intrinsics}
IsStarAlgebra(A) : AlgGen -> BoolElt
\end{intrinsics}

Decides if algebra has an involution, i.e. a $*$-algebra.

\index{Star}
\begin{intrinsics}
Star(A) : AlgGen -> Map
\end{intrinsics}

Returns involution of given $*$-algebra.

\begin{example}[StarAlgebra]

We demonstrate the functions dealing with involutions of nonassociative algebras.
\begin{code}
> A := OctonionAlgebra(Rationals(),-1,-1,-1);
> IsStarAlgebra(A);
true
>
> s := Star(A);
> A.1; // A.1 is the mult. id.
(1 0 0 0 0 0 0 0)
> A.1 @ s;
(1 0 0 0 0 0 0 0)
>
> A.2;
(0 1 0 0 0 0 0 0)
> A.2 @ s;
( 0 -1  0  0  0  0  0  0)
\end{code}
\end{example}

\subsection{Operations on power associative algebras}~
The following operations are defined for nonassociative algebras for which $x*(x*x)=(x*x)*x$.

\index{GenericMinimalPolynomial}\index{GenericMinimumPolynomial}
\begin{intrinsics}
GenericMinimalPolynomial(x) : AlgGenElt -> FldElt
GenericMinimumPolynomial(x) : AlgGenElt -> FldElt
\end{intrinsics}

The generic minimum polynomial of an element in a power associative algebra.

\index{GenericNorm}
\begin{intrinsics}
GenericNorm(x) : AlgGenElt -> FldElt
\end{intrinsics}

The generic norm of an element in a power associative algebra.

\index{GenericTrace}
\begin{intrinsics}
GenericTrace(x) : AlgGenElt -> FldElt
\end{intrinsics}

The generic trace of an element in a power associative algebra.

\index{GenericTracelessSubspaceBasis}
\begin{intrinsics}
GenericTracelessSubspaceBasis(A) : AlgGen -> Any
\end{intrinsics}

Given a power associative algebra return a basis for the elements of generic trace 0.

\begin{example}[TenGeneric]

The trace $x+\bar{x}$ of a quaternion doubles the rational component, producing
degenerate behavior in characteristic $2$.  The generic trace avoids this.
\begin{code}
> Q := QuaternionAlgebra(Rationals(), 1,1);
> Trace(Q!1);
2
> GenericTrace(Q!1);
1
> Q := QuaternionAlgebra(GF(2), 1,1);
> Trace(Q!1);
0
> GenericTrace(Q!1);
1
\end{code}

The generic minimum polynomial of an element $x$ in power associative algebra
need only be a factor of the minimal polynomial of its right regular matrix $yR_x:=x*y$.

\begin{code}
> J := ExceptionalJordanCSA(GF(5));
> p := GenericMinimumPolynomial(J.3+J.12);
> Rx := AsMatrices(Tensor(J), 2,0);     // yR_x = y*x.
> q := MinimalPolynomial(Rx[3]+Rx[12]);
> Degree(p);
3
> Degree(q);
6
> q mod p;
0
\end{code}
\end{example}

\section{Compositions algebras}

\index{CompositionAlgebra}
\begin{intrinsics}
CompositionAlgebra(K, a) : Fld, [FldElt] -> AlgGen
CompositionAlgebra(K, a) : Fld, [RngIntElt] -> AlgGen
\end{intrinsics}

Constructs the composition algebra with specified parameters.  The algebra returned
has an involution.
The method is modestly intentional choosing Magma's favored representation of
the individually classified algebras according to Hurwitz's theorem.  In the case of
fields the type returned is an algebra with involution, possibly the identity.

\index{OctonionAlgebra}
\begin{intrinsics}
OctonionAlgebra(K, a, b, c) : Fld, FldElt, FldElt, FldElt -> AlgGen
OctonionAlgebra(K, a, b, c) : Fld, RngIntElt, RngIntElt, RngIntElt -> AlgGen
\end{intrinsics}

Octonion algebra with involution given by the specified parameters.
This builds the Cayley-Dickson algebra over the quaternion algebra
$\left(\frac{a,b}{K}\right)$.  In particular, Magma's implementation
of quaternion algebras is applied.

\index{SplitOctonionAlgebra}
\begin{intrinsics}
SplitOctonionAlgebra(K) : Fld -> AlgGen
\end{intrinsics}

Returns the split octonion algebra over the field $F$.

\begin{example}[TenTriality]
The following example demonstrates some of the mechanics by exploring
the concept of triality \cite{Schafer}*{III.8}.

The Cartan-Jacobson theorem asserts that for fields of characteristic other
than 2 and 3, the derivation algebra of an octonion algebra is of Lie type
$G_2$.

\begin{code}
> O := OctonionAlgebra(GF(7),-1,-1,-1);
> L := DerivationAlgebra(O);   // Derivations as an algebra.
> SemisimpleType(L);
G2
\end{code}

Cartan's triality obtains $G_2$ from $D_4$ by relaxing to
derivations of the octonions as a generic tensor, rather than as an
algebra.
This is done computationally by changing the category of the octonion product
from an algebra to a tensor.

\begin{code}
> T := Tensor(O);
> T := ChangeTensorCategory(T,HomotopismCategory(2));
> M := DerivationAlgebra(T);  // Derivations as a tensor.
> SemisimpleType(M/SolvableRadical(M));
D4
\end{code}
\end{example}

\section{Jordan algebras}

\index{JordanTripleProduct}
\begin{intrinsics}
JordanTripleProduct(J) : AlgGen -> TenSpcElt
\end{intrinsics}

Returns the tensor describing the Jordan triple product.

\index{JordanSpinAlgebra}
\begin{intrinsics}
JordanSpinAlgebra(F) : TenSpcElt -> AlgGen
JordanSpinAlgebra(F) : Mtrx -> AlgGen
\end{intrinsics}

Returns the special Jordan algebra of spin type for given symmetric form.

\begin{example}[JordanBasic]
Jordan algebras have suggestive analogues of
commutative associative algebras, but experimenting shows serious
differences.

\begin{code}
> F := IdentityMatrix(Rationals(),2);
> J := JordanSpinAlgebra(F);
> T := Tensor(J);
> R := AsMatrices( T, 2,0);
> R[1];   // Is J.1 the identity?
[1 0 0]
[0 1 0]
[0 0 1]
> J.2*J.2 eq J.1;  // J.2^2=1?
true
> J.2*J.3 eq 0;  // Yet J.2 is a zero-divisor.
true
> e := (1/2)*(J.1+J.2);
> e^2 eq e;  // An idempotent of J?
true
\end{code}

Pierce decompositions in Jordan algebras have the usual 0 and 1 eigenspaces
but an additional 1/2-eigenspace emerges as well.

\begin{code}
> Re := (1/2)*(R[1]+R[2]);
> Eigenvalues(Re);
{ <1, 1>, <1/2, 1>, <0, 1> }
\end{code}
\end{example}

\index{ExceptionalJordanCSA}
\begin{intrinsics}
ExceptionalJordanCSA(O) : AlgGen -> AlgGen
ExceptionalJordanCSA(K) : Fld -> AlgGen
\end{intrinsics}

The exception central simple Jordan algebra over the given octonions.
If a field is supplied instead then the split octonion algebra over the
field is used.

\begin{example}[ChevalleyShaferF4]
In characteristic not $2$ or $3$, the exceptional central
simple Jordan algebra can be used to construct
the exceptional Lie algebra of type $F_4$.

\begin{code}
> J := ExceptionalJordanCSA(Rationals());
> T := Tensor(J);
> T := ChangeTensorCategory(T, HomotopismCategory(3));
> D := DerivationAlgebra(T);
> D2 := Codomain(Induce(D, 2));		// Represent D on U2.
> F4 := D2*D2;			              // Commutator.
> SemisimpleType(F4);
F4
> F4;                             // F4 represented on a 27-dim module.
Matrix Lie Algebra of degree 27 over Rational Field
\end{code}
\end{example}

\backmatter

\begin{bibdiv}
\begin{biblist}

\bib{Schafer}{book}{
   author={Schafer, Richard D.},
   title={An introduction to nonassociative algebras},
   series={Pure and Applied Mathematics, Vol. 22},
   publisher={Academic Press, New York-London},
   date={1966},
   pages={x+166},
   review={\MR{0210757}},
}

\end{biblist}
\end{bibdiv}

\printindex

\end{document}


This section covers basic performance tests, i.e. how specific algorithms scale
with grid resolution and with polynomial degree, on a \emph{single compute node}.

% --------------------------------------------------------------------------------
\section{Solver Performance - Poisson/Stokes problems}
\label{sec:SolverPerformancePoisson}
% --------------------------------------------------------------------------------
Two groups of solver are compared:
\begin{itemize}
\item
Direct Solvers: directs sparse methods, such as PARDISO\footnote{
\url{http://www.pardiso-project.org/}}
and MUMPS\footnote{
\url{http://mumps.enseeiht.fr/}}
are compared.
Their performance also serves as a comparative baseline.

%\item
%Iterative Algorithms without preconditioning, resp. low-impact, generic preconditioning:
%This includes solver libraries such as \code{monkey} (BoSSS-specific, supports GPU)
%as well as
%HYPRE\footnote{
%\url{https://computation.llnl.gov/projects/hypre-scalable-linear-solvers-multigrid-methods}}
%(native library, used via wrappers).

\item
Iterative Algorithms with \ac{dg}-specific preconditioners, such as aggregation multigrid
and multi-level additive Schwarz
\end{itemize}

The scaling and performance is profiled subsequent sections. For Performance profiling we stick to our working horse: the kcycle-Schwarz algorithm (with optional p-two-grid as block solver).
The performance profile of the krylov V-cycle with Schwarz pre and post-smoother is investigated. A direct solver (PARDISO) is used to solve the deepest coarse system. One may choose another direct solver for the coarse system, e.g. MUMPS. In practise PARDISO is more robust to ill-conditioned system, therefore in this performance analysis investigation we will stick to PARDISO as solver, wherever a direct solver is needed.
NOTE: the p-two-grid used in Schwarz or as a standalone preconditioner, the coarse system is solved by a direct solver.

We distinguish four phases of every solver-scenario:
\begin{itemize}
	\item MatrixAssembly: assemble Block matrix
	\item Aggregation basis init: create multigrid sequence, contains information about the transformation at the multigrid levels
	\item Solver Init: hand over/assemble relevant data for the chosen solver, e.g. operator matrix etc.
	\item Solver Run: solves the equation system: operator matrix, vector of dg coordinates and given RHS
\end{itemize}
Matrix assembly and aggregation init is discritization specific, whereas, Solver init and run ist specific for the used solver.

\subsection{Introduction of solvers}

\subsubsection{linear Solver: p-two-grid}
\label{sec:ptg_gmres}

The p two grid algorithm can be used as left preconditioner for the well known GMRES-algorithm. Or as block solver of the Schwarz blocks in the Orthonormalization-multigrid algorithm, described in \ref{alg:OrthoMG}.

\subsubsection{linear Solver: V-krylov-cycle with Schwarz smoother}
\label{sec:kcycle}

The orthonormalization multigrid is a combination of a v-cycle of a geometric multigrid (or algebraic as the agglomeration of cells is graph based) with an additive Schwarz smoother and a projection method onto the history of residual contributions of parts of the algorithm (smoother and coarse grid correction). A schematic of the solver can be checked out in \ref{fig:SolverScheme}.
For more details on the solvers check \cite{OpenSoftwarePDE}.

\begin{figure}[!h]
	\begin{center}
		\includegraphics[scale=0.65]{./apdx-NodeSolverPerformance/solver-scheme.png}
		\input{./apdx-NodeSolverPerformance/solver-scheme.png}
	\end{center}
	\caption{
		scheme of Orthonormalization Multigrid. The block solver is usually a direct solver like PARDISO. Note: for the smoother it is sufficient to solve the system approximately. This gives rise to approximate solutions of the Schwarz blocks, like a ILU algorithm.
	}
	\label{fig:SolverScheme}
\end{figure}

\subsection{DG-Poisson test problem}
\label{sec:ConstantDiffusionCoefficient}
The stationary 3D-problem:
\begin{equation}
\left\{ \begin{array} {rclll}
- \Delta T   & = & g_{\domain}
& \text{in}\ \Omega = (0,10) \times (-1,1) \times (-1,1)  &  \\
% ----
T   & = & g_D = 0
& \text{on}\ \Gamma_D = \{ (x,y,z) \in \real^3; \ x = 0 \}
& \text{Dirichlet-boundary} \\
% ----
\nabla T \cdot \vec{n}_{\partial \domain} & = & g_N
& \text{on}\ \Gamma_N = \partial \Omega \setminus \Gamma_D
& \text{Neumann-boundary}
\end{array} \right.
\label{eq:ContantCoeffPoissonBenchmark}
\end{equation}
where $g_{\domain}=-sin(x)$.
is investigated on a non-uniform, Cartesian grid
(equidistant in $z$, sinus-spacing in $x$ and $y$ direction).
The large $\Gamma_N$ makes the problem harder for non-preconditioned
iterative methods. See Figure \ref{fig:ConstantCoeffRuntimes} for results.

\subsection{DG-Poisson: scaling of solvers}

\graphicspath{{./apdx-NodeSolverPerformance/PoissonConstCoeff/plots/}}

\begin{figure}[!h]
	\begin{center}
		\input{./apdx-NodeSolverPerformance/PoissonConstCoeff/plots/ConstCoeffPoissonScaling.tex}
	\end{center}
	\caption{
		Solver wallclock-time vs. degrees-of-freedom, for different polynomial degrees $k$,
		for problem/Equation (\ref{eq:ContantCoeffPoissonBenchmark}).
	}
	\label{fig:ConstantCoeffRuntimes}
\end{figure}

\newpage

\subsubsection{DG-Poisson: krylov-cycle Profiling}

\begin{figure}[!h]
	\begin{center}
		\input{./apdx-NodeSolverPerformance/PoissonConstCoeff/plots/ConstCoeffPoissonexp_Kcycle_schwarz.tex}
	\end{center}
	\caption{
		Investigation of runtime of different code parts of the V-kcycle with additive Schwarz (p-two-grid as block solver) smoother. wallclock-time vs. degrees-of-freedom, for different polynomial degrees $k$,
		for problem/Equation (\ref{eq:ContantCoeffPoissonBenchmark}).
	}
	\label{fig:SIP_blockJacobianPCG}
\end{figure}
\newpage

\subsubsection{DG-Poisson: preconditioned GMRES Profiling}

\begin{figure}[!h]
	\begin{center}
		\input{./apdx-NodeSolverPerformance/PoissonConstCoeff/plots/ConstCoeffPoissonexp_gmres_levelpmg.tex}
	\end{center}
	\caption{
		Investigation of runtime of different code parts of the preconditioned GMRES algorithm. Wallclock-time vs. degrees-of-freedom, for different polynomial degrees $k$,
		for problem/Equation (\ref{eq:ContantCoeffPoissonBenchmark}).
	}
	\label{fig:SIP_SchwarzPGC}
\end{figure}
\newpage

\subsection{Xdg-Poisson test problem}
\label{sec:XdgPoisson}

\newcommand{\frakA}{\mathfrak{A}}
\newcommand{\frakB}{\mathfrak{B}}
\newcommand{\nOmega}{\vec{n}_{\partial \Omega } }
%\newcommand*{\jump}[1]{\left\llbracket {#1} \right\rrbracket}
\newcommand{\frakI}{\mathfrak{I}}
\newcommand{\nI}{\vec{n}_\frakI}

The test problem can be considered as stationary 3 dimensional heat equation with source-term and with two phases:
\begin{equation}
\left\{ \begin{array}{rll}
- \mu \Delta u                   & = f               & \text{ in } \Omega \setminus \frakI , \\
\jump{u}                         & = 0               & \text{ on } \frakI ,                  \\
\jump{\mu \nabla u \cdot \nI}    & = 0               & \text{ on } \frakI ,                  \\
u                                & = g_\text{Diri}   & \text{ on } \Gamma_\mathrm{Diri} ,    \\
\nabla u \cdot \nOmega           & = g_\text{Neu}    & \text{ on } \Gamma_\mathrm{Neu} .     \\
\end{array}
\right.
\label{eq:XdgPoissonBenchmark}
\end{equation}
with a constant diffusion coefficient in each subdomain

\begin{equation}
\mu (\vec{x}) =
\left\{ \begin{array}{ll}
\mu_\frakA & \text{for } \vec{x} \in \frakA, \\
\mu_\frakB & \text{for } \vec{x} \in \frakB. \\
\end{array} \right.
\label{eq:DiscDiffKoeff}
\end{equation}

where $\mu_1=1$ (inner) and $\mu_2=1000$ (outer) characterize the two phases. is investigated on a uniform, equidistant Cartesian grid. See \ref{fig:XdgRuntimes} for results.

\graphicspath{{./apdx-NodeSolverPerformance/XDGPoisson/plots/}}

\subsubsection{Xdg-Poisson: scaling of solvers}

\begin{figure}[!h]
	\begin{center}
		\input{./apdx-NodeSolverPerformance/XDGPoisson/plots/XdgPoissonScaling.tex}
	\end{center}
	\caption{
		Solver runtime vs. degrees-of-freedom, for different polynomial degrees $k$,
		for problem/Equation (\ref{eq:XdgPoissonBenchmark}).
	}
	\label{fig:XdgRuntimes}
\end{figure}
\newpage

\subsubsection{Xdg-Poisson: krylov-cycle Profiling}

\begin{figure}[!h]
	\begin{center}
		\input{./apdx-NodeSolverPerformance/XDGPoisson/plots/XdgPoissonexp_Kcycle_schwarz.tex}
	\end{center}
	\caption{
		Investigation of runtime of different code parts of the block Jacobian PCG. Solver runtime vs. degrees-of-freedom, for different polynomial degrees $k$,
		for problem/Equation (\ref{eq:XdgPoissonBenchmark}).
	}
	\label{fig:Xdg_blockJacobianPCG}
\end{figure}

\subsubsection{Xdg-Poisson: preconditioned GMRES Profiling}

\newpage
\begin{figure}[!h]
	\begin{center}
		\input{./apdx-NodeSolverPerformance/XdgPoisson/plots/XdgPoissonexp_gmres_levelpmg.tex}
	\end{center}
	\caption{
		Investigation of runtime of different code parts of the Schwarz PCG. Solver runtime vs. degrees-of-freedom, for different polynomial degrees $k$,
		for problem/Equation (\ref{eq:XdgPoissonBenchmark}).
	}
	\label{fig:Xdg_SchwarzPGC}
\end{figure}
\newpage

\subsection{Xdg-Stokes test problem}

As a test case for a two-phase stokes problem with Xdg approach, we choose an ellipsoid within a closed cube. The body is not touching the cube. There is no gravitational force and the boundaries are considered as walls ($\vec{u}_\mathrm{Diri}=\vec{0}$).

\newcommand{\divergence}[1]{{\mathrm{div}\left({#1}\right)}}
\newcommand{\normI}{{\vec{n}_{\frakI}}}

\begin{equation}
\left\{ \begin{array}{rll}

\nabla p
-
\divergence{\mu ( \nabla \vec{u} + (   \nabla \vec{u})^T ) }
& = 0
& \text{ in } \Omega \setminus \frakI = (-1,1)^3 , \\
\textrm{div}(\vec{u})  &=  0
& \text{ in } \Omega \setminus \frakI , \\
\jump{\vec{u}} & =  0
& \text{ on } \frakI \\
\jump{
	p \nI
	- \mu ( \nabla \vec{u} + (   \nabla \vec{u})^T ) \cdot \nI
}
& =
\sigma \kappa \normI
& \text{ on } \frakI(t), \\
u  & = \vec{u}_\mathrm{Diri}
& \text{ on } \Gamma_\mathrm{Diri} .    \\
\end{array} \right.
\label{eq:XdgStokes-Benchmark}
\end{equation}
with piece-wise constant density and viscosity for both phases, i.e.
\begin{equation}
\rho(\vec{x}) = \left\{ \begin{array}{ll}
\rho_\frakA & \textrm{for } \vec{x} \in \frakA \\
\rho_\frakB & \textrm{for } \vec{x} \in \frakB \\
\end{array} \right.
\quad \textrm{and} \quad
\mu(\vec{x}) = \left\{ \begin{array}{ll}
\mu_\frakA & \textrm{for } \vec{x} \in \frakA \\
\mu_\frakB & \textrm{for } \vec{x} \in \frakB \\
\end{array} \right.
.
\label{eq:defRhoAndMu}
\end{equation}
Furthermore, $\sigma$ denotes surface tension and $\kappa$ denotes the
mean curvature of $\frakI$. The body (ellipsoid) is defined by a level-set function:

\begin{equation}
(x/(\beta*r))^2 + (y/r)^2 +(z/r)^2-1=0
\end{equation}

where $\beta=0.5$ is the spherical aberration and $r=0.5$ the radius. The physical parameters are:

\begin{table}[h]
	\centering
	\begin{tabular}{l|c}
		$\rho_A$ & 1e-3  $kg / cm^3$\\
		$\rho_B$ & 1.2e-6  $kg / cm^3$\\
		$\mu_A$ & 1e-5 $kg / cm / sec$\\
		$\mu_B$ & 17.1e-8 $kg / cm / sec$\\
		$\sigma$ & 72.75e-3 $kg / sec^2$\\
	\end{tabular}
\end{table}

The surface tension is inducing a velocity field around the ellipsoid. This test case is non-physical due to the static body. A more realistic body would reshape to compensate the surface tension, which leads to oscillation of the body.

\graphicspath{{./apdx-NodeSolverPerformance/XDGStokes/plots/}}

\subsubsection{Xdg-Poisson: scaling of solvers}

\begin{figure}[!h]
	\begin{center}
		\input{./apdx-NodeSolverPerformance/XDGStokes/plots/XdgStokesScaling.tex}
	\end{center}
	\caption{
		Solver runtime vs. degrees-of-freedom, for different polynomial degrees $k$,
		for problem/Equation (\ref{eq:XdgStokes-Benchmark}).
	}
	\label{fig:XdgStokes-scaling}
\end{figure}

The size of Schwarzblocks was set to 10.000 DOF. It is known, that this raises the number of iterations and therefore the number of Schwarz blocks shall be constant for next study ...

\newpage

\subsubsection{Xdg-Poisson: krylov-cycle Profiling}

\begin{figure}[!h]
	\begin{center}
		\input{./apdx-NodeSolverPerformance/XDGStokes/plots/XdgStokesexp_Kcycle_schwarz.tex}
	\end{center}
	\caption{
		Investigation of runtime of different code parts of the block Jacobian PCG. Solver runtime vs. degrees-of-freedom, for different polynomial degrees $k$,
		for problem/Equation (\ref{eq:XdgStokes-Benchmark}).
	}
	\label{fig:XdgStokes-kcylce}
\end{figure}

\cleardoublepage

\section{Solver Performance - Navier-Stokes problems}
\label{sec:SolverPerformanceNSE}
Different solver strategies are conducted to solve the fully coupled incompressible Navier-Stokes equations. At the moment the following strategies can be examined:
\begin{itemize}
	\item Linearizsation of the NSE with: Newton(Gmres) or Picard
	\item Solving the linear problem with a Gmres approach or the direct solver MUMPS
	\item Preconditioning with Additive-Schwarz domain decomposition (with coarse solve on the coarsest multigrid level) and direct solver MUMPS for the Blocks (Automatic)
	\item Preconditioning with Additive-Schwarz kcycle Blocks on the coarsest multigrid level (with coarse solve on the coarsest multigrid level) and direct solver MUMPS for the Blocks
\end{itemize}
\subsection{Driven Cavity 3D}
The problem
\begin{equation}
\left\{ \begin{array} {rclll}
\rho_f\Big(\frac{\partial \vec{u}}{\partial t}+ \vec{u} \cdot \nabla \vec{u}\Big) +\nabla p - \mu_f \Delta \vec{u} & = & \vec{f}
& \text{and}\   &  \\
% ----
\nabla \cdot \vec{u} & = & 0
& \text{in}\ \Omega = (-0.5,0.5) \times (-0.5,0.5) \times (-0.5,0.5)  & \\
\vec{u}_D & = & \{1,0,0 \}
& \text{on}\ \Gamma_D = \{ (x,y,0z) \in \real^3; \ z = 0.5 \}
& \text{Dirichlet-boundary}\\
\vec{u}_W & = & 0
& \text{on}\ \Gamma_W = \partial \Omega \setminus \Gamma_D
& \text{Dirichlet-boundary}\\
\vec{u}_0(x,y,z) & = & \{1,0,0\}
& \text{in}\ \Omega = (-0.5,0.5) \times (-0.5,0.5) \times (-0.5,0.5)
& \text{Initial Condition}
\end{array} \right.
\label{eq:NavierStokesCavityBenchmark}
\end{equation}
is investigated on different cartesian grids. The physical parameters of the fluid are choosen to be $\rho_f=1$ and $\mu_f=0.0025$ which renders down to a Reynoldsnumber of 400.

\graphicspath{{./apdx-NodeSolverPerformance/NavierStokesDrivenCavity/plots/}}

\begin{figure}[h!]
	\begin{center}
		\input{./apdx-NodeSolverPerformance/NavierStokesDrivenCavity/plots/NodePerformance.tex}
	\end{center}
	\caption{
		Solver runtime vs. DoFs, for polynomial degree $k=2/1$,
		for problem/Equation (\ref{eq:NavierStokesCavityBenchmark}).
	}
	\label{fig:DrivenCavity}
\end{figure}