\documentclass{article}

\usepackage{hyperref}
\usepackage{amsmath}
\begin{document}

\subsubsection{Sparse-Estimation}
% Following \url{https://en.wikipedia.org/wiki/Forward%E2%80%93backward_algorithm}
  \begin{align*}
    T_{ij} &= P(S_{t+1}=j |S_t=i)\\
    O_j &= \text{diag}(P(X_t=j | S_t=0), P(X_t=j | S_t=1))\\
    f(t) &= f(t)\left[TO_{o(t)}\right]^{L(t)}\\
    f(t) &= \hat{f}(\tau_{t+1}-1)= P(\hat{o}_1,\hat{o}_2,,,\hat{o}_{\tau_{t+1}-1},\hat{X}_{\tau_{t+1}-1}|\pi_0)\\
    b(t) &= \hat{b}(\tau_{\tau_{t+1}}-1) = P(\hat{o}_{\tau_{t+1}},\hat{o}_{\tau_{t+1}+1},,,\hat{o}_{\hat{N}},\hat{X}_{\tau_{t+1}-1}|\pi_0)\\
    b(t-1) &= \left[TO_t\right]^{L(t)}b(t)\\
  \end{align*}

  \subsubsection{Xi-sum}
  \begin{align*}
    \hat{\xi}_{ij}(\tau-1) &= \hat{f}_{i}(\tau-1)T_{ij}O_{\hat{o}(\tau), jj}\hat{b}_j(\tau)\\
    \hat{\xi}(\tau-1) &= T_{ij} \circ (\hat{b}(\tau)\hat{f-1}(\tau))^TO_{\hat{o}(\tau)}\\
    \sum_{\tau=1}^{\hat{N}-1} \xi_\tau &= \sum_{\tau=1}^{\hat{N}-1} T_{ij} \circ (\hat{b}(\tau)\hat{f}(\tau-1))^TO_{\hat{o}(\tau)}\\
    &= T_{ij} \circ \sum_{\tau=1}^{\hat{N}-1} (\hat{b}(\tau)\hat{f}(\tau-1))^TO_{\hat{o}(\tau)}\\
    &= T_{ij} \circ \sum_{k=0}^{N-1} \sum_{\tau=t_k}^{t_{k+1}-1} (\hat{b}(\tau)\hat{f}(\tau-1))^TO_{\hat{o}(\tau)}\\
    &= T_{ij} \circ \sum_{k=0}^{N-1} \left[\sum_{\tau=t_k}^{t_{k+1}-1} (\hat{b}(\tau)\hat{f}(\tau-1))^T\right]O_{o(t_k)}\\
    &= T_{ij} \circ \sum_{k=0}^{N-1} \left[\sum_{\tau=t_k}^{t_{k+1}-1} (M_{k}^{t_{k+1}-1-\tau}\hat{b}(t_{k+1}-1)\hat{f}(t_k-1)M_k^{\tau-t_k})^T\right]O_{o(t_k)}\\
    &= T_{ij} \circ \sum_{k=0}^{N-1} \left[\sum_{t=0}^{L(k)-1} (M_{k}^{L(k)-1-t}b(k)f(k-1)M_k^{t})^T\right]O_{o(k)}\\
  \end{align*}
  \subsubsection{Update}
  \begin{align*}
    \hat{\gamma}(\tau) &= \hat{f}_{\tau} \circ \hat{b}_{\tau}^T/(\hat{f}_{\tau}\hat{b}_{\tau})\\
    \hat{\Gamma^\circ} &= \text{diagonal}{\hat\Gamma}\\
&= \sum_\tau \hat{f}_{\tau} \circ \hat{b}_{\tau}^T/(\hat{f}_{\tau}\hat{b}_{\tau})\\
    \hat{\Gamma} &= K \sum_\tau \hat{b}_{\tau}\hat{f}_{\tau}\\
    &= K \sum_t \sum_{n<L(t)} \hat{b}_{\tau_t+n} \hat{f}_{\tau_t+n}\\
    &= K\sum_t \sum_{n<L(t)}  \hat{b}_{\tau_t+n}\hat{f}_{\tau_t+n}\\
    &= K\sum_t \sum_{n<L(t)} \left[TO_{o(t)}\right]^{L(t)-(n+1)}b(t)f(t-1)\left[TO_{o(t)}\right]^{n+1}\\
    &= K\sum_t \gamma(t)\\
  \end{align*}
  \begin{align*}
   \gamma(t) &= \sum_{n<L(t)} \left[TO_{o(t)}\right]^{L(t)-(n+1)}b(t)f(t-1)\left[TO_{o(t)}\right]^{n+1}\\
   TO_{o(t)} &= P_{o(t)}D_{o(t)}P_{o(t)}^{-1}\\
   A(t) &= P_{o(t)}^{-1}b(t)f(t-1)P_{o(t)}\\
   \gamma(t) &= \sum_{n<L_t}P_{o(t)}D_{o(t)}^{L_t-(n+1)}A(t)D_{o(t)}^{n+1}P_{o(t)}^{-1}\\
   \gamma(t) &= P_{o(t)}\sum_{n<L_t}\left[D_{o(t)}^{L_t-(n+1)}A(t)D_{o(t)}^{n+1}\right]P_{o(t)}^{-1}\\
  &= P\sum_{n<L} \left[ \begin{pmatrix} d_1^{L-(n+1)} &0\\ 0 & d_2^{L-(n+1)}\\ \end{pmatrix} A \begin{pmatrix} d_1^{n+1} &0\\ 0 & d_2^{n+1}\\ \end{pmatrix}\right]P^{-1}\\
  &= P\sum_{n<L} \left[ \begin{pmatrix} d_1^L & d_1^{L-n-1}d_2^{n+1}\\ d_2^{L-n-1}d_1^{n+1} & d_2^L\\ \end{pmatrix} \circ A \right]P^{-1}\\
  &= P\sum_{n<L} \left[ \begin{pmatrix} d_1^L & d_1^L(d_2/d_1)^{n+1}\\ d_2^L(d_1/d_2)^{n+1} & d_2^L\\ \end{pmatrix} \circ A \right]P^{-1}\\
  &= P\left[ \begin{pmatrix} Ld_1^L & (d_2/d_1)\frac{d_1^{L+1}-d_2^{L+1}}{d_1-d_2}\\ (d_1/d_2)\frac{d_1^{L+1}
         -d_2^{L+1}}{d_1-d_2} & L d_2^L\\ \end{pmatrix} \circ A \right]P^{-1}\\
  \end{align*}
% L+1 prob not right
  \begin{align*}
  \gamma(t) &= \sum_{n<L_t}(TO_{o(t)})^{L_t-n}b_{t+1}f_t(TO_{o(t)})^n\\
  \gamma(t) &= \sum_{n<L_t}P_{o(t)}D_{o(t)}^{L_t-n}P_{o(t)}^{-1}b_{t+1}P_{o(t)}D_{o(t)}^{n}P_{o(t)}^{-1}\\
  &= P\sum_{n<L_t}\left[D^{L_t-n}AD^{n}\right]P^{-1}\\
  &= P\sum_{n<L} \left[ \begin{pmatrix} d_1^{L-n} &0\\ 0 & d_2^{L-n}\\ \end{pmatrix} A \begin{pmatrix} d_1^n &0\\ 0 & d_2^n\\ \end{pmatrix}\right]P^{-1}\\
  &= P\sum_{n<L} \left[ \begin{pmatrix} d_1^L & d_1^{L-n}d_2^n\\ d_2^{L-n}d_1^n & d_2^L\\ \end{pmatrix} \circ A \right]P^{-1}\\
  &= P\sum_{n<L} \left[ \begin{pmatrix} d_1^L & d_1^L(d_2/d_1)^n\\ d_2^L(d_1/d_2)^n & d_2^L\\ \end{pmatrix} \circ A \right]P^{-1}\\
  &= P\left[ \begin{pmatrix} (L+1)d_1^L & d_1^L\frac{1-(d_2/d_1)^{L+1}}{1-d_2/d_1}\\ d_2^L\frac{1-(d_1/d_2)^{L+1}}{1-d_1/d_2} & (L+1) d_2^L\\ \end{pmatrix} \circ A \right]P^{-1}\\
  &= P\left[ \begin{pmatrix} (L+1)d_1^L & \frac{d_1^{L+1}-d_2^{L+1}}{d_1-d_2}\\ \frac{d_1^{L+1}
        -d_2^{L+1}}{d_1-d_2} & (L+1) d_2^L\\ \end{pmatrix} \circ A \right]P^{-1}\\
\end{align*}
\end{document}


%!TEX root = main.tex

\chapter{Modulation}
\label{chap:modulation}

\begin{figure}[H]
	\begin{center}
		\includegraphics[width = 14cm]{img/surface-modulation_3.jpg}
		\caption{``Surface Modulation'' by Richard Sweeney}
		\label{fig:Surface Modulation}
	\end{center}
\end{figure}

\begin{center}
\begin{figure}[h!]
\tikzset{concept/.append style={fill={none}}}
\begin{tikzpicture}
  \path[mindmap,concept color=black,text=black]
    node[concept] {Modulation}
    [clockwise from=0]
    child[concept color=red!50!black] {
      node[concept] {AM}
      % }
      [clockwise from=90]
      child { node[concept] {Envelopes} }
      child { node[concept] {AM} }
      child { node[concept] {Ring Modulation} }
      % child { node[concept] {pro\-gramming languages} }
      % child { node[concept] {software engineer\-ing} }
    }
    child[concept color=blue] {
      node[concept] (fm) {FM}
      [clockwise from=-30]
    }
    child[concept color=red] { node[concept] (pm){Phase Modulation} }
    child[concept color=orange] { node[concept] (sd) {Sound design Challenge} }
    child[concept color=green] { node[concept] (conv) {Convolution} };

\begin{pgfonlayer}{background}
    \draw [circle connection bar]
      (fm) edge (sd)
      (fm) edge (pm);
  \end{pgfonlayer}

\end{tikzpicture}
\caption{Lecture Contents}
\end{figure}
\end{center}

% \section{Notizen}

% kürzer. Modulation als sub einheit einplanen.
% Sounddesign challg. halbe stunde ok..

% Passt garnicht: (weil eher additive synth)
% https://www.youtube.com/watch?v=oKv9S6mxnXE

% Convolution anreissen?
% Notation durchgehen.

% Aliasing besprochen?
% \href{https://www.youtube.com/watch?v=GBtHeR-hY9Y}{Water experiment}
% (youtube \glqq{}The Secret to Levitation\grqq{})

% AM, tremolo

% Envelopes in pd

% FM, vibrato

% sounddesign chall.

% Hü

\section{Convolution}
\label{sub:conv}
Before talking about modulation, we should quickly look at what's convolution\footnote{'Faltung' in German}.\\
Convolution is a mathematical operation, widely used in signal processing but also in other fields. Convolution of two functions, say, $x$ and $h$ is typically denoted as $x*h$. \textbf{Note that $*$ does not mean multiplication here, it means convolving $x$ with $h$}. Obviously this can lead to confusion in some cases, but notation problems are not uncommon in sciences.\footnote{e.g while mathematicians use $i$ as the imaginary number, electrical engineering folks use $j$ (their $i$ is electrical current.).}\\

We will define Convolution of a function $x$ with a convolution kernel $h$ of length $N$as\footnote{You will find different definitions online. The given definition seems more intuitive but is not very common. Here is a common one:$y(n) = \sum_{k=-\infty}^{\infty} x(k) h (n-k)$}:

\begin{equation}
y(n) = \sum_{k=0}^{N-1} x(n-k) h (k)
\end{equation}
In words: for every output sample $y(n)$ take the current input sample $x(n)$ times the first sample of the convolution Kernel $h(0)$ plus take the previous input sample $x(n-1)$ times the second sample of the convolution kernel $h(1)$ plus the third last input sample $x(n-2)$ times the third sample of the convolution kernel $h(2)$ and so on... until the $Nth$ sample of $h$. It can be viewed as a a weighted average of the previous input samples, with weighting coefficients found in $h$.

We will look at convolution in a later chapter. If you  are eager to know more about convolution can have a look at
\href{https://youtu.be/_vyke3vF4Nk?t=25m14s}{this}\footnote{https://youtu.be/\_vyke3vF4Nk?t=25m14s}. \link{http://www.songho.ca/dsp/convolution/convolution2d\_example.html}{Here} is an example of convolution in 2D. Convolution in 2D is actually a bit easier to understand and is usually done to achieve a blur effect on an image or video. But why mention convolution if it is not explained at this point? Amplitude modulation and convolution just have a very close relationship, an that is:
\important{Multiplying two signals in the time domain is equivalent to convolution in the frequency domain and convolution in the time domain is equivalent to multiplication in the frequency domain.
}

\section{AM} % (fold)
\label{sub:AM}
In this section we will try amplitude modulation. Amplitude modulation is used for radio communication (so you'll need to understand this as a technician, modulation techniques are extremely important and this is maybe the simplest one) but also in sound design. We will try to understand the problem from different perspectives at once:
\begin{itemize}
	\item Doing some math
	\item listening to it
	\item brining it in context to beating waves
	\item seeing it as convolution in the frequency domain
\end{itemize}

Amplitude modulation means modulating the amplitude of a signal(surprise!). Modulating means changing over time by another signal. So we have some signal, say, a sine wave, and change its amplitude with another signal, say, another sine wave. Taking this concept and reducing it radically, we end up with figure \ref{fig:simpleAM}.

\begin{figure}[H]
	\begin{center}
		\includegraphics{img/ringNaive.png}
		\caption{Simplest form of ``Ring modulation''.}
		\label{fig:simpleAM}
	\end{center}
\end{figure}
Of course, we get no sound in figure \ref{fig:simpleAM} because the frequencies are not initialized, but it shows the general principle. The caption of that figure says ``Ring Modulation''. Let's quickly get some vocabulary straight:\\
\begin{itemize}
	\item ``Amplitude modulation'' might mean any modulation of amplitude
	\item ``Ring Modulation'' means \textit{bi-polar} amplitude modulation.
	\item In sound design, ``Amplitude Modulation'' might specifically mean unipolar amplitude modulation.
\end{itemize}

And some more vocabulary to put what we are doing in a musical context:
\begin{itemize}
	\item Modulating the amplitude is called ``Tremolo'' in music \footnote{sadly, the fender Stratocaster's ``Tremolo Arm'' is used to control the pitch. Ignore Fender, they got it wrong. You can trust that most guitar players are confused because of this.}
	\item Modulating the pitch or frequency (``FM'') is called ``vibrato'' in a musical context.\footnote{Maybe think about it like this: The F in FM is a bit like the v in vibrato. Just to avoid confusion..}
\end{itemize}

Enough words, let's look at what AM looks like, look at figure \ref{fig:AMViz}.

\begin{figure}[h!]
	\centering
	\includegraphics[width=\textwidth]{AMviz.png}
	\caption[AM time domain]
	{Looking at AM in the time domain}
	\label{fig:AMViz}
\end{figure}

\begin{question}
	If we would listen to the signal depicted in the bottom plot of figure \ref{fig:AMViz}, what do you think we would hear? Try to imagine! If you can't, use pure data to test it! That's why we are using pd.
\end{question}
\begin{Answer}
	We would hear a 30Hz sine wave repeatedly rising and falling in amplitude.
\end{Answer}

\begin{question}
	Next question, same plot, same signal. So hopefully you found out that we hear a 30 Hz sine with rising and falling amplitude. At what frequency does the amplitude rise and fall? Remember we are modulating with a 1 Hz sine.
\end{question}
\begin{Answer}
	Two Hertz.
\end{Answer}

You can try out plotting what happens with different frequency settings in \link{https://colab.research.google.com/drive/1oO3ApcIfvcUoIqv9\_nDS-HNl8B4Gf5hl}{this Notebook}.

Maybe you remember from the waveshaping chapter that we actually calculated what frequencies should come out of AM. Also maybe you remember that:

\important{
Amplitude Modulation produces sum and difference frequencies of the input frequencies.
\begin{equation}
	cos(a)\cdot cos(b) = \frac{cos(a+b) + cos(a-b)}{2}
\end{equation}
}
But here we still hear the 30 Hz, just getting louder and softer, so what's wrong?\\
So, let's calculate this. We have two oscillators, $x_1(t) = cos(30t2\pi)$ and $x_2(t)=cos(t2\pi)$. We multiply them, ending up with:

\begin{equation}
	y(t) = cos(30t2\pi) \cdot cos(t2\pi)
\end{equation}
Ok, the above formula tells us this means:
\begin{equation}
	y(t) = \frac{cos(30t2\pi+t2\pi)+cos(30t2\pi-t2\pi)}{2}
\end{equation}
We can now simplify to:
\begin{equation}
	y(t) = \frac{cos((30+1)t2\pi)+cos((30-1)t2\pi)}{2} = \frac{cos(31t2\pi)+cos(29t2\pi)}{2}
\end{equation}

Hm, so we get out a 31Hz and 29Hz oscillator. What about that rising and falling in amplitude that we hear \textit{and} observe in the plot, surely there must be something wrong! Do you have a solution to this?\\

Let's use pure data to help us understand. Make two oscillators, one with 29Hz and one with 31Hz, what do you hear?

\begin{figure}[h!]
	\centering
	\includegraphics{beating.png}
	\caption[adding two oscillators]
	{Adding two oscillators with frequencies very close to each other}
	\label{fig:beating}
\end{figure}

If you listen to what is depicted in figure \ref{fig:beating}, you will in fact hear the same as if you do the 30Hz /1 Hz amplitude modulation (which indicates that our formula above is correct). What we hear is a phenomenon called beating(``Schwebung''). The phase of the two oscillators is canceling each other out at regular intervals because the frequencies are so close to each other. In fact the frequency of the beating $f_{beat}$ is always
\begin{equation}
	f{beat}=|f_1-f_2|
\end{equation}
So the difference of the two frequencies.

% \comm{vline and envelop description missing}
% \begin{figure}[H]
% 	\begin{center}
% 		\includegraphics[width = 14cm]{img/simpleEnv.png}
% 		\caption{caption}
% 		\label{fig:name}
% 	\end{center}
% \end{figure}

\section{FM} % (fold)
\label{sub:FM}

Frequency modulation can be used to modulate the frequency, meaning to vary the pitch of a sound, see figure \ref{fig:fmSlow}. But it can also be used to generate overtones and an overall richer spectrum, see figure \ref{fig:fmFast}. The two ``different versions'' only differ from each other by different parameter\footnote{We see which parameters can be adjusted in a minute. But if you want to know them now: modulation frequency, modulation amount and carrier frequency.} values being used.
We can think of Frequency Modulation (FM) as something of the form:

\begin{equation}
	y(t) = cos(cos(b))
\end{equation}

This really is a simplification, but the overall structure of the formula is correct.
% FM is mathematically very similar to phase modulation.
Looking at figure \ref{fig:fmIdea} we can see the very basic idea implemented in pure data.
\begin{figure}[H]
	\begin{center}
		\includegraphics{img/FMgeneral.png}
		\caption{The General Idea of FM}
		\label{fig:fmIdea}
	\end{center}
\end{figure}

Since the frequencies of the oscillators are not set, we won't hear anything when building this patcher. But looking at it may reveal that the idea simply is to control the frequency of an oscillator using another oscillator.\\
We can expand the patcher by adding some math to make it more usable, as in figure \ref{fig:fmNaive}.

Naive parameters are ${f_c}$ (Carrier Frequency), ${f_m}$ (Modulator Frequency), and ${A_m}$ (modulation Amount).

\begin{figure}[H]
	\begin{center}
		\includegraphics{img/FMnaive.png}
		\caption{Naive Implementation with Direct Parametrization.}
		\label{fig:fmNaive}
	\end{center}
\end{figure}

\important{

The output frequencies will be
\begin{equation}
	f_c \pm n \cdot f_m
\end{equation}

for $n$ being all integers. So the carrier frequency plus and minus integer multiples of the modulation frequency. We get many (theoretically infinitely) many overtones with different amplitudes this way.
}
\bgInfo{
	The amplitudes of the different frequencies are determined by Bessel functions which makes it so seemingly random.
		\centering
		\includegraphics[width=11cm]{bessel}
}

Typically, FM is controlled via \textit{Index}, \textit{Ratio}, and fundamental Frequency.
The Index, ${I}$ is given by Modulation Depth and Modulator Frequency.

% =============Something is wrong with this.============================

% \begin{mdframed}[backgroundcolor=black!10,rightline=false,leftline=false]
\bgInfo{
\begin{equation}
I = \frac{A_m}{f_m} % verified via http://computermusicresource.com/FM.synthesis.html
\end{equation}

A more controllable Implementation will generate the naive parameters from a Ratio, ${R}$, the Carrier Frequency and the Index:
\begin{equation}
	f_m = \frac{f_c}{R}
\end{equation}

\begin{equation}
	A_m = I \cdot f_m
\end{equation}

\begin{figure}[H]
	\begin{center}
		\includegraphics{img/FMcorrect.png}
		\caption{FM with Index and Ratio}
		\label{fig:fmComplete}
	\end{center}
\end{figure}

But there are also different implementations such as:
\begin{figure}[H]
	\centering
	\includegraphics[width=5cm]{fmMillerPuckette}
	\caption[miller Puckette FM]
	{Fm implementation from \cite{miller_puckette_theory_2006}}
	\label{fig:label}
\end{figure}

% ===========FIND EN EXPLANATION FOR THE ABOVE FM implementations=============

% \end{mdframed}
% =========================================================================
}

\begin{figure}[h!]
	\centering
	\includegraphics[width=\textwidth]{FMslow}
	\caption[FM visualization, low freq]
	{Modulator frequency: 1Hz, Crarrier Frequency 50 Hz, Modulation Amount 10. Please ignore the labeling of the Y axis of the spectrum plot($10^0$). It is wrong. The y axis goes from 0 to nyquist=22050Hz.}
	\label{fig:fmSlow}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[width=\textwidth]{FMfast}
	\caption[FM visualization, hi freq]
	{Modulator frequency: 100Hz, Crarrier Frequency 1000 Hz, Modulation Amount 0.5. Please ignore the labeling of the Y axis of the spectrum plot($10^0$). It is wrong. The y axis goes from 0 to nyquist=22050Hz.}
	\label{fig:fmFast}
\end{figure}

\section{Key Points}
\begin{itemize}
	\item Make sure you understand the differences between AM and FM
	\item Make sure you recognize what form of modulation is present if you see a patcher or a simplfied formula (like: $sin(a) \cdot sin(b)$ what form of modulation is this?)
	\item make sure you could make such a simplified formula if you see a patch and vice versa.
	\item make sure you know what frequencies result from an amplitude modulation
	\item make sure you know what frequencies result from a frequency modulation
\end{itemize}

% \section{Hausübung}
% \label{sub:Hausuebung}
% Andy Farnell, \href{http://aspress.co.uk/ds/pdf/pd_intro.pdf}{pd intro} chapter 6, lesen


\chapter{Conclusion and Future Work}\label{sec:summary} \label{sec:future_work}
%I first experimented with white-box model inference of MicroPilot's source code using existing state of the art tools.
%However, the algorithms' implementations showed quite a number of scalability issues when tested with such a large scale software system. I was able to mitigate or even solve some of the issues through various methods and tricks, but still they were not usable as a robust model inference technique. These attempts were published in ICPC '19 (co-located with ICSE) in negative results track. \footnote{I referenced that paper \cite{mashhadi2019empirical} in this thesis but did not include it.}
%Thereafter, considering the partner company's needs, I took a higher level black-box approach in modeling the software.

In this thesis, I developed a novel method for inferring black box models for autopilot software systems.
My method at its core is a deep neural network that combines convolutions and recurrent cells: hybrid CNN-RNN model.
This design is inspired by deep neural network architectures that showed good performance in other fields (such as speech recognition, sleep phase detection, and human activity recognition).
It can be used for both CPD and state classification problems in multivariate time series.

This method be used as a black-box state model inference for variety of use cases such as testing, debugging, and anomaly detection in control software systems, where there are several input signals that control output states.

I have trained and evaluated this neural network on two case studies of a UAV autopilot softwares, one from the FOSS\footnote{Free (as in freedom) and Open-Source software.} community and one from our industry partner.
It showed promising results in inferring a behavioural model from the autopilot execution data; showing significant improvement in both change point detection and state classification as compared with several baselines on 10 comparison metrics.

% I created a tool that automatically generated fuzz test scenarios for fixed wing aircraft that use Paparazzi. It could also execute the tests and collect telemetry data for training the model down the way.
% I managed to replicate the study and get results as good as (in some cases even better) the first study on MicroPilot. In addition, I created a hyper-parameter tuning pipeline that significantly improved the results on Paparazzi's data; the default hyper-parameter values were [manually] tuned for MicroPilot.

Some potential extensions to this work include:
(a) Examining if and how transfer learning can be employed to make this method work with smaller datasets and also reduce the labeling overhead.
(b) Adapting this method to work with other data types such as images in video surveillance systems, or sensor data + stream of images in self-driving cars, could be another interesting line of work for continuing this work
(c) Using the inferred model to perform a downstream task such as test generation or validation.
(d) This approach at its current form is an offline method which is more useful for a postmortem analysis. To make it beneficial in more settings, it could be expanded into being an online (or even real-time) method.


\chapter{FEniCS}

The \fenics project started in 2003 and the aim was create software that automates a finite element discretisation and solution of differential equations. The core libraries used within \fenics are DOLFIN \cite{LoggWells2010a,LoggWellsEtAl2012a}, FFC \cite{KirbyLogg2006a,LoggOlgaardEtAl2012a,OlgaardWells2010b}, FIAT \cite{Kirby2012a,Kirby2004a}, Instant, UFC \cite{AlnaesLoggEtAl2009a,AlnaesLoggEtAl2012a} and UFL \cite{AlnaesEtAl2012,Alnaes2012a}. Along the these core packages \fenics has a few extra optional packages that can be found here \url{http://fenicsproject.org/applications/}.

\section{Overview}

One of the key ideas of \fenics was to be able to write an easy to use software package for the solution of partial differential equations (PDEs). The main underlying code base for \fenics is written in C++. and Python

\fenics supports large range of a different finite element function space. Thus allowing \fenics to be used from electromagnetic problems to fluid. The full list of support elements can be found in table~\ref{tab:FunctionSpace}.
\begin{table}[h!]
    \begin{center}
        \begin{tabular}{ l  l }
            Name    &Usage\\
            \hline
            Argyris*    &``ARG''\\
            Arnold-Winther* &``AW''\\
            Brezzi-Douglas-Fortin-Marini*   &``BDFM''\\
            Brezzi-Douglas-Marini   &``BDM''\\
            Bubble  &``B''\\
            Crouzeix-Raviart    &``CR''\\
            Discontinuous Lagrange  &``DG''\\
            Hermite*    &``HER''\\
            Lagrange    &``CG''\\
            Mardal-Tai-Winther* &``MTW''\\
            Morley* &``MOR''\\
            Nedelec 1st kind H(curl)    &``N1curl''\\
            Nedelec 2nd kind H(curl)    &``N2curl''\\
            Quadrature  &``Q''\\
            Raviart-Thomas  &``RT''
        \end{tabular}
        \caption{Avaliable finite element function space (*only partly supported)}
        \label{tab:FunctionSpace}
    \end{center}
\end{table}

This breath introduction to \fenics will go through how to set up a very simple test case, namely the Poisson equation. For this simple problem we will consider both Dirichlet and Neumann boundary conditions and show how to set up the PDE in its variational form.

\section{Poisson example}

For a simple example to see how FEniCS works consider the Poisson equation. Defining $\Omega \subset \R^n$ to be the domain and the boundary as $\partial \Omega = \Gamma_D \cup \Gamma_N$ where $\Gamma_D$ and $\Gamma_N$ correspond to the Dirichlet and Neumann boundaries respectively. Thus, the Poisson equation with corresponding boundary conditions reads as:
\begin{equation} \label{eq:poisson}
 \left. \begin{aligned}
-\Delta u &= f \quad \mbox{in } \Omega\\
 u &= 0 \quad \mbox{in } \Gamma_D\\
\nabla u \cdot n &= g \quad \mbox{in } \Gamma_N
    \end{aligned}
 \right.
 \qquad \text{}
\end{equation}
For this example we will be considering the following function, domains and boundaries to be:
\begin{itemize}
\item $f = (x-\nicefrac{1}{2})^2-4(y-\nicefrac{1}{2})^4$
\item $g = 5x$
\item $\Omega = [0,1]\times[0,1]$
\item $\Gamma_D = \{(0,y) \cup (1,y) \quad | \quad  0\leq y\leq 1\}$
\item $\Gamma_N = \{(x,0) \cup (x,1) \quad | \quad  0\leq x\leq 1\}$
\end{itemize}
Defining the following trial and test function spaces  $\mathcal{V}$ and $\bar{\mathcal{V}}$ as
\begin{equation} \label{eq:PoissonFuncSpace}
 \left. \begin{aligned}
    \mathcal{V}&=H^1_{u_0}(\Omega)=\left\{\,{u}\in H^1(\Omega)\,:\,\text{${u}=u_{0}$ on $\partial\Omega$}\,\right\}, \\
    \bar{\mathcal{V}}&=H^1_0(\Omega)=\left\{\,{u}\in H^1(\Omega)\,:\,\text{${u}={0}$ on $\partial\Omega$}\,\right\}.
 \end{aligned}
 \right.
 \qquad \text{}
\end{equation}
Thus the variational formulation of (\ref{eq:poisson}) depends on finding $u \in \mathcal{V}$ such that
\begin{equation}
\label{eq:PoissonWeak}a(u,v) = L(v) \quad \forall v\in\bar{\mathcal{V}},
\end{equation}
where
\begin{equation}
a(u,v) = \int_{\Omega} \nabla u \cdot \nabla v \, dx \quad \mbox{and} \quad L(v) = \int_{\Omega} fv dx +\int_{\Gamma_N}gv\,ds.
\end{equation}

\subsection{Defining mesh and function space}

For this example we want to consider a uniform unit square mesh. This is created by using the following python code
\begin{pythoncode}
mesh = UnitSquareMesh(32,32)
\end{pythoncode}
Once the mesh is create we no turn to the function space. For this model problem we want to consider the function space $\mathcal{V}$  and $\bar{\mathcal{V}}$ defined by \eqref{eq:PoissonFuncSpace}. Therefore, the discrete finite element space use is the following space
$$\uu{V}_h  = \{\, \uu{u}\in H_1( \Omega)\, :\, \uu{u}|_K \in {\mathcal P}_{1}(K), \, K \in{\mathcal T}_h \, \},$$
where ${\mathcal T}_h=\{K\}$ regular and quasi-uniform triangles. This function space is create in \fenics using the following command
\begin{pythoncode}
V = FunctionSpace(mesh,'CG',1)
\end{pythoncode}

\subsection{Defining subdomains}
Separating different parts of the boundary is easily done within FEniCS. For this our simple Possion model the boundary is split up into two parts. The top and bottom defines the Dirichlet boundary condition whilst the left and right portions of the boundary define the Neumann conditions. Before partitioning the domain into the Dirichlet and Neumann boundaries we need to define the different sections of the boundary. This is done using classes as follows:
\begin{pythoncode}
# Defining boundary classes
class Left(SubDomain):
    def inside(self, x, on_boundary):
        return near(x[0], 0.0)

class Right(SubDomain):
    def inside(self, x, on_boundary):
        return near(x[0], 1.0)

class Bottom(SubDomain):
    def inside(self, x, on_boundary):
        return near(x[1], 0.0)

class Top(SubDomain):
    def inside(self, x, on_boundary):
        return near(x[1], 1.0)

# Initialize sub-domain instances
left = Left()
top = Top()
right = Right()
bottom = Bottom()
\end{pythoncode}
Using the classes above we group the Dirichlet and Neumann boundaries together. These can then be used to set up the boundary conditions for our problem.
\begin{pythoncode}
boundaries = FacetFunction("size_t", mesh)
boundaries.set_all(0)
left.mark(boundaries, 1)
right.mark(boundaries, 1)
top.mark(boundaries, 2)
bottom.mark(boundaries, 2)
\end{pythoncode}

\subsection{Boundary Conditions and weak formulation}

The boundaries have been split into their separate domains so now we can think about defining the boundary conditions, bilinear and linear (\ref{eq:PoissonWeak}) forms.

First consider the homogeneous Dirichlet boundary:
$$  u = 0 \quad \mbox{in } \Gamma_D.$$
The Dirichlet boundary condition is created by using the  \code{DirichletBC} class. When defining Dirichlet boundary conditions on separate subdomains we need to use four inputs to the  \code{DirichletBC} class. The first one is the function space that the boundary condition applies to, the value on the boundary and then the last two arguments define which part of the boundary the boundary condition is defined on.
\begin{pythoncode}
u0 = Constant(0.0)
bc = DirichletBC(V, u0, boundaries,1)
\end{pythoncode}

To define the variational form of the problem we first need to determine the trial function  \code{u} and test function  \code{v} belonging to the function space $\mathcal{V}$. Furthermore, the source term $f$ and Neumann condition $g$ are required in the linear form $L(v)$. Both are defined using the  \code{Expression} class. Together with the trial and test functions the variational form can be created using the following code:
\begin{pythoncode}
# Define variational problem
u = TrialFunction(V)
v = TestFunction(V)
f = Expression("(pow(x[0]-0.5,2)-4*pow(x[1]-0.5,4))")
g = Expression("(5*x[0])")
a = inner(grad(u), grad(v))*dx
L = f*v*dx + g*v*ds(2)
\end{pythoncode}

\subsection{Assembly and solving}

Now the variational  (or weak form) have been stated the next step is to solve the problem. This can be done in two was within FEniCS
\begin{itemize}
    \item[1.] Assemble system and solve
    \item[2.] Solve without assembly.
\end{itemize}
First, consider assembling then solve the system. This can be done by using the {\code{assemble\_system}} class. This class takes the variational form as the inputs and the Dirichlet boundary condition.
\begin{pythoncode}
A, b = assemble_system(a,L,bc)
\end{pythoncode}
The final solving step is the same for both ways. To do this we first define a function {\code{u}} in the corresponding function space ($\mathcal{V}$) that will represent the solution. Next, calling the \code{solve} function with either the matrix \code{A} and vector \code{b} as the arguments or the variational form will compute the solution using sparse direct solvers which is the  default solving method for \fenics
\begin{pythoncode}
u = Function(V)
solve(A,u.vector(),b)
solve(a == L, u, bc)
\end{pythoncode}

\subsubsection{Optional solution parameters}

For simplicity above the \code{solve} used \fenics default parameters. However, this may not necessarily be the most efficient so may want to change these parameters. It is well known for example that multigrid is the most efficient solve for the Poisson problem. Here is how to set this up.
\begin{pythoncode}
solve(a == L, u, bc,
            solver_parameters=dict(linear_solver="cg",
            preconditioner="amg"))
\end{pythoncode}
For more optional parameters use the following commands:
\begin{pythoncode}
print list_krylov_solver_methods()
print list_krylov_solver_preconditioners()
print list_linear_algebra_backends()
print list_linear_solver_methods()
print list_lu_solver_methods()
\end{pythoncode}

Instead of using FEniCS's inbuilt solving function you can use other external packages. For instance you can link you your FEniCS code with Trilinos \cite{Trilinos-Users-Guide,Trilinos-Overview}, PETSc \cite{petsc-web-page,petsc-user-ref} or your own code.

\subsection{Visualisation}

Once you have solved your problem the next step is to either visualise or save the result. To output the solution in a VTK file you need to create a file with \code{.pvd} suffix. This enables you to use external software (for example ParaView \cite{}) to visualise the solution. Alternatively FEniCS's has an interface for the VTK visualisation tools. We do this by using the \code{plot} command.
\begin{pythoncode}
# Save solution in VTK format
file = File("poisson.pvd")
file << u

# Plot solution
plot(u, interactive=True)
\end{pythoncode}

\begin{figure}[h!]
  \centering
    \includegraphics[scale=.6]{../FEniCS/Figures/dolfin_plot_1}
\caption{Numerical solution to (\ref{eq:poisson})}
\end{figure}

% \bibliographystyle{plain}
% \bibliography{/home/mwathen/Dropbox/MastersResearch/MHD/THESIS/ref/ref}


\chapterp{3p}{Offline labelling of SWR segments}
\label{ch:offline}

To quantify the performance of a sharp wave-ripple (SWR) detection algorithm, we need an evaluation or `test' recording, annotated with the segments of time when actual SWR events were present. This section describes how such annotations can be made, and how our data specifically was annotated.

SWR's are an empirical phenomenon of hippocampal area CA1, `defined' by what their voltage traces look like. In other words, there is no ground truth available to know when SWR's occur. Scientists looking to annotate their LFP recordings with SWR segments therefore have to rely either on judgement calls by human labellers, or on an automated, offline SWR detection algorithm. The former could be considered more subjective, and is definitely more labour intensive than the latter -- especially if multiple scientists are consulted to obtain a consensus labelling. Most studies use an automated, offline algorithm to detect SWR segments (see \cref{apx:offline-detection-algos} for relevant quotes from a collection of such studies).

`Offline' here means that SWR detection happens after the recording has been completed, and that there are thus no real-time constraints on the detection algorithm. This means that 1) there are no hard bounds on algorithm execution time, and 2) that the algorithm can use information `from the future': when deciding whether a recording sample $\z_t$ belongs to an SWR segment, it can consider samples $\z_{t_f}$ that occured after $\z_t$ (i.e. $t_f > t$), instead of considering only past samples $\z_{t_p}$ (where $t_p \leq t$).

\begin{figure}
\img[1.14]{107_2--107_8}
\captionn{Steps for automated, offline SWR labelling}{See text for details. Each vertical scalebar indicates the same voltage range. $H[o_t]$ denotes the Hilbert transform of $o_t$. Note its phase lag of 90\si{\degree} with respect to $o_t$. In the second panel from the bottom, note the two threshold crossings of $T\low$ (marked with black dots) that did not result in a ripple segment, because the second threshold $T\high$ was not reached. The distribution of the envelope $n_t$ was estimated using the entire dataset (and not just the displayed fragment).}
\label{fig:offline-steps}
\end{figure}

\section{Overview of offline ripple detection}

The main steps of the offline SWR detection algorithm that most studies use -- such as the ones cited in \cref{apx:offline-detection-algos}, and the one of this thesis -- can be summarized as follows:
\begin{enumerate}
\item Use a single channel of input data; namely from an electrode in the pyramidal cell layer of CA1, where the ripple part of SWR's is most strongly present. (We will denote this voltage signal with $z_t$);
\item Band-pass filter the recording to retain only `ripple' frequencies. (We will denote the filter output as $o_t$);
\item Obtain the envelope of the band-pass filtered signal. (We will denote this envelope with $n_t$);
\item Calculate a `high' and a `low' threshold ($T\high$ and $T\low$) to apply to the envelope $n_t$, based on summary statistics of $n_t$ and two custom multipliers ($\alpha\high$ and $\alpha\low$);
\item Define ripple events as times when the envelope crosses the high threshold (i.e. $n_t > T\high$);
\item Define the start and end time of each such ripple as the closest times where the envelope falls back below the lower threshold (i.e. $n_t < T\low$).
\end{enumerate}

\Cref{fig:offline-steps} visualizes each step, as applied to a fragment of our dataset.

Note that this procedure only detects ripples, and not sharp waves.\footnote{Although interestingly, the sharp wave part of sharp wave-ripples was discovered before the ripple part \cite[p. 1]{Buzsaki2015}.}

\section{Validating parameter choices}
\label{sec:validating-parameter-choices}

Some of the above steps have free parameters (such as the ripple frequency band used for filtering, or the threshold multipliers $\alpha\high$ and $\alpha\low$). If not mentioned otherwise, our parameter choices were made as follows.

In a first pass of the offline detection algorithm, ripples were detected using a very broad-band filter and a low detection threshold. This ensured that all `true' SWR's were included in the detected events set (in addition to many spurious detections).

Next, five neuroscientists were independently asked to decide for each detected event whether it was a sharp wave-ripple or not. This was done through a custom-made web app, an example screen of which is shown in \cref{fig:labelface-UI}. Note that the labellers could factor multiple recording channels into their decision, including stratum radiatum channels displaying sharp wave activity. Only the events that were labelled as an SWR by at least three neuroscientists were retained.

Finally, when setting the parameters of the eventual offline ripple detection algorithm, the algorithm's output was compared to the decisions made by the neuroscientists. The parameters were then adjusted until the output of the algorithm matched the neuroscientists' decisions reasonably well.

The following subsections describe the detection steps in more detail, and compare our choices of methods and parameters to those made in the literature.

\begin{figure}
\img[1.1]{labelface-UI}
\captionn{User interface for SWR labelling}{Each event in the list at the top is represented by two voltage traces: one from an electrode in the pyramidal cell layer (the top trace), and one from an electrode in the stratum radiatum.
The large plot at the bottom gives more comprehensive view of the currently active event: all 16 channels are plotted (instead of only two), and the plot ranges from 1000 ms before to 1000 ms after the event.
In this large plot, the blue vertical line marks the currently active event. The grey vertical lines correspond to other detected events. (The scalebar at the beginning of this plot represents 1 mV). Users decide whether the currently active event is an SWR by clicking the buttons in the top-right corner or by using keyboard shortcuts. The source code for this labelling web app is available at \url{https://github.com/tfiers/labelface}.}
\label{fig:labelface-UI}
\end{figure}

\section{Band-pass filter design}

The wideband voltage signal $x_t$ was band-pass filtered between 100 and 200 Hz, using a linear time-invariant filter with zero output lag.
% todo: mention downsampling

Many other studies use a higher left bound, of about 140--150 Hz (see \cref{tab:bands}). Using such a high bound in our dataset however resulted in ripples that went undetected (even at low detection thresholds), although they were convicingly marked as SWR's by the neuroscientists.
% todo: plot peak-f distribution of marked SWR's

The band-pass filter was designed using the windowed-sinc method, with a Kaiser window (using SciPy's \texttt{firwin} and \texttt{kaiserord} functions) \cite{Roelandts2016,Jones2018a}. The transition width was chosen to be 10\% of the bandwidth (i.e. 10 Hz), and the attenuation to be 40 dB, resulting in an FIR filter of order 150 (at a 1000 Hz sampling frequency).

The filter was applied bidirectionally, resulting in a zero-lag output, and a total attenuation of 80 dB. \Cref{fig:offline-steps} shows an example of the filter output in green.

(We cannot easily compare our filter design method with the literature, as most studies do not mention anything about filter design besides the frequency band used).

\section{Envelope calculation}

Next, the instantaneous envelope $u_t$ of the filter output $o_t$ was calculated using its Hilbert transform\footnotemark{} $H[o_t]$:
%
\begin{equation}
\label{eq:Hilbert_envelope}
u_t = \sqrt{o_t^2 + H[o_t]^2},
\end{equation}

\footnotetext{To be precise, $H$ is a discrete approximation to the continuous Hilbert transform $\mathcal{H}$. There are multiple ways to make a discrete approximation to $\mathcal{H}$ \cite{Oppenheim2009}. Most of them make use of the discrete Fourier transform, which makes computing $H$ an efficient operation. We used SciPy's \texttt{hilbert} function (which, confusingly, does not return the Hilbert transform of its input, but rather its analytical signal), and zero-padded the input signal to the nearest power of 2 or 3 to benefit from the speedup brought by the fast Fourier transform algorithm.}

The Hilbert transform delays each frequency component of a signal by 90\si{\degree} (see the gray signal in \cref{fig:offline-steps})  \cite{Lyons2010}. This means that the envelope of a narrowband signal such as $o_t$ can be easily obtained as the magnitude of the so called `analytic signal' $o_t + j H[o_t]$, as in \cref{eq:Hilbert_envelope}.\footnotemark{}

\footnotetext{To see why, consider a local approximation of the narrowband signal $o_t$ by a sinusoid $a \cos{\omega t}$. Its Hilbert transform is then $a \sin{\omega t}$. From \cref{eq:Hilbert_envelope}, the envelope $u_t$ will be locally approximated by the magnitude of the original signal: $u_t = \sqrt{(a \cos{\omega t})^2 + (a \sin{\omega t})^2} = \abs{a}$.}

After calculating $u_t$, a final, smoothed envelope $n_t$ was obtained by convolving $u_t$ with a Gaussian kernel ($\sigma$ = 7.5 ms, support radius of $4 \sigma$). Compare the blue ($u_t$) and the red signal ($n_t$) in \cref{fig:offline-steps}.

Other studies (such as \cite{Nadasdy1999} and \cite{Csicsvari2000}) use a ``root-mean-square'' approach to calculate the envelope of the filter output $o_t$. In these studies, presumably, the squared signal $o_t^2$ is smoothed using some kernel (of unspecified type and bandwidth) to obtain the ``mean-square'' signal of which the square root is taken.

\section{Threshold calculation}

The two detection thresholds were calculated as follows:
\begin{align}
\label{eq:thresholds-symbolic}
T\high &= \alpha\high \times \median{n_t} \\
T\low  &= \alpha\low  \times \median{n_t}
\end{align}
%
where $\median{n_t}$ denotes the median of the smoothed envelope $n_t$. As per the procedure described in \cref{sec:validating-parameter-choices}, we set $\alpha\high = 6.2$ and $\alpha\low = 3.6$. At a median envelope magnitude $\median{n_t} = \SI{17.0}{\micro\volt}$, this results in thresholds $T\high = \SI{105.4}{\micro\volt}$ and $T\low = \SI{61.2}{\micro\volt}$.

Most studies calculate thresholds as follows: $T\high = \mean{n_t} + \beta\high \times \std(n_t)$. Here $\mean{n_t}$ denotes the mean of the envelope, $\std(n_t)$ denotes its standard deviation, and $\beta\high$ is a custom multiplier analogous to $\alpha\high$. For the non-negative, assymetric distributions of envelope signals, using both a measure of center and a measure of spread to define thresholds seems unnecessary (see the distribution of $n_t$ in \cref{fig:offline-steps}), which is why we chose to use only one measure (namely the median).
% One advantage of the median over the mean, is that the former is far less sensitive to outliers of than the latter. Our dataset did not seem to include any outliers, such as erroneous samples with improbably high magnitude, however.

The detection multiplier $\beta\high$ varies wildly between studies: from 1, over 3, 4, and 5, up until 7 (\cite{Csicsvari2000,Dutta2018,Behrens2005,Sadowski2016,Nadasdy1999}, respectively).\footnotemark{} It is clear that such different thresholds will give very different sensitivity-precision trade-offs for SWR detection (the lower thresholds yielding more false positive detections, and the higher thresholds yielding more missed true SWR events).

\footnotetext{Threshold multipliers cannot be compared precisely. Imagine two recordings with equally powerful ripples. Both recordings will then need an equal threshold $T\high$ to detect the same types of ripples. When one of the recordings has a different background `noise' level or a different ripple incidence rate, it will have different $\mean{n_t}$ and $\std(n_t)$ values. This means that $\beta\high$ needs to change to maintain an equal threshold $T\high$.}

To compare our thresholds to those in the literature, we calculate the $\beta$ multipliers corresponding to our chosen $\alpha$ multipliers. Given that our dataset has a mean envelope magnitude of 22.3 \uV{} and a standard deviation of 22.9 \uV{}, we find $\beta\high = 3.63$ and $\beta\low = 1.70$. These values fall near the center of those reported in the literature (see \cref{apx:offline-detection-algos}).
% todo: calculate ripple magnitude and background magnitude

\section{Segment post-processing}

Finally, two add-hoc rules were applied to the automatically detected ripple segments. First, segments with only a small gap between them (of less than 10 ms) were joined together. Then, segments of too short a duration (less than 25 ms) were eliminated.

This step is rarely done in other studies. An exception is e.g. \cite{Dutta2018}, were segments shorter than 15 ms were eliminated.

% todo:
% - two thresholds is standard (Ji & Wilson 2007, via Ego-Stengel 2009)
% - Ego-Stengel 2009 also joins small gaps.

% todo: standardized data


\section{Sequences and series}


\chapter{Alternative RL Algorithms}\label{alternativeRL}

\paragraph{Sarsa-Learning}

Sarsa Learning is very similar to Q-Learning, and it generalises to Deep Sarsa-Learning just as Q-Learning generalises to Deep Q-Learning. The difference is that unlike Q-Learning, which is an off-policy method, Sarsa is on-policy. This means that in its update equation:

\begin{equation} \label{eq:sarsa-learningUpdate}
Q(s_t,a_t) \longrightarrow Q(s_t,a_t) + \alpha[( r_t + Q(s_{t+1}, a')) - Q(s_t,a_t)]
\end{equation}

$a'$ is also sampled according to the $\epsilon$-greedy technique, and it is not chosen greedily to be the best estimate like in Q-Learning. Then, naturally, the next chosen action will be exactly $a'$.

Due to this difference, Sarsa is more stable during training, but also it converges more slowly as it is not directly learning the optimal (greedy) policy, but an $\epsilon$-greedy policy~\cite{sutton2018RLbook}. Hence, Sarsa is more suitable when performance during training matters, and bad decisions are penalised (e.g. a valuable robot gets broken), but this is not the case in our protocols.

\paragraph{Monte Carlo Methods}

While Q- and Sarsa-Learning update their estimates based on other estimates (from one step ahead), Monte Carlo methods only use actual rewards for the update. This way, initialisation of the estimates doesn't matter that much, so it is more robust. In particular, the update rule in a simple Monte Carlo method is

\begin{equation} \label{eq:monte-carloUpdate}
Q(s_t,a_t) \longrightarrow Q(s_t,a_t) + \alpha[G_t - Q(s_t,a_t)]
\end{equation}

The problem with this approach is slow training. The reason is partly that if any exploration action (the case with probability $\epsilon$) is taken after timestep $t$, then, $Q(s_t,a_t)$ cannot be updated, since the new estimate doesn't necessarily reflect the estimated optimal value. Overall, Monte Carlo methods are rarely used in practice but they can be combined with Q-Learning (see e.g. the recent~\cite{wang2018montecarloqlearning}), which is an option I do not consider any further.

\paragraph{Policy Gradient}

Policy gradient methods are in contrast with the methods outlined above because they do not learn state- or action-value functions, instead they directly learn an optimal (stochastic) policy. Briefly, these algorithms use (another) neural network that represents the policy, and therefore returns probabilities choosing a given action in a given state. This leads to using the outputs of the neural network directly while playing the games during training, not the $\epsilon$-greedy technique. An advantage of this method is that there is no sharp boundary between the currently best estimated action and the second best, unlike for $\epsilon$-greedy. \NOTE{A}{Maybe add an equation. The problem is that it is a bit out of nowhere without derivation, and the derivation is a bit long.}

From the several policy gradient related approaches, I implemented the so-called Actor-Critic method~\cite{grondman2012actorcritic} and it didn't provide superior results to Deep Q-Learning or Sarsa-Learning. As explained in~\cite{bhandari2019policygradientconvergence}, policy gradient methods might converge to a local maximum, not the global optimum policy, and they might requires more time to converge. Policy gradient methods are still an active area of research, and while their usecase in unknown environments with state aliasing (where a stochastic strategy is desired) is clear, they are not yet the most widely used in full-knowledge scenarios like ours (i.e.\ where the exact current state is known).


% !TeX spellcheck = en_US
% !TeX root = DynELA.tex
%
% LaTeX source file of DynELA FEM Code
%
% (c) by Olivier Pantalé 2020
%
%\selectlanguage{french}%

\chapter*{Notations\addcontentsline{toc}{chapter}{Notations}\markboth{Notations}{Notations}}

\LETTRINE{F}rom a general point of view, it is usual to observe that one of the main difficulties in the field of mechanics, as in other fields, is the non-homogeneity of notations between the various authors. It is then easy to make completely incomprehensible the slightest theory when one decides to change notation. As the notion of universal notation is not yet valid (even if certain conventions can be assimilated to universal concepts), then we present below the set of notations used throughout this document and in a broader way in all the other documents in that series.

\subsection*{Notations Conventions \vspace{-1ex}}

\begin{longtable}[l]{>{\raggedright}p{0.2\paperwidth}>{\raggedright}p{0.8\paperwidth}}
$a$ & Scalar\tabularnewline
$\overrightarrow{a}$ & Vector\tabularnewline
$\A$ & $2^{nd}$ order Tensor or matrix\tabularnewline
$\IiA$ & $3^{rd}$ order Tensor\tabularnewline
$\IIA$ & $4^{th}$ order Tensor\tabularnewline
\end{longtable}

\subsection*{Linear Algebra and Mathematical Operators \vspace{-1ex}}

\begin{longtable}[l]{>{\raggedright}p{0.2\paperwidth}>{\raggedright}p{0.8\paperwidth}}
$\overrightarrow{a}\cdot\overrightarrow{b}$ & Dot product of the vectors $\overrightarrow{a}$ and $\overrightarrow{b}$\tabularnewline
$\overrightarrow{a}\otimes\overrightarrow{b}$ & Tensor (or Dyadic) product of the vectors $\overrightarrow{a}$ and $\overrightarrow{b}$\tabularnewline
$\overrightarrow{a}\wedge\overrightarrow{b}$ & Vectorial product of the vectors $\overrightarrow{a}$ and $\overrightarrow{b}$\tabularnewline
$\A:\B$ & Double contracted product of the two tensors $\A$ et $\B$\tabularnewline
$\stackrel{\bullet}{\boxempty}$ & Time derivative of quantity $\boxempty$\tabularnewline
$\stackrel{\bullet\bullet}{\boxempty}$ & Second order time derivative of quantity $\boxempty$\tabularnewline
$\boxempty_{,\boxempty}$ & Partial derivative of quantity $\boxempty$ with respect to $_{\boxempty}$\tabularnewline
$\boxempty^{T}$ & Transpose of a matrix or a vector $\boxempty$\tabularnewline
$\tr\,\boxempty$ & Trace of a matrix or a tensor $\boxempty$ ($\tr\,\boxempty=\sum\boxempty_{ii}$)\tabularnewline
$\dev\,\boxempty$ & Deviatoric part of a tensor $\boxempty$ ($\dev\,\boxempty=\boxempty-\frac{1}{3}\tr\,\boxempty \Id$)\tabularnewline
$\delta_{ij}$ & Kronecker delta identity\tabularnewline
$\Id$ & Unity matrix or second order tensor\tabularnewline
$\IId$ & Unity fourth order tensor\tabularnewline
\end{longtable}

\subsection*{Basic Continuum Mechanics\vspace{-1ex}}

\begin{longtable}[l]{>{\raggedright}p{0.2\paperwidth}>{\raggedright}p{0.8\paperwidth}}
$\overrightarrow{x}=\left[\begin{array}{ccc}
x & y & z\end{array}\right]^{T}$ & Coordinates in the physical domain\tabularnewline
$\overrightarrow{u}=\left[\begin{array}{ccc}
u & v & w\end{array}\right]^{T}$ & Displacement field\tabularnewline
$\overrightarrow{\omega}=\left[\begin{array}{ccc}
\omega_{x} & \omega_{y} & \omega_{z}\end{array}\right]^{T}$ & Rotation field\tabularnewline
$\Om$ & Arbitrary body in the current configuration\tabularnewline
$\Gam$ & Boundary of an arbitrary body $\Om$ in the current configuration\tabularnewline
$\rho$ & Material density\tabularnewline
$E$ & Young's modulus of a material\tabularnewline
$\nu$ & Poisson's ratio of a material\tabularnewline
$K$ & Bulk modulus of a material\tabularnewline
$\lambda$ & Lamé's first parameter of a material\tabularnewline
$\mu=G$ & Lamé's second parameter / Coulomb's shear modulus\tabularnewline
$\overrightarrow{F}$ & External load vector\tabularnewline
$\overrightarrow{f}$ & External load vector\tabularnewline
$\Eps$ & Green-Lagrange strain tensor\tabularnewline
$\Sig$ & Cauchy stress tensor\tabularnewline
$\Dev$ & Deviatoric part of the Cauchy stress tensor\tabularnewline
$\Alp$ & Backstress tensor\tabularnewline
$\Fi$ & $\Fi=\Dev-\Alp$\tabularnewline
\end{longtable}

\subsection*{Constitutive laws\vspace{-1ex}}

\begin{longtable}[l]{>{\raggedright}p{0.2\paperwidth}>{\raggedright}p{0.8\paperwidth}}
$f$ & \tabularnewline
$\n$ & Direction of the plastic flow\tabularnewline
$\q$ & Heredity variables in an elastoplastic behavior\tabularnewline
$\overline{\sigma}$ & von Mises equivalent stress\tabularnewline
$\overline{\varepsilon}^{p}$ & Equivalent plastic strain\tabularnewline
$\stackrel{\bullet}{\overline{\varepsilon}^{p}}$ & Equivalent plastic strain rate\tabularnewline
$\Lambda$ & Norm of the plastic strain\tabularnewline
$\sigma^{v}$ & \tabularnewline
$\sigma_{0}^{v}$ & \tabularnewline
$\sigma_{\infty}^{v}$ & \tabularnewline
\end{longtable}

\subsection*{Large Deformations\vspace{-1ex}}

\begin{longtable}[l]{>{\raggedright}p{0.2\paperwidth}>{\raggedright}p{0.8\paperwidth}}
$\overrightarrow{X}=\left[\begin{array}{ccc}
X & Y & Z\end{array}\right]^{T}$ & Coordinates in the reference domain\tabularnewline
$\E$ & Green-Lagrange deformation tensor\tabularnewline
$\F$ & Deformation gradient tensor\tabularnewline
$\U,\ \V$ & Right and left pure deformation tensors\tabularnewline
$\R$ & Rotation tensor\tabularnewline
$\iL$ & Deformation speed tensor\tabularnewline
$\D$ &Symmetric part of the $\iL$ tensor\tabularnewline
$\W$ & Skew-symmetric part of the $\iL$ tensor\tabularnewline
\end{longtable}

\subsection*{Finite Element Data Structures\vspace{-1ex}}
\begin{flushleft}
\begin{longtable}[l]{>{\raggedright}p{0.2\paperwidth}>{\raggedright}p{0.8\paperwidth}}
$\N$ &Shape functions matrix\tabularnewline
$\overrightarrow{\xi}=\left[\begin{array}{ccc}
\xi & \eta & \zeta\end{array}\right]^{T}$ & Coordinates in the parent domain\tabularnewline
$\B$ & Derivatives of the shape functions\tabularnewline
$\boxempty^{e}$ & Quantity $\boxempty$ related to element $e$\tabularnewline
$\J$ & Jacobian matrix\tabularnewline
$\M$ & Mass matrix\tabularnewline
$\K$ & Stiffness matrix\tabularnewline
$\overline{\F}$ & External surfacic load vector\tabularnewline
$\F$ & External load vector\tabularnewline
$\overline{\f}$ & External volumic load vector\tabularnewline
$\q$ & Nodal unknowns vector\tabularnewline
$n_{g}$ & Number of nodes of the current element\tabularnewline
$n_{Q}$ & Number of integration points of the current element\tabularnewline
\end{longtable}
\par\end{flushleft}


\documentclass[../report/main.tex]{subfiles}

\begin{document}

% The asterix after \subsection disables section numbering
\subsection*{Algorithm Description}

This algorithm is called braindead tour because it requires little to no thinking.  It simply goes from city 1 to city 2 to ... to city n as ordered in the input file.

\subsection*{Algorithm Discussion}

This algorithm was implemented early on just as a way to test file I/O and to understand the provided helper programs such as tsp-verifier.  No real research was done for this algorithm because it wasn't considered a serious solution, but we are including it in this report for completeness.  For this implementation we used Python 3.5.  The algorithm calculates and sums the distances between each city in the order they exist in the input file (plus distance from the last city back to the first).  It runs very quickly in O(n) time.  The efficiency of the algorithm in terms of calculated tour vs optimal is entirely dependent on the ordering of the cities in the input file.  For the three example cases provide we saw routes as good as 9\% over optimal and as bad as 71 times optimal.

\subsection*{Algorithm Pseudo-code}

\begin{verbatim}
braindeadTour()
    cities = []
    read date from file into cities[] // each city has id, x, and y attributes

    totalDist = 0
    prevCity = cities[disc[0]]
    for i = 1 to len(cities)
        eachCity = cities[i]
        addDist = dist(eachCity, prevCity)
        totalDist += addDist
        prevCity = eachCity
    addDist = dist(prevCity, cities[disc[0]])
    totalDist += addDist

    write totalDist to file
    write contents of disc[] to file // 1 item per line

dist(cityOne, cityTwo)
    dx = cityOne['x'] - cityTwo['x']
    dy = cityOne['y'] - cityTwo['y']
    dxSq = dx ^ 2
    dySq = dy ^ 2
    return ((dxSq + dySq) ^ 0.5)
\end{verbatim}
\end{document}