\section{Coherent state}

Let
\begin{equation}
\widehat{b}^- = \widehat{a}^- - \alpha,\qquad
\widehat{b}^+ = \rbr{\widehat{b}^-}^\dagger = \widehat{a}^+ - \alpha^*.
\label{eq:displacement-b}
\end{equation}
One seeks the state $\Ket{\alpha}$ satisfying
\begin{equation}
\widehat{b}^-\Ket{\alpha} = 0.
\label{eq:coherent-state-def-1}
\end{equation}
Expanding $\widehat{b}^-$ in terms of $\widehat{x}$ and $\widehat{p}$,
\begin{equation}
\widehat{b}^- = \sqrt{\frac{m\Omega}{2}}\rbr{\widehat{x}-x_\alpha}
+ \frac{\ii}{\sqrt{2m\Omega}}\rbr{\widehat{p}-p_\alpha},
\end{equation}
where
\begin{equation}
x_\alpha = \sqrt{\frac{2}{m\Omega}}\Re\alpha,\qquad
p_\alpha = \sqrt{2m\Omega}\Im\alpha.
\end{equation}

One may define
\begin{align}
\widehat{y} & \coloneqq \widehat{x} - x_\alpha \widehat{1}
= \widehat{x} - \ii x_\alpha\sbr{\widehat{p},\widehat{x}}_-
= \ee^{-\ii x_\alpha \widehat{p}} \widehat{x} \ee^{+\ii x_\alpha \widehat{p}},
\label{eq:coh-y}\\
\widehat{k} & \coloneqq \widehat{p} - p_\alpha \widehat{1}
= \widehat{p} + \ii p_\alpha\sbr{\widehat{x},\widehat{p}}_-
= \ee^{+\ii p_\alpha \widehat{x}} \widehat{p} \ee^{-\ii p_\alpha \widehat{x}},
\label{eq:coh-k}
\end{align}
where in the last steps, Lemma \ref{lem:bch-sandwich} has been applied.
One can verify
\begin{equation}
\sbr{\widehat{y}, \widehat{k}}_- = \ii \widehat{1}.
\end{equation}

\subsection{Displacement operator}
Inspired by \cref{eq:coh-y,eq:coh-k}, one attempts
\begin{equation}
\rfun{\widehat{D}}{\alpha}\widehat{a}^-\rfun{\widehat{D}^{-1}}{\alpha},
\end{equation}
where
\begin{equation}
\rfun{\widehat{D}}{\alpha} \coloneqq
\ee^{+\alpha\widehat{a}^+ - \alpha^*\widehat{a}^-}
\equiv \ee^{\ii\rbr{-x_\alpha\widehat{p}+p_\alpha\widehat{x}}}
\end{equation}
is the \emph{displacement operator}. Note that
\begin{equation}
\rfun{\widehat{D}^{-1}}{\alpha} = \rfun{\widehat{D}}{-\alpha}
= \rfun{\widehat{D}^\dagger}{\alpha}.
\end{equation}

\begin{equation}
b^- = a^- - \alpha = a^- + \alpha\sbr{a^+, a^-}_-
= a^- + \sbr{\alpha a^+ - \alpha^* a^-, a^-}_-.
\end{equation}
Note that
\begin{equation}
\ad_{\alpha a^+ - \alpha^* a^-}^{\rbr{n}}a^- \equiv 0,\qquad\forall n \ge 2.
\end{equation}
By Lemma \ref{lem:bch-sandwich},
\begin{equation}
b^- = \ee^{+\alpha a^+ - \alpha^* a^-}a^-\ee^{-\alpha a^+ + \alpha^* a^-}
= \rfun{D}{\alpha} a^- \rfun{D}{\alpha}^{-1}.
\end{equation}
Now
\begin{equation}
a^-\Ket{0} \coloneqq 0 \eqqcolon \rfun{D^\dagger}{\alpha} b^- \Ket{0}
= a^- \rfun{D^\dagger}{\alpha} \Ket{\alpha}.
\end{equation}
With $\rfun{D}{\alpha}$ unitary in mind, one finds
\begin{equation}
\Ket{\alpha} = \ee^{\ii\theta}\rfun{D}{\alpha}\Ket{0}.
\end{equation}
One can fix the phase $\theta$ to be zero.

\subsection{Time evolution}

Compute
\begin{equation}
\rfun{\widehat{\mscrU}}{t} \rfun{\widehat{D}}{\alpha}
\rfun{\widehat{\mscrU}^\dagger}{t}.
\label{eq:tim-evo-disp-1}
\end{equation}

\Cref{eq:tim-evo-disp-1} simplifies to
\begin{equation}
\ee^{-\ii \Omega t a^+ a^-} \rfun{D}{\alpha}
\ee^{+\ii \Omega t a^+ a^-}
\label{eq:tim-evo-disp-2}
\end{equation}

One wishes to interchange $\ee^{-\ii \Omega t a^+ a^-}$ and
$\rfun{D}{\alpha}$. By Theorem \ref{thm:bch-brading}, one computes
\begin{align}
\ad_{-\ii \Omega t a^+ a^-}^{(1)}\rbr{\alpha a^+ - \alpha^* a^-}
&= \sbr{-\ii \Omega t a^+ a^- , \alpha a^+ - \alpha^* a^-}_-
\nonumber\\
&= -\ii\Omega t \rbr{\alpha a^+\sbr{a^-, a^+}_- - \alpha^*\sbr{a^+, a^-}_- a^-}
\nonumber\\
&= -\ii\Omega t \rbr{+\alpha a^+ + \alpha^* a^-} \nonumber\\
&=	+ \alpha\rbr{-\ii\Omega t}^1 a^+
	- \alpha\rbr{+\ii\Omega t}^1 a^-;
\end{align}
\begin{align}
\ad_{-\ii \Omega t a^+ a^-}^{(2)}\rbr{\alpha a^+ - \alpha^* a^-}
&= \sbr{-\ii \Omega t a^+ a^-,
	-\ii\Omega t \rbr{+\alpha a^+ + \alpha^* a^-}}_-
\nonumber\\
&= \Omega^2 t^2\rbr{-\alpha a^+ + \alpha^* a^-}
\nonumber\\
&= + \alpha\rbr{-\ii\Omega t}^2 a^+
- \alpha\rbr{+\ii\Omega t}^2 a^-.
\end{align}
By induction one can prove
\begin{equation}
\ad_{-\ii \Omega t a^+ a^-}^{(n)}\rbr{\alpha a^+ - \alpha^* a^-}
= + \alpha\rbr{-\ii\Omega t}^n a^+
- \alpha\rbr{+\ii\Omega t}^n a^-,
\end{equation}
so that \cref{eq:tim-evo-disp-1} transforms to
\begin{align}
&\phantom{{}={}}\rfun{\exp}{+\alpha\ee^{-\ii\Omega t}a^+
	-\alpha^*\ee^{+\ii\Omega t}a^-}
	\ee^{-\ii \Omega t a^+ a^-}
	\ee^{+\ii \Omega t a^+ a^-} \\
&= \rfun{\exp}{+\alpha\ee^{-\ii\Omega t}a^+
-\alpha^*\ee^{+\ii\Omega t}a^-} \\
&\equiv \rfun{D}{\alpha^t}.
\end{align}
by Theorem \ref{thm:bch-brading}, where
\begin{equation}
\alpha^t\coloneqq \alpha\ee^{-\ii\Omega t}.
\end{equation}

As a result, the time evolution of the coherent ground state is
\begin{equation}
\rfun{\mscrU}{t} \Ket{\alpha} = \rfun{\mscrU}{t} \rfun{D}{\alpha} \Ket{0}
= \rfun{D}{\alpha^t} \rfun{\mscrU}{t} \Ket{0}
= \ee^{-\ii\Omega t/2}\Ket{\alpha^t},
\end{equation}
in terms of the coherent parameter $\alpha$.

\subsection{Particle numbers}

\begin{namedthm}{Lemma}
\begin{equation}
\rfun{\widehat{D}}{\alpha} = \ee^{-\vbr{\alpha}^2/2}
\ee^{+\alpha\widehat{a}^+} \ee^{-\alpha^*\widehat{a}^-}.
\end{equation}
\end{namedthm}
\begin{proof}
Use Corollary \ref{thm:bch-merging}.
\end{proof}

\begin{namedthm}{Particle number representation of $\Ket{\alpha}$}
\begin{equation}
\Braket{n | \alpha} = \ee^{-\vbr{\alpha}^2/2}
\frac{\alpha^n}{\sqrt{n!}}.
\end{equation}
\end{namedthm}

\begin{namedthm}{$\widehat{a}$-particle number in $\Ket{\alpha}$}
\begin{equation}
\Braket{\alpha | \widehat{a}^+ \widehat{a}^- | \alpha} = \vbr{\alpha}^2.
\end{equation}
\end{namedthm}
\begin{proof}
\begin{equation}
\rfun{\widehat{D}^\dagger}{\alpha}\widehat{a}^-\rfun{\widehat{D}}{\alpha}
= \widehat{a}^- + \sbr{-\alpha \widehat{a}^+, \widehat{a}^-}_-
= \widehat{a}^- + \alpha;
\end{equation}
\begin{equation}
\rfun{\widehat{D}^\dagger}{\alpha}\widehat{a}^+
\widehat{a}^-\rfun{\widehat{D}}{\alpha}
=
\rfun{\widehat{D}^\dagger}{\alpha}\widehat{a}^+\rfun{\widehat{D}}{\alpha}
\rfun{\widehat{D}^\dagger}{\alpha}\widehat{a}^-\rfun{\widehat{D}}{\alpha}
=
\rbr{\widehat{a}^+ + \alpha^*}\rbr{\widehat{a}^- + \alpha}.
\end{equation}
\end{proof}

\begin{namedthm}{$\widehat{b}$-particle number in $\widehat{a}$-ground state}
\begin{equation}
\Braket{0 | \widehat{b}^+ \widehat{b}^- | 0} = \vbr{\alpha}^2
\end{equation}
\end{namedthm}

\begin{proof}
\begin{equation}
\widehat{b}^+ \widehat{b}^- =
\rbr{\widehat{a}^+ - \alpha^*}\rbr{\widehat{a}^- - \alpha}
\end{equation}
according to \ref{eq:displacement-b}.
\end{proof}

\begin{namedrem}{Remark}
Since $\Ket{0}$ is stationary, $\Braket{0 | \widehat{b}^+ \widehat{b}^- | 0}$
is a constant of motion.

\end{namedrem}

\subsection{Wave function}
Since $\sbr{+\ii p_\alpha\widehat{x}, -\ii x_\alpha\widehat{p}}_- =
\ii p_\alpha x_\alpha \widehat{1}$ is central, one has
\begin{align}
\Bra{x}\rfun{\widehat{D}}{\alpha}
&= \Bra{x}\ee^{\ii\rbr{+p_\alpha\widehat{x}-x_\alpha\widehat{p}}}
\nonumber\\
&= \Bra{x}\ee^{+\ii p_\alpha\widehat{x}}\ee^{-\ii x_\alpha\widehat{p}}
\ee^{-\sbr{+\ii p_\alpha\widehat{x}, -\ii x_\alpha\widehat{p}}_-/2}
\nonumber \\
&= \ee^{-\ii p_\alpha x_\alpha/2} \ee^{+\ii p_\alpha x}
\Bra{x} \ee^{-\ii x_\alpha\widehat{p}}
\nonumber \\
&= \ee^{-\ii p_\alpha x_\alpha/2} \ee^{+\ii p_\alpha x}
\Bra{x-x_\alpha},
\label{eq:displacement-x-bra}
\end{align}
using \cref{eq:displace-x-bra}. Similarly
\begin{align}
\Bra{x}\rfun{\widehat{D}}{\alpha}
&= \ee^{-\sbr{-\ii x_\alpha\widehat{p}, +\ii p_\alpha\widehat{x}}_-/2}
\ee^{-\ii x_\alpha\widehat{p}}\ee^{+\ii p_\alpha\widehat{x}} \Ket{x}
\nonumber \\
&= \ee^{+\ii p_\alpha x_\alpha/2} \ee^{+\ii p_\alpha x} \Ket{x+x_\alpha}.
\end{align}
Note the extra phase factors compared with
\cref{eq:displace-x-ket,eq:displace-x-bra}.

\Cref{eq:displacement-x-bra} solves the wave function of coherent ground state
\begin{align}
\Braket{x | \alpha} &= \Braket{x | \rfun{D}{\alpha} | 0}
\nonumber \\
&= \rbr{\frac{m\Omega}{\pp}}^{1/4} \cfun{\exp}%
{-\frac{1}{2}m\Omega\rbr{x-x_\alpha}^2}\cfun{\expi}{p_\alpha x - \frac{1}{2}
p_\alpha x_\alpha},
\end{align}
as well as its time evolution
\begin{align}
&\phantom{{}={}}\rbr{\frac{m\Omega}{\pp}}^{-1/4}\Braket{x | \rfun{\mscrU}{t} |
\alpha} \nonumber \\
&=\ee^{-\ii\Omega t/2}
\Braket{x | \rfun{D}{\alpha\ee^{-\ii\Omega t}} | 0}
\nonumber \\
&=\cfun{\exp}{-\frac{1}{2}m\Omega\rbr{x-x_\alpha^t}^2}
\cfun{\expi}{-\frac{1}{2}\Omega t + x p_\alpha^t
- \frac{1}{2} p_\alpha^t x_\alpha^t},
\end{align}
where
\begin{equation}
x_\alpha^t \coloneqq x_{\alpha^t},\qquad
p_\alpha^t \coloneqq p_{\alpha^t},
\end{equation}
or
\begin{equation}
\begin{pmatrix} \rbr{m\Omega}^{+1/2} x_\alpha^t \\
\rbr{m\Omega}^{-1/2} p_\alpha^t \end{pmatrix} \coloneqq
\begin{pmatrix}
\cos\Omega t & \sin\Omega t \\ -\sin\Omega t & \cos\Omega t
\end{pmatrix}
\begin{pmatrix} \rbr{m\Omega}^{+1/2} x_\alpha \\
\rbr{m\Omega}^{-1/2} p_\alpha\end{pmatrix}
\end{equation}

\subsection{Density matrix and Wigner function}

\begin{nameddef}{Density matrix in the particle number representation}
\begin{equation}
\Braket{n_1 | \rfun{\widehat{\rho}}{t} | n_2} = \ee^{-\vbr{\alpha}^2}
\frac{\rbr{\alpha^t}^{n_1} \rbr{\alpha^t{}^*}^{n_2}}{\sqrt{n_1!n_2!}}
= \ee^{-\vbr{\alpha}^2} \frac{\alpha^{n_1}
\rbr{\alpha^*}^{n_2}}{\sqrt{n_1!n_2!}}\ee^{-\ii\Omega\rbr{n_1-n_2}t}.
\end{equation}

\end{nameddef} % Density matrix in the particle number representation

\begin{nameddef}{Density matrix in the position representation}
\begin{align}
&\phantom{{}={}} \Braket{x_1 | \rfun{\widehat{\rho}}{t} | x_2} \nonumber \\
&= \rbr{\frac{m\Omega}{\pp}}^{1/2} \cfun{\exp}{-\frac{m\Omega}{2}\rbr{\rbr{
x_1-x_\alpha^t}^2 + \rbr{x_1-x_\alpha^t}^2}}\,
\ee^{+\ii p_\alpha^t\rbr{x_1-x_2}}
\end{align}
\end{nameddef} % Density matrix in the position representation

\begin{nameddef}{Wigner function}
\begin{equation}
\rfun{W}{x, p; t} = \frac{1}{\pp} \cfun{\exp}{-m\Omega \rbr{x-x_\alpha^t}^2
-\frac{1}{m\Omega}\rbr{p-p_\alpha^t}^2}.
\end{equation}
\end{nameddef} % Wigner function


\documentclass[10pt]{article}

\usepackage{amsmath,amscd}
\usepackage{amssymb,array}
\usepackage{amsfonts,latexsym}
\usepackage[mathscr]{euscript}
\usepackage{graphicx,subfig,wrapfig}
\usepackage{times}
\usepackage{psfrag,epsfig}
\usepackage{verbatim}
\usepackage{tabularx}
\usepackage[toc,page]{appendix}

\newcommand{\matlab}[1]{\texttt{#1}}
\newcommand{\setname}[1]{\textsl{#1}}
\newcommand{\Ce}{\mathbb{C}}
\newcommand{\Ree}{\mathbb{R}}
\newcommand{\p}{\begin{pmatrix}}
\newcommand{\pp}{\end{pmatrix}}
\newcommand{\bm}{\begin{bmatrix}}
\newcommand{\bb}{\end{bmatrix}}
\newcommand{\eul}[1]{e^{#1}}

\begin{document}

\title{ \vspace{-30mm}Systems Bioengineering 3\\Homework 12}
\author{Greg Kiar}

\maketitle
\begin{enumerate}
\item
%q1
\begin{enumerate}
\item For the following evaluation of average time, we make use of the fact that the transcript follows an exponential trend.
\begin{align*}
E(T) = t_{avg} &= \int_0^{\infty} t e^{-\alpha t} dt \\
&= \alpha \begin{bmatrix} \frac{-t}{\alpha} - \int_0^{\infty} \frac{1}{\alpha} e^{-\alpha t} dt \end{bmatrix} \\
&= \alpha \begin{bmatrix} \frac{-t}{\alpha}e^{-\alpha t} + \frac{-1}{\alpha^2} e^{-\alpha t}\end{bmatrix}_{0,t} \\
t_{avg} &= \frac{1}{\alpha} = 10 \text{min}
\end{align*}
At equilibrium, $\beta = \alpha n$, $\therefore \mu_n = 10$ at equilibrium. The variance is $10$ since for independent stochastic events $\sigma_n^2 = \mu_n$.
\item We can see that the ODE of this system is: $\dot{X}(t) = \beta - \alpha X(t)$. Solving this model using the equation: \begin{align*} \dot{X} &= \frac{d}{dX} ln(X(t)) \\ 0 &= \frac{d}{dX}\begin{pmatrix} ln(\frac{\beta}{\alpha} - n) \end{pmatrix} \\
n &= \frac{\beta}{\alpha} = 10 \end{align*}
\item Source code for this part was provided by the TAs. It has been modified slightly and is attached in an appendix.
\item When starting the simulation at $n=10$ and running it for $10,000$ min, the mean and variance were $\mu = 10.1287$ and $\sigma^2 = 9.8604$, respectively. These values are near but not exactly expected values. This makes sense because the discrete stochastic system has a finitely small timestep. If the timesteps were infinitely small or the trial infinitely long, the values would approach the theoretical values.
\end{enumerate}

\item \begin{enumerate}
\item Solving $\dot{X}(t) = \beta - \alpha X(t)$ for the case prior to equilibrium allows us to use the Laplace transform. The solution of this system becomes \begin{align*} X(t) = \frac{\beta}{\alpha} (1 - e^{-\alpha t}) \end{align*}
\item Shown below is the solution for $1000$ randomly generated trajectories each for $100 min$. \\ \includegraphics[scale=0.2]{sbe3hw12q2b.png}
\item It can be seen by the figure below, a plot of the variance, that the variance follows the same exponential trend as the random variable $n$. From this, we can set an approximate equation for the variance to be $\sigma^2 = \frac{\beta}{\alpha} (1 - e^{-\alpha t}) $. From this we can see that the variance is at a maximum as $t \to \infty$. \includegraphics[scale=0.2]{sbe3hw12q2c.png}
\end{enumerate}

\item \begin{enumerate}
\item Similarly to the equation found in 2a), we can solve $ \dot{X}(t) = \frac{\beta}{b} - \alpha X(t) $, where $b=5$ here. The result for $X(t)$ is as follows: \begin{align*} X(t) &= \frac{\beta}{b \alpha} (1 - e^{-\alpha t})\end{align*}
We can see from this result that the burst size, $b$, affects the forward rate of the system. The forward rate$ = \frac{\beta}{b}$.
\item Simulating the stochastic and ODE models of this system for $10,000 min$ with the inital condition $X(0) = n(0) = 2$, the equilibrium value, the discrete model produced the values shown below. \begin{align*} \text{Discrete} && \mu &= 12.0428 && \sigma^2 = 30.3180 \\ \end{align*}
\item For the stochastic system with larger burst sizes, the mean and variance both increase. When testing this for different burst sizes, an approximately linear relationship is observed for both the mean and variance values. This relationship does not change as $\beta$ or $\alpha$ increase and decrease simultaneously to one another. The approximate expressions are as follows: \begin{align*} \mu &= \frac{\beta}{\alpha}\\ \sigma^2 &= \frac{2 \beta}{b \alpha} \end{align*}
\end{enumerate}
\end{enumerate}
\newpage
\begin{appendices}
\section{Trajectories}
\input{get_trajectory}
\section{Example main script}
\input{Prob2_forStudents}
\end{appendices}
\end{document}


\subsection{Hodge stars}

The Hodge star operator is a generalisation of cross product.
In 3d space if we have a plane, we can get a vector perpendicular and visa versa.
Generally, we are in \(n\)-dimensional space and we input \(k\) vectors and get out \(n-k\) vectors.

\subsection{Hodge duals}


% file: problems/euclid.tex

\section{Extra Problem: $\euclid(m, n)$}  \label{section:euclid}

Prove the recursive Euclid algorithm as shown in Algorithm~\ref{alg:euclid} for
computing the greatest common divisor (gcd) of two natural numbers are totally correct.

\begin{marginfigure}%
  \includegraphics[width=0.60\linewidth]{figs/euclid}
  \label{fig:euclid}
\end{marginfigure}

\input{algs/euclid}

\subsection{Proof}

\marginnote{Also pay attention to the way how to write a mathematical induction proof.}

We prove the partial correctness of \euclid{} by strong mathematical induction on $n$,
with $m$ any fixed natural number.

\begin{description}
  \item[Basis:] $n = 0$. We have that
    \marginnote{Make sure you understand each of these three ``='''s:
      \begin{enumerate}[(1)]
	\item By $n = 0$;
	\item By the property of $\gcd$;
	\item By the \euclid{} algorithm.
      \end{enumerate}
    }
    \[
      \gcd(m,n) = \gcd(m,0) = m = \euclid(m,0).
    \]
  \item[Inductive Hypothesis:]
    Suppose that $n \ge 1$ and
    \[
      \gcd(m, k) = \euclid(m, k), \;\forall 0 \le k \le n - 1.
    \]
  \item[Inductive Step:]
    We need to prove that ($n \ge 1$)
    \[
      \gcd(m, n) = \euclid(m, n).
    \]

    According to \euclid{}, we have
    \[
      \euclid(m,n) = \euclid(n, m \bmod n).
    \]
    Since $(m \bmod n) < n$, by the inductive hypothesis, we have
    \[
      \euclid(n, m \bmod n) = \gcd(n, m \bmod n).
    \]
    Therefore, it suffices to prove that
    \[
      \boxed{\gcd(m,n) = \gcd(n, m \bmod n).}
    \]
    For notational convenience, we denote
    \[
      d = \gcd(m,n), \quad d' = \gcd(n, m \bmod n).
    \]
    Because $d, d' \ge 0$, it is sufficient to obtain $d = d'$
    by showing that $d \mid d'$ and $d' \mid d$:

    \begin{itemize}
      \item $d \mid d'$.
      \item $d' \mid d$.
    \end{itemize}
\end{description}


\paragraph{Adversarial accuracy:} check if data point in test set is classified correctly \emph{and} network is robust in region around point (e.g., using PGD in $\epsilon$ $L_\infty$-ball). Often have tradeoff with standard accuracy.
\paragraph{Opt. Prob.:} $\displaystyle \argmin_\theta \E_{(x, y) \sim D} \left[ \max_{x' \in S(x)} \loss(\theta; x', y)\right]$
\subsection*{PGD training}
\begin{enumerate}
    \item Select mini-batch $B$
    \item $\displaystyle B_\text{max} \gets \{\argmax_{x' \in S(x)} \loss(\theta; x', y) \lvert x \in B\}$
    \item $\displaystyle \theta \gets \theta - \frac{1}{\lvert B_\text{max} \rvert} \sum_{(x, y) \in B_\text{max}} \nabla_\theta \loss(\theta; x, y)$
\end{enumerate}
\subsection*{TRADES}
$$ \argmin_\theta \E_{(x, y) \sim D} \left[ \loss(\theta; x, y) + \lambda \cdot \max_{x' \in S(x)} \loss(\theta; x', f_\theta(x))\right]$$


% !TeX root = ../diss.tex

\chapter{Preparation}

There were two main areas of preparation for my project. One was to research the Smith-Waterman (SW) algorithm, and the ways it has been modified and implemented by others.
The other area was researching the different architectures, programming environment and tools to use.

\section{Overview of the Smith-Waterman algorithm}
\label{sec:SW_Overview}
Sequence alignment algorithms are used to find similar regions between two strings.
This involves predicting where symbols in the two strings correlate with each other (both matching and mismatching symbols), or where one sequence has some symbols that have been inserted into that sequence or deleted from the other.
This can be represented as follows, where symbols in the same column correspond to one another, and dashes represent insertions or deletions (gaps):
\begin{center}
\begin{tabular}{c}
\begin{lstlisting}[basicstyle=\ttfamily\linespread{0.8}]
AC-T
ACGT
\end{lstlisting}
\end{tabular}
\end{center}

The Smith-Waterman algorithm \cite{SW_Original} is a local alignment algorithm; it can produce alignments that span a subset of the input sequences.
This is useful when aligning long segments of DNA which contain non-coding regions or multiple different genes (which occurs when aligning against an entire chromosome).
When aligning a single gene against a chromosome from a different species, a local alignment algorithm can find the relevant region whilst excluding the rest of the longer sequence in the alignment.

\subsection{The Dynamic Programming approach}
\label{sec:SW_DP}

The Smith-Waterman algorithm \cite{SW_Original} finds alignments using dynamic programming.
For sequences $A=a_1 a_2 \ldots a_N$ and $B=b_1 b_2 \ldots b_M$, a scoring matrix $H$ is constructed where cell $H_{i,j}$ gives the score of the best local alignment that ends with $a_i$ and $b_j$.
Using a similarity function $s(a,b)$ which gives the similarity of two sequence symbols, and gap score list $W_k$ which gives the penalty for a deletion of length $k$, $H_{i,j}$ is defined as follows:
$$ H_{i,j} = \max \begin{cases}
H_{i-1,j-1} + s(a_i, b_j) & \text{(Including both symbols in alignment)} \\
\max_{k \geq 1}(H_{i-k,j} - W_k) & \text{(Skipping $k$ symbols from $a$)} \\
\max_{l \geq 1}(H_{i,j-l} - W_l) & \text{(Skipping $k$ symbols from $b$)} \\
0 & \text{(Starting alignment here)}
\end{cases}$$

The grid is initialised using $\forall i\leq N . H_{i,0}=0$ and $\forall j \leq M . H_{0,j}=0$.

For example, using sequences {\ttfamily GACT} and {\ttfamily ACGT} and the following definitions of $s(a,b)$ and $W_k$:
$$s(a, b)= \begin{cases}
5 &  a = b \\
-4 & a \neq b
\end{cases}
\qquad \qquad \qquad W_k = -k
$$

\begin{center}
$H = $ \begin{tabular}{|c|ccccc|} \hline
& & {\ttfamily A} & {\ttfamily C} & {\ttfamily G} & {\ttfamily T} \\ \hline
& $0$ & $0$ & $0$ & $0$ & $0$ \\
{\ttfamily G} & $0$ & $0$ & $0$ & $3$ & $2$ \\
{\ttfamily A} & $0$ & $3$ & $2$ & $2$ & ${\color{darkorange} 1}$ \\
{\ttfamily C} & $0$ & $2$ & ${\color{darkblue}6}$ & $5$ & $4$ \\
{\ttfamily T} & $0$ & $1$ & $5$ & $4$ & $8$ \\ \hline
\end{tabular}
\end{center}

Taking two examples from this:
\begin{align*}
{\color{darkblue}H_{3,2}} = 6 &= \max \begin{cases}
    3 + s(\text{\ttfamily C}, \text{\ttfamily C}) =6\\
    \max(2-1, 0 - 2, 0-3) =1 \\
    \max(2-1, 0 - 2) =1\\
    0
\end{cases} \\
{\color{darkorange} H_{2,4}} = 1 &= \max \begin{cases}
    3 + s(\text{\ttfamily A}, \text{\ttfamily T}) =1 \\
    \max(2-1, 0 - 2) =1 \\
    \max(2-1, 2 - 2, 3-3, 0-4) =1 \\
    0
\end{cases}
\end{align*}
Note that in the case of $H_{2,4}$ (and others) there are multiple ways of getting to the maximum value (from above, the left, or diagonally in this case), and this leads to multiple alignments that have the same optimal score.

\subsection{Back-tracing}
\label{sec:SW_Back_tracing}
Stored with each grid cell is a ``pointer'' which locates the previous cell in the alignment ending with that cell, where pointer $p \in \{Above_k, Left_k, Diagonal, Nil\}$. $Nil$ represents this cell not being part of an alignment, because all paths to that cell are worse than the threshold $0$. $Left_k$ represents skipping $k$ symbols from $a$, likewise for $Above_k$ skipping $k$ symbols from $b$. In the general case, this matrix of pointers $P$ is found using:
$$ P_{i,j} = \begin{cases}
    Diagonal, & H_{i,j} = H_{i-1,j-1} + s(a_i, b_j) \\
    Left_k, & H_{i,j} = \max_{k \geq 1}(H_{i-k,j} - W_k) \\
    Above_k, & H_{i,j} = \max_{l \geq 1}(H_{i,j-l} - W_l) \\
    Nil, & H_{i,j} = 0
\end{cases}$$
\pagebreak

An example of $P_{i,j}$ is as follows (where $\cdot$ represents $Nil$):
\begin{center}
$P = $     \begin{tabular}{|p{0.035\textwidth}|p{0.035\textwidth}p{0.035\textwidth}p{0.035\textwidth}p{0.035\textwidth}p{0.035\textwidth}|} \hline
& & \hfil {\ttfamily A} & \hfil {\ttfamily C} & \hfil {\ttfamily G} & \hfil {\ttfamily T} \\ \hline
& \hfil $\cdot$ & \hfil $\cdot$ & \hfil $\cdot$ & \hfil $\cdot$ & \hfil $\cdot$ \\
\hfil {\ttfamily G} & \hfil ${\color{darkorange} \cdot}$ & \hfil $\cdot$ & \hfil $\cdot$ & \hfil $\nwarrow$ & \hfil $\leftarrow_1$ \\
\hfil {\ttfamily A} & \hfil $\cdot$ & \hfil ${\color{darkorange} \nwarrow}$ & \hfil $\leftarrow_1$ & \hfil $\uparrow_1$ & $\leftarrow_1$ \\
\hfil {\ttfamily C} & \hfil $\cdot$ & \hfil $\uparrow_1$ & \hfil ${\color{darkorange} \nwarrow}$ & \hfil ${\color{darkorange} \leftarrow_1}$ & $\leftarrow_2$ \\
\hfil {\ttfamily T} & \hfil $\cdot$ & \hfil $\uparrow_2$ & \hfil $\nwarrow$ & \hfil $\leftarrow_1$ & \hfil ${\color{darkorange} \nwarrow}$ \\ \hline
\end{tabular}
\end{center}

A running maximum score is maintained during the production of the grid, and the cell $H_{i,j}$ with the highest score corresponds to the highest-scoring alignment, which finishes with $a_i$ and $b_j$.
Starting here, the alignment can be found from end to start by following the pointers in the grid. The best path through $P_{i,j}$ is highlighted in {\color{darkorange}orange}, and gives alignment:
\begin{center}
\begin{tabular}{c}
\begin{lstlisting}[basicstyle=\ttfamily\linespread{0.8}]
AC-T
ACGT
\end{lstlisting}
\end{tabular}
\end{center}

\subsection{Algorithmic complexity}
\label{sec:SW_Complexity}
For sequences of lengths $N$ and $M$, this algorithm requires $O(NM)$ space to store the two matrices, each with $(N+1)\times(M+1)$ elements.
The runtime is $O(N^2 M)$, where without loss of generality $N\geq M$.
Each cell needs to consider the cell to its diagonal (constant time), and all of the cells above it (of order $M$ cells), and all cells to its left (of order $N$ cells).
This gives a runtime for evaluating a cell as $O(N+M)$, which is $O(N)$ given $N\geq M$.
There are $NM$ cells to be evaluated, so producing the grid takes $O(N^2 M)$ time.

If the gap evaluation scheme did not maximise over a row, and instead performed constant time operations, then the complexity is $O(NM)$.
This is discussed in \cref{sec:SW_gaps}.

Each back-tracing step requires constant time (a matrix lookup, and some pointer comparisons) to produce each element in the alignment, and the length of alignment is bounded by $N+M$, where the alignment is wholly made up of gaps.
Therefore, this step requires $O(N)$ space and time, where $N\geq M$.

Overall, the space requirements of the algorithm as presented is $O(NM)$, and the time requirements are either $O(NM)$ or $O(N^2 M)$ depending on how gaps are evaluated.
Improvements to this are discussed in \cref{sec:SW_Linear_Prep}.

\subsection{Scoring alignments}
\label{sec:SW_Scoring}

The general approach to scoring alignments is given in the paper written by Smith and Waterman \cite{SW_Original}, but the exact details are not specified.
There are two areas that need definition: how to compare a pair of sequence symbols, and how to score gaps in the alignment.

\subsubsection{Sequence similarity functions}
\label{sec:SW_similarity_functions}
The original paper requires similarity function $s(a,b)$ which gives the similarity of two sequence symbols. One simple implementation might be:
$$s(a, b) = \begin{cases}
m,   & a =    b \\
\mu, & a \neq b
\end{cases}$$

For two constants $m$ and $\mu$ where typically $m>0$ and $\mu<0$.
The implementation Smith-Waterman in the European Molecular Biology Open Software Suite (EMBOSS) \cite{EMBOSS} uses an approach equivalent to using $m=5$ and $\mu=-4$ for DNA alignments by default.

However, the biological processes that leads to one symbol being substituted for another cannot be entirely expressed in these two constants, especially when some symbols are more likely than others to replace a given symbol.
This is particularly problematic in proteins, which naturally are made up of 22 different types of amino acids.

A more granular approach is to base $s(a,b)$ on a substitution matrix, which encodes a score for each different pairing of symbols.
This has performance penalties (\cref{sec:C_scoring_eval,sec:CUDA_scoring_eval,sec:SV_expected_v_actual,sec:FPGA_utilisation,sec:SV_Fmax}), because the first approach can use direct equality testing and hard-coded constants, whereas using a substitution matrix for proteins requires a lookup from a $22\times22$ matrix.

Discussions of which substitution matrices are best for which applications are beyond the scope of this project, and they are all implemented in the same way.
I chose to base investigations into the performance implications into this on the BLOSUM50 matrix for proteins \cite{BLOSUM}, which is used for alignments between sequences that are not closely related.

\subsubsection{Scoring gaps}
\label{sec:SW_gaps}
The original paper \cite{SW_Original} references a gap weight vector $W_k$, which defines the score penalty of a gap of length $k$, where gaps jump either $k$ elements to the left, or above.
However, it makes the algorithm run in $O(N^2 M)$ time for $N\geq M$ (\cref{sec:SW_Complexity}), so other approaches are used instead.

One solution to this problem was suggested by Waterman and Smith \cite{SW_Metrics} in an earlier paper about global alignment: to define $W_k=kw_1$ for some gap penalty $w_1$.
Using the recursive definition of $H_{i,j}$ the score using a $Left$ gap is:
$$\max_{k \geq 1}(H_{i-k,j} -W_k) = H_{i-1,j} - w_1$$
Gotoh proposed a different way of solving this problem \cite{Gotoh}.
The constant gap penalty models the cost of inserting or deleting symbols in a sequence by making this penalty proportional to the length of the insertion or deletion.
This is not ideal, because as Gotoh observes, ``long gap[s] can be produced by a single mutational event''.
Instead he proposes what is later called the affine-gap method, where $W_k=uk+v$ for some gap penalties $u$ and $v$.
The penalty of starting to insert or delete a sequence is given by $u$, and $v$ accounts for the different lengths of such a sequence.

This approach requires more space than the original Smith-Waterman algorithm because choosing whether to open a gap has an impact on scores later in the alignment.
When evaluating a mismatched cell in the dynamic-programming grid it is impossible to know whether to pay the mismatch penalty because a sequence of matches immediately follow, or if a long sequence of mismatches follow and it would be less expensive to open up a gap to cover them instead.
Therefore, extra space is needed to keep track of scores when the decision to open a gap is and also is not taken.

To achieve this, three grids are constructed instead of one.
This triples the memory requirements in practice but does not change the asymptotic memory requirements.
The main score grid $H_{i,j}$ is calculated using a similar approach to before, representing the score of the best alignment using sequences $a_1\ldots a_i$ and $b_1 \ldots b_j$.
Additionally, there is grid $F_{i,j}$ which represents the score of the best alignment between $a_1\ldots a_i$ and $b_1 \ldots b_j$ provided it ended with an open left-gap.
$G_{i,j}$ is similar for above gaps.
This gives grid definitions:
\begin{align*}
H_{i,j} &= \max(H_{i-1,j-1} + s(a_i, b_j), F_{i,j}, G_{i,j}, 0) \\
F_{i,j} &= \max(H_{i-1,j} + u + v, F_{i-1,j} + v) \\
G_{i,j} &= \max(H_{i,j-1} + u + v, G_{i,j-1} + v)
\end{align*}

This is modified from the original paper somewhat for notational consistency with the rest of the dissertation.

Pointer grids need to be stored for all three grids, where the back-tracer will start in grid $H$ (and this is the only grid where the running maximum is kept), but it will trace through $F$ and $G$ as well to determine where a gap started.

\subsection{Linear space modification to Smith-Waterman algorithm}
\label{sec:SW_Linear_Prep}
\subsubsection{Overview}
\label{sec:SW_Linear_Overview}

A common approach to improving the runtime of the Smith-Waterman algorithm is to reduce the amount of space used by increasing the amount of time spent in doing arithmetic.
Hirschberg’s algorithm \cite{Hirschberg} pioneered this approach for the Longest Common Subsequence problem.
Myers and Miller apply this approach to local alignments \cite{MyersMiller}, which is summarised below.

The purpose of this algorithm is to correctly and efficiently split the alignment into small chunks, manageable for the quadratic space implementation which is where the back-tracing is done.
It recursively divides a large grid into two smaller grids.
\Cref{fig:Linear_Space_Structure} shows this, where the optimal alignment is shown in orange.
The grid has been divided into two smaller grids by the first pass of the algorithm, a grid with blocks $1$ and $2$, and the grid with blocks $3$ and $4$.
The grey segments do not contain the alignment and can be ignored.
The black arrows represent the direction the blocks are evaluated in, discussed further on.

\Cref{lst:pseudo_sw} is a pseudo-code representation of the quadratic space Smith-Waterman algorithm, to be used in this linear space algorithm.
Parameter \lstinline{fixedTop} specifies whether the alignment can start from anywhere in the grid, or just the top left corner (and is implemented by including or removing the options of $0$ and $Nil$ from the maximising functions).
This is necessary because the optimal alignment inside a subproblem may not start at the top-left, but the overall optimal alignment may pass through multiple sub-grids where the top-left of each grid must be connected to the bottom-right of the next grid.
Parameter \lstinline{fixedBottom} is similar, specifying whether to start back-tracing at the best cell in the grid, or the bottom-right cell.

\begin{figure}
    \centering
    \includegraphics[width=(0.4\textwidth)]{figs/linear_space_structure.pdf}
    \caption{The divided dynamic programming matrix when using the linear space approach, after the first iteration}
    \label{fig:Linear_Space_Structure}
\end{figure}

\begin{lstlisting}[basicstyle=\linespread{0.9}\ttfamily\footnotesize, label={lst:pseudo_sw},captionpos=b,caption={Pseudo-code representation of quadratic space implementation of SW algorithm}]
fun sw(seq1, seq2, fixedTop, fixedBottom):
    Dynamic programming approach using quadratic space
    if fixedTop:
        Only allow alignments to start at the top-left
    else:
        Allow alignments to start anywhere
    if fixedBottom:
        Return the alignment ending at the bottom-right grid cell
    else:
        Return the alignment ending at the best cell in the grid
\end{lstlisting}

To split a large grid into two smaller grids, the scores of the middle column of cells are found by starting from the top-left and working towards the middle, and also by starting from the bottom right and working towards the middle.
\Cref{lst:pseudo_sw_col} is a representation of this.
In \cref{fig:Linear_Space_Structure}, blocks $1$ and $3$ are evaluated from their top-lefts to their middle columns, and blocks $2$ and $4$ from the bottom-right, and the direction of evaluation represented by the black arrows.
These scores are found using a linear space.
These two columns are added together, and each cell of the column gives the score for the best alignment that passes through that cell.
The optimal alignment might cross the middle column or be in one of these half-grids.

The middle columns are found using $O(N)$ space for sequences of lengths $N$ and $M$, and maintains $O(NM)$ time complexity, albeit with a greater constant factor.
This uses the fact that only the immediate neighbours are needed to calculate the score of a cell, so the values of a previous column can be discarded once the values in the next column have been computed. This data dependency is shown in \cref{fig:Linear_Space_Dependencies}.
However, discarding this information leads to recomputation of the cells in the smaller subproblems that are recursively evaluated.

Maxima are calculated whilst computing each half, and compared to the maximum cell in the summed columns.
If the overall maximum is in one of the half-grids, restart the algorithm between that cell and the top-left or bottom-right, if on the left or right sides respectively.
Otherwise, the algorithm is rerun between the end cells and the maximum middle cell.
This recursively breaks up the alignment until it is small enough to tackle directly using the quadratic space algorithm (\cref{sec:SW_DP,sec:SW_Back_tracing,sec:SW_Complexity}), and then joins them back together.
\begin{figure}[h]
    \centering
    \includegraphics[width=(0.4\textwidth)]{figs/linear_space_deps.pdf}
    \caption{Data dependencies when using the linear space approach}
    \label{fig:Linear_Space_Dependencies}
\end{figure}
\begin{lstlisting}[basicstyle=\ttfamily\linespread{0.9}\footnotesize, label={lst:pseudo_sw_col},captionpos=b,caption={Pseudo-code representation of linear space algorithm which finds middle columns}]
    fun sw_col(seq1, seq2, direction, fixedStart):
        direction in {forwards, backwards}
        direction determines which half of grid is evaluated
            - Forwards: from top left, to column (seq2/2)
            - Backwards: from bottom right, to column (seq2/2)
        fixedStart similar to fixedTop of sw()
        Calculate grid column by column, returning:
            - Values of column (seq2/2)
            - Location of best cell, and score
    \end{lstlisting}
\begin{minipage}{\linewidth}
\begin{lstlisting}[basicstyle=\linespread{0.9}\ttfamily\footnotesize, label={lst:pseudo_sw_linear},captionpos=b,caption={Pseudo-code representation of the linear space Smith-Waterman algorithm}]
fun sw_linear(seq1, seq2, fixedTop, fixedBottom):
    if seq1 and seq2 are small:
        return sw(seq1, seq2, fixedTop, fixedBottom)
    else:
        (forwardCol, bestForwards) = sw_col(seq1, seq2, forwards, fixedTop)
        (backwardCol, bestBackwards) = sw_col(seq1, seq2,
                                              backwards, fixedBottom)
        middleCol = forwardCol + backwardCol # Using vector addition
        bestMiddle = max(middleCol)

        # Using list splicing: seq1[x:] = [seq1[x], seq1[x+1], ... seq1[n]]
        if best path is bottom-right to bestBackwards and not fixedTop:
            return sw_linear(seq1[bestForwards.i:], seq2[bestForwards.j:],
                             fixedTop, fixedBottom)
        else if best path is top-left to bestForwards and not fixedBottom:
            return sw_linear(seq1[0:bestForwards.i], seq2[0:bestForwards.j],
                             fixedTop, fixedBottom)
        else use middle cell:
            alignStart = sw_linear(seq1[0:bestMiddle.i],seq2[0:len(seq2/2)],
                                   fixedTop, true)
            alignEnd = sw_linear(seq1[bestMiddle.i:],seq2[len(seq2/2):],
                                 true, fixedBottom)
            return alignStart @ alignEnd
\end{lstlisting}
\end{minipage}

\subsubsection{Back-tracing}
\label{sec:SW_Linear_Back_tracing}

The process of building the final alignment, back-tracing, is done in parts and later joined, in accordance with the divide-and-conquer approach of this algorithm.
When the large grid has been recursively divided to small enough grids, the quadratic space algorithm (\cref{sec:SW_DP,sec:SW_Back_tracing}) is used to find an alignment.
This is because producing an alignment requires a grid of pointers taking $O(NM)$ space, so when the grid is small enough for a full grid of pointers to be made, a full grid of scores can also be made.
The resulting aligned sequences from each of these small alignments can be joined together to make the overall alignment.

\subsubsection{Runtime complexity}
\label{sec:SW_Linear_Complexity}
This additional arithmetic does not impact the runtime complexity.
For sequences of lengths $N$ and $M$ the linear space approach evaluates $2NM$ grid cells (each taking $O(1)$ time).

The amount of work done in each iteration halves. The first step evaluates every grid cell, taking $NM$ evaluations (over two halves of size $\frac{N}{2} \times M$).
If the alignment is in one half, then the next area to evaluate has at least halved.
If the alignment is split across the middle column, some point at position $m$ will be chosen on that column.
Then two grids will be evaluated of area $\frac{N}{2} \times m$ and $\frac{N}{2} \times (M-m)$ which sum to $\frac{NM}{2}$ for all possible $m$.
Using this repeated halving of work, the total number of grid evaluations required is:
$$ \text{Cells evaluated } =  NM + \frac{NM}{2} + \frac{NM}{4} + \cdots \leq NM \sum_{i=0}^\infty 2^{-i} = 2NM $$

\section{Overview of architectures and programming languages used}
\label{sec:Architecture_prep}
The objective of my project was to implement the Smith-Waterman algorithm using different hardware architectures (general purpose CPUs, GPUs, and FPGAs).
As part of my preparation I learnt about these architectures so I could produce implementations that were best suited for these different architectures.

\subsection{Central Processing Units and C}
\label{sec:CPU_prep}
Modern general-purpose Central Processing Units (CPUs) can have very high clock speeds, multiple processing cores, and have rich Instruction Set Architectures (ISAs).
The CPU in my laptop, used by this project, is an Intel Core i7-8750H.
This has six processing cores, and has a maximum clock frequency of $\SI{4.1}{\giga\hertz}$ \cite{i7-8750H}.
This clock frequency is significantly higher than the other platforms I used, however other implementations will be able to do more work per clock cycle, using increased parallelism or specialisation.

I chose to write my programs for the CPU in C. The major reason I chose to do so was that the GPU APIs available (CUDA and OpenCL) are APIs for C/C++, so for the sake of interoperability it made sense to implement my project in either C or C++.
This would allow me to later reuse segments of code for the GPU implementation, especially the control logic and testing framework.
Also, I had previous experience with C from Part IB and a summer internship, so I chose to use C.

The focus of my project is exploring parallelism in Smith-Waterman, and with a multicore CPU I could implement a solution that was multi-threaded.
This is not a native feature in the C language, so I used the Native POSIX Thread Library, part of the GNU C Library.
This allows multiple threads to be run on multiple processing cores, where all of the scheduling is handled by the library and the host operating system.

\subsection{Graphics Processing Units and CUDA}
\label{sec:GPU_prep}
Graphics Processing Units (GPUs) are designed to exploit vector-level parallelism in programs. The GPU in my laptop, used by this project, is an Nvidia GTX 1050 Ti.
It has $768$ CUDA cores in it \cite{1050-Ti}; it can compute up to $768$ different things in parallel.
This is much more parallelised than the CPU, though the device has a significantly lower clock speed of $\SI{1392}{\mega\hertz}$.

I chose to use CUDA, Nvidia’s proprietary API for programming their GPUs.
An alternative was OpenCL, but I chose CUDA because it has a more helpful profiling tool, which suggests possible sources of poor performance, and I used this to guide development.

The highly parallel nature of CUDA causes its execution model to be different to a single core of a CPU.
In CUDA, a body of work is a grid, which is a group of blocks, where a block is a group of threads.
The GPU has a collection of streaming multiprocessors (SMs, in my case $24$) which execute different blocks, and each of these SMs execute $32$ threads at once.
These groups of threads are called warps and are subsets of blocks.
Threads in a warp execute in lockstep, with a single instruction for all $32$ threads in the warp.
The overall number of CUDA cores is $24 \times 32=768$.

Units of code to be run on the GPU are called kernels, which are similar to C functions. When a kernel is called the number of blocks and threads in a block are specified.
There is a hard limit to the size of a block of $1024$ threads \cite{CUDA_Guide}, therefore programs implemented in CUDA cannot require synchronisation over an arbitrary number of threads.

All of the threads are given the same arguments and execute the same blocks of code (though different paths may take different branches through the code).
All that distinguishes them are two variables, their \lstinline{threadId} and \lstinline{blockId}.
All threads within a block can be synchronised using the execution barrier \lstinline{__syncthreads()}.
However, there is no synchronisation between blocks, even when executing the same kernel, and they can execute in parallel to each other.

\subsection{Field Programmable Gate Arrays and SystemVerilog}
\label{sec:FPGA_prep}
An extension to my project was to implement the Smith-Waterman algorithm on an FPGA using a hardware description language (HDL).
I chose to use SystemVerilog as my HDL, because I had studied it in Part IB.
Field Programmable Gate Arrays are programmable circuits, principally made up of programmable lookup tables, flip-flops, integrated memory blocks and programmable routing between the features on the chip.
These devices can be programmed to make custom circuitry on the FPGA chip.

This is appealing because it allows me to define a custom accelerator optimised just for this problem, able to evaluate a cell in the dynamic programming grid in just one clock cycle.
My other implementations require numerous clock cycles to do this same work.
The device is clocked significantly slower than the CPU or GPU, with the default clock frequency of my device being $\SI{50}{\mega\hertz}$, and the theoretical maximum of $\SI{550}{\mega\hertz}$.
However, being able to do more work per clock cycle can reduce the impact of a slower clock frequency.

Although an implementation on an FPGA will require much more power and be significantly slower than the same logic in an ASIC, working with an FPGA was possible for this project and manufacturing an ASIC would cost far too much money.

I used a Terasic DE1-SoC board \cite{DE1-SoC} loaned from the Computer Architecture Group.
This has an Altera Cyclone V SE FPGA SoC \cite{CycloneV}, which has $\SI{85000}{}$ logic elements; a fairly small FPGA, but also not the smallest produced. This device was significantly older than the CPU and GPU it was compared against. This FPGA uses a 28nm process and was made in 2012 \cite{CycloneV}, whereas the CPU and GPU are from 2018 and use a 14nm process \cite{i7-8750H} \cite{1050-Ti}.
A smaller process node might improve the maximum frequency of a design implemented on an FPGA, and there would be more resources on the device at a similar price, which could be used to do more work in parallel.

The FPGA SoC also has a dual-core ARM Cortex-A9 on the same package as the FPGA \cite{CycloneV}, referred to as the Hard Processing System, HPS.
The ARM core can control the accelerator using an interconnect between the HPS and FPGA.
In particular, it provides a memory-mapped Avalon streaming interface, where the HPS requests to read or write to the FPGA as if it were memory and the FPGA’s responses to these requests can be defined in HDL.

A major design consideration for my work on the FPGA was memory capacity, due to the quadratic memory requirement of the Smith-Waterman algorithm.
There are several different types of memory on the FPGA \cite{CycloneV} and development board \cite{DE1-SoC} I was using, suited for different things.
This is tabulated in \cref{tab:FPGA_Memories}.
For the SDRAM memories, the throughput given (marked \dag) is the peak throughput, but average throughput will be lower as selecting rows takes time.
The ways that these memories were used was discussed in \cref{sec:Memory_in_SV}.

\begin{table}[h]
    \centering
    \begin{tabular}{|p{0.2\textwidth}|p{0.4\textwidth}p{0.1\textwidth}p{0.2\textwidth}|} \hline
        Name & Description & Capacity & Throughput \\ \hline
        ALM Registers & On-chip; all registers are directly accessible by routing elements & $\SI{15.67}{\kibi\byte}$ & $\SI{15.67}{\kibi\byte\cycle}$  \\ \hline
        M10K Blocks & On-chip; made up of $\SI{10}{\kibi\bit}$ blocks units where $\SI{40}{\bits}$ can be accessed in a cycle, which are pipelined with one delay-slot between request and action.  & $\SI{49.63}{\kibi\byte}$ & $\SI{1.94}{\kibi\byte\cycle}$ \\ \hline
        FPGA SDRAM & Off-chip; requires additional hardware to interface with it, and multiple delay slots as rows and columns are selected. & $\SI{64}{\mebi\byte}$
         & $\SI{2}{\byte\cycle}^\dag$ \\ \hline
        HPS DDR3 & Off-chip; requires additional hardware to interface with it, and multiple delay slots as rows and columns are selected. & $\SI{1}{\gibi\byte}$ & $\SI{8}{\byte\cycle}^\dag$ \\ \hline
        \end{tabular}

    \caption{Types of memory on the Terasic DE1-SoC development board}
    \label{tab:FPGA_Memories}
\end{table}

\FloatBarrier

\subsection{Licencing}
\label{sec:Licencing_prep}

The majority of tools and libraries I used for my project are open source, and the rest are available for private use.
\begin{itemize}
\item GNU Compiler Collection; particularly the C compiler---GPLv3
\item GNU C Library, particularly Native POSIX Thread Library---LGPL
\item GNU Debugger; for C and CUDA code---GPLv3
\item Gprof; C profiler---GPLv3
\item JetBrains CLion; C IDE---student license
\item Visual Studio Code; general source-code editor---MIT License
\item CUDA Toolkit; including a compiler, debugger, memory-checking suite and profiler---proprietary, freely distributed
\item Quartus Prime Lite Edition; for simulating and synthesising HDL designs---proprietary, freely distributed
\item Git; for version control of all source code and dissertation text---GPLv2
\item LaTeX; for typesetting this dissertation---LPPL
\item BibTeX; for references in this dissertation---GPLv3
\end{itemize}

\section{Development methodology}
\label{sec:Methodology_prep}

The plan for the project was outlined in the project proposal (\cref{sec:Proposal}), sequentially implementing designs for each platform.
By doing this, if any unexpected difficulty arose in the later stages of the project, then there would still be completed implementations from earlier in the project to evaluate.
The implementations were ordered in ascending difficulty and my descending confidence, to this end, with the first three required for the core success criteria.
In order these were:
\begin{enumerate}
\item Implementing the Smith-Waterman algorithm in single-threaded C code.
\item Modifying the C code to make it multi-threaded.
\item Implementing the Smith-Waterman algorithm in CUDA.
\item Implement the Smith-Waterman algorithm in SystemVerilog and simulate design.
\item Instantiate Verilog implementation on an FPGA.
\end{enumerate}

I tested throughout development and had two main testing areas: correctness and performance. Any errors were likely to propagate between implementations where similar approaches were used, especially between C and CUDA, so it was important to catch errors as they arose instead of letting them propagate.
I used an external implementation \cite{EMBOSS} to check my first C programs.
I tested the rest of my work using this implementation, because the reference implementation would only give one correct solution when there might be more than one for a given alignment problem, whereas my programs could decide whether a given alignment was valid and optimal.

\section{Starting Point}
\label{sec:Starting_Point}

I started with reasonable experience with C, having studied it in Part IB and writing some utility programs in C during a summer job. I had no experience with CUDA, or GPU programming in general, and had written a small amount of SystemVerilog during Part IB, but never having attempted a project of this scale in it before.
Therefore, in my project plan I assigned more time for the GPU and FPGA implementations compared to the CPU implementation, based on my relative levels of experience.

The only code used that was not my own was SystemVerilog used for administrative purposes.
I used a template from the Part IB ECAD+Arch course for instantiating a SystemVerilog design on the development board I was using, which correctly arranged IO pins.
My supervisor, Peter Rugg, gave me the Verilog he had used to build an Avalon streaming interface between the HPS and the FPGA, to transfer data between my hardware design and the OS on the HPS.
I based my streaming interface on this, and I am grateful for his assistance.

Sequence alignment and Smith-Waterman in particular are covered in detail by the first few lectures of the Part II Bioinformatics course, and is presented in \cref{sec:SW_Overview}.
This course runs in Michaelmas term.
The linear space approach (\cref{sec:SW_Linear_Prep}) is described for global alignment in that course, and I read a paper \cite{MyersMiller} for its application to local alignments.


% Created 2020-04-15 mié 17:46
% Intended LaTeX compiler: pdflatex
\documentclass[presentation,aspectratio=1610]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{khpreamble}
\usepackage{pgfplots}
\usepackage{pdfpages}
\usepackage{circuitikz}
\usepgfplotslibrary{groupplots}
\usetikzlibrary{positioning}
\renewcommand*{\not}[1]{\ensuremath{\bar{#1}}}
\renewcommand*{\not}[1]{\ensuremath{\overline{#1}}}
\usetheme{default}
\author{Kjartan Halvorsen}
\date{\today}
\title{Boolean algebra, logic diagrams and truth tables}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={Boolean algebra, logic diagrams and truth tables},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.3.6)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Boolean algebra, minterms and maxterms}
\label{sec:org605a59f}
\begin{frame}[label={sec:orgb501097}]{AND and OR}
\(a, b \in \{0,1\}\)
\begin{columns}
\begin{column}{0.5\columnwidth}
\begin{block}{AND}
\begin{center}
\begin{tabular}{|cc|c|}
\hline
\(a\) & \(b\) & \(a\) AND \(b\), \(ab\)\\
\hline
0 & 0 & 0\\
0 & 1 & 0\\
1 & 0 & 0\\
1 & 1 & 1\\
\hline
\end{tabular}
\end{center}

\begin{center}
\begin{tikzpicture}
  \draw (0,0) to[switch, label=$a$, o-] (2,0) to[switch, label=$b$, -o] (4, 0);
\end{tikzpicture}
\end{center}
\alert{Closed circuit \(\Leftrightarrow\) 1}

\alert{Open circuit \(\Leftrightarrow\) 0}

\begin{center}
\includegraphics[width=0.5\linewidth]{../../figures/and-gate.pdf}
\end{center}
\end{block}
\end{column}

\begin{column}{0.5\columnwidth}
\begin{block}{OR}
\begin{center}
\begin{tabular}{|cc|c|}
\hline
\(a\) & \(b\) & \(a\) OR \(b\), \(a+b\)\\
\hline
0 & 0 & 0\\
0 & 1 & 1\\
1 & 0 & 1\\
1 & 1 & 1\\
\hline
\end{tabular}
\end{center}

\begin{center}
\begin{tikzpicture}
  \draw (0,0) to[switch, label=$a$, o-o] (4,0);
  \draw (1,0) to[short] (1,-1) to[switch, l_=$b$, ] (3, -1) to[short] (3, 0);
\end{tikzpicture}
\end{center}

\begin{center}
\includegraphics[width=0.5\linewidth]{../../figures/or-gate.pdf}
\end{center}
\end{block}
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label={sec:org1573679}]{NAND and NOR}
\(a, b \in \{0,1\}\)
\begin{columns}
\begin{column}{0.5\columnwidth}
\begin{block}{NAND}
\begin{center}
\begin{tabular}{|cc|c|}
\(a\) & \(b\) & \(a\) NAND \(b\), \(\overline{a\cdot{}b}\)\\
\hline
0 & 0 & 1\\
0 & 1 & 1\\
1 & 0 & 1\\
1 & 1 & 0\\
\hline
\end{tabular}
\end{center}

\begin{center}
\includegraphics[width=0.5\linewidth]{../../figures/nand-gate.pdf}
\end{center}
\end{block}
\end{column}

\begin{column}{0.5\columnwidth}
\begin{block}{NOR}
\begin{center}
\begin{tabular}{|cc|c|}
\(a\) & \(b\) & \(a\) NOR \(b\), \(\overline{a+b}\)\\
\hline
0 & 0 & \\
0 & 1 & \\
1 & 0 & \\
1 & 1 & \\
\hline
\end{tabular}
\end{center}

\begin{center}
\includegraphics[width=0.5\linewidth]{../../figures/nor-gate.pdf}
\end{center}
\end{block}
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label={sec:org2d0b9ef}]{Boolean algebra, contd}
\(x, y, z \in \{0,1\}\)

\begin{center}
\begin{tabular}{r|c|c|}
 & Property & Dual\\
\hline
Properties of 0 and 1 & \(x+0=x\) & \(x\cdot 0=0\)\\
 & \(x+1=1\) & \(x \cdot 1 = x\)\\
Idempotency & \(x+x=x\) & \(x\cdot x = x\)\\
Complementarity & \(x+\not{x}=1\) & \(x\cdot \not{x}=0\)\\
Involution & \(\not{\not{x}}=x\) & \\
Commutative & \(x+y=y+x\) & \(x\cdot y = y\cdot x\)\\
Associative & \((x+y) + z = x + (y+z)\) & \((xy)z = z(yz)\)\\
Distributive & \(x\cdot (y+z) = xy + xz\) & \(x+yz=(x+y)(x+z)\)\\
\hline
\end{tabular}
\end{center}
\end{frame}

\begin{frame}[label={sec:orge5065a4}]{Boolean algebra, contd}
\(x, y \in \{0,1\}\)

\begin{center}
\begin{tabular}{r|c|c|}
 & Theorem & Dual\\
\hline
Absorption & \(x+xy=x(1+y)=x\) & \(x(x+y)=x\)\\
Logic adjacency & \(xy + x\not{y} = x(y+\not{y}) =x\) & \((x+y)(x+\not{y}) = x\)\\
De Morgan's & \(\not{x+y}=\not{x}\cdot{}\not{y}\) & \(\not{xy} = \not{x} + \not{y}\)\\
\hline
\end{tabular}
\end{center}
\end{frame}

\begin{frame}[label={sec:orgd14be63}]{DeMorgan's theorem}
\begin{center}
\includegraphics[width=0.4\linewidth]{../../figures/Demorganlaws.png} From wikipedia
\end{center}
\end{frame}
\begin{frame}[label={sec:orgbb8fe29}]{Simplify functions}
\begin{enumerate}
\item \(f = (a+b)(a+c)\)
\item \(f = a + \not{a}b\)
\end{enumerate}
\end{frame}

\begin{frame}[label={sec:orgd9a80c9}]{Logic diagram \(\rightarrow\) function}
Determine the function represented by the logic diagrams
\begin{center}
\includegraphics[width=0.7\linewidth]{../../figures/exercise-gate-4.pdf}
\end{center}
\end{frame}

\begin{frame}[label={sec:org42b2888}]{Function \(\rightarrow\) logic diagram}
Draw the diagram corresponding to the boolean function
\begin{enumerate}
\item \(f = (a+b)(a+c)\)
\item \(f = a + \not{a}b\)
\end{enumerate}
\end{frame}

\begin{frame}[label={sec:org98e40c3}]{Group exercise}
\begin{enumerate}
\item Enter breakout room
\item One of you downloads and shares this presentation
\item Work together on the problems in the previous three slides
\begin{enumerate}
\item Simplify functions
\item Determine function from logic diagram
\item Draw logic diagram from function
\end{enumerate}
\end{enumerate}
\end{frame}
\end{document}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS630: Database Management Systems
% Copyright 2014 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

For questions 1 to 3, you are given the following relational schema.

\begin{terminal}
Students (@*\underline{sid}*@:integer, sname:string, age:integer)
Grades (@*\underline{sid}*@:integer, @*\underline{cid}*@:integer, grade:integer)
Courses (@*\underline{cid}*@:integer, cname:string, credits:integer)
\end{terminal}

The meaning of attributes is as follows.

\begin{enumerate}[topsep=0pt,itemsep=0ex,partopsep=5pt,parsep=0pt,label=-]
\item \texttt{sid} : unique student identifier, primary key in table \texttt{Students}
\item \texttt{cid} : unique course identifier, primary key in table \texttt{Courses}
\item \texttt{sname} : student name
\item \texttt{age} : student age
\item \texttt{cname} : course name
\item \texttt{credits} : number of credits for a course
\item \texttt{grade} : the grade obtained by student identified by \texttt{sid} for course identified by \texttt{cid}; \texttt{sid} and \texttt{cid} are foreign keys referring to the \texttt{sid} and \texttt{cid} fields in the \texttt{Students} and \texttt{Courses} tables, respectively.
\end{enumerate}

\section*{Question 1}

Write \textbf{relational algebra} expressions for the following queries given the schema above:

\begin{enumerate}[label=(\alph*)]

\item Find the grades that students of age 20 obtained in courses with 4 credits.

\textbf{Solution:}

$$ \pi_{G.grade}(((\sigma_{S.age=20}S)\Join G)\Join (\sigma_{C.credits=4}C)) $$

\item Find the names of students who took a course named \textit{Calculus} and did not get a \textit{C} grade in any course.

\textbf{Solution:}

$$ \rho(S1,\pi_{G.sid}((\sigma_{C.cname='Calculus'}C)\Join G)) $$
$$ \rho(S2,\pi_{G.sid}(\sigma_{G.grade='C'}G)) $$
$$ \pi_{S.sname}((S1-S2)\Join S) $$

\item Find the ages of students who got an \textit{A} in some course with 3 credits or who got a \textit{B} in any course.

\textbf{Solution:}

$$\rho(S1,\pi_{S.sid}((\sigma_{G.grade='A'}((\sigma_{C.credits=3}C)\Join G))\Join S)) $$
$$\rho(S2,\pi_{G.sid}(\sigma_{G.grade='B'}G)) $$
$$ \pi_{S.age} ((S1\cup S2)\Join S) $$

\item Find the maximum age among students who took \textit{Calculus}.

\textbf{Solution:}

$$ \rho(S1,(((\sigma_{C.cname='Calculus'}Courses)\Join Grades)\Join Students) ) $$
$$ \rho(S2(1\rightarrow sid2,2\rightarrow sname2, 3\rightarrow age2),S1) $$
$$ \rho(S3(1\rightarrow sid), \pi_{S1.sid}(S1\Join_{S1.age<S2.age2} S2)) $$
$$ \pi_{Students.age}((Students - S3)\Join Students ) $$

\end{enumerate}


%Chapter "Computational aspects"
%
\graphicspath{ {./img/Computational/} }
\chapter{Computational aspects}
\label{chap: Computational Aspects}
\section{Global assembly}
In this section we will discuss the fundamental process of building the system of algebraic equations through the addition or assembly of elemental coefficient matrices as described by \cref{eq:assem} where $K^G$ is the global coefficient matrix; $k^i$ is the local elemental matrix for the $i$-th element; $\assem_{i=1}^{Numel}$ is the assembly operator and $i$ is an element index ranging between $1$ and the total number of elements $Numel$ that conform the finite element model. Notice that the assembly operator is analogous to the sum operator commonly used in the representation of a series of $N$ terms but it contains information indicating the position of each single term from the element coefficient matrix within the global system.
\begin{equation}\label{eq:assem}
{K^G}=\assem_{i=1}^{Numel} k^i.
\end{equation}

In this section we will describe the fundamental algorithmic steps to perform that process. We first summarize the basic general steps and then illustrate the process through a simple example involving a $2D$ mesh of 4-noded elements. The section is complemented with the Python scripts {\bf GLOBAL.py} and the secondary module {\bf assemutil.py}.

\subsection*{The assembly algorithm}
The process of assembly of the elemental coefficient matrices into the global system involves (i) the identification of the active degrees of freedom (or equation numbers) assigned to each node in the mesh (ii) the identification of the relationship between the elemental degrees of freedom and the global degrees of freedom and (iii) the computation of the coefficient matrix for each element in the model.

The first step is easily accomplished by assigning a boundary condition flag to the degrees of freedom existing at each node. Here we use a $0$ value to indicate an active degree of freedom and a $-1$ to indicate a prescribed degree of freedom. This information is stored in a boundary condition array $IBC$, of dimension $nn \times MDIM$, where $nn$ corresponds to the  total number of nodal points and $MDIM$ represents the problem dimensionality. The $IBC$ array is first input by the user and later modified by the program in a process where the boundary condition flag is read and equations are counted and assigned according to the result. If the boundary condition flag is equal to $0$ the program assigns an equation number while it produces a $0$ value if the flag is equal to $-1$.

In the second step the nodes conforming each element are stored in a connectivity array  $IELCON$ of dimension $Numel \times MxNNel$ where $Numel$ is the number of elements in the model and $MxNNel$ is the maximum number of nodes in a given element. Thus each row in the $IELCON$ array stores the nodal point data for the element. Each entry in the $IELCON$ array can now be directly translated into equation numbers using the processed $IBC$ array. This results in the discrete version of the assembly operator $\assem_{i=1}^{Numel}$ also called the assembly list or the $DME$ operator in our codes.

In the final step the mesh is covered one element at a time and each elemental coefficient matrix is assembled into the global matrix as indicated by the $DME$ operator. The actual computation of the elemental matrix is conducted by an element based subroutine, called here $UEL$, which may be different for each element in the mesh according to different kinematic o material assumptions. The complete process is summarized in \cref{algo:overall} where we describe for completeness additional steps involved in the finite element algorithm. We have introduced additional global and elemental arrays $RHS^G$ and $rhs^i$ respectively, storing the element nodal excitation and the resulting vector of global excitation. This vector is assembled simultaneously, and using the same data, with the global coefficient matrix. In the final step in the finite element algorithm the system of equations is solved. Notice that the essential boundary conditions were already considered during the assembly process and as a result the global algebraic system of equations is ready to be solved. Finally, after the system has been properly solved the nodal results are scattered through the elements in a process which is inverse to the assembly operation but that uses the same information contained in the $DME$ operator.

\begin{algorithm}[H]\label{algo:overall}
\SetAlgoLined
\KwData{Finite element model}
\KwResult{Field function}
\BlankLine
READ $IBC$ and $IELCON$ arrays ;\\
Compute modified $IBC$ array ;\\
Compute $DME$ operator (using $IBC$ and $IELCON$ arrays);\\
$K^G \leftarrow 0.0$;\\
\BlankLine
\For{$i \leftarrow 1$ to $Numel$}{
    Call UEL(element parameters;$K^i$) ;\\
    $K^G \leftarrow K^G+K^i$ (Assemble each $k^i$ into $K^G$ according to the $DME$ operator);\\
    $RHS^G \leftarrow RHS^G+rhs^i$ (Assemble each $rhs^i$ into $RHS^G$);\\
	\BlankLine
	}
\BlankLine
Solve ${K^G}{U^G} = RH{S^G}$;\\
Scatter solution to the elements
\caption{Summarized algorithm for the finite element method}
\end{algorithm}
\newpage

\subsection*{Sample problem}
Consider the mesh shown in \cref{fig:quad}. The problem parameters in this case are $nn=9$, $Numel=4$ and $MxNNel=4$.

\begin{figure}[h]\label{fig:quad}
\centering
\includegraphics[width=10cm]{mesh2.pdf}
\caption{Finite element mesh of 4-noded elements. The global matrix can be assembled using the python scripts {\bf GLOBAL.py} and the secondary module {\bf assemutil.py}. The input files required to run the script are called {\bf nodes.txt} and {\bf eles.txt}}
\end{figure}

The array of boundary conditions as input from the user corresponds to
\[IBC = \begin{bmatrix}
-1 & -1\\
-1 & -1\\
-1 & -1\\
-1 & 0\\
0 & 0\\
0 & 0\\
-1 & -1\\
-1 & -1\\
-1 & -1
\end{bmatrix}\]

while its code-modified version reads
\[IBC = \begin{bmatrix}
0 & 0\\
0 & 0\\
0 & 0\\
0 & 1\\
2 & 3\\
4 & 5\\
0 & 0\\
0 & 0\\
0 & 0
\end{bmatrix}\]

Notice that the program changes each $-1$ entry to a $0$ valued entry and assigns an equation number to the position containing an original $0$ entry during an equation-assigning and equation-counting operation. Similarly, the element connectivity for the current example read
\[IELCON = \begin{bmatrix}
1 &2 &5 &4\\
2 &3 &6 &5\\
4 &5 &8 &7\\
5 &6 &9 &8
\end{bmatrix}\]

The so-called $DME$ operator is actually the same $IELCON$ but translated into equation numbers for each element. Accordingly in the current example we have
\[DME = \begin{bmatrix}
0 &0 &0 &0 &2 &3 &0 &1\\
0 &0 &0 &0 &4 &5 &2 &3\\
0 &1 &2 &3 &0 &0 &0 &0\\
2 &3 &4 &5 &0 &0 &0 &0
\end{bmatrix}\]

The definition of the element connectivity and its subsequent translation into degrees of freedom is carried out according to a local element definition like the one shown in \cref{fig:locdof}

\begin{figure}[H]\label{fig:locdof}
\centering
\includegraphics[width=8cm]{localdof.pdf}
\caption{Local definition for a 4-noded element.}
\end{figure}

As a result the $(i,j)$ entry in the $DME$ corresponds to the global equation number associated with the local degree of freedom $j$ of the $i$ element. For instance the value of $4$ stored at position $(2,5)$ in the current $DME$ indicates that the global equation $4$ corresponds to the local equation $5$ (column index) in element $2$ (row index).

The assembly process is then conducted by identifying the relation between the entries in each row of the $DME$ operator and the list of local degrees of freedom for the reference element shown in \cref{fig:locdof}. Accordingly, for element 2 it follows that the assembly of row $5$ of the local stiffness matrix proceeds as follows
\begin{align*}
K_{4,4}^G & \leftarrow  K_{4,4}^G + k_{5,5}^2 \\
K_{4,5}^G & \leftarrow  K_{4,5}^G + k_{5,6}^2 \\
K_{4,3}^G & \leftarrow  K_{4,3}^G + k_{5,8}^2
\end{align*}

In \cref{algo:overall} it must be noticed that the local stiffness matrix for each $i$ element is obtained by the call to the local element subroutine $UEL$. In the script GLOBAL.py these elemental routines produce a fictitious matrix filled out with ones and the actual computation of the elemental coefficient matrix is discussed later.

\section{Sparse assembly}
In Finite Elements is common to have stiffness and mass matrices that are sparse, i.e., matrices in which most of the elements are zero.

For instance, in a regular mesh formed with bilinear quadrilaterals the number of nonzero entries is given by the expression
\[\text{storage} = 9 n_x n_y - 9 n_x - 3 n_y + 4\, ,\]
where $n_x$ is the number of nodes in the $x$ coordinate and $n_y$ is the number of nodes in the $y$ coordinate. Figure \ref{fig:sparse_storage} presents the needed storage for a mesh with $n_x=n_y$ for different sizes.
\begin{figure}[H]
\centering
\includegraphics[width=6 in]{sparse_storage.pdf}
\caption{Nonzero entries in a sparse matrix for a structured mesh of bilinear elements.}
\label{fig:sparse_storage}
\end{figure}

\section{Python implementations}

\section{Commercial codes and user subroutines}


\documentclass{article}
\usepackage[utf8]{inputenc}

\title{cycle}
\author{nguyenngockhanh.pbc }
\date{April 2021}

% cases
\usepackage{amsmath}
% symbol
\usepackage{amssymb}

\newtheorem{definition}{Definition}
\newtheorem{theorem}{Theorem}

\begin{document}

    \maketitle

    \textbf{Is there an natural number function that always has a cycle and the cycle length is unbounded?}

    \section{Problem}\label{sec:problem}

    Firstly, let define precisely the notations we are using in this text.

    \begin{definition}(Natural Number Set)
        \begin{equation}
            \label{eq:natural}
            \mathbb{N} = \{ 1, 2, 3, \dots \}
        \end{equation}
    \end{definition}

    \begin{definition}(Natural Number Function)
        All functions that have the domain and codomain $\mathbb{N}$
        \begin{equation}
            \label{eq:function}
            f: \mathbb{N} \to \mathbb{N}
        \end{equation}
    \end{definition}

    Denote $f^m(n)$ where $n \in \mathbb{N}$ be the value of $m$-times recursive call of function $f$ to input $n$.
    More formally,

    \begin{gather*}
        f^1(n) = f(n),\\
        f^{m+1}(n) = f(f^m(n))\\
    \end{gather*}

    \begin{definition}(Cycle number of a Natural Number Function)
        Let $f: \mathbb{N} \to \mathbb{N}$.
        A cycle set of $n$ is the set of all numbers $m \in \mathbb{N}$ such that $f^m(n) = n$

        \begin{equation}
            \label{eq:cycle_set}
            c^{(f)}(n) = \{m : m \in \mathbb{N} \land f^m(n) = n \}
        \end{equation}
        if cycle set is non-empty, define the cycle number as the smallest number in the cycle set
        \begin{equation}
            \label{eq:cycle_number}
            c^{(f)}_{\min}(n) = \min{c^{(f)}(n)}
        \end{equation}
    \end{definition}

    In this text, we ignore the trivial case where cycle number is 1.

    Our main theorem is stated as follows

    \begin{theorem}(Cycle)
        There exists a natural number function such that (1) it has a non-trivial cycle number for every input and (2) the set of all cycle numbers is unbounded.

        \begin{equation}
            \label{eq:cycle_theorem}
            \exists f : \mathbb{N} \to \mathbb{N}, (
            \forall n \in \mathbb{N},
            c^{(f)}_{\min}(n) > 1
            ) \land (
            \forall m_0 \in \mathbb{N},
            \exists n \in \mathbb{N},
            c^{(f)}_{\min}(n) \geq m_0
            )
        \end{equation}
    \end{theorem}

    In other words, this function partitions the natural number set into infinitely number of finite subsets where the cardinality of them are unbounded.
    Each cycle is associated with a subset.

    \section{Proof}\label{sec:proof}

    A simple construction satisfies those properties is as follows:

    Suppose we found partition on $\mathbb{N}$ of ($1^*$) infinitely many finite subsets where ($1^{**}$) each of them has the cardinality of at least 2, ($2^*$) the cardinality of these subsets is unbounded.

    \begin{equation}
        \label{eq:partition}
        \mathbb{P} = \{ P_i \}_{i=1}^\infty = \{ P_1, P_2, P_3, \dots \}
    \end{equation}

    Where we order all elements in each $P_i$, so that for every $P_i$, we have a minimum element, a maximum element and a function to yield the successor element if the input is not the maximum element namely

    \begin{equation}
        \label{eq:succ_func}
        succ_i : P_i \backslash \{ \max{P_i} \} \to P_i \backslash \{ \min{P_i} \}
    \end{equation}

    Define a function $f_i: P_i \to P_i$ that returns minimum element of $P_i$ if the input is the maximum element of $P_i$, otherwise return its successor.

    \begin{equation}
        \label{eq:func_sub}
        f_i(n)=
        \begin{cases}
            \min P_i &\text{if $n = \max P_i$}.\\
            succ_i(n) &\text{otherwise}.
        \end{cases}
    \end{equation}

    This function has a cycle number of the cardinality of $P_i$
    Since $\mathbb{P}$ is a partition, these $P_i$ are disjoint and their union is $\mathbb{N}$.
    We define the function $f: \mathbb{N} \to \mathbb{N}$

    \begin{equation}
        \label{eq:func}
        f(n)= f_i(n) \text{ if } n \in P_i
    \end{equation}

    ($1^* \land 1^{**} \to 1$) For every input $n$, the cycle number is $|P_i| \geq 2$ where $P_i$ is the associated subset. ($2^* \to 2$) Since these subsets are unbounded in size, the set of all cycle numbers of $f$ is also unbounded.

    In order to finish the proof, we will show a partition on $\mathbb{N}$ that satisfies ($1^*$), ($1^{**}$) and ($2^*$).

    Let $S_i$ be the set of all natural numbers in the range $[2^i, 2^{i+1})$ for $i=0, 1, 2, \dots$.
    e.g. $S_0 = \{ 1 \}$, $S_1 = \{ 2, 3 \}$, $S_2 = \{ 4 , 5, 6, 7 \}$, $S_3 = \{ 8, 9, \dots, 15 \}$, etc.

    Our partition is in the form

    \begin{equation}
        \label{eq:partition_concrete}
        \mathbb{P} = \{\{ S_0 \cup S_1 \}\} \cup \{ S_i \}_{i=2}^\infty
    \end{equation}

\end{document}


\documentclass{standalone}
\begin{document}
	\chapter{Classification of numbers}

	\begin{itemize}
		\item{Natural numbers: $\mathbb{N}$; (1, 2, 3, 4 \ldots)}
		\subitem{This set includes every number which is both positive and whole.}
		\item{Integer numbers: $\mathbb{Z}$; (-2, -1, 0, 1, 2, \ldots)}
		\subitem{The integer number set includes every negative and positive whole numbers, similarly to $\mathbb{N}$}
		\item{Rational numbers: $\mathbb{Q}$; (-1, 2 , $\frac{1}{2}$)}
		\subitem{A number is s.t.b rational if expressed in the form $\frac{p}{q};p, q \in \mathbb{Z}$.}
		\item{Irrational numbers: $\mathbb{Q'}$; ($\pi, e, \sqrt{2}, \sqrt{5}, \ldots$))}
		\subitem{If a number is not classified as any of the above, it is referred to as irrational.}
		\item{Real numbers: $\mathbb{R}$}
		\subitem{Anything mentioned above inclusively represent the set of Real numbers}\\
	\end{itemize}

	\emph{We can additionally refer to positive or negative numbers in any set by using the notation:}\\
	\begin{center}
		$\mathbb{R}^+$ and $\mathbb{R}^-$
	\end{center}
	%TODO: insert image of sets

	\newpage
	\end{document}


\documentclass[../../../thesis.tex]{subfiles}

\begin{document}

    \section{Localization}

        We will briefly explain the localization of a category at a set. We will look at local objects and localizations by adjoint functors. This section is based on \cite{Krause21}.

        \subsection{Definition and Local Objects}

            The localization of a category at some set of morphisms is a universal category where these morphisms have been inverted. In this manner, we will think of the localization as adding formal inverses to the morphisms in this set to the category.

            \begin{definition}
                Let $\mathcal{C}$ be a category and $S \subseteq Mor(\mathcal{C})$ a set of morphisms in $\mathcal{C}$. We define the localization of $\mathcal{C}$ at $S$ as a category $\mathcal{C}[S^{-1}]$ together with a functor $Q : \mathcal{C} \rightarrow \mathcal{C}[S^{-1}]$ satisfying the properties:
                \begin{enumerate}
                    \item For any morphism $s\in S$, $Qs$ is an invertible morphism.
                    \item Any functor $F : \mathcal{C} \rightarrow \mathcal{D}$ satisfying $Fs$ is invertible if $s\in S$ factors through $Q$. Thus there exists a unique functor $\bar{F} : \mathcal{C}[S]^{-1} \rightarrow \mathcal{D}$ and a natural isomorphism $\alpha : F \implies \bar{F} \circ Q$ relating the functors.
                \end{enumerate}

                \begin{center}
                    \begin{tikzcd}
                        \mathcal{C} \ar{rd}{Q} \ar{rr}{F} & & \mathcal{D} \\
                        & \mathcal{C}[S^{-1}] \ar[dashed]{ur}{\bar{F}}
                    \end{tikzcd}
                \end{center}
            \end{definition}

            Informally, the category $\mathcal{C}[S^{-1}]$ may be constructed as the category with the same objects as $\mathcal{C}$. The morphisms are constructed with paths of morphisms in $\mathcal{C}$, where we also allow formal inverse paths $s^{-1}$ for any morphism in $S$. The constant paths would be the new identity morphisms, while every path from an object $X$ to $Y$ represents a morphism.

            \begin{remark}
                Beware whenever localizing a locally small category. Adding more morphisms to this category may make some set of morphisms between objects big.
            \end{remark}

            By the universality of the functor $Q$ we have the following lemma.

            \begin{lemma}
                Let $\mathcal{D}$ be a category, then the pre-composition functor is fully faithful.
                \begin{align*}
                    \_ \circ Q : Fun(\mathcal{C}[S^{-1}],\mathcal{D}) & \rightarrow Fun(\mathcal{C}, \mathcal{D}) \\
                    F & \mapsto F \circ Q
                \end{align*}
                Moreover, we may identify $Fun(\mathcal{C}[S^{-1}],\mathcal{D})$ as the full subcategory of functors in $Fun(\mathcal{C}, \mathcal{D})$ which sends every morphism in $S$ to an isomorphism.
            \end{lemma}

            We proceed to define $S$-local objects, and informally, these are all the objects which does not change after the localization process.

            \begin{definition}
                An object $Y\in \mathcal{C}$ is called $S$-local ($S$-closed or $S$-orthogonal) if for any $s\in S$, the map $\mathcal{C}(s, Y)$ is a bijection. Define $S\perp$ to be the full subcategory of $S$-local objects.
            \end{definition}

            The following lemma gives an equivalent definition for "localness".

            \begin{lemma}
                An object $Y\in \mathcal{C}$ is $S$-local if and only if the canonical map $q$ is a natural isomorphism.
                \begin{align*}
                    q_{X,Y} : \mathcal{C}(X,Y) \rightarrow \mathcal{C}[S^{-1}](X,Y)
                \end{align*}
            \end{lemma}

        \subsection{Localizing at Adjoint Functors}

            Given two functors $F : \mathcal{C} \rightarrow \mathcal{D}$ and $G : \mathcal{D} \rightarrow \mathcal{C}$ such that they form an adjoint pair $F \dashv G$, we want to see how they react with localizations.

            \begin{proposition}
                Let $F \dashv G$ be an adjoint pair as above, and define $S = $\startset{$s\in Mor\mathcal{C}$ $\mid$ $Fs$ is invertible}. Draw out the following diagram, with the relations $F = \bar{F}\circ Q$.
                \begin{center}
                    \begin{tikzcd}
                        & \mathcal{C} \ar{ld}[description]{Q} \ar[bend right = 20]{dd}[description]{F} \\
                        \mathcal{C}[S^{-1}] \ar{rd}[description]{\bar{F}} \\
                        & \mathcal{D} \ar[bend right = 20]{uu}[description]{G}
                    \end{tikzcd}
                \end{center}
                The following are equivalent:
                \begin{enumerate}
                    \item $G$ is fully faithful.
                    \item The counit $\varepsilon : FG \implies Id_\mathcal{D}$ is invertible for any object $X\in \mathcal{D}$.
                    \item The functor $F$ induces an equivalence $\bar{F}:\mathcal{C}[S^{-1}] \rightarrow \mathcal{D}$.
                \end{enumerate}
            \end{proposition}

            In the light of the last proposition, we may define "short exact sequences" of pre-additive categories.

            \begin{definition}
                Suppose there is a diagram of additive functors like below.
                \begin{center}
                    \begin{tikzcd}[column sep = large]
                        \mathcal{A} \ar[bend left = 30, tail]{r}{E} \ar[phantom]{r}{\bot} & \mathcal{B} \ar[bend left = 30, two heads]{r}{F} \ar[bend left = 30, two heads]{l}{E_\rho} \ar[phantom]{r}{\bot} & \mathcal{C} \ar[bend left = 30, tail]{l}{F_\rho}
                    \end{tikzcd}
                \end{center}
                It is called a localization sequence whenever the following holds:
                \begin{enumerate}
                    \item $(E,E_\rho)$ and $(F,F_\rho)$ are adjoint pairs.
                    \item $E$ and $F_\rho$ are fully faithful.
                    \item $ImE \simeq KerF$.
                \end{enumerate}
            \end{definition}

\end{document}


\chapter{Classification}
Linear regression discussed in the previous chapter assumes a quantitative response value Y but often classification is needed to solve a problem. Consider the binary classification problem: Predicting whether a tumor is malignant or not based on the size of the tumor. Linear regression could be used by converting the wanted qualitative response variable into a quantitative response. The quantitative response can be converted into a qualitative response by saying that $p(maligant = true | tumorsize) > 0.5$. The model would then classify a tumor as malignant if the probability is higher than 0.5.

A problem though arises from this, since the model will make estimates outside the interval $[0,1]$. Because some predictions are outside of this interval, then the prediction can not be thought of as probabilities. For example measuring a tumor size of 0 would predict a negative value. Another problem with linear regression is that it can not be extended to a classification problem with more than two classes. It can only be used for binary classification. The reason for this is that there might not be a natural ordering between the different classes and the gap between some classes might be bigger than others. This would make it difficult to make an accurate model using a linear regression \citep[pp. 129-130]{ISLR}. Instead this chapter presents different methods that are more suited for classification.

\section{Logistic Regression}
\label{chp:logreg}
Logistic regression is a classification method. It avoids the problem with values outside the interval of $[0,1]$. In logistic regression a logistic function is used and can be seen in (3.1).

\begin{equation}
	p(X) = \dfrac{e^{\beta_0 + \beta_1 X}}{1 + e^{\beta_0 + \beta_1 X}}
\end{equation}

By taking the exponential of $\beta_0 + \beta_1 X$ the result will always be equal to or higher than 0. By dividing by the same equation + 1 the prediction will never be higher than 1. In the equation $\beta_0$ and $\beta_1$ is similar to what is used in linear regression. $\beta_0$ describes the probability when the predictor is 0 and $\beta_1$ is the regression coefficient that is multiplied  with the predictor. The equation can be extended to multiple logistic regression using multiple predictors. Instead the equation looks as shown in (3.2).

%\begin{center}
%	$p(X) = \dfrac{e^{\beta_0 + \beta_1 X_1 + ... + \beta_p X_p}}{1 + e^{\beta_0 + \beta_1 X_1 + ... + \beta_p X_p}}$
%\end{center}

\begin{equation}
	p(X) = \dfrac{e^{\beta_0 + \beta_1 X_1 + ... + \beta_p X_p}}{1 + e^{\beta_0 + \beta_1 X_1 + ... + \beta_p X_p}}
\end{equation}

%\todo{måske indsæt en figur der viser grafer med linear og logistic}

\subsection{Lab 4.6.1 - The Stock Market Data}
In this lab the correlation between the predictors and todays return value is examined. Todays return value describes the direction. If the number is negative the stock went down and if it is positive the stock went up. The available predictors are 5 lagging indicators \emph{Lag1, Lag2, Lag3, Lag4} and \emph{Lag5} and a \emph{Volume} which describes the number of shares traded in billions.

With the python code shown in Listing \ref{lst:correlation} a matrix with the pairwise correlation among the predictors and todays return is printed. Numpy is used for calculating the correlation matrix. Pandas is used for creating a table based on the calculated correlation matrix.

\begin{lstlisting}[language=Python, label=lst:correlation, caption=Printing correlation matrix]
cor = np.corrcoef(preparedData)
labels = ['Year', 'Lag1', 'Lag2', 'Lag3', 'Lag4', 'Lag5', 'Volume',
'Today']
df = pd.DataFrame(cor, columns=labels, index=labels)
print df
\end{lstlisting}

The printed correlation matrix is illustrated in figure \ref{fig:lab461}. Here it is shown that there is very little correlation between the different Lag variables and todays return. This also makes sense because the variables are lagging indicators and therefore it only changes after the economy has already changed. For example if \emph{Lag1} describes how the market went up yesterday then it is less likely to go up today. An interesting correlation can be seen between \emph{Year} and \emph{Volume}. This indicates that each year the average number of traded stocks increases.

\myFigure{461.png}{Pairwise correlation matrix between predictors and todays return}{fig:lab461}{1}
\FloatBarrier

\myFigure{461_plot.png}{Correlation between Volume and Year. Higher index describes forward in time}{fig:lab461Plot}{0.45}

\emph{Volume} is plotted in figure \ref{fig:lab461Plot}. Here the increase in traded stock is obvious. When choosing predictors in our logistic regression model it must be noted that a correlation matrix not necessarily describes the best predictors, at least not if multiple predictors are used. When choosing predictors it is important to choose the predictors that actually are associated with the response variable. The following lab will therefore examine the data to find good predictors.

\subsection{Lab 4.6.2 - Logistic Regression}
\label{sec:lab462}
In this lab a logistic regression model is made based on the stock market data. The predictors were examined in Lab 4.6.1 and some of the predictors will be chosen to predict the direction of the stock market. The direction can either be \emph{up} or \emph{down} and hence this is a binary classification problem.

In the exercise the p-values will first be examined. This is done by fitting a logistic model with the predictors \emph{Lag1, Lag2, Lag3, Lag4, Lag5} and \emph{Volume} to predict the \emph{direction}. The statsmodels python library is used to print a summary of the fitted model. This is shown in listing \ref{lst:summary}.

\begin{lstlisting}[language=Python, label=lst:summary, caption=Printing summary]
formula = 'Direction ~ Lag1+Lag2+Lag3+Lag4+Lag5+Volume'
model = smf.glm(formula=formula, data=data, family=sm.families.Binomial())
print(model.fit().summary())
\end{lstlisting}

\myFigure{Lab462Pvalue.PNG}{Summary of data with associated p-value describing good predictors}{fig:lab462Pvalue}{0.6}
\FloatBarrier

The printed summary is illustrated in figure \ref{fig:lab462Pvalue}. Here it is  seen that the lowest p-value is \emph{Lag1} but it still has a value of 0.145, which is still quite high and it presents no evidence of any association between \emph{Lag1} and \emph{Direction}. But note here that only using the predictors with the lowest p-value could yield the most accurate result when predicting the \emph{Direction}. The reason for this is that some predictors have so little association with the \emph{Direction} that they will only make the prediction less accurate. This will also be shown in the following examples.

In Listing \ref{lst:AllPredictors} the training data is divided into a training set and a test set. Then all the data from 2004 and before is used to train our logistic model and then it is tested on the data from 2005. In this implementation all predictors are used to fit the logistic model.

\begin{lstlisting}[language=Python, label=lst:AllPredictors, caption=Logistic regression using all predictors]
X_train = data[:'2004'][['Lag1', 'Lag2', 'Lag3', 'Lag4', 'Lag5', 'Volume']]
Y_train = data[:'2004']['Direction']

X_test = data['2005':][['Lag1', 'Lag2', 'Lag3', 'Lag4', 'Lag5', 'Volume']]
Y_test = data['2005':]['Direction']

lr = linear_model.LogisticRegression()
lr.fit(X_train,Y_train)
\end{lstlisting}

Based on the test data it is examined how well our logistic model predicts the \emph{Direction}. In Listing \ref{lst:LRscore} a score method is called for the logistic regression model. This function returns the mean accuracy of the model. The score shows that the model only predicts correctly $48.8\%$ of the time. Based on this result the model needs to be approved upon.
\begin{lstlisting}[language=Python, label=lst:LRscore, caption=Printing mean accuracy]
print(lr.score(X_test, Y_test))
\end{lstlisting}

In Listing \ref{lst:OnlyLag1Lag2} only the predictors \emph{Lag1} and \emph{Lag2} are now used. \emph{Lag1} is included because it has the lowest p-value. \emph{Lag2} is also used as predictor because it has a very low correlation with \emph{Lag1} and also the third lowest p-value.
%It has a very low correlation with \emph{Lag1} and also the third lowest p-value.
Note though that the p-value of \emph{Lag2} is only 0.006 higher than the second lowest p-value, which describes \emph{Volume}. \emph{Volume} also has almost twice as high correlation with \emph{Lag1} and this might lead to a better result using \emph{Lag1} and \emph{Lag2} for prediction.

\begin{lstlisting}[language=Python, label=lst:OnlyLag1Lag2, caption=Logistic regression using only Lag1 and Lag2 as predictors]
X_train = data[:'2004'][['Lag1', 'Lag2']]
Y_train = data[:'2004']['Direction']

X_test = data['2005':][['Lag1', 'Lag2']]
Y_test = data['2005':]['Direction']

lr = linear_model.LogisticRegression()
lr.fit(X_train,Y_train)
\end{lstlisting}

Using only \emph{Lag1} and \emph{Lag2} as predictors gave a better result. Running the score function in Listing \ref{lst:LRscore} showed a mean accuracy of $56\%$. By only using \emph{Lag1} and \emph{Lag2} the model has been improved. This combination of predictors also showed the most accurate for predicting the \emph{Direction}. Using p-values for choosing predictors is important and provides information about the association when multiple predictors  are used.

\section{Linear Discriminant Analysis}
\label{chp:lindisana}

In chapter \ref{chp:logreg} it was described how logistic regression could be used for binary classification. It was also stated how it could be extended to  a classification problem with more than two classes. In practice logistic regression is though rarely used for classification problems with more than two classes \citep[pp. 137]{ISLR}. Furthermore a problem with logistic regression is that it does not perform very stable under some circumstances. One big problem with logistic regression is that it might not converge if the classes are well-separated  \citep{convergencefailure}. An example could be the data in the table below. Here we see that the classes are well-separated as $X<0$ then $Y=0$ and if $X>0$ then $Y=1$. Because of this perfect separation the algorithm will never converge because it can keep optimizing.
%\begin{center}
%	\begin{tabular}{rll}
%		\multicolumn{1}{c}{\textbf{Y}} &
%		\multicolumn{1}{c}{\textbf{X}} \\ \hline
%		0     &  -2  \\[0.05cm]
%		0     &  -1  \\[0.05cm]
%		0     &  -5  \\[0.05cm]
%		1     &    3  \\[0.05cm]
%		1     &    1  \\[0.05cm]
%		1     &    6  \\[0.05cm]
%	\end{tabular}
%\end{center}

\begin{center}
	\begin{tabular}{l|llllll}
%		\multicolumn{1}{c}{\textbf{Y}} &
%		\multicolumn{1}{c}{\textbf{X}} \\ \hline
		X & -2 & -1 & -5 & 3 & 1 & 6 \\[0.05cm]
		Y &  0 &  0 &  0 & 1 & 1 & 1  \\[0.05cm]
	\end{tabular}
\end{center}

Instead a method called Linear Discriminant Analysis \emph{LDA} can be used. In logistic regression we directly model $Pr(Y|X)$. With LDA $P(X|Y)$ is modeled and prediction is done by using Bayes theorem to achieve $P(Y|X)$ and find the most probable class given the predictors. The following variables are used:

\begin{itemize}
	\item Let $\pi_k$ represent the prior probability $P(Y=k)$ for K classes.
	\item Let $f_k(x)$ represent the probability $P(X=x|Y=k)$. So $f_k(x)$ is the probability of the predictors when the class Y is given.
\end{itemize}

By inserting these in Bayes formula and normalizing to get $P(Y=k|X=x)$ the following can be written.

\begin{equation}
P(Y=k|X = x) = \dfrac{\pi_k f_k(x)}{\sum_{l=1}^{K} \pi_l f_l(x)}
\end{equation}

For the equation the two variables $\pi_k$ and $f_k(x)$ still needs to be estimated. $\pi_k$ can be estimated by computing the fraction of the training data that belongs to the k'th class, while $f_k(x)$ can be estimated as a Gaussian. This means that the equation differs when using only 1 predictor to using multiple. When only using one predictor and taking log of the equation it can be rewritten as the following

\begin{equation}
\delta_k(x) = x  \dfrac{\mu_k}{\sigma^2} - \dfrac{\mu_k^2}{2\sigma^2} + log(\pi_k)
\end{equation}

Then the equation is calculated for each class k and is classified for the k that results in the highest probability. The equation for multiple predictors instead uses a mean vector and a covariance matrix and can be written as follows:

\begin{equation}
\delta_k(x) = x^T \Sigma^{-1} \mu_k -  \frac{1}{2} \mu_k^{T} \Sigma^{-1} \mu_k + log (\pi_k)
\end{equation}

The means of the Gaussian for each class are simply calculated by summing all values with a specific class and dividing by the total number. The covariance matrix is calculated as a common covariance for all classes.

Note here that LDA assumes the observation to have a Gaussian distribution. If this is the case then LDA is likely to outperform Logistic regression, while if it is not, then Logistic regression is likely to outperform LDA \citep{comparelogandlda}.

\subsection{Lab 4.6.3 - Linear Discriminant Analysis}
First a linear discriminant analysis model is created with the python library sklearn. The model is created based on the two predictors \emph{Lag1} and \emph{Lag2}.

\begin{lstlisting}[language=Python, label=lst:ldaModel, caption=Creating linear discriminant analysis model sklearn]
X_train = data[:'2004'][['Lag1', 'Lag2']]
Y_train = data[:'2004']['Direction']

X_test = data['2005':][['Lag1', 'Lag2']]
Y_test = data['2005':]['Direction']

lda = LinearDiscriminantAnalysis()
fit = lda.fit(X_train,Y_train)
\end{lstlisting}

After creating the model the calculated prior probability, means and coefficients can be read. The prior probability for \emph{Down} is $\pi_1=0.492$ while the prior probability for \emph{Up} is $\pi_2=0.508$. So based on the data there are more days where the stock market went up than down.

The means $\mu_k$ are described by a matrix. The matrix describes a separate mean for both \emph{Lag1} and \emph{Lag2} and for each predictor a mean for \emph{Up} and \emph{Down}. The mean value for \emph{Lag1} and \emph{Lag2} has a tendency to be positive on days when the stock went up while they are negative when the stock went down.

\begin{center}
	$\begin{matrix} 	 & Lag1 & Lag2	\\ 	 Down & 0.0428 & 0.0339 \\ 	 Up & -0.0395 & -0.0313 \end{matrix} $
\end{center}

The coefficients are estimated to $-0.0554$ and $-0.0443$. These are the multipliers for \emph{Lag1} and \emph{Lag2}.

%\todo{Er det vigtigt at vise hvordan vi printer disse værdier?}
%\begin{lstlisting}[language=Python, label=lst:ldaProperties, %caption=printing lda prior mean and coefficients]
%print "Prior probabilities:"
%print lda.priors_
%
%print "Mean:"
%print lda.means_
%
%print "Coefficients:"
%print lda.coef_
%\end{lstlisting}

In listing \ref{lst:ldaScore} the score is printed. The score show that the LDA model predicted correctly in $56\%$ of the predictions on the test data.
\begin{lstlisting}[language=Python, label=lst:ldaScore, caption=Printing LDA score]
print(lda.score(X_test, Y_test))
\end{lstlisting}

This shows how linear discriminant analysis performs very similar to logistic regression on the stock market data.

\section{Quadratic Discriminant Analysis}
Similar to LDA the Quadratic discriminant analysis \emph{QDA} also assumes that the observations are drawn from a Gaussian distribution. The difference lies in the fact that LDA assumes a shared covariance for the observations for all classes K, while QDA assumes a different covariance for the observations for each class.

\myFigure{lda_vs_qda.PNG}{Comparison of LDA and QDA}{fig:qdaCompare}{0.6}

In figure \ref{fig:qdaCompare} the difference between LDA and QDA is illustrated. It shows the linear decision boundary of LDA, while the QDA can learn quadratic boundaries.

\subsection{Lab 4.6.4 - Quadratic Discriminant Analysis}
Again the stock market data is used for prediction with \emph{Lag1} and \emph{Lag2} as predictors. The QDA model is estimated with sklearn. This is shown in listing \ref{lst:qdaModel}.

\begin{lstlisting}[language=Python, label=lst:qdaModel, caption=Creating QDA model]
X_train = data[:'2004'][['Lag1', 'Lag2']]
Y_train = data[:'2004']['Direction']

X_test = data['2005':][['Lag1', 'Lag2']]
Y_test = data['2005':]['Direction']

qda = QuadraticDiscriminantAnalysis()
fit = qda.fit(X_train,Y_train)
\end{lstlisting}

In listing \ref{lst:qdaScore} the score for the QDA model is printed. The print shows that on the test data the prediction was correct in $59.9\%$ of the cases. With the stock market data it actually performed better than both logistic regression and linear discriminant analysis. This might indicate that the relationship for the data set is better captures with a quadratic form.
\begin{lstlisting}[language=Python, label=lst:qdaScore, caption=Printing qda score]
print(qda.score(X_test, Y_test))
\end{lstlisting}

\section{K-Nearest Neighbors}
K-nearest neighbors or \emph{KNN} is a classification technique that classifies observations depending on distance to k other observations.

Say you have a  data set with two classes of observations A and B, with 10 observations each. Then a test observation is desired to be classified with $k=5$, which means the five nearest observations will be examined. If three of the examined observations are A and the other two are B then the estimated probabilities are $3/5$ for class A and $2/5$ for class B.

\subsection{Lab 4.6.5 - K-Nearest Neighbors}

In this lab exercise the method K-nearest neighbors is used. To perform this method, sklearn has a function called \emph{KNeighborsClassifier}. This takes one parameter, which is number of k. After this the function fit is called, that fits the data.
The code is shown in listing \ref{lst:kneighbor}.

\begin{lstlisting}[caption={Python KNN function, where k is set to 1}, label=lst:kneighbor, mathescape=true]
X_train = data[:'2004'][['Lag1', 'Lag2']]
Y_train = data[:'2004']['Direction']

X_test = data['2005':][['Lag1', 'Lag2']]
Y_test = data['2005':]['Direction']

knn = KNeighborsClassifier(n_neighbors=1)
fit = knn.fit(X_train, Y_train)
\end{lstlisting}

The function is used to predict the market's movement from the dates in 2005. If the value k is set to 1, 50\% of the predictions are correctly predicted which is not very good.
If k is change to 3, the predictions turn out to be correct in 53\% of the cases. So increasing the number of k from 1 to 3 made the model a bit better. All kinds of different values can be chosen for k, but it is important not to change the number too often because the model then will be customized to the training data which is not good.

If the results are compared with the results that QDA produces, the QDA provides better results than KNN.