\documentclass{article}

% Packages
\usepackage[a4paper]{geometry}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{progproving}
\usepackage{listings}

\geometry{top=25mm,bottom=25mm,inner=25mm,outer=20mm}
\allowdisplaybreaks

\begin{document}
\title{ProgProving package : Example}

\author{BERG Lucas}

\maketitle

\section{Introduction}
    This document is an example for the ProgProving package. We will prove the following function :

    \begin{lstlisting}[language=C]
int sum(int array[], int length) {
    int total = 0;
    int i = 0;
    while (i < length) {
        total = total + array[i];
        i = i + 1;
    }
    return total;
}
    \end{lstlisting}

\section{Specifications}
    First, let's write the specifications of the function.

    \subsection{Header}
        \begin{lstlisting}[language=C]
int sum(int array[], int length)
        \end{lstlisting}

    \subsection{Environment}
        /

    \subsection{Preconditions}
        \begin{flalign*}
            & \pre = \state{
                0 \leq length = \text{size of array[]}
            } &
        \end{flalign*}

    \subsection{Postconditions}
        \begin{flalign*}
            & \post = \state{
                & length = length_0 \wedge \\
                & array = array_0 \wedge \\
                & sum = \sum^{length-1}_{j=0} array[j]
            } &
        \end{flalign*}

\section{Proof using the strongest postcondition (sp)}
    \subsection{Definitions}
        \begin{flalign*}
            & \init \equiv
                \begin{aligned}
                    & total := 0; \\
                    & i := 0;
                \end{aligned}
            & \\
            & \iter \equiv
                \begin{aligned}
                    & total := total + array[i]; \\
                    & i := i + 1;
                \end{aligned}
            & \\
            & \term \equiv  sum := total; & \\
            & \text{Loop condition} : \cond \equiv i < length & \\
            & \text{Loop preconditions} : \lpre = \state{
                & 0 \leq length = \text{size of array[]} \wedge \\
                & total = 0 \wedge \\
                & i = 0
            } &
        \end{flalign*}

    \subsection{Find the loop invariant}
        First, we need to find the loop invariant. In most cases, the invariant will describe the state of the loop using i-1. In our case, the invariant is :
        \begin{flalign*}
            & \inv = \state{
                & 0 \leq i \leq length \wedge \\
                & total = \sum^{i-1}_{j = 0} array[j] \wedge \\
                & length = length_0 \wedge \\
                & array = array_0
            } &
        \end{flalign*}

    \subsection{Proof that $\inv$ is an invariant}
        Now, we have to prove $\{ \inv \wedge \cond \} \iter \{ \inv \}$. If we can prove that, this means that the invariant is correct.
        First, let's define $\inv \wedge \cond$
        \begin{flalign*}
            & \inv \wedge \cond = \state{
                & 0 \leq i \leq length \wedge \\
                & total = \sum^{i-1}_{j = 0} array[j] \wedge \\
                & length = length_0 \wedge \\
                & array = array_0
            } \wedge \state{
                i < length
            } = \state{
                & 0 \leq i < length \wedge \\
                & total = \sum^{i-1}_{j = 0} array[j] \wedge \\
                & length = length_0 \wedge \\
                & array = array_0
            } &
        \end{flalign*}

        Then, we can do a sp to check if $\inv$ is an invariant.

        \begin{flalign*}
                & \ppsp{\iter}{\inv \wedge \cond} & \\
            %
            %
            %
            =   & \ppsp{
                    \begin{aligned}
                        & total := total + array[i]; \\
                        & i := i + 1;
                    \end{aligned}
                }{
                    \state{
                        & 0 \leq i < length \wedge \\
                        & total = \sum^{i-1}_{j = 0} array[j] \wedge \\
                        & length = length_0 \wedge \\
                        & array = array_0
                    }
                } & \\
            %
            %
            %
            =   & \ppsp{
                    i := i + 1;
                }{
                    \state{
                        & 0 \leq i < length \wedge \\
                        & total = \sum^{i-1}_{j = 0} array[j] \wedge \\
                        & length = length_0 \wedge \\
                        & array = array_0
                    }[total/total']
                    \wedge
                    \state{
                        total := total + array[i]
                    }[total/total']
                } & \\
            %
            %
            %
            =   & \ppsp{
                    i := i + 1;
                }{
                    \state{
                        & 0 \leq i < length \wedge \\
                        & total' = \sum^{i-1}_{j = 0} array[j] \wedge \\
                        & length = length_0 \wedge \\
                        & array = array_0
                    }
                    \wedge
                    \state{
                        total = total' + array[i]
                    }
                } & \\
            %
            %
            %
            =   & \ppsp{
                    i := i + 1;
                }{
                    \state{
                        & 0 \leq i < length \wedge \\
                        & total' = \sum^{i-1}_{j = 0} array[j] \wedge \\
                        & length = length_0 \wedge \\
                        & array = array_0
                    }
                    \wedge
                    \state{
                        total - array[i] = total'
                    }
                } & \\
            %
            %
            %
            =   & \ppsp{
                    i := i + 1;
                }{
                    \state{
                        & 0 \leq i < length \wedge \\
                        & total - array[i] = \sum^{i-1}_{j = 0} array[j] \wedge \\
                        & length = length_0 \wedge \\
                        & array = array_0
                    }
                } & \\
            %
            %
            %
            =   & \ppsp{
                    i := i + 1;
                }{
                    \state{
                        & 0 \leq i < length \wedge \\
                        & total = (\sum^{i-1}_{j = 0} array[j]) + array[i] \wedge \\
                        & length = length_0 \wedge \\
                        & array = array_0
                    }
                } & \\
            %
            %
            %
            =   & \ppsp{
                    i := i + 1;
                }{
                    \state{
                        & 0 \leq i < length \wedge \\
                        & total = \sum^{i}_{j = 0} array[j] \wedge \\
                        & length = length_0 \wedge \\
                        & array = array_0
                    }
                } & \\
            %
            %
            %
            =   & \state{
                    & 0 \leq i < length \wedge \\
                    & total = \sum^{i}_{j = 0} array[j] \wedge \\
                    & length = length_0 \wedge \\
                    & array = array_0
                }[i/i'] \wedge \state{
                    i := i + 1
                }[i/i'] & \\
            %
            %
            %
            =   & \state{
                    & 0 \leq i' < length \wedge \\
                    & total = \sum^{i'}_{j = 0} array[j] \wedge \\
                    & length = length_0 \wedge \\
                    & array = array_0
                } \wedge \state{
                    i = i' + 1
                } & \\
            %
            %
            %
            =   & \state{
                    & 0 \leq i' < length \wedge \\
                    & total = \sum^{i'}_{j = 0} array[j] \wedge \\
                    & length = length_0 \wedge \\
                    & array = array_0
                } \wedge \state{
                    i - 1 = i'
                } & \\
            %
            %
            %
            =   & \state{
                    & 0 \leq i-1 < length \wedge \\
                    & total = \sum^{i-1}_{j = 0} array[j] \wedge \\
                    & length = length_0 \wedge \\
                    & array = array_0
                } & \\
            %
            %
            %
            =   & \state{
                    & 0 < i \leq length \wedge \\
                    & total = \sum^{i-1}_{j = 0} array[j] \wedge \\
                    & length = length_0 \wedge \\
                    & array = array_0
                } & \\
            %
            %
            %
            & \Rightarrow \inv &
        \end{flalign*}

        All right, the invariant is correct so we can move on to the next step.

    \subsection{Proof that $\inv$ is strong enough}
        The next step is to prove that the invariant is true before the loop so we need to prove $\lpre \Rightarrow \inv$.

        \begin{flalign*}
            \lpre = & \state{
                        & 0 \leq length = \text{size of array[]} \wedge \\
                        & total = 0 \wedge \\
                        & i = 0
                    } & \\
            %
            %
            %
                  = & \state{
                        & 0 = i \leq length = \text{size of array[]} \wedge \\
                        & total = \sum^{-1}_{j = 0} array[j] = 0
                    } & \\
            %
            %
            %
                    & \Rightarrow \inv
        \end{flalign*}

        All right, the invariant is strong enough so we can move on to the next step.

    \subsection{Proof that the program will return the postconditions after the loop}
        The next step is to prove that the program will return the postconditions after the loop so we need to prove $\{ \inv \wedge \neg \cond \} \term \{ \post \}$.
        First, let's define $\inv \wedge \neg \cond$
        \begin{flalign*}
            \inv \wedge \neg \cond & = \state{
                & 0 \leq i \leq length \wedge \\
                & total = \sum^{i-1}_{j = 0} array[j] \wedge \\
                & length = length_0 \wedge \\
                & array = array_0
            } \wedge \state{
                i \geq length
            } = \state{
                & i = length \wedge \\
                & total = \sum^{i-1}_{j = 0} array[j] \wedge \\
                & length = length_0 \wedge \\
                & array = array_0
            } & \\
            & = \state{
                & total = \sum^{length-1}_{j = 0} array[j] \wedge \\
                & length = length_0 \wedge \\
                & array = array_0
            } &
        \end{flalign*}

        Then, we can do a sp.

        \begin{flalign*}
                & \ppsp{\term}{\inv \wedge \neg \cond} & \\
            %
            %
            %
            =   & \ppsp{
                    sum := total;
                }{
                    \state{
                        & total = \sum^{length-1}_{j = 0} array[j] \wedge \\
                        & length = length_0 \wedge \\
                        & array = array_0
                    }
                } & \\
            %
            %
            %
            =   & \state{
                    & total = \sum^{length-1}_{j = 0} array[j] \wedge \\
                    & length = length_0 \wedge \\
                    & array = array_0
                } \wedge \state{
                    sum := total;
                } & \\
            %
            %
            %
            =   & \state{
                    & total = \sum^{length-1}_{j = 0} array[j] \wedge \\
                    & length = length_0 \wedge \\
                    & array = array_0
                } \wedge \state{
                    sum = total;
                } & \\
            %
            %
            %
            =   & \state{
                    & sum = \sum^{length-1}_{j = 0} array[j] \wedge \\
                    & length = length_0 \wedge \\
                    & array = array_0
                } & \\
            %
            %
            %
            & \Rightarrow \post &
        \end{flalign*}

        Everything is proved so the program is correct.
\end{document}


\subsubsection{Second Order Variation of Parameters}
\noindent
We'll modify our second order equation of have a 1 as the coefficient of the $y''$ term by dividing to get an equation of the form
\begin{equation*}
	y'' + py' + qy = g(x)
\end{equation*}
Just like for undetermined coefficients, we'll find homogeneous and particular solutions $y = y_h + y_p$. Since the equation is second-order, the solution to the homogeneous equation will yield two fundamental solutions $y_1$ and $y_2$ where $y_h = C_1y_1 + C_2y_2$.\\

\noindent
So, we can write $y$ as
\begin{equation*}
	y(x) = A(x)y_1 + B(x)y_2
\end{equation*}
where
\begin{equation*}
	\begin{cases}
		A'y_1  + B'y_2  = 0 \\
		A'y_1' + B'y_2' = g(x)
	\end{cases}
\end{equation*}
We will then solve this system to solve for $A'$ and $B'$ and integrate.

\ifodd\includeHigherOrderExamples\input{./higherOrder/nonHomeg/variationParameters_secondOrder_example.tex}\fi


% !TEX root = Main.tex
\appendix
\appendixpage
\addappheadtotoc
\section{The benzene molecule}\label{benzex}
\begin{wrapfigure}[7]{r}{.3\textwidth}
	\vspace{-2.3em}
	\centering
	\begin{tikzpicture}
		\chemfig{1*6(-2-3-4-5-6-)}
	\end{tikzpicture}
	\caption{Indices of a benzene molecule}\label{benz}
\end{wrapfigure}
As an example the Hamiltonian of benzene is considered. In \cref{benz} one can see the indices of a benzene molecule. Remember that \(\bra{\phi_{\pi}(1)}\hat{H}\ket{\phi_{\pi}(1)} = 0\) and \cref{V}, the Hamiltonian reads:
\begin{align}
	\mqty{                            \\ \\ \\ \vb{H} = V_{pp\pi}\\ \\ \\} \ \mqty{						&  \mqty{1 & 2 & 3 & 4 & 5 & 6} \\
		\mqty{1                           \\ 2 \\ 3 \\ 4 \\ 5 \\ 6} &	\mqty*(0 & 1 & 0 & 0 & 0 & 1 \\
	1 & 0 & 1 & 0 & 0 & 0             \\
	0 & 1 & 0 & 1 & 0 & 0             \\
	0 & 0 & 1 & 0 & 1 & 0             \\
	0 & 0 & 0 & 1 & 0 & 1             \\
	1 & 0 & 0 & 0 & 1 & 0)}\label{BH}
\end{align}
As a helping aid, \cref{BH} shows the atomic indices of the atom on the top and to the left of the matrix. This will give an understanding of how to work with such matrices.
The structure of the benzene molecule is rotationally symmetric and rotating the indices one sixth must yield the same Hamiltonian. Consider the energy eigenvector:
\begin{align}
	\phi = \mqty(c_1 & c_2 & c_3 & c_4 & c_5 & c_6)
\end{align}
There must exist an operator that rotates the indices as such:
\begin{align}
	C_6\phi = \mqty(c_2 & c_3 & c_4 & c_5 & c_6 & c_1)
\end{align}
The rotated Hamiltonian is the same, and thus \(C_6\) and \(\vb{H}\) commute. The rotated vector must be an eigenvector with the same energy and it should be possible to find simultaneous eigenvectors to \(C_6\) and \(\vb{H}\).
\begin{align}
	C_6\phi = \mqty(c_2 & c_3 & c_4 & c_5 & c_6 & c_1) = \lambda\mqty(c_1 & c_2 & c_3 & c_4 & c_5 & c_6)
\end{align}
This operator \(C_6\) is represented with the matrix:
\begin{align}
	\vb{C}_6 = \mqty*(0 & 1 & 0 & 0 & 0 & 0  \\
	0                   & 0 & 1 & 0 & 0 & 0  \\
	0                   & 0 & 0 & 1 & 0 & 0  \\
	0                   & 0 & 0 & 0 & 1 & 0  \\
	0                   & 0 & 0 & 0 & 0 & 1  \\
	1                   & 0 & 0 & 0 & 0 & 0)
\end{align}
It can quickly be shown that the normalised eigenvectors to \(C_6\) are
\begin{align}
	\phi_n = \frac{1}{\sqrt{6}}\mqty(\lambda_n^0 & \lambda_n^1 & \lambda_n^2 & \lambda_n^3 & \lambda_n^4 & \lambda_n^5), \quad \lambda_n = \exp{-i2\pi n / 6}, \quad n = 0,1,2,3,4,5
\end{align}
These eigenvectors are also eigenvectors for \(\vb{H}\) with the eigenvalues:
\begin{align}
	\varepsilon_n = \lambda_n + \lambda_{n-1} = 2 \cos{n\pi/3}
\end{align}
Thus thanks to the rotational symmetry it was possible to find the eigenvectors and eigenenergies for the Hamiltonian.
\section{Additional figures}\label{appfigs}
\begin{figure}[h]
	\centering
	\begin{subfigure}[b]{0.45\textwidth}
		\includegraphics[width=\textwidth]{Figures/BetaimrealTE8.eps}
		\caption{Figure showing a plot of the Green's function at the \nth{8} site}
		\label{4th}
	\end{subfigure}
	~ %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc.
	%(or a blank line to force the subfigure onto a new line)
	\begin{subfigure}[b]{0.45\textwidth}
		\includegraphics[width=\textwidth]{Figures/BetaimrealTE11.eps}
		\caption{Figure showing a plot of the Green's function at the \nth{11} site}
		\label{7th}
	\end{subfigure}
	\caption{Two plots showing how the Green's function changes as the site is changed. The \nth{8} and \nth{11} sites are corresponding to atoms of those indices (8, 11) in \cref{inlinepointplot}. Note how the LDOS changes (imaginary part) for the different sites.}\label{siteLDOSplot}
\end{figure}
\begin{figure}[h]
	\centering
	\begin{subfigure}[b]{0.3\textwidth}
		\includegraphics[width=\textwidth]{Figures/FabNPGBS.eps}
		\caption{Normal NPG}
		\label{Fabbs}
	\end{subfigure}
	~ %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc.
	%(or a blank line to force the subfigure onto a new line)
	\begin{subfigure}[b]{0.3\textwidth}
		\includegraphics[width=\textwidth]{Figures/paraNPGBS.eps}
		\caption{Para NPG}
		\label{parabs}
	\end{subfigure}
	~ %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc.
	%(or a blank line to force the subfigure onto a new line)
	\begin{subfigure}[b]{0.3\textwidth}
		\includegraphics[width=\textwidth]{Figures/metaNPGBS.eps}
		\caption{Meta NPG}
		\label{metabs}
	\end{subfigure}
	\caption{Plot showing band structures in the energy range \SI{-1.5}{\electronvolt} to \SI{1.5}{\electronvolt} for normal, para and meta NPG. The are plotted between symmetry points \(X\) and \(Y\) with respect to the origin \(\Gamma\)}\label{allbands}
\end{figure}

% \section{Project overview}
% A Gantt chart is provided on the next page. \textbf{Not Updated.}
% \newpage
% \begin{turnpage}
% \setcounter{myWeekNum}{6}
% \ganttset{%
% 	calendar week text={\myWeek{}}%
% }
% \begin{figure}\vspace{-10mm}
% \begin{ganttchart}[
% 		hgrid,
% 		vgrid={*{6}{draw=none}, dotted},
% 		x unit=.15cm,
% 		%	y unit title=.6cm,
% 		%	y unit chart=.6cm,
% 		inline,
% 		milestone inline label node/.append style={left=5mm},
% 		milestone/.append style={xscale=3},
% 		time slot format=isodate,
% 		time slot format/start date=2019-02-04
% 	]{2019-02-04}{2019-05-31}
% 	\gantttitlecalendar{year, month=shortname, week}\\
% 	\ganttgroup{Report writing}{2019-02-25}{2019-05-31}\\
% 	\ganttgroup[inline = false]{Course 33442}{2019-02-04}{2019-03-31}\\
% 	\ganttbar{Ch. 1 \& 2}{2019-02-04}{2019-02-17}\\
% 	\ganttlinkedbar[link bulge=2]{Ch. 3}{2019-02-18}{2019-02-24}\\
% 	\ganttlinkedbar[link bulge=2,bar inline label node/.style={right=15pt}]{Ch. 4 \& 5}{2019-02-25}{2019-03-03}\\
% 	\ganttgroup[inline = false]{Python code}{2019-03-04}{2019-03-31}\\
% 	\ganttbar{Py TB scripts}{2019-02-18}{2019-03-17}\\
% 	\ganttlinkedbar[link bulge=2, bar inline label node/.style={right=45pt}]{Small NPG systems simulations}{2019-03-10}{2019-03-31}\\
% 	\ganttmilestone{Proof of Concept with Python}{2019-03-31}\\
% 	\ganttgroup[inline = false]{Large scale TB}{2019-04-01}{2019-04-28}\\
% 	\ganttbar[bar inline label node/.style={left=10pt}]{SISL \& TBtrans tutorial}{2019-04-01}{2019-04-05}\\
% 	\ganttlinkedbar[link bulge=2, bar inline label node/.style={right=50pt}]{Setup NPG variations}{2019-04-06}{2019-04-28}\\
% 	\ganttgroup[inline = false]{Generate data}{2019-04-28}{2019-05-31}\\
% 	\ganttmilestone{Hand in report}{2019-05-31}
% \end{ganttchart}
% \end{figure}
% \end{turnpage}
% \clearpage
% \global\pdfpageattr\expandafter{\the\pdfpageattr/Rotate 90}


\documentclass{article}

\usepackage{mathrsfs,amssymb,amsmath}

\begin{document}

\section{If $p$ is a seminorm on $\mathscr{X}$, $\mathscr{M}$ is a linear manifold in $\mathscr{X}$, and $\bar{p}: \mathscr{X} / \mathscr{M} \mapsto [0,\infty]$ is defined by $\bar{p}(x + \mathscr{M}) = \inf\{p(x+y):y \in \mathscr{M}\}$, then $\bar{p}$ is a seminorm on $\mathscr{X} / \mathscr{M}$}.

Triangle Inequality: Must show that $\bar{p}(x_1+x_2+M) \le \bar{p}(x_1+\mathscr{M}) + \bar{p}(x_2+\mathscr{M})$

$\inf\{p(x_1 + x_2 + y) : y \in \mathscr{M}\} \le \inf\{p(x_1 + y) : y \in \mathscr{M}\} + \inf\{p(x_2 + y ): y \in \mathscr{M}\}$.

Apply the triangle inequality somehow inside the infimums to expand into a larger expression that is still "less than or equal to" all the way through???

Absolute homogeneity: Must show that $\bar{p}(\alpha x + \mathscr{M}) = |\alpha|\bar{p}(x + \mathscr{M})$

$\inf\{p( \alpha x +y) : y \in \mathscr{M}\} = \inf\{p( \alpha x + \alpha y) : y \in \mathscr{M}\} = \inf\{  | \alpha | p( x +y) : y \in \mathscr{M}\}  =  | \alpha | \inf\{p( x +y) : y \in \mathscr{M}\} $

The first equality is justified since both $y$ and $\alpha y$ are in $\mathscr{M}$, and the second is since $p$ is itself a seminorm.

\section{Show that if $\mathscr{M} \le \mathscr{X}$ and $\mathscr{M}$ is topologically complimented in $\mathscr{X}$, then $\mathscr{M}^{\bot}$ is topologically complimented in $\mathscr{X}^*$ and that its complement is weak-star and linearly homeomorphic to $\mathscr{X}^* / \mathscr{M}^{\bot}$}

\end{document}


\paragraph{Activation functions} We consider 21 activation functions, 6 of which are ``novel'' and proposed in \citet{Ramach:2018}. The functional form of these 6 is given in Table \ref{table:functions}, together with the \sigmoid{} function.
\begin{table}[!htb]
  \centering
  \begin{tabular}{ll}
  \toprule
    \sigmoid & $f(x)=\sigma(x)=1/(1+\exp(-x))$\\
    %\mytanh & \\
    \swish & $f(x)=x\cdot \sigma(x)$\\
    \maxsig & $f(x)=\max\{x,\sigma(x)\}$\\
    \cosid & $f(x)=\cos(x)-x$\\
    \minsin & $f(x)=\min\{x,\sin(x)\}$\\
    \arctid & $f(x)=\arctan(x)^2-x$\\
    \maxtanh & $f(x)=\max\{x,\tanh(x)\}$\\
    \midrule
    \lrelua & $f(x)=\max\{x,0.01x\}$ \\
    \lrelub & $f(x)=\max\{x,0.3x\}$ \\
    {\small \pentan} & $f(x)=\begin{cases}\tanh(x) & x>0,\\ 0.25\tanh(x) & x\le 0\end{cases}$\\
    \bottomrule
  \end{tabular}
  \caption{Top: \sigmoid{} activation function as well as 6 top performing activation functions from \citet{Ramach:2018}. Bottom: the LReLU functions with different parametrizations as well as \pentan{}.}
  \label{table:functions}
\end{table}

The remaining 14 are: \mytanh, \mysin, \relu, \lrelua, \lrelub, \maxouta, \maxoutb, \maxoutc, \prelu, \linear, \elu{}, \cube, \pentan, \selu{}. We briefly describe
%a few of these in greater detail:
them:
\lrelua{} and \lrelub{} are the so-called leaky relu (LReLU) functions \cite{Maas:2013}; the idea behind them is to avoid zero activations/derivatives in the negative region of \relu{}. Their functional form is given in Table \ref{table:functions}. \prelu{} \cite{He:2015} generalizes the LReLU functions by allowing the slope in the negative region to be a %n arbitrary
learnable parameter. The maxout functions \cite{Goodfellow:2013} are different in that they introduce additional parameters and do not operate on a single scalar input. For example, \maxouta{} is the operation that takes the maximum of two inputs: $\max\{\mathbf{xW}+\mathbf{b},\mathbf{xV}+\mathbf{c}\}$, so the number of learnable parameters is doubled. \maxoutb{} is the analogous function that takes the maximum of three inputs.
%: $\max\{\mathbf{xW}+\mathbf{b},\mathbf{xV}+\mathbf{c},\mathbf{xU}+\mathbf{d}\}$, etc.
As shown in \citet{Goodfellow:2013}, maxout can approximate any convex function. \mysin{} is the standard sine function, proposed in neural network learning, e.g., in \citet{Parascandolo:2016}, where it was shown to enable faster learning on certain tasks than more established functions. \pentan{} \cite{Xu:2016} has been defined in analogy to the LReLU functions, which can be thought of as ``penalizing'' the identity function in the negative region.
The reported good performance of \pentan{} on CIFAR-100 \cite{Krizhevsky:2009}
%, an image classification dataset with 100 classes,
lets the authors speculate that the slope %and offset
of activation functions near the origin may be crucial for learning.
\linear{} is the identity function, $f(x)=x$. \cube{} is the function $f(x)=x^3$, proposed in \citet{Chen:2014} for an MLP used in dependency parsing. \elu{} \cite{Clevert:2015} has been proposed as (yet another) variant of \relu{} that assumes negative values, making the mean activations more zero-centered. \selu{} is a scaled variant of \elu{} used in \citet{Klambauer:2017} in the context of so-called self-normalizing neural nets.

\paragraph{Properties of activation functions}
\begin{table*}[!htb]
\centering
\footnotesize
\begin{tabular}{llll}
  \toprule
  Property & Description & Problems & Examples \\ \midrule
  derivative & $f'$ & $>1$ exploding gradient (e) &  \sigmoid{} (v), \mytanh{} (v), \cube{} (e)\\
  & & $<1$ vanishing (v) & \\
  zero-centered & range centered around zero? &   if not, slower learning & \mytanh{} ($+$), \relu{} ($-$) \\
  saturating & finite limits & vanishing gradient in the limit & \mytanh{}, \pentan{}, \sigmoid{}\\
  monotonicity & $x>y\implies f(x)\ge f(y)$ & unclear & exceptions: \mysin{}, \swish{}, \minsin{}
  \\ \bottomrule
 \end{tabular}
 \caption{Frequently cited properties of activation functions}.
 \label{table:properties}
\end{table*}

Many properties of activation functions have been speculated to be crucial for successful learning.
Some of these are listed in Table \ref{table:properties}, together with brief descriptions and illustrations.
%%%First, most activation functions are monotonic, probably as a historical
%One important factor appears to be the \textbf{derivative} of the activation functions, because small derivatives may lead to vanishing gradients and large derivatives to exploding gradients. Since the relu function \cite{Glorot:2011} has a derivative of 1 in the positive region, which is  putatively the optimal value, it has been claimed to be much more suitable than its predecessors such as \mytanh{} and particularly \sigmoid{}. However, \relu{} has a derivative of zero in the negative region and additionally is not \textbf{zero-centered}, %which makes functions such as leaky relu appear more suitable.
%which is the reason why functions such as LReLU and \elu{} have been proposed.
%Another frequently cited property is whether activation functions are \textbf{saturating} or not, that is, how they behave in the limits. A non-saturating function like \relu{} tends towards infinity as the pre-activation goes towards $+\infty$, while functions like \sigmoid{} and \mytanh{} converge toward a finite limit. This means that the derivatives of saturating functions may become very small, implying again a vanishing gradient problem.
%Most activation functions proposed are also \textbf{monotonic}, that is, larger $x$ implies larger activation. This is probably a historical relic: originally, neural networks were described by metaphors adapted from neuro-science, where a larger pre-activation of a unit was associated with a larger probability of that unit ``firing''. By contrast, many of the activation functions found by automatic search in \citet{Ramach:2018} are non-monotonic, including \swish{}. In particular, automatic search yielded many periodic functions, such as sine and cosine, usually in connection with the raw pre-activation $x$.
%\citet{Ramach:2018} also find that ``simpler'' activation functions tend to outperform more complicated ones, and functions using division perform badly, because of numerical problems when the denominator approaches zero.

Graphs of all activation functions can be found in the appendix.


\problemname{Greedily Increasing Subsequence}
Given a permutation $A = (a_1, a_2, \dots, a_N)$ of the integers $1, 2, \dots, N$, we define the \emph{greedily increasing subsequence} (GIS) in the following way.

Let $g_1 = a_1$. For every $i > 1$, let $g_i$ be the leftmost integer in $A$ that is strictly larger than $g_{i-1}$.
If there for a given $i$ is no such integer, we say that the GIS of the sequence is the sequence $(g_1, g_2, ..., g_{i - 1})$.

Your task is to, given a permutation $A$, compute the GIS of $A$.

\section*{Input}
The first line of input contains an integer $1 \le N \le 10^6$, the number of elements of the permutation $A$.
The next line contains $N$ distinct integers between $1$ and $N$, the elements $a_1, \dots, a_N$ of the permutation $A$.

\section*{Output}
First, output a line containing the length $l$ of the GIS of $A$.
Then, output $l$ integers, containing (in order) the elements of the GIS.

\section*{Explanation of sample 1}
In this case, we have the permutation  $2, 3, 1, 5, 4, 7, 6$.
First, we have $g_1 = 2$.
The leftmost integer larger than $2$ is $3$, so $g_2 = 3$.
The leftmost integer larger than $3$ is $5$ ($1$ is too small), so $g_3 = 5$.
The leftmost integer larger than $5$ is $7$, so $g_4 = 7$.
Finally, there is no integer larger than $7$.
Thus, the GIS of $2, 3, 1, 5, 4, 7, 6$ is $2, 3, 5, 7$.


\documentclass{article}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{natbib}
\usepackage{graphicx}

\setlength\parindent{0pt}

\title{Retrograde and anterograde linear models of pathological spread along directed structural connectomes}
\author{Eli J. Cornblath}

\begin{document}

\maketitle

\section*{Introduction}

Linear diffusion models are a highly promising tool for investigating the mechanisms of neurodegenerative disease progression, which is thought to be driven by transsynaptic spread throughout structural connectomes \cite{Raj2012,Pandya2017,Pandya2019,Henderson2019,Mezias2020}.

There are two possible directions for transsynaptic spread. In retrograde spread, misfolded proteins travel backwards from distal axons towards the soma. In anterograde spread, the opposite process occurs-- misfolded proteins starting in the soma of a neuron travel down the axon to distal regions.\\

In this brief document, I will describe the simple version of the model used in Ref. \cite{Henderson2019} to capture anterograde spread between to brain regions (equivalently, neurons or network nodes). Here, we use a matrix $W$ and the equation

\begin{equation}
\dot{x}=Wx
\label{eq1}
\end{equation}
to instantiate an anterograde spreading process whereby pathology in node $A$ spreads from neuron soma in $A$ along axons that terminate in node $B$. In our model

\begin{equation}
W=
\begin{bmatrix}
W_{A\rightarrow A} & W_{B\rightarrow A}\\
W_{A\rightarrow B} & 	W_{B\rightarrow B}
\end{bmatrix},
\label{eq2}
\end{equation}
where the element $W_{A\rightarrow B}$ indicates the strength of the axonal projections initiating from the somas in region $A$ and terminating in region $B$, and $	W_{A\rightarrow A} = 	W_{B\rightarrow B} = 0$. \\

Suppose at the beginning of model time, we seed 1 unit of pathology in region $A$ and represent it in the vector $x$:
\begin{equation}
x = \begin{bmatrix}
1 \\ 0
\end{bmatrix}
\label{eq3}
\end{equation}

\noindent The general form of equation \ref{eq1} is solved by the dot product between the $i$th row of $W$ and the columns of $x$ to generate $\dot{x}_i$, as in
\begin{equation}
\dot{x} =
\begin{bmatrix}
W_{1,1} & W_{1,2} \\
W_{2,1} & W_{2,2}
\end{bmatrix}
\begin{bmatrix}
x_{1,1} \\
x_{2,1}
\end{bmatrix}
=
\begin{bmatrix}
W_{1,1}x_{1,1} + W_{1,2}x_{2,1} \\
W_{2,1}x_{1,1} + W_{2,2}x_{2,1}
\end{bmatrix}.
\label{eq4}
\end{equation}

\noindent We can substitute in our values into equation \ref{eq4} and solve for $\dot{x}$, which yields

\begin{equation}
\dot{x}=
\begin{bmatrix}
W_{A\rightarrow A} & W_{B\rightarrow A}\\
W_{A\rightarrow B} & 	W_{B\rightarrow B}
\end{bmatrix}
\begin{bmatrix}
1 \\ 0
\end{bmatrix}
\label{eq5}
\end{equation}
\begin{equation}
=
\begin{bmatrix}
(1\times W_{A\rightarrow A}) + (0\times W_{B\rightarrow A}) \\
(1\times W_{A\rightarrow B}) + (0\times W_{B\rightarrow B})
\end{bmatrix}
=
\begin{bmatrix}
0 \\
W_{A\rightarrow B}
\end{bmatrix} .
\label{eq6}
\end{equation}

In this equation, we now observe that pathology in node $B$, represented by $x_{2,1}$, will change over time at a rate determined by the strength of axonal projections from neuron somas in $A$ terminating in $B$, which are reflected in $W_{A\rightarrow B}$. This process reflects anterograde spread as intended through the design of $W$ in equation \ref{eq2}, and as we implemented in Ref. \cite{Henderson2019}. \\

Note that in equation \ref{eq1}, we define $\dot{x}$, which is the rate of change of pathology at each node. However, it is often more intuitive to solve for the amount of pathology at each node, represented by the vector $x$. Integration of equation \ref{eq1} yields

\begin{equation}
x=e^{Wt}x_o
\end{equation}

where $x_o$ is the initial state of $x$, and $e$ is the natural exponent.
\section*{Acknowledgments}

I would like to thank Jason Z. Kim for his input on this document.

\bibliographystyle{plain}
\bibliography{\string ~/Dropbox/Cornblath_Bassett_Projects/library.bib}
\end{document}


\section{Sequential Inversion}\label{sec:sequential}
The DCI framework relies on evaluating the ratio function $r(\param)$ in $\dimD$--dimensional QoI space, so we turn our attention to addressing the challenges associated with the growth of this space.
As $\dimD$ increases, we must approximate a push-forward distribution with perhaps a fixed number of samples (from model evaluations) $\nsamps$, which represents a considerable source of error since the convergence rate for kernel density estimation with Gaussian kernels is $\mathcal{O} (N^{2+\dimD})$.

For example, consider a time-dependent problem for which hundreds of spatial sensors are providing streams of data.
Approximating a $100$-dimensional space with $\nsamps = 1E3$ or $1E4$ samples (as we have been using for demonstrations), poses a problem for any density approximation method.
However, either of these values for $\nsamps$ are generally sufficient to estimate a one-dimensional distribution.
In some sense, approximating a QoI at each location over time is reasonable, but doing so for all of them simultaneously is not.
To this end, we propose an approach to solving the parameter estimation problem by performing inversion through a sequence of scalar-valued QoIs rather than employ a vector-valued approach.

Any choice of dimension below $\dimD$ would suffice, but this sequential-scalar-valued approach provides a starting place and admits a simplicity in exposition.
By choosing a dimension of one, the focus of the examples is restricted to solely the order in which the QoIs are inverted; it avoids the additional complexity of enumerating the combinations of QoI when dimensions can vary.
We also choose to use a linear map for convenience so that we can use the analytical solutions presented in Chapter~\ref{chapter:mud} without concern for approximation error.
Furthermore, we omit measurement error from polluting the observations so that all the inverse contours intersect at a point.
In the event that there is measurement error, each contour will be displaced, so the collection of contours will form a convex hull whose volume is proportional to the approximation error.
By omitting measurement error, we simulate scalar-valued QoI which are constructed with sufficient number of measurements so as to ameliorate the impact of misidentifying each contour's location in $\pspace$.

With each iteration in the sequence of inverse problems, we explain measurements that constitute a single QoI at the expense of accuracy in others.
By contrast, the vector-valued approach seeks accuracy in all of the directions of observations simultaneously.
This trade off is all about efficiency, since 1-dimensional problems are computationally ``cheap,'' we can iterate through many more of them for the same computational cost.
By the time we finish iterating through all available QoI, the estimate obtained from $Q^{(1)}$ may have drifted significantly away from its solution contour through the sequence of inverting through $Q^{(1)}, Q^{(2)}, \dots Q^{(100)}$.
To address this, we perform multiple passes through the set of QoI.
Borrowing from other sequential algorithms, these ``epochs'' will allow us to iterate until the solution stops changing by some predefined relative threshold, representing a lack of ``learning'' through continued effort.

\subsection{Motivating Linear Example}
We study the following motivating two-dimensional example with QoI defined by $10$ equispaced rotations of the unit vector $[0, 1]$ through the first two Euclidean quadrants.
We first plot the result of a single epoch in the left panel of Fig.~\ref{fig:iterative-linear-demo}.

\begin{figure}
  \centering
  \includegraphics[width=0.475\linewidth]{examples/iterative/10D-firstepoch.png}
  \includegraphics[width=0.475\linewidth]{examples/iterative/10D-fewepochs.png}

  \caption{
  Dotted lines show the solution contours for each row of the operator $A$.
  (Left): First epoch for iterating through 10 QoI.
  (Right): Three more epochs allows our estimate to get much closer to the true value.
  }
  \label{fig:iterative-linear-demo}
\end{figure}

The spiral shape is a result of the underlying geometry of this QoI map defined by rotations. The successive rows are so similar to each other that very little is ``learned'' between each iteration; the projection doesn't cover a large distance in $\pspace$.
At the end of these epochs, the estimate in the right panel of \ref{fig:iterative-linear-demo} is still far off from the true parameter value (the intersection of the contours).

To further underscore the lack of mutually distinct information in successive rows of the QoI, we choose two pairs of indices from among the ten available in order to define two QoI maps, the contours for which we plot in different colors in Fig.~\ref{fig:iterative-linear-demo-pair}.
We solve a total of ten 1-D inverse problems for each of them (five epochs) to match the budget of the previous example in the left panel of \ref{fig:iterative-linear-demo} (with ten maps and one epoch).

\begin{figure}
  \centering
  \includegraphics[width=0.475\linewidth]{examples/iterative/10D-fewepochs-pair.png}
  \includegraphics[width=0.475\linewidth]{examples/iterative/10D-fewepochs-pair-alt.png}
  \caption{
  Iterating through five epochs of two QoI, each formed by picking two of the ten available rows of $A$ at random.
  The random directions chosen on the left exhibit more redundancy than those on the right, so the same amount of iteration results in less accuracy.
  }
  \label{fig:iterative-linear-demo-pair}
\end{figure}

We observe that in Fig~\ref{fig:iterative-linear-demo-pair}, that there is much greater accuracy in estimating the true parameter value than in the case of Fig~\ref{fig:iterative-linear-demo}.
The reason for this difference is that there is more mutually distinct information between successive iterations of a pair of random rows of $A$ than there is between adjacent rows, as measured by the angle between the solution contours.

\subsection{Connection to Skewness}
If we are careful with how we construct maps or choose an iteration strategy, we can achieve considerably more accurate solutions with the same computational cost.
Had the choice of QoI components corresponded to a pair of rows that were orthogonal, the initial mean would converge to the reference value in a single epoch (two iterations), since there is no redundancy in information whatsoever.
This is equivalent to saying that we have an incentive to select rows that induce a QoI map with unit skewness.

\begin{figure}
  \centering
  \includegraphics[width=0.475\linewidth]{examples/iterative/10D-firstepoch-pair-smart.png}
  \includegraphics[width=0.475\linewidth]{examples/iterative/10D-firstepoch-rand.png}
  \caption{
  (Left): Iterating through a single epoch with a QoI formed by picking rows of $A$ which exhibit mutual orthogonality.
  (Right): Iterating through the rows of $A$ at a random order for a single epoch results in considerably more accuracy than doing so in the original order of rows of $A$.
  }
  \label{fig:iterative-linear-demo-smart}
\end{figure}

We show this in the left half of Figure~\ref{fig:iterative-linear-demo-smart} for two QoI maps with orthogonal pairs of components.
If instead no a priori analysis of the rows of $A$ and iteration through the available QoI at random is the chosen ordering, more accuracy is achieved with only ten iterations.
We show this in the right-half of Figure \ref{fig:iterative-linear-demo-smart}, which exhibits a more accurate estimate compared to \ref{fig:iterative-linear-demo} at the same computational cost.

\subsection{Comparisons and Convergence Results}
To make these results more concrete, we propose the following example:
We limit ourselves to solving 100 inverse problems (i.e. up to ten epochs for this map), with the \emph{only} difference between approaches being the order in which the rows of $A$ are used.
First, we use the QoI as they are presented: in order with respect to increased rotation angle (which defines the rows of $A$).
Next, we shuffle the rows of $A$ and then perform ten epochs using this permuted map.
Lastly, we create an ordering based on a random shuffling of ten sets of indices representing the rows of $A$.
The latter approach is similar to the second in that the same problems are solved the same number of times overall, but it lifts the restriction that a row must only be used once in each successive set of ten iterations (equal computational effort).

\begin{figure}
  \centering
  \includegraphics[width=0.95\linewidth]{examples/iterative/10D-convergence-comparison.png}
  \caption{
  Twenty different initial means are chosen and iterated on for three approaches.
  Individual experiments are transparent and the mean error is shown as solid lines.
  In the \emph{Ordered} approach, we iterate through the rows of $A$ as they are given to us for ten epochs.
  \emph{Shuffled QoI} refers to establishing a different random ordering of the rows of $A$ for each trial, and then
  using this ordering for ten epochs.
  Finally, in the \emph{Random QoI} approach, we choose a QoI at random for each of 100 iterations, where the ordering still ensures each row gets used ten times, representing the same overall set of inverse problems solved as the other two.
  }
  \label{fig:iterative-convergence-comparison}
\end{figure}

In Figure~\ref{fig:iterative-convergence-comparison}, it is shown that using the rows of $A$ sequentially performs very poorly (the error struggles to get past a single decimal place of accuracy), which aligns with ``spiraling'' seen in Figure~\ref{fig:iterative-linear-demo} where the first few epochs are plotted.
Shuffling the rows but requiring that every tenth iteration to use the same row (i.e., ensure same ordering for each epoch), leads to a considerable improvement by which sixteen decimal places of accuracy are achieved in under $100$ iterations.
In a few instances, the shuffled approach stumbles on an ordering that accelerates convergence, likely due to orthogonal pairs of rows in the shuffled order.
These cases exhibit the kind of behavior seen in the left panel of Fig~\ref{fig:iterative-linear-demo-smart}; in other words, sometimes random shuffling finds the ``smart'' rows to iterate through.
Since the ordering has no dependence on iteration number in the approach where we use random rows, we have more opportunities to find these successive orthogonal pairings, and so we see that on average, it takes fewer iterations to achieve the same accuracy.

%
% \subsection{Iterated Solutions for a PDE Example}\label{sec:iterated-nonlinear}
%
% Batch-updates is the connection to make here.
% We are going to set up the heatrod example here but place measurement devices throughout and record the measurements at several intervals in time, the point being here that the dimension of the QoI will be higher than the input space but it's okay because we're iterating.
%
% Some systems will be more informative early in time, others late, so the best thing to do is not really something we're going to answer, we're just going to show how this approach \emph{could} work in a situation like this where the data is streaming in over time.


% !TEX root = main.tex

%-------------------------------------------------
\section{The likelihood ratio test}\label{sec:slrt}

\begin{definition}
The \emph{likelihood ratio} statistic $\lambda:D\to\R$ for testing $H_0:\theta\in\Theta_0$ against $H_1:\theta\in\Theta_1$ is defined by
\[
\lambda(\mathbf{x}) = \frac{\max\{L(\theta;\mathbf{x}):\theta\in\Theta_0\}}{\max\{L(\theta;\mathbf{x}):\theta\in\Theta_1\}}
\]
\end{definition}

% simple case
If $H_0$ and $H_1$ are both simple hypotheses, say $H_0:\theta=\theta_0$ and $H_1:\theta=\theta_1$, the likelihood ratio reduces to
\[
\lambda(\mathbf{x}) = \frac{L(\theta_0;\mathbf{x})}{L(\theta_1;\mathbf{x})}
\]

If $\lambda(\mathbf{x})$ is small then $L(\theta_1;\mathbf{x})$ is large relative to $L(\theta_0;\mathbf{x})$ which indicates that $\theta_1$ is more likely than $\theta_0$ of being the true parameter value, and as a result we might be inclined to reject $H_0$.

% definition: likelihood ratio test
\begin{definition}
The \emph{likelihood ratio test} (LRT) of $H_0:\theta\in\Theta_0$ against $H_1:\theta\in\Theta_1$ is defined by the critical region
\[
C = \left\{\mathbf{x}:\lambda(\mathbf{x}) \leq k\right\}
\]
where $k$ is chosen according to the required size of the test.
\end{definition}

If $H_0$ and $H_1$ are both simple hypotheses, this is sometimes called the \emph{simple likelihood ratio test} (SLRT).

% example
\begin{example}
Let $X_1,\ldots,X_n$ be a random sample from the $\text{Exponential}(\theta)$ distribution, where $\theta>0$ is an unknown rate parameter. Derive an explicit form for the critical region of the likelihood ratio test for testing the simple hypothesis $H_0:\theta=\theta_0$ against the simple alternative $H_1:\theta=\theta_1$, where $\theta_1 > \theta_0$.
\begin{solution}
The PDF is $f(x;\theta) = \theta e^{-\theta x}$ so the likelihood function is
\[
L(\theta;\mathbf{x}) = \prod_{i=1}^n \theta e^{-\theta x_i} = \theta^n \exp\left(-\theta\sum_{i=1}^n x_i\right).
\]
Hence the likelihood ratio is
\[
\lambda(\mathbf{x})
	= \frac{L(\theta_0;\mathbf{x})}{L(\theta_1;\mathbf{x})}
	= \frac{\theta_0^n \exp\left(-\theta_0\sum_{i=1}^n x_i\right)}{\theta_1^n \exp\left(-\theta_1\sum_{i=1}^n x_i\right)}
	= \left(\frac{\theta_0}{\theta_1}\right)^n \exp\left(-(\theta_0-\theta_1)\sum_{i=1}^n x_i\right).
\]
The critical region for the SLRT is therefore
\begin{align*}
C = \{\mathbf{x}:\lambda(\mathbf{x}) \leq k\}
	& = \left\{\mathbf{x}:\left(\frac{\theta_0}{\theta_1}\right)^n\exp\left(-(\theta_0-\theta_1)\sum_{i=1}^n x_i\right) \leq k\right\} \\
	& = \left\{\mathbf{x}: n\log\left(\frac{\theta_0}{\theta_1}\right) - (\theta_0-\theta_1)\sum_{i=1}^n x_i \leq \log k \right\} \\
%	& = \left\{\mathbf{x}: \sum_{i=1}^n x_i \leq \frac{\log k - n\log(\theta_0/\theta_1)}{\theta_0-\theta_1} \right\}\\
	& = \left\{\mathbf{x}: \sum_{i=1}^n x_i \leq \frac{\log k + n\log(\theta_1/\theta_0)}{\theta_1-\theta_0} \right\}\\
	& = \left\{\mathbf{x}: \sum_{i=1}^n x_i \leq k' \right\} \quad\text{or alternatively}\quad \left\{\mathbf{x}: \frac{1}{n}\sum_{i=1}^n x_i \leq k'' \right\}
\end{align*}
where we have used the fact that $\theta_1 > \theta_0$. The critical value $k'$ (or $k''$) is then chosen according to the required size of the test.
\end{solution}
\end{example}

% example
\begin{example}
Let $X_1,\ldots,X_n$ be a random sample from the distribution of $X$, whose PDF is given by
\[
f(x;\theta) = \begin{cases}
	\theta x^{\theta-1}	& 0\leq x\leq 1 \\
	0					& \text{otherwise,}
\end{cases}
\]
where $\theta\geq 1$ is an unknown scalar parameter. Construct a likelihood ratio test of the null hypothesis $H_0:\theta=1$ against the alternative $H_1:\theta>1$.
\begin{solution}
The PDF is $f(x;\theta) = \theta e^{-\theta x}$ so the likelihood function is
\[
L(\theta;\mathbf{x})
	= \textstyle\prod_{i=1}^n \theta x_i^{\theta-1}
	= \theta^n\left(\prod_{i=1}^n x_i\right)^{\theta-1}.
\]
The likelihood ratio is
\[
\lambda(\mathbf{x})
	= \frac{L(1;\mathbf{x})}{L(\theta;\mathbf{x})}
	= \frac{1}{\theta^n\big(\prod_{i=1}^n x_i\big)^{\theta-1}}
	= \theta^{-n}\left(\prod_{i=1}^n x_i\right)^{1-\theta}.
\]
The critical region is
\begin{align*}
C = \{\mathbf{x}:\lambda(\mathbf{x}) \leq k\}
	= \left\{\mathbf{x}:\theta^{-n}\left(\prod_{i=1}^n x_i\right)^{1-\theta} \leq k\right\}
	& = \left\{\mathbf{x}: -n\log\theta + (1-\theta)\sum_{i=1}^n \log x_i \leq \log k \right\} \\
	& = \left\{\mathbf{x}: (1-\theta)\sum_{i=1}^n\log x_i \leq \log k + n\log\theta\right\}\\
	& = \left\{\mathbf{x}: -\sum_{i=1}^n \log x_i \leq -\frac{\log k + n\log\theta}{1-\theta}\right\} \\
\end{align*}
Thus a likelihood ratio test of $H_0:\theta=1$ against $H_1:\theta>1$ is given by the critical region
\[
C = \{\mathbf{x}: T(\mathbf{x}) \leq k'\} \qquad\text{where}\quad T(\mathbf{x}) = -\sum_{i=1}^n \log x_i,
\]
and $k'>0$ is chosen according to the required size of the test.
%\bigskip
%\textbf{Remarks}
%\bit
%\it The null hypothesis $H_0:\theta=1$ corresponds to $X\sim\text{Uniform}[0,1]$.
%\it As $\theta$ increases away from $1$, the probability mass moves away from $x=0$ and towards $x=1$.
%\it Observations that are close to $1$ yield small positive values of $-\log x_i$.
%\it Observations that are close to $0$ yield large positive values of $-\log x_i$.
%\eit
%Small values of $T(\mathbf{x}) = -\sum_{i=1}^n \log x_i$ therefore indicate that many observations are close to $1$, leading us to reject the claim that $X\sim\text{Uniform}[0,1]$. Compare this test to a test based on the sample mean of the $X_i$:
%\[
%C' = \left\{\mathbf{x}: \frac{1}{n}\sum_{i=1}^n x_i \geq k''\right\}
%\]
%A preponderance of observations close to $1$ will lead us to reject $H_0$. However the LRT is a more sensitive test: $-\log(x)$ \emph{amplifies} the contribution of observations that are close to zero, and \emph{diminishes} the contribution of observations that are close to $1$.
\end{solution}
\end{example}


%!TEX root = ../Tibt.tex

\exercise{2.7}

\subsubsection*{Point (a)}

One has:
\begin{eqnarray*}
    \textrm{linear regression:} \quad && \hat{f}(x_0) = x_0^T \hat{\beta} = x_0^T (\mathbf{X} ^{T} \mathbf{X}) ^{-1} \mathbf{X} ^{T} Y \\
    && l_i(x_0, \mathcal{X}) = \left( \mathbf{X} (\mathbf{X} ^{T}\mathbf{X}) ^{-1} x_0\right)_i \\
    \textrm{k-nn:} \quad && l_i(x_0, \mathcal{X}) = \frac{1}{k}\, \mathbf{I} \left( x_i \textrm{ is among the } k \textrm{ closest neighbors of } x_0 \right)
\end{eqnarray*}

\subsubsection*{Point (b)}

Notice that:
\begin{eqnarray*}
    \mathbb{E}_{\mathcal{Y}|\mathcal{X}} \left[ \hat{f}(x_0) \right] & = & \bm{l}^{T}(x_0; \mathcal{X})\,\bm{f}(X),\\
    \textrm{Var}_{\mathcal{Y}|\mathcal{X}}\left( \hat{f}(x_0) \right) & = & \bm{l}^{T}(x_0; \mathcal{X})\, \textrm{Cov}(\bm{\epsilon}) \, \bm{l}(x_0; \mathcal{X}) = \sigma^2 \, ||\bm{l}(x_0; \mathcal{X})||^2
\end{eqnarray*}
where we denoted $(\bm{l}(x_0; \mathcal{X}))_i \equiv l_i(x_0; \mathcal{X})$ and $(\bm{f}(X))_i \equiv f(x_i)$. Hence:
\begin{eqnarray*}
    \mathbb{E}_{\mathcal{Y}| \mathcal{X}} \left[ \left( f(x_0) - \hat{f}(x_0) \right)^2 \right]  & = & \textrm{Bias}^2_{\mathcal{Y}| \mathcal{X}}(y_0) + \textrm{Var}_{\mathcal{Y}| \mathcal{X}}(y_0)\\
    \textrm{Bias}^2_{\mathcal{Y}| \mathcal{X}}(y_0) & \equiv & \left( f(x_0) - \bm{l}^{T}(x_0; \mathcal{X})\,\bm{f}(X) \right)^2\\
    \textrm{Var}_{\mathcal{Y}| \mathcal{X}}(y_0) & \equiv & \sigma^2 ||\bm{l}(x_0; \mathcal{X})||^2
\end{eqnarray*}
The first term represents the bias, while the second represents the variance of the estimator as the training responses vary for fixed $\mathcal{X}$.

\subsubsection*{Point (c)}

For this part can go down a similar road, using now the independence of $X$ and $\epsilon$:
\begin{eqnarray*}
    \mathbb{E}_{\mathcal{Y},\mathcal{X}} \left[ \hat{f}(x_0) \right] & = & \mathbb{E}_{\mathcal{X}} \left[ \bm{l}^T(x_0; \mathcal{X}) \, \bm{f}(X) \right],\\
    \textrm{Var}_{\mathcal{Y},\mathcal{X}}\left( \hat{f}(x_0) \right) & = & \textrm{Var}_{\mathcal{X}} \left( \bm{l}^T(x_0; \mathcal{X}) \, \bm{f}(X) \right) +
    \sigma ^2 \mathbb{E}_{\mathcal{X}} \left[ ||\bm{l}(x_0; \mathcal{X})||^2  \right],
\end{eqnarray*}
Hence:
\begin{eqnarray*}
    \mathbb{E}_{\mathcal{Y}, \mathcal{X}} \left[ \left( f(x_0) - \hat{f}(x_0) \right)^2 \right] & = & \textrm{Bias}^2_{\mathcal{Y}, \mathcal{X}}(y_0) + \textrm{Var}_{\mathcal{Y}, \mathcal{X}}(y_0)\\
    \textrm{Bias}^2_{\mathcal{Y}, \mathcal{X}}(y_0) & \equiv &\left( f(x_0) - \mathbb{E}_{\mathcal{X}} \left[ \bm{l}^T(x_0; \mathcal{X}) \, \bm{f}(X) \right]  \right)^2\\
    \textrm{Var}_{\mathcal{Y}, \mathcal{X}}(y_0) & \equiv & \textrm{Var}_{\mathcal{X}} \left( \bm{l}^T(x_0; \mathcal{X}) \, \bm{f}(X) \right) +
    \sigma ^2 \mathbb{E}_{\mathcal{X}} \left[ ||\bm{l}(x_0; \mathcal{X})||^2  \right]
\end{eqnarray*}

\subsubsection*{Point (d)}

Combining the equations above one can see that:
\begin{eqnarray*}
    \textrm{Bias}^2_{\mathcal{Y}, \mathcal{X}}(y_0) + \textrm{Var}_{\mathcal{Y}, \mathcal{X}}(y_0) = \mathbb{E}_{\mathcal{X}} \left( \textrm{Bias}^2_{\mathcal{Y}| \mathcal{X}}(y_0) + \textrm{Var}_{\mathcal{Y}| \mathcal{X}}(y_0) \right)
\end{eqnarray*}
which is a simple consequence of the conditional expectations identity:
\begin{eqnarray*}
    \mathbb{E}_{\mathcal{Y}, \mathcal{X}} \left[ \left( f(x_0) - \hat{f}(x_0) \right)^2 \right] =  \mathbb{E}_{\mathcal{X}} \left[ \mathbb{E}_{\mathcal{Y}| \mathcal{X}} \left[ \left( f(x_0) - \hat{f}(x_0) \right)^2 \right]  \right]
\end{eqnarray*}
{\color{red} This might be the relationship the authors wanted us to find.}


\paragraph{Statistical test}
for testing hypothesis $H$ against $K$, defined on sample $\sample$, is a function:
\begin{gather*}
\psi: \mathcal{H} \rightarrow \{0, 1\}
\end{gather*}
where $\psi = 1$ means that we reject $H$ (i.e. we accept $K$) and $\psi = 0$ means that we do not reject $H$.

Usually, $\psi$ has the following form:
\begin{gather*}
\psi(\sample) =
\begin{cases}
1 \mbox{ if } T(\sample) \in \mathcal{K}_\alpha \\
0 \mbox{ if } T(\sample) \not\in \mathcal{K}_\alpha
\end{cases}
\end{gather*}
where $T$ is a test statistic, and $\mathcal{K}_\alpha$ is a critical region.

\paragraph{Null hypothesis}
is usually denoted by $H$, and it is a hypothesis that we test. In most of the cases it is some
phenomenon about which we are not sure whether it really happens, and we perform a test in order to
determine the reality.

\paragraph{Alternative hypothesis}
is usually denoted by $K$, and it is a hypothesis opposite to the null hypothesis.

\paragraph{Critical region}
is a set of all outcomes of a test, for which we reject the null hypothesis. Denoted by $\mathcal{K}_\alpha$.

\paragraph{Type I error}
is made when $H$ is rejected when in fact it is true. Probability of this error is denoted by $\alpha_\psi$.

\paragraph{Type II error}
is made when $H$ is not rejected when in fact it is false. Probability of this error is denoted by $\beta_\psi$.

\paragraph{Significance level}
denoted with $\alpha$ is the upper bound for probability of making type I error. It is fixed in
advance.
\begin{gather*}
\begin{cases}
\alpha_\psi \leq \alpha \\
\beta_\psi \rightarrow \min
\end{cases} \equiv
\begin{cases}
P(\psi = 1 | H) \leq \alpha \\
P(\psi = 0 | K) \rightarrow \min
\end{cases}
\end{gather*}

``Significance'' in statistics does not have the usual everyday meaning.

\paragraph{Power of a test}
is a probability of rejecting false hypothesis.
\begin{gather*}
\Pi = P(\psi = 1 | K) = 1 - P(\psi = 0 | K) = 1 - \beta_\psi
\end{gather*}

\paragraph{Uniformly most powerful test}
(or U.M.P.T.) is such test $\psi^*$, defined for hypothesis $H: \theta \in \Theta_H$ against $K: \theta \in
\Theta_K$, on the significance level $\alpha$, that satisfies the following:
\begin{gather*}
\forall \theta \in \Theta_K : \mathbb{E}_\theta \psi^*(X) \geq \mathbb{E}_\theta \psi(X)
\end{gather*}

\paragraph{Neyman-Pearson Lemma}
states, that if we consider a testing problem $H: f_0$ against $K: f_1$, defined as follows:

Let $\alpha \in (0,1)$ and let $\psi$ denote a test that:
\begin{gather*}
\psi(X) = \begin{cases}
1 \mbox{ if } f_1(X) > k f_0(X) \\
0 \mbox{ if } f_1(X) < k f_0(X)
\end{cases}
\label{eq:neymanpearson1} \tag{A}
\end{gather*}

where $k$ is a constant, $k > 0$, and it satisfies:
\begin{gather*}
\mathbb{E}_{f_0} \psi(X) = \alpha
\label{eq:neymanpearson2} \tag{B}
\end{gather*}

\noindent \ldots then we may conclude the following:
\begin{enumerate}

  \item Test $\psi$ satisfying \eqref{eq:neymanpearson1} and \eqref{eq:neymanpearson2} is the
  U.M.P.T. for $H$ against $K$ on the significance level $\alpha$.

  \item If $\alpha$ is the U.M.P.T. for $H$ against $K$ on the significance level $\alpha$, the test
  $\psi$ should satisfy \eqref{eq:neymanpearson1} and \eqref{eq:neymanpearson2}.

\end{enumerate}

\paragraph{Monotonic likelihood ratio}
for a family $P = \{ f_\theta : \theta \in \Theta \}$ exists if there is a statistic $T$
such that $\frac{ p_{\theta_1}(X) }{ p_{\theta_0}(X) }$ for $\theta_1 > \theta_0$ is a nondecreasing
function of $T(X)$.

\paragraph{Karlin-Rubin theorem}
let $P = \left\{ f_\theta : \theta \in \Theta \right\}$ denote a family with the monotonic
likelihood ratio with respect to statistic $T$. Consider the hypothesis $H: \theta \leq \theta_0$
against $K: \theta > \theta_0$. Then, for any $\alpha \in (0,1)$ there exists the U.M.P.T on the
significance level $\alpha$ of the form:
\begin{gather*}
\psi(\sample) = \begin{cases}
1 \mbox{ if } T(\sample) > C_\alpha \\
0 \mbox{ otherwise}
\end{cases}
\end{gather*}

\paragraph{Unbiasedness of a hypothesis test}
is determined by checking if for a test $\psi$ the following holds:
\begin{gather*}
\begin{cases}
\forall \theta \in \Theta_H : \mathbb{E}_\theta \psi(X) \leq \alpha \\
\forall \theta \in \Theta_K : \mathbb{E}_\theta \psi(X) \geq \alpha
\end{cases}
\end{gather*}

\paragraph{Consistent hypothesis test}
is such test for H against K, for which the following holds:
\begin{gather*}
\forall \theta \in \Theta_K \limit{n}{\infty} \mathbb{E}_\theta \psi(\sample) = 1
\end{gather*}

\paragraph{Two categories}
of hypothesis tests are: parametric and non-parametric.

\paragraph{Parametric test}
is a test, in which we have parameters that uniquely define the distribution to which the sample
under study belongs.

\paragraph{Non-parametric statistical test}
is a test that does not assume that sample belongs to any specific distribution. Such test is also
called a distribution-free test.

\paragraph{p-value}
is an indicator for rejection or acceptance of null hypothesis. When it is not smaller than
significance level of the test, we accept our hypothesis.


\label{ch:four}
\label{ch:four}

\section{Mutual information}\label{mutInfo}
	The mutual information measures the amount of information that $X$ and $Y$ \textit{share}.
    It can be used as a measure of correlation (or dependency) between random variables.
    Intuitively we can use the mutual information --- along with a check of privacy against Eve --- to measure how much shared key two random variables $X$ and $Y$ can hold.
    \begin{definition}
	Let $X$ and $Y$ be two jointly distributed random variables. Then the mutual information of the random variables is the relative entropy --- a measure of distance between probability distributions --- between the joint distribution $P_{XY}(x,y)$ and the product distribution $P_X(x)\cdot P_Y(y)$.
	\begin{equation}
		\I(X;Y) = \sum_{y\in\mathcal{Y}}\sum_{x\in\mathcal{X}} p(x,y) \log\left(\frac{p(x,y)}{p(x)p(y)}\right)
	\end{equation}
	or equivalently, showing its relation to the entropies of the random variables
	\begin{equation}
		\I(X;Y) = \Ent(X) - \Ent(X|Y) = \Ent(X,Y) - \Ent(X|Y) - \Ent(Y|X) = \Ent(Y) - \Ent(Y|X)
	\end{equation}
	This relation can be seen more directly in Fig. \ref{fig:mutual_info}.
    \end{definition}
	Mutual information is nonnegative and bounded by the entropy of random variable $X$
	\begin{equation}
		0 \leq \I(X;Y) \leq \min \left(\Ent(X), \Ent(Y)\right)
	\end{equation}
	In this sense the mutual information can also be interpreted as how much information $X$ gives about $Y$, thus being bounded by its own entropy.
	\begin{figure}[ht]
		\centering
		\input{images/mutual-info}
		\caption{Representation of mutual information $\I(X;Y)$ in relation with entropies $\Ent(X)$ and $\Ent(Y)$ and joint entropy $\Ent(X,Y)$ of the random variables .
		\label{fig:mutual_info}}
	\end{figure}

\section{An eavesdropper that can choose the best channel to listen to}
Additional information on a third random variable $Z$ can increase or decrease the mutual information \cite{CT12}.
    The \emph{conditional mutual information} $\I(X;Y|Z)$ is the expected value of the mutual information of $X$ and $Y$ given a realization of a third variable $Z$.
    In a context of key exchange, we can interpret this as the remaining correlation between honest parties after the observations of an attacker Eve.
    What if Eve tried to minimize this, i.e. tried to find the best viewpoint possible over the communication between Alice and Bob?
    \begin{definition}\cite{MW99, RW03}
    	Let $P_{XYZ}$ be a discrete probability distribution. Then the intrinsic information between $X$ and $Y$ given $Z$ is
    \begin{equation} \label{intrininfo}
    	\intrinfo{X}{Y}{Z}:= \inf_{Z\rightarrow \bar{Z}} \I(X;Y | \bar{Z})
    \end{equation}
    \end{definition}
    The infimum is taken over all possible channels applied to $Z$ (the choice of a channel can be seen as the choice of a point of view for Eve).

    The intrinsic information is an upper bound to the secret-key rate, although not tight \cite{RW03}.
    \begin{equation} \label{eq:bkeyinfo}
    	\keyrate{X}{Y}{Y} \leq \intrinfo{X}{Y}{Y}
    \end{equation}
    Refer to the next chapter to see an analysis of the gap between the two measures.
    The amount of secret bits Alice and Bob can extract from the distribution is then bounded by how much the attacker Eve can disrupts their conditional mutual correlation.
    Intrinsic information is also a lower bound to another measure, \emph{information of formation}, which is the amount of initial secret bits between Alice and Bob required to create the distribution $P_{XYZ}$ with LOPC.

\section{When correlation is unusable}
Setting the bound in Eq. \ref{eq:bkeyinfo} we can see that not always factoring out the adversary can be enough to be able to produce a key.
For example we could have
\begin{align*}
	\keyrate{X}{Y}{Z} & = 0 \\
	\intrinfo{X}{Y}{Z} & > 0
\end{align*}
meaning that there exists some sort of mutual correlation between Alice and Bob, but they share no key.
Whether this case is possible is the question of bound information expressed at the beginning of this work.
\begin{definition}\cite{GisWolf00, RW03}
Let $P_{XYZ}$ be a joint probability distribution for parties Alice, Bob and Eve.
For such distribution let
\begin{equation}
	\intrinfo{X}{Y}{Z} > 0
\end{equation}
and
\begin{equation}
\keyrate{X}{Y}{Z} = 0
\end{equation}
hold.
Then $P_{XYZ}$ is said to have \emph{bound information}.
\end{definition}

Recalling the intuition from quantum mechanics, we now pose the case of the existence of bound entanglement.
As stated before (section \ref{distillation}), quantum distillation extract from an entangled mixed state a set of quasi-pure entangled states.
Pure entangled states can be used as a resource to produce a key for Alice and Bob \cite{Ekert91}.
There are, furthermore, entangled mixed states that are non-distillable, i.e. no pure entanglement can be extracted from them \cite{3H98}.
Bound entanglement is a kind of correlation between Alice and Bob --- that can become inaccessible to Eve --- but nevertheless of no use for generating a secret (quantum) key.
So in the quantum regime this questions has already been answered.


\section{Hausdorff distance}
\label{hausdorff_distance_chapter}
\nblink{brats/15a\_hausdorff\_distance.ipynb}

In the previous section, we showed differences between the reference neural network output and the output of images where a part of them has been masked by a circle.
Showing this difference for all the applied masks is impractical. A single visualization which shows all the output segment changes would be helpful.
A first step is getting a single number for the difference between two segments: The unchanged reference segment vs. output segment from masked image or any generated segment vs. the ground truth.

A way to calculate the similarity or difference between two matrices, e.g. the output segments of the neural network, is a distance function.

We choose the Hausdorff distance function because it has a specific property that is helpful for our requirements: A slightly moved object is still considered more similar to
an object with a completely different shape, even when the changed pixel count is exactly the same.

% The formula for the Hausdorff distance is:
% $ _{\mathrm {H} }(X,Y)=\max\{\,\sup _{x\in X}\inf _{y\in Y}d(x,y),\,\sup _{y\in Y}\inf _{x\in X}d(x,y)\} $
% $X$ and $Y$ are the two sets/matrices which are compared. $d(x,y)$ is the distance between two points. $inf$ and $sup$ are the Infimum and supremum: When comparing two
% ordered sets A and B, the infinum of A in comparison to B is the biggest item in A that is still smaller than all items of B. The inverse is the supremum,
% the smallest item in A that is still bigger
% \begin{figure}[H]
% \centering
% \includegraphics[width=8cm]{chapters/06_hdm/images/inf_sup.png}
% \caption{Visualization of the Infimum and supremum \cite{hausdorffdistanceimage2}}
% \label{inf_sup}
% \end{figure}

Intuitively, the Hausdorff distance searches the two maximal distances between sets (A to B and B to A) and returns the higher one as the distance.
The maximal distance from set A to set B is the biggest distance between a pixel from set A to a pixel in set B, but the distance from the pixel from set A to the pixel B has to be the smallest distance from pixel A to any pixel on the set B.

A visual explanation of the Hausdorff distance is given in Figure \ref{hausdorff_distance}.

\begin{figure}[H]
\centering
\includegraphics[width=6cm]{chapters/06_hdm/images/hausdorff_distance.png}
\caption{Visualization of the Hausdorff distance. The Hausdorff distance from set Y to set X is the maximal distance of a point in set Y to a point in set X, but this distance still has to be the smallest distance from this specific point on set Y to any point on set X \cite{hausdorffdistanceimage}.}
\label{hausdorff_distance}
\end{figure}

A naive implementation of the algorithm in Python is given in Listing \ref{hausdorff_distance_python}.

\begin{listing}[H]
\begin{minted}{python}
def inner_hausdorff(X, Y):
    minimal_distances = []
    for x in X:
        distances = []
        for y in Y:
            distances.append(distance(x,y))
        min_distance = min(distances)  # infinum
        minimal_distances.append(min_distance)
    return max(minimal_distances) # supremum

def hausdorff_distance(X, Y):
    return max(inner_hausdorff(X, Y), inner_hausdorff(Y, X))
\end{minted}
\caption{Naive implementation of the Hausdorff distance in Python}
\label{hausdorff_distance_python}
\end{listing}

\clearpage

\subsection{Examples}
The following visualizations show samples of shapes compared with the Hausdorff distance function.

\begin{figure}[H]
    \centering
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_original.png}
        \caption{Original shape}
    \end{subfigure}%
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_moved1.png}
        \caption{Shape moved slightly to the right}
    \end{subfigure}
    \caption{Hausdorff distance between the left and the right figure: 476.}
    \label{hdm_moved1}
\end{figure}

Figure \ref{hdm_moved1} shows a slightly moved circle, the Hausdorff distance between the images is quite low with 476.

\begin{figure}[H]
    \centering
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_original.png}
        \caption{Original shape}
    \end{subfigure}%
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_moved2.png}
        \caption{Shape moved to the right}
    \end{subfigure}
    \caption{Hausdorff distance between the left and the right figure: 1656. }
    \label{hdm_moved2}
\end{figure}

Figure \ref{hdm_moved2} shows a shape that is moved to the right, showing a bigger Hausdorff distance than in Figure \ref{hdm_moved1} with 1565.

\begin{figure}[H]
    \centering
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_original.png}
        \caption{Original shape}
    \end{subfigure}%
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_hole.png}
        \caption{Shape moved to the right}
    \end{subfigure}
    \caption{Hausdorff distance between the left and the right figure: 840. }
    \label{hdm_hole}
\end{figure}

Figure \ref{hdm_hole} shows the shape at the same position but with a hole in the middle.

\begin{figure}[H]
    \centering
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_original.png}
        \caption{Original shape}
    \end{subfigure}%
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_square.png}
        \caption{Shape moved to the right}
    \end{subfigure}
    \caption{Hausdorff distance between the left and the right figure: 1197. }
    \label{hdm_square}
\end{figure}

Figure \ref{hdm_square} shows the shape transformed into a square. The distance between the shapes is much bigger compared the the shape with a hole in it in Figure \ref{hdm_hole}, even when the
changed count of pixels is similar.

\begin{figure}[H]
    \centering
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_original.png}
        \caption{Original shape}
    \end{subfigure}%
    \begin{subfigure}{.35\textwidth}
        \centering
        \includegraphics[width=\linewidth]{chapters/06_hdm/images/hdm_smaller_circles.png}
        \caption{Shape moved to the right}
    \end{subfigure}
    \caption{Hausdorff distance between the left and the right figure: 1353.}
    \label{hdm_smaller_circles}
\end{figure}

Figure \ref{hdm_smaller_circles} shows the shape completely replaced by three smaller circles. The Hausdorff distance is the highest of all the sample images with 1353.


% !TEX encoding = UTF-8 Unicode
\section{BLAS - Basic Linear Algebra Subprograms}
%%%%%%%%%%%%%%%%
	\begin{frame}{BLAS - Basic Linear Algebra Subprograms}{Levels}
		\begin{itemize}
			\item copy
			\item dot
			\item saxpy \\ \small $ z = \alpha x + y $ \normalsize
			\item gaxpy \\ \small $ z = Ax + y $ \normalsize
			\item matmul \\ \small  $ c = AB $ \normalsize
		\end{itemize}
		Level-1, Level-2, Level-3 $ \Rightarrow O(n), O(n^2), O(n^3)$
	\end{frame}
	\begin{frame}{Levels}
		\begin{enumerate}[a)]
			\item Level 1 BLAS vector-vector operations
			$y \leftarrow y + \alpha x$; dot products
			\begin{figure}
				\includegraphics[height=1cm]{img/13/level1}
			\end{figure}
			\item Level 2 BLAS matrix-vector operations
			$y \leftarrow y + Ax$
			\begin{figure}
				\includegraphics[height=1cm]{img/13/level2}
			\end{figure}
			\item Level 3 BLAS matrix-matrix operations
			$A \leftarrow A + B \cdot C$
			\begin{figure}
				\includegraphics[height=1cm]{img/13/level3}
			\end{figure}
		\end{enumerate}
	\end{frame}
	\begin{frame}{Arithmetic/Memory references}
		Arithmetic is performed at the top of the memory hierarchy!!!
		\begin{figure}
			\includegraphics[height=5cm]{img/13/arithmeticmemref}
		\end{figure}
	\end{frame}
	\begin{frame}{Arithmetic/Memory references}
		\begin{tabular}{ | l | l | l | l | l |}
		\hline
		BLAS Level & 		& flops 	&memref 	& $\frac{flops}{mem ref}$ \\ \hline
		1 & $y \leftarrow y + \alpha x$ 	& $2n$ 	& $3n$ 	& $\frac{2}{3} $ \\ \hline
		2 & $y \leftarrow y + Ax$ 		& $2n^2$ 	& $n^2$ 	& 2 			\\ \hline
		3 & $C \leftarrow C + AB$ 	& $2n^3$ 	& $4n^2$ 	& $\frac{n}{2}$ 	\\ \hline
		\end{tabular} \\

		For level 1: 2 vector loads, 2 vector operations, 1 vector store\dots

		Higher level: Increase of granularity $\Rightarrow$ lower synchronization cost
	\end{frame}
	\begin{frame}{BLAS character}
		\begin{itemize}
			\item \textit{clarity} - code $\rightarrow$ shorter, easier to read
			\item \textit{modularity} - programmers have larger building blocks
			\item \textit{performace} - manufacturers tuned machine-spec. BLAS
			\item \textit{program portability} - machine dependencies confined to the BLAS
		\end{itemize}
	\end{frame}
	\begin{frame}{BLAS naming conventions}
	\[
		\underbrace{1}_{} \underbrace{2}_{} \underbrace{3}_{} \underbrace{4}_{} \underbrace{5}_{} \underbrace{6}_{}
	\]
	1 $\rightarrow$ fortran data type of the matrix \\
	2, 3 $\rightarrow$ kind of the matrix involved \\
	4, 5, 6 $\rightarrow$ type of the operation
	\end{frame}


\subsection{MLE of the Gaussian distribution}

The parameters are the population means and covariance matrix.

The MLE estimator for the mean is the sample mean.

The MLE estimator for the covariance matrix is the unadjusted sample covariance.