% !TEX root = main.tex

%-------------------------------------------------
\section{Bias and mean squared error}\label{sec:}

%We now consider various notions of what constitutes a good estimator.
%\bigskip

%Let $X$ be a random variable and let $F_X(x;\theta)$ denote its CDF, where $\theta\in\Theta$ is an unknown parameter. Let $X_1,X_2,\ldots,X_n$ be a random sample from the distribution of $X$, and let $T(X_1,X_2,\ldots,X_n)$ be an estimator of the unknown parameter $\theta$. Intuitively, we would like the value $T(x_1,x_2,\ldots,x_n)$ to be `close' to $\theta$ for `most' sample realisations $(x_1,x_2,\ldots,x_n)$.

Let $X$ be a random variable and let $F_X(x;\theta)$ denote its CDF where $\theta\in\Theta$ is an unknown parameter. Let $\mathbf{X}=(X_1,X_2,\ldots,X_n)$ be a random sample from the distribution of $X$ and let $T(\mathbf{X})$ be an estimator of $\theta$. Intuitively, we would like the estimate $T(\mathbf{x})$ to be ``close'' to the true value $\theta$ for ``most'' sample realisations $\mathbf{x}\in\R^n$.

%-----------------------------
\subsection{Bias}

\begin{definition}
The \emph{bias} of an estimator $T$ is the expected difference between $T(\mathbf{X})$ and the true parameter value $\theta$,
\[
\bias(T;\theta) = \expe\big[T(\mathbf{X}) - \theta\big].
\]
If $\bias(T;\theta)=0$ for all $\theta\in\Theta$ then $T$ is said to be an \emph{unbiased} estimator of $\theta$.
\end{definition}

% example: sample variance
\begin{example}
Let $\mathbf{X}=(X_1,X_2,\ldots,X_n)$ be a random sample from the $N(\mu,\sigma^2)$ distribution and consider the sample mean estimator of $\sigma^2$ defined by
\[
T(\mathbf{X}) = \frac{1}{n}\sum_{i=1}^n (X_i-\bar{X})^2 \quad\text{where}\quad \bar{X} = \frac{1}{n}\sum_{i=1}^n X_i.
\]
\ben
\it Show that $T$ is a biased estimator of $\sigma^2$ and find its bias.
\it Show how can $T$ be modified to produce an unbiased estimator of $\sigma^2$.
\een
\begin{solution}
\ben
\it % <<<
Recall that $\expe(\Xbar) = \mu$ and $\var(\Xbar) = \sigma^2/n$. First,
\[
\expe(T)	= \expe\left(\frac{1}{n}\sum_{i=1}^n (X_i-\Xbar)^2\right)
			= \frac{1}{n}\sum_{i=1}^n \expe(X_i^2) - \expe(\Xbar^2).
\]
Since $\expe(X_i^2)	= \var(X_i)+\expe(X_i)^2$ and $\expe(\Xbar^2) = \var(\Xbar)+\expe(\Xbar)^2$, we have
\[
\expe(X_i^2) = \sigma^2 + \mu^2 \quad\text{and}\quad \expe(\Xbar^2) = \frac{\sigma^2}{n} + \mu^2.
\]
so
\[
\expe(T)
	= \frac{1}{n}\left[ n(\sigma^2+\mu^2)-n\left(\frac{\sigma^2}{n} + \mu^2\right)\right]
	= \left(\frac{n-1}{n}\right)\sigma^2.
\]
Thus $T$ is a biased estimator of $\sigma^2$, and its bias is
\[
\bias(T;\sigma^2) = \expe(T - \sigma^2) = \expe(T) - \sigma^2 = -\frac{\sigma^2}{n}.
\]
We say that $\hat{\sigma}^2$ is a \emph{negatively biased} estimator of $\sigma^2$ (it underestimates the true value).
\it % <<<
Let
\[
S^2(\mathbf{X}) = \left(\frac{n}{n-1}\right)T(\mathbf{X}) = \frac{1}{n-1}\sum_{i=1}^n (X_i-\bar{X})^2
\]
It is easy to see that $S^2$ is an unbiased estimator of $\sigma^2$. This is called the \emph{sample variance}.
\een
\end{solution}
\end{example}

% exercise: uniform
\begin{exercise}
Let $\mathbf{X}=(X_1,X_2,\ldots,X_n)$ be a random sample from the $\text{Uniform}(0,\theta)$ distribution, where $\theta>0$ is unknown.
\ben
\it Find the bias of $T_1(\mathbf{X}) = 2\Xbar$ as an estimator of $\theta$.
\it Find the bias of $T_2(\mathbf{X}) = \max\{X_1,X_2,\ldots,X_n\}$ as an estimator of $\theta$.
\it How can $T_2$ be modified to produce an unbiased estimator of $\theta$?
\een
\begin{answer}
\ben
\it % <<<
$T_1$ is an \emph{unbiased} estimator of $\theta$ because
\[
\bias(T_1)
	= \expe(T_1 - \theta)
	= \expe(T_1) - \theta
	= 2\expe(\Xbar) -\theta
	= 2(\theta/2) - \theta
	= 0.
\]
\it % <<<
Because the $X_i$ are independent, the CDF of $T_2$ is
\[
\prob(T_2 \leq t) = \left\{\begin{array}{ll}
	0				& t \leq 0, \\
	(t/\theta)^n 	& 0 < t < \theta, \\
	1				& t \geq \theta.
\end{array}\right.
\]
A simple calculation shows that
\[
\displaystyle\expe(T_2) = \left(\frac{n}{n+1}\right)\theta.
\]
Hence $T_2$ is a \emph{negatively biased} estimator of $\theta$:
\[
\bias(T_2) = \expe(T_2 - \theta) = \left(-\frac{1}{n+1}\right)\theta.
\]
\it % <<<
A \emph{bias correction} can be applied to $T_2$ to give an unbiased estimator $T_3 = \displaystyle\left(\frac{n+1}{n}\right)T_2$.
\een
\end{answer}
\end{exercise}
%-----------------------------
\subsection{Mean squared error}
\bit
\it The bias of $T$ as an estimator of $\theta$ is the \emph{mean estimation error} $\expe\big[T-\theta\big].$
\it The magnitude of the error can be quantified by the \emph{mean squared estimation error} $\expe\big[(T-\theta)^2\big]$.
\eit

% definition
\begin{definition}
The \emph{mean squared error} of $T$ as an estimator of $\theta$ is
\[
\mse(T;\theta) = \expe\big[\big(T(\mathbf{X})-\theta\big)^2\big].
\]
\end{definition}

The accuracy of an unbiased estimator can be quantified by its variance. For biased estimators, the MSE also takes the bias into account. The following result is easily proved by writing $T-\theta$ as $[T-\expe(T)]+[\expe(T)-\theta]$ then expanding the square and applying the linearity of expectation.
% theorem
\begin{theorem}
$\mse(T;\theta) = \var(T) + \bias(T;\theta)^2$.
\end{theorem}
%\begin{proof}
%\begin{align*}
%\mse(T,\theta) = \expe\big[(T-\theta)^2\big]
%	& = \expe\big[(T-\expe(T) + \expe(T)-\theta)^2\big] \\
%	& = \expe\big[(T-\expe(T))^2 + 2(T-\expe(T))(\expe(T)-\theta)+ (\expe(T)-\theta)^2 \big] \\
%	& = \expe\big[(T-\expe(T))^2\big] + 2\expe(T-\expe(T))(\expe(T)-\theta)+ (\expe(T) - \theta)^2  \\
%	& = \expe\big[(T-\expe(T))^2\big] + \big[\expe(T) - \theta\big]^2 \\
%	& = \var(T) + \bias(T,\theta)^2.
%\end{align*}
%\end{proof}
As the following example shows, a biased estimator with small variance is often ``better'' than an unbiased estimator with large variance.

% example
\begin{example}
Let $X_1,X_2,\ldots,X_n$ be a random sample from a distribution with mean $\mu$ and variance $\sigma^2$. The sample mean $\bar{X}$ is an unbiased estimator of $\mu$, while the following statistic is a biased estimator of $\mu$:
\[
T(X_1,X_2,\ldots,X_n) = \frac{1}{n+1}\sum_{i=1}^n X_i.
\]
\ben
\it Find $\mse(\bar{X};\mu)$ and $\mse(T;\mu)$.
\it Find a condition involving $\mu$ and $\sigma$ under which $\mse(T;\mu) < \mse(\bar{X};\mu)$.
\een

\begin{solution}
\begin{enumerate}
\item
\begin{align*}
\mse(\bar{X};\mu)
	= \expe\big[(\bar{X}-\mu)^2\big]
	& = \var(\bar{X}) + \expe(\bar{X}-\mu)^2
	= \var(\bar{X})
	= \frac{\sigma^2}{n}. \\
\mse(T;\mu)
	= \expe\big[(T-\mu)^2\big]
	& = \var(T) + \expe(T-\mu)^2 \\
	& = \var\left(\frac{X_1+X_2+\ldots+X_n}{n+1}\right) + \left(\frac{n}{n+1}\mu - \mu\right)^2 \\
	& = \frac{n\sigma^2}{(n+1)^2} + \frac{\mu^2}{(n+1)^2}
\end{align*}
\item
Comparing $\mse(T;\mu)$ with $\mse(\bar{X};\mu)$:
\begin{align*}
\mse(T;\mu) - \mse(\bar{X};\mu)
	& = \left(\frac{n\sigma^2}{(n+1)^2} + \frac{\mu^2}{(n+1)^2}\right) - \frac{\sigma^2}{n} \\
	& = \frac{\mu^2}{(n+1)^2} - \left(\frac{1}{n} - \frac{n}{(n+1)^2}\right)\sigma^2 \\
	& = \frac{1}{(n+1)^2}\left[\mu^2 - \left(\frac{2n+1}{n}\right)\sigma^2\right].
\end{align*}

This shows that
\[
\mse(T;\mu) < \mse(\bar{X};\mu) \text{\quad whenever\quad} \mu^2 < \left(\frac{2n+1}{n}\right)\sigma^2.
\]
\end{enumerate}
\end{solution}
\end{example}

%----------------------------------------
\begin{exercise}
\begin{questions}

\question
Let $X_1, X_2, ...,X_n$ be a random sample of observations from the $\text{Uniform}[\theta,\theta+1]$ distribution, where $\theta$ is an unknown parameter.
\ben
\it Show that the sample mean $\bar{X}$ is a biased estimator of $\theta$, and find its bias.
\it Find the variance and mean squared error of $\bar{X}$.
\it How can $\bar{X}$ be modified to produce an unbiased estimator of $\theta$?
\een
\begin{answer}
\ben
\it % <<<
The sample mean is biased because $\expe(\bar{X}) = \theta +\frac{1}{2} \neq \theta$. The bias is given by
\[
\bias(\bar{X}) = \expe(\bar{X}-\theta) = \expe(\bar{X})-\theta = \frac{1}{2}.
\]
\it % <<<
The variance and mean squared error of $\bar{X}$ as an estimator of $\theta$ are
\begin{align*}
\var(\bar{X})
	& = \frac{\var(X)}{n} = \frac{1}{12n}. \\
\mse(\bar{X})
	&  = \var(\bar{X}) + \bias(\bar{X})^{2} = \frac{1}{12n} + \frac{1}{4} = \frac{(1+3n)}{12n}.
\end{align*}
\it % <<<
$\bar{X} -\frac{1}{2}$ is an unbiased estimator of ${\theta}$, because $\expe(\bar{X} -\frac{1}{2}) =\expe(\bar{X}) - 1/2 = \theta$.
\een
\end{answer}

\question
Let $X$ be a continuous random variable with the following PDF, where $\alpha>0$ is known but $\theta>0$ is unknown.
\[
f(x;\theta) = \begin{cases}
	\displaystyle \frac{\alpha x^{\alpha-1}}{\theta^{\alpha}}	& 0\leq x\leq \theta, \\[2ex]
	0															& \text{otherwise.}
\end{cases}
\]
A random sample $X_1,X_2,\ldots,X_n$ is taken from the distribution of $X$. An estimator of $\theta$ is provided by
\[
T = \max\{X_1,X_2,\ldots,X_n\}.
\]
\begin{parts}
\part % << (1)
Show that
\[
\expe(T) = \displaystyle\left(\frac{n\alpha}{n\alpha+1}\right)\theta
\quad\text{and}\quad
\var(T) = \displaystyle\left(\frac{n\alpha}{n\alpha+2}\right)\theta^2-\left(\frac{n\alpha}{n\alpha+1}\right)^2\theta^2.
\]
\begin{answer}
To find $\expe(T)$ and $\var(T)$ we first need to find the distribution of $T$. Let $F_X$ denote the CDF of $X$.
\[
F_X(x;\theta) = \begin{cases}
	0 & x < 0 \\
	\displaystyle\left( \frac{x}{\theta } \right) ^{\alpha }  & 0 \leq x \leq \theta \\
	1 & x > \theta.
\end{cases}
\]
It is easy to show that the CDF of $T=\max\{X_1,X_2,\ldots,X_n\}$ is
\[
F_{T}(v) = \prob(T\leq t) = \big[F_X(t)\big]^{n}.
\]
In this case,
\[
F_{T}(t;\theta) =
\begin{cases}
	0	& t < 0, \\[1ex]
	\displaystyle\left( \frac{t}{\theta } \right) ^{n\alpha }  & 0 \leq t \leq \theta, \\[1ex]
	1	& t > \theta,
\end{cases}
\]
Hence the PDF of $T$ is
\[
f_{T}(t;\theta) = \begin{cases}
	\displaystyle\frac{n\alpha }{\theta ^{n\alpha } } t^{(n\alpha -1)} & 0 \leq t \leq \theta \\[2ex]
	0	& \text{otherwise.}
\end{cases}
\]
The expected value and variance of $T=\max\{X_1,X_2,\ldots,X_n\}$ are computed as follows:
\begin{align*}
\expe(T)
	& = \frac{n\alpha}{\theta^{n\alpha}} \int_{0}^{\theta}  t^{n\alpha }\,dt
	  = \frac{n\alpha}{\theta^{n\alpha}} \left[\frac{t^{n\alpha +1} }{(n\alpha +1)} \right] _{0}^{\theta }
	= \frac{n\alpha }{(n\alpha +1)} \theta, \\
\expe(T^2)
	& = \int_{0}^{\theta }\frac{n\alpha }{\theta ^{n\alpha } }  t^{(n\alpha +1)}\,dt
	= \frac{n\alpha }{\theta ^{n\alpha } } \left[ \frac{t^{n\alpha +2} }{n\alpha +2} \right] _{0}^{\theta }
	= \frac{n\alpha }{(n\alpha +2)} \theta ^{2}, \\
\var(T)
	& = \frac{n\alpha }{(n\alpha +2)} \theta ^{2} -\frac{n^{2} \alpha ^{2} }{(n\alpha +1)^{2} } \theta ^{2}.
\end{align*}
\end{answer}

\part % << (2)
Show that $T$ is a biased estimator of $\theta$.
\begin{answer}
$T$ is a biased estimator of ${\theta}$ because
\[
\expe(T) = \left(\frac{n\alpha }{n\alpha +1}\right) \theta,
\]
so $\expe(T)\neq\theta$. The bias is
\[
\bias(T)
	= \expe(T-{\theta})
	= \expe(T)-{\theta}
	= \frac{n\alpha }{(n\alpha +1)} \theta -\theta =\frac{-\theta }{(n\alpha +1)}
\]
\end{answer}

\part % << (3)
Find a multiple of $T$ that yields an unbiased estimator of $\theta$.
\begin{answer}
The estimator $\displaystyle\left(\frac{n\alpha +1}{n\alpha}\right)T$ is an unbiased estimator of $\theta$ because
\[
\expe\left[\left(\frac{n\alpha +1}{n\alpha}\right)T\right]
	= \left(\frac{n\alpha +1}{n\alpha}\right)\expe(T)
	= \left(\frac{n\alpha +1}{n\alpha}\right)\frac{n\alpha }{(n\alpha +1)} \theta
	= \theta.
\]
\end{answer}

\part % << (4)
Find the mean squared error of $T$.
\begin{answer}
The mean squared error of $T$ as an estimator of $\theta$ is
\begin{align*}
\mse(\hat{\theta })
	& = \var(\hat{\theta }) + \bias(T)^{2} \\
	& = \frac{n\alpha }{(n\alpha +2)} \theta ^{2} -\frac{n^{2} \alpha ^{2} }{(n\alpha +1)^{2} } \theta ^{2} + \frac{\theta ^{2} }{(n\alpha +1)^{2} } \\
	& = \frac{2\theta ^{2} }{(n\alpha +2)(n\alpha +1)}
\end{align*}
\end{answer}

\end{parts}

\end{questions}
\end{exercise}


\section{Countably Infinite Types} \label{infinite}
We have now built up a substantial amount of theory relating to finite types.
In this section, we will look at the \emph{countable} types: we will see that
there is a parallel kind of classification of predicates to the finiteness
predicates, with some notable differences.
\subsection{Countability}
For our first countability predicate, we will mirror split enumerability:
\begin{definition}[Split Countability]
  A type is ``split countable'' if there is a \emph{stream} which contains all
  of its elements.
  \begin{agdalisting*}
    \ExecuteMetaDataInline[agda/Cardinality/Infinite/Split.tex]{split-count}
  \end{agdalisting*}
\end{definition}
The similarity to split enumerability should be clear: the only difference
between the two definitions, in fact, is the type of the container.

For countability we use \emph{streams}: these are basically infinite lists.
To a Haskeller, normal lists themselves often fulfil this purpose, but in a
total language like Agda, we need a totally different type.
Lists, as an inductive type, are not permitted to be infinitely large.
\begin{definition}[Streams]
  In Agda the type of streams can be given as a container:
  \begin{agdalisting*}
    \ExecuteMetaDataInline[agda/Container/Stream.tex]{stream-def}
  \end{agdalisting*}
  Although this definition is so simple it is more common to define it without
  reference to the usual container machinery:
  \begin{agdalisting*}
    \ExecuteMetaDataInline[agda/Codata/Stream.tex]{stream-def}
  \end{agdalisting*}
  By inlining the definition of the container primitives it's clear that the two
  types are isomorphic, but by defining streams in these terms we're able to
  use things like the membership function on containers.
\end{definition}

In the previous sections we saw different flavours of finiteness which were
really just different flavours of relations to \AgdaFunction{Fin}.
Unsurprisingly, given the definition of streams, we will see in this subsection
that different flavours of countability are really just different flavours of
relations to \Nat.
Case in point: our definition of split enumerability is definitionally equal to
a split surjection from \Nat.
\begin{agdalisting*}
  \ExecuteMetaDataInline[agda/Cardinality/Infinite/Split.tex]{split-surj}
\end{agdalisting*}
From this we can derive decidable equality, just like we could with split
enumerability. \todo{Explain better?}

We also have equivalents to manifest enumerability, or even cardinal finiteness,
in the countable setting: they are less interesting than split countability,
however.
\subsection{Closure}
The closure proofs are where countability begins to differ from split
enumerability.
We will see one closure proof that stays the same, one that is absent, and one
that is additional.
\paragraph{Instances for Finite Types}
Before we move on to the proper closure proofs, it is worth pointing out that
all (non-empty) finite types are also countable.
Split countability, like split enumerability, does not disallow duplicates: this
means that for any non-empty type we can simply repeat one of its elements
infinitely to produce a countability proof.
\paragraph{Closure Under \(\Sigma\)}
The proof that split enumerability was closed under \AgdaDatatype{\(\Sigma\)}
was quite straightforward:
we were able to use the ``normal'' pattern of taking the Cartesian product of
two lists in order to generate the finite support list for
\AgdaDatatype{\(\Sigma\)}.
Unfortunately this doesn't work for infinite types: the reason for which can be
seen in \Cref{pairings}.

% \todo{Explain this notion of exploration better:
%   \begin{itemize}
%     \item Talk about how we have to figure out a surjection from the natural
%       numbers.
%     \item This is like trying to figure out a pattern which hits every number.
%     \item Or like a way to map some type to the numbers.
%     \item When doing this in some kind of closure-like proof, we can pretend
%       that the input types (i.e. \(A\) and \(B\) in \(A \times B\)) are simply
%       \(\mathbb{N}\).
%   \end{itemize}
% }

\input{figures/pairing-functions}

The depth-first pattern is what we used previously: this explores the first list
exhaustively before exploring anything other than the first element of the
second list.
This clearly won't work for streams, as it would mean that nothing other than
the first element of the second type could be found in the entire support
stream.

So instead we use the second pattern: breadth-first search.
The way we actually code this pattern up is a little complex: we treat the
search space as having several ``levels''.
Each item in each input list has a level (its position in that input list); the
output level for two items is the \emph{sum} of those levels.
In pseudo-set-builder notation:
\begin{equation*}
  (\mathit{xs}\times\mathit{ys})_n =
    \left[ (\mathit{xs}_i , \mathit{ys}_j) \vert i \leftarrow \left[ 0 \ldots n \right] ; j \leftarrow \left[ 0 \ldots n \right] ; i + j = n  \right]
\end{equation*}
And then this is flattened to give the output support list.

One last detail of this function before we actually provide it: we use yet
another definition of lists in its implementation, instead of the normal lists,
as it is useful for termination proofs.
\begin{agdalisting*}
  \begin{multicols}{2} \centering
    \ExecuteMetaDataInline[agda/Data/List/Kleene.tex]{plus-def} \columnbreak
    \ExecuteMetaDataInline[agda/Data/List/Kleene.tex]{star-def}
  \end{multicols}
\end{agdalisting*}
This definition of lists interleaves the definition of non-empty lists with the
definition of possibly-empty lists.
This makes it much easier to switch between the two without conversion
functions.

Finally, we can provide the function which actually performs the breadth-first
search two streams.
\begin{agdalisting*}
  \ExecuteMetaDataInline[agda/Cardinality/Infinite/Split.tex]{sigma-sup}
\end{agdalisting*}
The corresponding cover proof is relatively straightforward, so we don't include
it here.
\paragraph{Closure Under The Kleene Star}
One of the more useful types we can prove to be countably infinite is the
\emph{list}.
This proof is significantly more complex than the previous, however: we need to
find a pattern which eventually covers any given element of type
\AgdaDatatype{List}\;\(A\), given \AgdaFunction{\(\aleph\)!}\;\(A\).
Again we will tackle this pattern by first treating the exploration space as a
series of finite \emph{levels}: we can then concatenate all of these levels
together to a final exploration pattern.
The next trick is to figure out how to define those levels: we'll do it by
saying the lists of a given level \(n\) each must sum (after incrementing the
indices of each element) to \(n\).
That means that the 0th level consists only of the empty list, the 1st level
consists of the list \(\left[ 0 \right]\), the 2nd of \(\left[ 0 , 0
\right]\) and \(\left[ 1 \right]\), and so on.
Again, in pseudo set-builder notation:
\begin{equation*}
  \Nat\star_i \coloneqq \left[ \left[ \mathit{xs}_{j - 1} \mid j \in \mathit{js} \right] \mid \mathit{js} : \AgdaDatatype{List}\;\Nat ; \AgdaFunction{sum}\;\mathit{js} = i ; 0 \notin \mathit{js}  \right]
\end{equation*}
\todo{Include code?}
\paragraph{No Closure Under \(\Pi\)}
One closure proof that is certainly not possible is closure under \(\Pi\):
it is not the case that functions from countable types are themselves countable.
While this is a pretty basic fact in computer science, we include it here to
show how simple its proof in Agda can be:
\begin{agdalisting*}
  \ExecuteMetaDataInline[agda/Cardinality/Infinite/Split.tex]{cantor-diag}
\end{agdalisting*}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../paper"
%%% End:


\chapter{Convolutional Neural Networks}
\label{appx:simulation}

Convolutional Neural Networks for image classification \cite{krizhevsky2012imagenet} take an image as an input, process it, and output category to which that image belongs. The processing part consists of a series of layers through which image is propagated in order to learn features, which in turn determine to which class an image belongs. (\textcolor{red}{\autoref{fig:cnn1}}).

\begin{figure}[h]
	\centering
	\includegraphics[scale=2]{cnn.jpg}
	\caption{Convolutional Neural Network consisting of two Convolutional layers, two Max-Pooling layers, Flatten layer and two Fully-Connected (Dense) layers, source \cite{alom2019state}}
	\label{fig:cnn1}
\end{figure}

The most commonly used layers in CNN architectures are convolutional layer, max-pooling layer, flatten layer, dense layer, and dropout layer.

\section{Convolutional Layer}

The convolutional layer is the building block of the CNN architecture. Its primary purpose is to extract features from an input image, such as edges, lines, curves, colors. As we go deeper inside the network, it starts identifying more complex features, such as shapes, objects. This layer consists of multiple filters (feature extractors, usually 3$\times$3 matrices) whose parameters need to be learned.
\section{Max-Pooling Layer}

The max-pooling layer is located after a series of convolutional layers in CNN architecture. It is a downsampling method that reduces dimensionality, thus decreasing the number of parameters and computational power needed in order to train the network, while retaining important features and patterns. It is achieved by applying a max filter to non-overlapping subregions (usually 2$\times$2 matrices), thus reducing the size of each feature map by a factor of 2.

\section{Flatten Layer}

The output of the convolutional base of the network (series of convolutional and max-pooling layers) is a two-dimensional matrix, and before feeding that data to the classification top of the network, it needs to be transformed. Flatten layer reshapes the output matrix to a vector, thus removing all dimensions but one in the process, making the data prepared for the series of fully-connected layers.

\section{Fully-Connected Layer}

After the high-level features of the image have been detected, a series of fully-connected (dense) layers are attached to the top of the network in order to classify an image into a label. Dense layers consist of a huge number of nodes (neurons), which provide a way of learning non-linear combinations of features outputted by a convolutional base, and determine which features most correlate to a particular class.

\section{Dropout Layer}

The fully-connected layer contains the most parameters in the network, and as a result neurons develop co-dependency amongst each other during training, which leads to overfitting the data (not generalizing well on new, unseen images). In order to prevent that, dropout layers are positioned right after dense layers in CNN architecture as a means of regularizing the network. Dropout consists of randomly ignoring (dropping out) fraction of neurons of fully-connected layer, which in turn makes network learn more robust features, and achieve better performance.
\clearpage


\section{Strong Markov Property}
\begin{definition}
    A random variable $T=\Omega\to\{0,1,\ldots\}\cup\{\infty\}$ is a stopping time if the event $\{T=n\}$ only depends on $X_0,\ldots,X_n$.
\end{definition}
\begin{example}
    (a) The first passage time $T_j=\inf\{n\ge 1:X_n=j\}$ is a stopping time.\\
    (b) The hitting time $H^A$ of a subset $A\subset I$ is a stopping time.\\
    (c) (non-example) The last exit time of a subset $A\subset I$, defined by $L^A=\sup\{n\ge 0:X_n\in A\}$ is in general not a stopping time.
\end{example}
\begin{theorem}[Strong Markov Property]\label{strong_markov}
    Let $(X_n)_{n\ge 0}$ be $\operatorname{Markov(\lambda,P)}$ and $T$ be a stopping time for $(X_n)$.
    Then, conditional on $T<\infty$ and $X_T=i$, the sequence $(X_{T+n})_{n\ge 0}$ is $\operatorname{Markov}(\delta_i,P)$ and independent of $X_0,\ldots,X_T$.
\end{theorem}
\begin{proof}
    Let $B$ be an event determined by $X_0,\ldots,X_T$, then $B\cap\{T=m\}$ is determined by $X_0,\ldots,X_m$.
    So
    \begin{align*}
        &\phantom{=}\mathbb P[\{X_T=j_0,\ldots,X_{T=n}=j_n\}\cap B\cap\{T=m\}\cap\{X_T=i\}]\\
        &=\mathbb P[X_T=j_0,\ldots,X_{T=n}=j_n]\mathbb P[B\cap\{T=m\}\cap\{X_T=i\}
    \end{align*}
    Summing over $m$ gives
    \begin{align*}
        &\phantom{=}\mathbb P[\{X_T=j_0,\ldots,X_{T=n}=j_n\}\cap B\cap\{T<\infty\}\cap\{X_T=i\}]\\
        &=\mathbb P[X_T=j_0,\ldots,X_{T=n}=j_n]\mathbb P[B\cap\{T<\infty\}\cap\{X_T=i\}]
    \end{align*}
    If $\mathbb P[T<\infty,X_T=i]>0$, we then have
    \begin{align*}
        &\phantom{=}\mathbb P[\{X_T=j_0,\ldots,X_{T=n}=j_n\}\cap B|T<\infty,X_T=i]\\
        &=\mathbb P[X_T=j_0,\ldots,X_{T=n}=j_n]\mathbb P[B|T<\infty,X_T=i]
    \end{align*}
    As desired.
\end{proof}
\begin{example}[Gambler's Ruin Continued]
    Back to our previous example where for $i>0$, $p_{i,i+1}=p$ and $p_{i,i-1}=q=1-p$ and $p_{0,0}=1$.
    We previously found the hitting probability of $\{0\}$.
    We now want to find the distribution of time to hit it starting from $1$.
    Let $H_j=\inf\{n\ge 0:X_n=j\}$ and
    $$\phi(s)=\mathbb E[s^{H_0}]=\mathbb E[s^{H_0}1_{H_0<\infty}]=\sum_{n=0}^\infty s^n\mathbb P[H_0=n]$$
    where $s\in [0,1)$.
    We claim that $\mathbb E_2[s^{H_0}]=\phi(s)^2$.
    To show this, we shall use the strong Markov property.
    Conditional on $H_1<\infty$ under $\mathbb P_2$, we can write $H_0=H_1+\tilde{H}_0$ where $\tilde{H}_0$ is the time it takes after $H_1$ to reach state $0$.
    Since $H_1$ is a stopping time, by applying Theorem \ref{strong_markov} on $H_1$ we get $\tilde{H}_0$ is independent of $H_1$ as it only depends on $(X_{H_1+n})_{n\ge 0}$, so
    \begin{align*}
        \mathbb E_2[s^{H_0}]&=\mathbb E_2[s^{H_1}|H_1<\infty]\mathbb E_2[s^{\tilde{H}_0}|H_1<\infty]\mathbb P[H_1<\infty]\\
        &=\mathbb E_2[s^{H_1}1_{H_1<\infty}]\mathbb E_2[s^{\tilde{H}_0}|H_1<\infty]\\
        &=\phi(s)^2
    \end{align*}
    Our second claim is that $ps\phi(s)^2-\phi(s)+qs=0$.
    Conditional on $X_1=2$, we have $H_0=1+\bar{H}_0$ where $\bar{H}_0$ is the time takes after $1$ step to reach $0$.
    Then $\bar{H}_0$ under $\mathbb P[\cdot|X_2=2]$ is the same distribution as $H_0$ under $\mathbb P_2$.
    So
    \begin{align*}
        \phi(s)&=\mathbb E_1[s^{H_0}]\\
        &=p\mathbb E_1[s^{H_0}|X_1=2]+q\mathbb E_1[s^{H_0}|X_1=0]\\
        &=p\mathbb E_1[s^{1+\bar{H}_0}|X_1=2]+qs\\
        &=ps\mathbb E_1[s^{\bar{H}_0}|X_1=2]+qs\\
        &=ps\mathbb E_2[s^{H_0}]+qs\\
        &=ps\phi(s)^2+qs
    \end{align*}
    which shows the claim.
    This means that
    $$\phi(0)=0,\phi(s)=\frac{1\pm\sqrt{1-4pqs^2}}{2ps},s>0$$
    since $\phi(s)\le 1$ for all $s$ and $\phi$ is continuous, only the minus case is possible in the quadratic formula, so we get, for $s>0$,
    $$s\mathbb P[H_0=1]+s^2\mathbb P[H_0=2]+\cdots=\phi(s)=\frac{1-\sqrt{1-4pqs^2}}{2ps}=qs+pq^2s^3+\cdots$$
    In particular, $\mathbb P[H_0=1]=q,\mathbb P[H_0=2]=0,\ldots$.
    In addition, as $s\to 1^-$, we have $\phi(s)\to\mathbb P_1[H_0<\infty]$, so
    $$\mathbb P_1[H_0<\infty]=\frac{1-\sqrt{1-4pq}}{2p}=\begin{cases}
        1\text{, if $p\le q$}\\
        q/p\text{, if $p>q$}
    \end{cases}$$
    which is our previous result.
    Also, in the case $p\le q$,
    $$\mathbb E_1[H_0]=\mathbb E_1[H_01_{H_0<\infty}]=\lim_{s\to 1^-}\phi^\prime(s)=\lim_{s\to 1^-}\frac{p\phi(s)^2+q}{1-2ps\phi(s)}=\frac{1}{1-2p}=\frac{1}{q-p}$$
\end{example}


\section{Neural Networks: Regression \& Multinomial Classification}
supervised learning.
\subsection{Terminology}
\paragraph{Training Example} has form ($x_n, y_n$), with $x_n$ as input vector, $y_n$ expected/true output vector.

\paragraph{Loss/Cost Function} maps values of one or more variables onto a number representing \textbf{loss/cost}.

Learning $\rightarrow$ minimizing a loss function.

\paragraph{Risk Function} \textbf{expectation} of the \textbf{loss function}. In neural network, we minimizes our risk by  \textbf{minimizing the empirical risk function -- average loss of all training examples}.

\paragraph{Activation Function}
\begin{itemize}
	\item linear activation
	\item sigmoid activation: \textbf{focus of this lecture},  $\sigma(x) = \frac{e^x}{1 + e^x}$
	\item Perceptron activation
	\item ReLU activation
\end{itemize}
\paragraph{Epoch} one \textbf{forward pass} and one \textbf{backward pass} of \textbf{all} training examples. One pass = forward + backward pass.
\begin{itemize}
	\item Forward Pass: calculate the output of \textbf{all training } through the neural network.
	\item Backward Pass: Backpropagation
\end{itemize}

\paragraph{Backpropagation} calculate a \textbf{gradient} that is needed in \textbf{calculation of weights} to be used in network. It describes how a \textbf{single training example} starting from \textbf{output neurons} determines the goal for the neurons on the next layer and \textbf{steps backwards recursively}.

\paragraph{Multi-Layer Feed-Forward Networks} represent arbitrary \textbf{non-linear} functions. It consists of
\begin{itemize}
	\item input layer
	\item hidden layer with \textbf{activation}
	\item output layer with \textbf{activation}
\end{itemize}
\textbf{Weights and biases} need to be adapted in the neural network.

$\rightarrow$ updated by \textbf{backpropagation (gradient descent)}.

\subsection{Multi-Layer Feed-Forward Network}
\subsubsection{Setup of a Neural Network}
\begin{figure}[H]
	\centering
	\includegraphics[width=0.7\textwidth]{nn.png}
\end{figure}

\subsubsection{Process}
Goal of training: \textbf{minimizes loss function, minimizes empirical risk}.
\\ \ \\
Assume a network with 1 hidden layer, 1 output layer:

\begin{enumerate}[label= \protect \circled{\arabic*} ]
	\item \textbf{Forward Pass}: from input layer to output layer (with \textbf{sigmoid} activation).

	If we are asked to perform, this calculation can done \textbf{matrix-wise}!! No need to separate each observation.

	\begin{align*}
		z^{[1]} &= W^{[1]}\cdot a^{[0]} + b^{[1]} =  W^{[1]}\cdot x + b^{[1]}\\
		a^{[1]} &= g^{[1]}(z^{[1]}) = \sigma(z^{[1]}) \\
		z^{[2]} &= W^{[2]}\cdot a^{[1]} + b^{[2]} \\
		a^{[2]} &= g^{[2]}(z^{[2]}) = \sigma(z^{[2]})
	\end{align*}

	\item \textbf{Loss Function}: calculate the loss of the network output according to the loss function $l(y, \hat{y})$

	If we evaluate the model using \textbf{cross-entropy loss}: the calculation for $y \ln\hat{y}$ is a dot product(element-wise multiplication).
	$$l(y,\hat{y}) = - [y \ln \hat{y} + (1-y)\ln(1 - \hat{y})]$$
	\item \textbf{Empirical Risk}: calculate the empirical risk $\mathcal{L}$ by \textbf{averaging} the loss.
	$$\mathcal{L}(y,\hat{y}) = \frac{1}{n} \cdot \Sigma l(y,\hat{y})$$

	\item \textbf{Backpropagation}: readapt the \textbf{weights and biases} using \textbf{gradient descent}.

	example in updating layer 2:
	\begin{align*}
		W^{[2]}_{t+1} &= W^{[2]}_t - \alpha \cdot dW =W^{[2]}_t - \alpha \cdot \frac{\partial L}{\partial W^{[2]}} \\
		b^{[2]}_{t+1} &= b^{[2]}_t - \alpha \cdot db =b^{[2]}_t - \alpha \cdot \frac{\partial L}{\partial b^{[2]}}
	\end{align*}
	\begin{itemize}
		\item $\frac{\partial L}{\partial W^{[2]}}, \frac{\partial L}{\partial b^{[2]}}$: chain rule
		$$\frac{\partial L_n}{\partial W} = \frac{\partial L_n}{\partial a_{n}} \cdot \frac{\partial a_{n}}{\partial z_{n}} \cdot \frac{\partial z_n}{\partial W}$$
	\end{itemize}
	\begin{align*}
		L_n &= \frac{1}{2} (y_{n} - g(w_{kl}a_{kn} + b_l))^2 = \frac{1}{2} (y_n - a_{ln})^2, \quad &\frac{\partial L_n}{\partial a_{ln}} &= -(y_n - a_{ln})\\
		a_{ln} &= g(z_{ln}) , \quad &\frac{\partial L_n}{\partial z_{ln}} &= g'(z_{ln}) \\
		z_{ln} &= w_{kl}a_{kn} + b_l, \quad &\frac{\partial L_n}{\partial w_{ln}} &= a_{kn}
	\end{align*}

	If the activation is a sigmoid activation: $\sigma(x) = \frac{e^x}{1 + e^x}$, $\sigma'(x) = \sigma(x)(1 - \sigma(x))$.
	$$dW^{[2]} = -(y- a^{[2]})\cdot a^{[1]^{T}} = (a^{[2]} - y) \cdot a^{[1]^{T}}$$
\end{enumerate}

\subsubsection{Trainable Parameters}
Number of trainable parameters: the \textbf{free parameters} of the neural network are \textbf{weights and biases}-- $W^{[1]}, b^{[1]}, W^{[2]}, b^{[2]}, \dots$.

$\rightarrow$ define the \textbf{dimension} of these parameters, add up all possible trainable elements.

(eg: $W^{[1]}$ is 2x2-matrix, therefore 4 trainable parameters)
\subsection{Gradient Descent for Backpropagation}
\begin{itemize}
	\item Goal: given any function f, find $x^* = \arg\min_{x} f(x)$
	\item \textbf{Gradient} at position x is defined as the \textbf{partial derivative}:
	$$\nabla f(x) = \begin{bmatrix}
	\frac{\partial f(x)}{x_1} \\ \vdots \\ \frac{\partial f(x)}{x_d}
	\end{bmatrix}$$
	\begin{itemize}
		\item Interpretation: in d-dimensional space, gradient points in \textbf{direction of steepest ascent} of f at point x.
	\end{itemize}
	$\rightarrow$ to \textbf{minimize loss function} $\rightarrow$ \textbf{descent}, the opposite direction $\mathbf{- \nabla f(x)}$.

\end{itemize}
\subsubsection{General Process}
\begin{enumerate}[label= \protect \circled{\arabic*} ]
	\item choose an \textbf{initial point}
	\item choose a \textbf{step size (either fixed or dynamic)}.
	\item take a step in the \textbf{direction opposite the gradient}.
	\begin{itemize}
		\item fixed step size:
		$$\begin{bmatrix}
		x_n \\y_n
		\end{bmatrix} = \begin{bmatrix}
		x_{n-1} \\ y_{n-1}
		\end{bmatrix} - \alpha \cdot \nabla f(x_{n-1}, y_{n-1})$$
		\item dynamic step size:
		$$\begin{bmatrix}
		x_n \\y_n
		\end{bmatrix} = \begin{bmatrix}
		x_{n-1} \\ y_{n-1}
		\end{bmatrix} - \alpha_{n} \cdot \nabla f(x_{n-1}, y_{n-1})$$
	\end{itemize}
	\item repeat till convergence.
\end{enumerate}

Convergence to optimum: depends on the \textbf{step size}.
\begin{itemize}
	\item too small: would converge eventually, but takes long time.
	\item too large: value \textbf{oscillates}, doesn't converge.
	\item would stall if $\nabla f(x) = 0$.
	\item can stuck at saddle point.
\end{itemize}
Criteria:
\begin{itemize}
	\item function f is \textbf{convex}
	\item step size $\alpha$ is square summable, but not summable.
\end{itemize}
$\rightarrow$ Alternative: introduce \textbf{momentum}.

\subsubsection{Process with Momentum Introduced}
\begin{itemize}
	\item Idea: uses an exponential averaging of gradients to \textbf{make sudden changes in direction less likely}.
\end{itemize}
\begin{align*}
	d_n &= \beta \cdot d_{n-1} + \alpha \cdot\nabla f(x_{n-1})\\
	x_n &= x_{n-1} - d_n
\end{align*}


Let us begin with a simple example, displaying some of the
difficulties we may have.

\begin{Problem}{unbounded-inverse}
  On the space $\ell_2(\R)$ define the operator $A$ by its eigenvalue
  decomposition
  \begin{align}
    A: \ell_2(\R) &\to \ell_2(\R)\\
    e_k & \mapsto \tfrac1k e_k.
  \end{align}
  Here, $\{e_k\}$ is the orthogonal basis of unit vectors of the form
  \begin{gather}
    \arraycolsep0.1em
    \begin{array}{cccccccc}
      e_k =(0&,\ldots,&0&,&1&,&0&,\ldots)^\transpose.\\
      &&&&\uparrow\\
      &&&&k
    \end{array}
  \end{gather}
  \begin{enumerate}
  \item Show that this operator does not have a bounded inverse, albeit
    its eigenvalues are positive.
  \item Show that the range of $A$ is not closed in $\ell_2(\R)$
  \end{enumerate}
\begin{solution}
  \begin{enumerate}
  \item For each $e_k$, the inverse is $A^{-1} e_k = k e_k$. In particular, $A$ is injective.
    On the other hand, it holds
    \begin{gather}
      \lim_{k\to\infty}\frac{\norm{A^{-1}e_k}}{\norm{e_k}}=\lim_{k\to\infty}k=\infty
    \end{gather}
      and the inverse cannot be bounded.
  \item We have to construct a convergent sequence in the range of $A$
    such that the pre-image of the sequence does not converge.
    \begin{enumerate}
    \item Choose
      \begin{gather}
        v_n = \sum_{k=1}^n \frac1k e_k.
      \end{gather}
      \item $v_n$ is a Cauchy-sequence, since
        \begin{gather}
          \norm{v_m-v_n}^2 = \norm*{\sum_{k=m}^n \frac1k e_k}^2
          = \sum_{k=m}^n \frac1{k^2}\norm*{e_k}^2
          \le \frac1{m^2} \sum_{k=1}^\infty \frac1{k^2}
          = \frac{\pi^2}{6} \frac1{m^2}.
        \end{gather}
      \item We conclude that $v=\lim_{n\to\infty}v_n$ exists in the
        closure of the range of $A$.
      \item There holds
        \begin{gather}
          v_n = A \sum_{k=1}^n e_k =: A u_n.
        \end{gather}
      \item Due to the injectivity of $A$ for $v$ to be in the range of $A$,
            $u_n$ has to converge in $\ell_2(\R)$.
      \item The sequence $u_n$ is not a Cauchy sequence, since
        \begin{gather}
          \norm{v_m-v_n}^2 = \norm*{\sum_{k=m}^n e_k}^2
          = \sum_{k=m}^n \norm*{e_k}^2
          = n-m.
        \end{gather}
    \end{enumerate}
  \end{enumerate}
\end{solution}
\end{Problem}

\section{Finite-dimensional problems}
\begin{intro}
  So far, our power horse for well-posedness was the Lax-Milgram
  lemma, which can be applied under the conditions
  \begin{xalignat}2
    a(u,v) &\le M \norm{u}\norm{v} & \forall u,v&\in V\\
    a(u,u) &\ge \ellipa \norm{u}^2 & \forall u&\in V.
  \end{xalignat}
  The second condition can also be rewritten in terms of the
  \putindex{Rayleigh quotient} as
  \begin{gather}
    0 < \ellipa = \inf_{u\in V}\frac{a(u,u)}{\norm{u}^2}.
  \end{gather}
  Restricting this to a finite dimensional space, the notation usually
  changes from
  \begin{gather}
    a(u,v) = f(v)
    \qquad\text{to}\qquad
    v^\transpose A u = v^\transpose f,
  \end{gather}
  where $A\in \R^{n\times n}$ is the matrix associated with the
  bilinear form. The bound for the Rayleigh quotient means nothing but
  that the real parts of all eigenvalues of $A$ are bounded from below
  by $\ellipa$. Thus, a matrix $A$ for which we can apply the
  Lax-Milgram lemma is positive definite. And the statement of the
  lemma in finite dimension is, that a positive definite matrix is
  invertible. We know from linear algebra that this is true, but we
  also know that the condition is all but necessary.
\end{intro}

\begin{intro}
  Why did we replace this clear theorem by the weaker Lax-Milgram
  lemma, when we studied elliptic partial differential equations?  For
  the first condition, it should be noted that spectral properties of
  operators between spaces of infinite dimension are much harder to
  obtain. Further, we do not need information on the whole spectrum,
  but only on the eigenvalue closest to zero. Therefore, we used a
  simple estimate in order to avoid discussing the spectrum at
  all. But, there is an important difference between
  Theorem~\ref{Theorem:la-invertible} and the
  estimate~\eqref{eq:infsup:elliptic}: the assumption of the theorem
  is qualitative, $\lambda \neq 0$, while the assumption of Lax-Milgram
  is quantitative,
  \begin{gather}
    \Re\lambda \ge \ellipa> 0.
  \end{gather}
  The following problem shows why such a change is necessary.
\end{intro}

\begin{Problem}{lax-milgram-not-applicable}
  Find an invertible, symmetric matrix $A\in \R^{2\times 2}$ and a
  vector $v\in \R^2$ such that $v^\transpose A v=0$ and thus the Lax-Milgram
  lemma is inconclusive.
\begin{solution}
  \begin{gather}
    A =
    \begin{pmatrix}
      1 & 0 \\ 0 & -1
    \end{pmatrix}
  \end{gather}
\end{solution}
\end{Problem}

The question of well-posedness in finite dimensions can be answered by:

\begin{Theorem}{la-invertible}
  A matrix $A\in\R^{n\times n}$ is invertible if and only if one of
  the following equivalent conditions holds:
  \begin{enumerate}
  \item all its (possibly complex) eigenvalues are nonzero,
  \item all its singular values are nonzero,
  \item for each nonzero $v\in\R^n$ holds $Av\neq 0$.
  \end{enumerate}
\end{Theorem}

\begin{intro}
  We focus on the second and third conditions, respectively, in
  Theorem~\ref{Theorem:la-invertible}.
  But, the problem above tells us that we
  will run into trouble, if we do not quantify this. Therefore, we
  start our attempt by requiring:
  \begin{gather}
    \norm{Au}^2 \ge \ellipa \norm{u}^2 \qquad\forall u\in V.
  \end{gather}
  But while this is a condition we can easily write down for matrices
  and operators, it does not work that well for bilinear forms. Thus,
  we first look at the singular value decomposition.
\end{intro}

\input{svd}

\begin{Definition}{ker-range-rn}
  \index{ker}
  Let $A: V\to W$ be a linear operator. Then, we define
  the \define{kernel} and the \define{range} of $A$ as
  \begin{align}
    \ker A &= \bigl\{ v\in V \big| Av = 0\bigr\} \\
    \range A &= \bigl\{ w\in W \big| \;\exists\,v\in V: Av=w\bigr\}.
  \end{align}
\end{Definition}

\begin{Definition}{orthogonal1}
  Let $V\subset\R^n$ be a subspace. We define the \define{orthogonal
    complement} of $V$ as
  \begin{gather}
    \label{eq:infsup:4}
    \ortho{V} = \bigl\{w\in \R^n \big| \;\forall\,v\in V \scal(w,v) = 0 \bigr\}.
  \end{gather}
\end{Definition}

\begin{Lemma}{ker-coker-rn}
  Let $A\in \R^{m\times n}$ and $A^\transpose$ its transpose. Then, there holds
  \begin{gather}
    \label{eq:infsup:5}
    \begin{split}
      \ker A &= \ortho{\range{A^\transpose}}\\
      \range A &= \ortho{\ker{A^\transpose}}\\
      \ker{A^\transpose} &= \ortho{\range A}\\
      \range{A^\transpose} &= \ortho{\ker{A}}
    \end{split}
  \end{gather}
\end{Lemma}

\begin{proof}
  % First, we note that
  % \begin{gather}
  %   \R^n = \ker A \oplus \ortho{\ker A},
  %   \qquad
  %   \R^m = \ker{A^\transpose} \oplus \ortho{\ker{A^\transpose}}.
  % \end{gather}
  Let $A=U\Sigma V^\transpose$ be the singular value decomposition of $A$ and
  $r$ be the number of nonzero singular values. Then, the first $r$
  vectors of $U$ span the range of $A$ and the last $n-r$ vectors of
  $V$ span its kernel. Furthermore,
  \begin{gather}
    A^\transpose = \bigl(U\Sigma V^\transpose\bigr)^\transpose = V \Sigma U^\transpose.
  \end{gather}
  Therefore, the first $r$ vectors of V span the range of
  $A^\transpose$ and the last $n-r$ vectors of $U$ span its
  kernel. The lemma follows since $U$ and $V$ are orthogonal.
\end{proof}

\begin{Corollary}{ker-coker-iso}
  Let $A\in \R^{m\times n}$ and $A^\transpose$ its transpose. Then, the
  restrictions $A\colon \ortho{\ker A} \to \range A$ and $A^\transpose\colon
  \ortho{\ker{A^\transpose}} \to \range{A^\transpose}$ are isomorphisms.

  The linear system $Ax=f\in\R^m$ has at least one solution if and
  only if $f\in \range A$. If $x\in\R^n$ is such a solution, then
  every $y\in\R^n$ with $y-x\in\ker A$ is a solution as well.
\end{Corollary}

\begin{proof}
  We note that $\dim \range A = \dim \range{A^\transpose}$. Thus, by
  \slideref{Lemma}{ker-coker-rn} the dimensions of domain and range of
  each of the restricted operators are equal, say $\dim \range A =
  r$. The singular value decomposition of the operators is
  \begin{gather}
    A = U\Sigma V^\transpose \qquad A^\transpose = V\Sigma U^\transpose,
  \end{gather}
  where all matrices are in $\R^{r\times r}$ and
  \begin{gather}
    \Sigma = \diag(\sigma_1,\dots,\sigma_r),
  \end{gather}
  and all singular values are positive. Thus, $A$ and $A^\transpose$ are invertible.
\end{proof}

\begin{Corollary}{svd-infsup}
  Let $r=\dim \ortho{\ker A}$. Then, for the smallest nonzero singular
  value there holds
  \begin{gather}
    \label{eq:infsup:6}
    \sigma_r
    = \inf_{v\in \ortho{\ker A}} \sup_{w\in \R^m} \frac{w^\transpose A v}{\norm{v}\norm{w}}
    = \inf_{w\in \ortho{\ker{A^\transpose}}} \sup_{v\in \R^n} \frac{w^\transpose A v}{\norm{v}\norm{w}}.
  \end{gather}
\end{Corollary}

\begin{proof}
  Since the Cauchy-Schwarz inequality turns into an equation if and
  only if the two vectors are coaligned, there holds for any $v\in \R^n$:
  \begin{gather}
    \sup_{w\in\R^m}\frac{w^\transpose A v}{\norm{w}} = \frac{v^\transpose A^\transpose A v}{\norm{Av}} = \frac{\norm{Av}^2}{\norm{Av}}.
  \end{gather}
  Therefore,
  \begin{gather}
    \inf_{v\in \ortho{\ker A}} \sup_{w\in \R^m}
    \frac{w^\transpose A v}{\norm{v}\norm{w}}
    = \inf_{v\in \ortho{\ker A}} \frac{\norm{Av}}{\norm{v}}.
  \end{gather}
  Now, let $v = \sum \alpha_i v_i$ where $v_i$ are the columns of $V$
  in the SVD of $A$. Then,
  \begin{gather}
    \norm{Av}^2 = \norm*{A\sum_{i=1}^r \alpha_i v_i}^2
    = \norm*{\sum_{i=1}^r \sigma_i \alpha_i u_i}^2
    = \sum_{i=1}^r \sigma_i^2 \alpha_i^2.
  \end{gather}
  The quotient
  \begin{gather}
    \frac{\sum_{i=1}^r \sigma_i^2 \alpha_i^2}{\sum_{i=1}^r \alpha_i^2}
  \end{gather}
  clearly has its minimum if $\alpha_1 = \dots=\alpha_{r-1} = 0$.
\end{proof}

\begin{Definition}{infsup1}
  A bilinear form $a(\cdot,\cdot)$ on $V\times W$ is said to admit the
  \define{inf-sup condition} or is called \define{inf-sup stable}, if
  there holds
  \begin{gather}
    \label{eq:infsup:1}
    \inf_{u\in V} \sup_{w\in W} \frac{a(u,w)}{\norm{u}_V\norm{w}_W}
    \ge \ellipa > 0.
  \end{gather}
\end{Definition}

\begin{remark}
  In this finite dimensional exposition, is clear that $V$ and $W$
  must have the same dimension, and thus $V=W=\R^n$. This will be
  different, when we consider infinite dimensional spaces and indeed
  consider different spaces $V$ and $W$.
\end{remark}

\begin{todo}
  The last statement uses the wrong spaces. Either restrict the whole
  argument here to $\R^n$ or think about dual spaces.
\end{todo}

\begin{Lemma}{infsup2}
  The following statements are equivalent to the inf-sup
  condition~\eqref{eq:infsup:1}:
  \begin{gather}
    \label{eq:infsup:2}
    \forall u\in V \;\exists w\in W \;:\;
    a(u,w) \ge \ellipa \norm{u}_V\norm{w}_W
  \end{gather}
  \begin{gather}
    \label{eq:infsup:3}
    \forall u\in V
    \;\exists w\in W \;:\;
    \left\{
    \arraycolsep0.3ex
    \begin{array}{rcl}
      \norm{w}_W &\le&\norm{u}_V\\
      a(u,w) &\ge& \ellipa \norm{u}_V^2
    \end{array}
    \right.
  \end{gather}
  \begin{gather}
    \label{eq:infsup:3a}
    \forall u\in V
    \;\exists w\in W \;:\;
    \left\{
    \arraycolsep0.3ex
    \begin{array}{rcl}
      \ellipa \norm{w}_W &\le&\norm{u}_V\\
      A^\transpose w &=& u
    \end{array}
    \right.
  \end{gather}
\end{Lemma}

\begin{Problem}{inf-sup-equivalence}
  Prove \slideref{Lemma}{infsup2}.
\begin{solution}
  We have to prove the following statements are equivalent to the inf-sup condition:
  \begin{align}
    \forall u\in V \;\exists w\in W \;:\; a(u,w) \ge \ellipa \norm{u}_V\norm{w}_W     \tag{1}
  \end{align}
  \begin{align}
   \begin{aligned}
    \forall u\in V \;\exists w\in W \;:\;
    \begin{cases}
      \norm{w}_W \le\norm{u}_V\\
      a(u,w) \ge \ellipa \norm{u}_V^2
    \end{cases}
   \end{aligned}
   \tag{2}
  \end{align}
  \begin{align}
   \begin{aligned}
    \forall u\in V \;\exists w\in W \;:\;
    \begin{cases}
      \ellipa \norm{w}_W \le \norm{u}_V\\
      Aw = u
      \end{cases}
  \end{aligned}
      \tag{3}
  \end{align}
  The inf-sup condition reads
  \begin{align}
   \inf_{u\in V} \sup_{w\in W} \frac{a(u,w)}{\norm{u}_V\norm{w}_W} \ge \ellipa \tag{IS}
  \end{align}

  $(IS)\Rightarrow(3)$\\
  The inf-sup condition is equivalent to $A: ker(A)^\perp\to V^*$ being an isomorphism.
  By the Riesz representation theorem there exists for a given $u\in V$ a $w\in W$ such that
  $Aw=J u$ where $J$ is the Riesz map. Hence, it holds
  \begin{align}
a(u,w)=\langle A w, u\rangle = \langle J u, u\rangle = \norm{u}_V^2.
  \end{align}
 Due to $w\in ker(A)^\perp$, $A^{-1}$ is bounded and
 \begin{align}
 \norm{w}_W=\norm{A^{-1}u}_V\leq \frac{1}{\ellipa}\norm{u}_V.
  \end{align}

  $(3)\Rightarrow(2)$ \\
  Define $\tilde{w}=\ellipa w$. Then,
  \begin{align}
a(u,\tilde{w})=\ellipa a(u,w) = \ellipa \norm{u}_V^2
  \end{align}
  and
  \begin{align}
\norm{\tilde{w}}_W=\ellipa \norm{w}_W\leq\norm{u}_V
  \end{align}

  $(2)\Rightarrow(1)$ \\
  \begin{align}
  a(u,w) \ge \ellipa \norm{u}_V^2 \ge \ellipa \norm{u}_V \norm{w}_W
  \end{align}

  $(1)\Rightarrow(IS)$ \\
  \begin{align}
  &\forall u\in V \exists w\in W: a(u,w)\ge \ellipa \norm{u}_V\norm{w}_W\\
  &\Rightarrow \forall u\in V: \sup_{w\in W} \frac{a(u,w)}{\norm{w}_W}\ge \ellipa \norm{u}_V\\
  &\Rightarrow \inf_{u\in V}\sup_{w\in W} \frac{a(u,w)}{\norm{w}_W\norm{u}_V}\ge \ellipa
  \end{align}
\end{solution}
\end{Problem}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Infinite dimensional Hilbert spaces}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{intro}
  In the previous section, we derived quantitative conditions to
  ensure the invertibility of a matrix $A$ or its restriction to its
  cokernel $\ortho{\ker A}$. The arguments there have a natural
  extension to infinite dimensional Hilbert spaces, which we will
  derive in this section. We already saw in
  \slideref{Problem}{unbounded-inverse} that we may run into trouble
  if the range of $A$ is not closed. On the other hand, it turns out
  that most notions of linear algebra related to orthogonality can be
  maintained in Hilbert spaces if closed subspaces are considered.
  We begin by citing the most important results.

  The presentation here is neither complete nor self-contained. It
  just highlights some of the important facts. In particular, we
  assume the validity of the \putindex{Riesz representation theorem}
  and the existence of a \putindex{Schauder basis} a priori.
\end{intro}

\begin{Notation}{musical-isomorphisms}
  The \putindex{Riesz representation theorem} induces isomorphisms
  between a Hilbert space $V$ and its dual $V^*$. We call them
  \define{Riesz isomorphism}s. Following custom in differential
  geometry, we denote them as \define{musical isomorphism}s
  \begin{xalignat}2
    I_\flat\colon V&\to V^*
    &I_\sharp\colon V^*&\to V\\
    v&\mapsto v^\flat
    &\phi&\mapsto \phi^\sharp.
  \end{xalignat}
  \index{sharp@$\sharp$}\index{flat@$\flat$}\index{b@$\flat$}
\end{Notation}

\begin{Problem*}{riesz-h10}{Riesz Isomorphisms}
  Let $V=H^1_0(\domain)$ on the domain $\domain=(0,1)$ be equipped with the
  inner product
  \begin{gather}
    \scal(u,v)_V = \int_0^1 u'v'\dt.
  \end{gather}
  \begin{enumerate}
  \item Show that for any function $f\in L^2(\domain)$ the functional
    $\phi_f$ defined by
    \begin{gather}
      \phi_f(v) = \int_0^1 f v \dt
    \end{gather}
    is in $V^*$, even if $f\not\in V$.
  \item Discuss this for the function $f\equiv 1$ and compute $f^\sharp$.
  \end{enumerate}
\end{Problem*}

\begin{Example}{solution-pressure}
  When we look at computing the pressure in Stokes' equations, we have
  to solve a weak formulation of the form: find $p\in Q$ such that
  \begin{gather}
    \form(\div \vv,p) = g(v) \qquad \forall \vv\in \vV.
  \end{gather}
  Here $\vg\in \vV^*$ consists of the parts of the momentum equation not
  containing the pressure. If we define $B^\transpose$ by
  \begin{gather}
    \scal(B^\transpose q,\vv)_{V^*\times V} = \form(\div \vv,q)
    \qquad \forall \vv\in \vV, q \in Q,
  \end{gather}
  then the operator form of the problem posed in $\vV^*$ is
  \begin{gather}
    B^\transpose p = g
  \end{gather}
  Since this equation involves two different spaces, we require an
  extension of \slideref{Lemma}{ker-coker-rn}.
\end{Example}

\begin{Definition}{polar-orthogonal}
  Let $W\subset V$ be a subspace of a Hilbert space $V$. We define its
  \define{orthogonal complement} $\ortho{W}\subset V$ and its
  \define{polar space} $\polar{W}\subset V^*$ by
  \begin{gather}
    \label{eq:infsup:7}
    \begin{aligned}
    \ortho{W} &= \bigl\{v\in V &\big|&& \scal(v,w)_{V} &= 0
    &\forall\,w&\in W\bigr\},
    \\
    \polar{W} &= \bigl\{f\in V^* &\big|&& \scal(f,w)_{V^*\times V} &= 0
    &\forall\,w&\in W\bigr\}.
    \end{aligned}
  \end{gather}
  For a subspace $U\subset V^*$, we define its polar space
  \begin{gather}
    \dualpolar{U} = \bigl\{v\in V \quad\big|\quad \scal(u,v)_{V^*\times V} = 0
    \quad\forall\,u\in U\bigr\}
  \end{gather}
\end{Definition}

\begin{Problem}{polar-orthogonal}
  Show that $\polar{W} = I_\flat \ortho{W}$ and $\dualpolar{U} = I_\sharp \ortho{U}$.
\end{Problem}

\begin{Lemma}{orthogonal-closed}
  The polar space $\polar{W}$ and the orthogonal complement $\ortho{W}$ of a
  subspace $W\subset V$ are both closed. So is the polar space $\dualpolar{U}$
  of a subspace $U\subset V^*$.
\end{Lemma}

\begin{proof}
  Consider the mapping
  \begin{align}
    \Phi_w\colon V^* &\to \R,\\
    \phi&\mapsto \scal(\phi,w)_{V^*\times V}.
  \end{align}
  For any $w$, the kernel of $\Phi_w$ is closed as
  the pre-image of a closed set. $\polar{W}$ is closed since it is the
  intersection of these kernels for all $w\in W$.

  The inner product is continuous on $V\times V$. Therefore, the
  mapping
  \begin{align}
    \Psi_w\colon V &\to \R,\\
    v&\mapsto \scal(v,w),
  \end{align}
  is continuous. The argument continues as above. Similar for $\dualpolar{U}$.
\end{proof}

\begin{Theorem}{orthogonal-complement}
  Let $W$ be a subspace of a Hilbert space $V$ and $\ortho{W}$ its
  orthogonal complement. Then, $\ortho{W} =
  \ortho{\overline{W}}$. Furthermore, there holds
  \begin{gather}
    V = W \oplus \ortho{W}
    \qquad\Longleftrightarrow\qquad
    \text{$W$ is closed.}
  \end{gather}
\end{Theorem}

\begin{proof}
  Clearly, $\ortho{\overline{W}} \subset \ortho{W}$ since
  $W\subset\overline{W}$. Let now $u\in \ortho{W}$. Then, $\phi =
  \scal(u,\cdot)$ is a continuous linear functional on $V$. Therefore,
  if a sequence $w_n \subset W$ converges to $w\in \overline{W}$, we
  have
  \begin{gather}
    \scal(u,w) = \lim_{n\to\infty} \scal(u,w_n) = 0.
  \end{gather}
  Hence, $u\in \ortho{\overline{W}}$ and $\ortho{W} = \ortho{\overline{W}}$.

  Now, the ``only if'' follows by the fact, that if $W$ is not
  closed, there is an element $w\in \overline{W}$ but not in $W$ such that
  $\scal(w,u)=0$ for all $u\in \ortho{W}$. Thus, $w\not\in \ortho{W}$ and
  consequently $w\not\in \ortho{W} \oplus W$.

  Let now $W$ be closed. We show that there is a unique decomposition
  \begin{gather}
    \label{eq:infsup:8}
    v = w + u,\qquad w\in W, \;u\in \ortho{W},
  \end{gather}
  which is equivalent to $V = W \oplus \ortho{W}$. Uniqueness follows,
  since
  \begin{gather}
    v = w_1+u_1 = w_2+u_2
  \end{gather}
  implies that for any $y\in V$
  \begin{gather}
    0 = \scal(w_1-w_2+u_1-u_2,y) = \scal(w_1-w_2,y) + \scal(u_1-u_2,y).
  \end{gather}
  Choosing $y=u_1-u_2$ and $w_1-w_2$ in turns, we see that one of the
  inner products vanishes for orthogonality and the other implies that
  the difference is zero.

  If $v\in W$, we choose $w=v$ and $u=0$. For $v\not\in W$, we prove
  existence by considering that due to the closedness of $W$ there holds
  \begin{gather}
    d=\inf_{w\in W} \norm{v-w} >0.
  \end{gather}
  Let $w_n$ be a minimizing sequence. Using the parallelogram identity
  \begin{gather}
    \norm{a+b}^2+\norm{a-b}^2 = 2\norm{a}^2+2\norm{b}^2,
  \end{gather}
  we prove that $\{w_n\}$ is a Cauchy sequence by
  \begin{align}
    \norm{w_m-w_n}^2 &= \norm{(v-w_n)-(v-w_m)}^2\\
    &= 2\norm{v-w_n}^2+2\norm{v-w_m}^2-\norm{2v-w_m-w_n}^2\\
    &= 2\norm{v-w_n}^2+2\norm{v-w_m}^2-4\norm*{v-\frac{w_m+w_n}2}^2\\
    &\le 2\norm{v-w_n}^2+2\norm{v-w_m}^2-4d^2,
  \end{align}
  since $(w_m+w_n)/2\in W$ and $d$ is the infimum. Now we use the
  minimizing property to obtain
  \begin{gather}
    \lim_{m,n\to\infty}\norm{w_m-w_n}^2 = 2d^2-2d^2 -4d^2=0.
  \end{gather}
  By completeness of $V$, $w=\lim w_n$ exists and by the closedness of
  $W$, we have $w\in W$. Let $u=v-w$. By continuity of the norm, we
  have $\norm{u}=d$. It remains to show that $u\in \ortho{W}$. To this
  end, we introduce the variation $w+\epsilon \tilde w$ with $\tilde
  w\in W$ to obtain
  \begin{align}
    d^2 &\le \norm{v-w-\epsilon \tilde w}^2\\
    &= \norm{u}^2-2\epsilon\scal(u,\tilde w)+\epsilon^2 \norm{\tilde w},
  \end{align}
  implying for any $\epsilon>0$
  \begin{gather}
    0\le-2\epsilon\scal(u,\tilde w)+\epsilon^2 \norm{\tilde w},
  \end{gather}
  which requires $\scal(u,\tilde w) = 0$.
\end{proof}

\begin{Definition}{orthogonal-projection}
  Let $V$ be a Hilbert space and $W\subset V$ be a closed
  subspace. For a vector $v\in V$, let $v=w+u$ be the unique
  decomposition with $w\in W$ and $u\in \ortho{W}$. Then we call $w$ and
  $u$ the \define{orthogonal projection}s of $v$ into $W$ and $\ortho{W}$,
  respectively. We write
  \begin{gather}
    \Pi_W v = w, \qquad \Pi_{\ortho{W}} v = u.
  \end{gather}
\end{Definition}

% \begin{Lemma}{polar-orthogonal-hilbert}
%   Let $V$ be a Hilbert space and $W\subset V$ be a closed
%   subspace. Then, the polar space $\polar{W}\subset V^*$ and the orthogonal
%   space $\ortho{W}$ can be isometrically identified by \putindex{Riesz
%     representation}.
% \end{Lemma}

% \begin{proof}
%   For every $f$ in the
%   dual of $\ortho{W}$, define $g\in V^*$ by
%   \begin{gather}
%     \scal(g,v)_{V^*\times V} =
%     \scal(f,\Pi_{\ortho{V}}v)_{(\ortho{V})^*\times \ortho{V}}.
%   \end{gather}
%   Clearly, $g(v)=0$ for $v\in W$, therefore $g\in \polar{W}$.
% \end{proof}

% \begin{Corollary}

% \end{Corollary}

\begin{Theorem*}{closed-range}{Closed Range Theorem}
  Let $V,W$ be Hilbert spaces and $A\colon V\to W$ a continuous linear
  operator. Then, the following statements are equivalent:
  \begin{gather}
    \label{eq:infsup:9}
    \begin{split}
      \range A &\text{ is closed in } W,\\
      \range{A^\transpose} &\text{ is closed in } V^*,\\
      \range A &= \dualpolar{\ker{A^\transpose}},\\
      \range{A^\transpose} &= \polar{\ker A}.
    \end{split}
  \end{gather}
\end{Theorem*}

\begin{remark}
  This is the famous \emph{\putindex{closed range theorem}} by Banach.
  It actually holds under weaker assumptions, for instance $V,W$ only
  Banach spaces. The proof can be found for instance
  in~\cite[p.~205--209]{Yosida80}.
\end{remark}

\begin{Theorem*}{open-mapping}{Open Mapping Theorem}
  Let $A\colon V\to W$ be continuous and surjective. Then, the image
  $A(U)\subset W$ of any open set $U\subset V$ is open.
\end{Theorem*}

\begin{remark}
  This is the \emph{\putindex{open mapping theorem}} by Banach. The
  proof can be found for instance in~\cite[p.75--76]{Yosida80}.
\end{remark}

\begin{Lemma}{closed-infsup}
  Let $A\colon V\to W$ be continuous. Then, $\range A$ is closed in
  $W$ if and only if there exists $\ellipa>0$ such that
  \begin{gather}
    \label{eq:infsup:10}
    \forall w\in \range A\;
    \exists v\in V\quad
    Av = w
    \;\wedge\;
    \ellipa \norm{v}_V \le \norm{w}_W.
  \end{gather}
\end{Lemma}

\begin{proof}
  We first show that the inf-sup condition~\eqref{eq:infsup:10}
  implies $\range A$ closed. To this end, let $\{w_n\}$ be a Cauchy
  sequence in $\range A$ converging to a point $w\in W$. By the assumption,
  there
  is a sequence $\{v_n\}$ in $V$ such that $Av_n = w_n$ and
  $\ellipa \norm{v_n} \le \norm{w_n}$. Hence, using
  \begin{gather}
    \norm{v_m-v_n}_V \le \frac1\ellipa \norm{w_m-w_n}_W,
  \end{gather}
  we realize that $\{v_n\}$ is a Cauchy sequence in $V$. Therefore, $v_n\to v\in
  V$ and due to continuity of $A$ we obtain $Av=w$ and thus $w\in
  \range A$.

  Conversely, let $\range A$ be closed in $W$. Thus, it is a Banach
  space and the \putindex{open mapping theorem} applies to $A\colon
  V\to\range A$. We map the open unit ball $B_1(0)\subset V$ and
  obtain that $A(B_1(0))$ is open in $\range A$, implying that there
  is an open ball $B_\delta(0) \subset A(B_1(0))$. This is sufficient
  to construct $v$:

  Let $w\in\range A$. Then,
  \begin{gather}
    \tilde w = \frac\delta2 \frac{w}{\norm{w}}
    \in B_\delta(0) \subset A(B_1(0)).
  \end{gather}
  Hence, there is $v\in V$ with $\norm{v}<1$ such that $Av=\tilde w$,
  which proves the lemma.
\end{proof}

\begin{Theorem}{infsup-well-equivalence}
  Let $a(\cdot,\cdot)$ on $V\times W$ be a bounded bilinear form % such that
%  \begin{gather}
%    a(v,w) \le M \norm{v}_V \norm{w}_W,
%  \end{gather}
  and $A\colon V\to W^*$ its associated operator.
  Then, the following statements are equivalent:
  \begin{enumerate}
  \item There exists $\ellipa>0$ such that
    \begin{gather}
      \label{eq:infsup:11}
      \inf_{w\in W}\sup_{v\in V}
      \frac{a(v,w)}{\norm{v}_V\norm{w}_W}
      \ge \ellipa.
    \end{gather}
  \item The operator $A^\transpose\colon W\to \polar{\ker A}$ is an isomorphism and
    \begin{gather}
      \label{eq:infsup:12}
      \norm{A^\transpose w}_{V^*} \ge \ellipa \norm{w}_{W} \qquad\forall w\in W.
    \end{gather}
  \item The operator $A\colon \ortho{\ker A}\to W^*$ is an isomorphism
    and
    \begin{gather}
      \label{eq:infsup:13}
      \norm{Av}_{W^*} \ge \ellipa\norm{v}_V\qquad \forall v\in \ortho{\ker A}.
    \end{gather}
  \end{enumerate}
\end{Theorem}

\begin{proof}
  First, we show the equivalence of the first two statements. Let us
  use equivalently to the inf-sup condition~\eqref{eq:infsup:11}
  \begin{gather}
    \norm{A^\transpose w}_{V^*}
    = \sup_{v\in V}\frac{\scal(A^\transpose w,v)}{\norm{v}_V}
    = \sup_{v\in V}\frac{a(v,w)}{\norm{v}_V}
    \ge \ellipa\norm{w} \qquad
    \forall w\in W.
  \end{gather}
  Thus, equations~\eqref{eq:infsup:11} and~\eqref{eq:infsup:12} are
  equivalent and we have already proven that the second statement
  implies the first. It remains to show the $A^\transpose$ is an isomorphism
  from $W$ onto $\polar{\ker A}$. Equation~\eqref{eq:infsup:12} implies that
  $A^\transpose\colon W \to \range{A^\transpose}$ is an isomorphism and its inverse is
  bounded by $1/\ellipa$ (multiply both sides by $A^{-1}$). Using
  \slideref{Lemma}{closed-infsup}, we obtain that $\range{A^\transpose}$ is
  closed in $V^*$ and the \putindex{closed range theorem} settles the
  issue.

  In order to prove equivalence of the second and third statement, we
  use the result of \slideref{Problem}{polar-orthogonal} to isometrically
  identify $(\ortho{\ker A})^*$ with $\polar{\ker A}$. Thus, $A$ is an
  isomorphism from $\ortho{\ker A}$ onto $W^*$ if and only if $A^\transpose$ is an
  isomorphism from $W$ onto $(\ortho{\ker A})^* = \polar{\ker A}$. and
  \begin{gather}
    \norm{A}_{W^*\to \ortho{\ker A}} = \norm{A^\transpose}_{\polar{\ker A}\to W}.
  \end{gather}
\end{proof}

\begin{Corollary}{infsup-well-posedness1}
  Let $a(\cdot,\cdot)$ on $V\times W$ be a bounded bilinear form. %such that
  %\begin{gather}
  %  a(v,w) \le M \norm{v}_V \norm{w}_W.
  %\end{gather}
  Let the inf-sup-condition
  \begin{gather}
    \inf_{w\in W}\sup_{v\in V}
    \frac{a(v,w)}{\norm{v}_V\norm{w}_W}
    \ge \ellipa > 0
  \end{gather}
  hold.  Then, the problem finding $w\in W$ such that
  \begin{gather}
    a(v,w) = f(v) \qquad\forall v\in V,
  \end{gather}
  has a unique solution for $f\in \polar{\ker A}$ and
  \begin{gather}
    \norm{w}_W \le \frac1\ellipa \norm{f}_{V^*}.
  \end{gather}
  The opposite implication holds true.
\end{Corollary}

\begin{remark}
  \slideref{Corollary}{infsup-well-posedness1} exhibits an asymmetry
  between the left and right argument. In particular, we obtain a
  unique solution only for the adjoint operator $A^\transpose$, which is
  exactly what we need, when we compute say a pressure from the
  divergence of a velocity field. In general, we consider the
  restriction of $f$ to the polar set of the kernel in the above
  well-posedness result detrimental and would prefer a result that
  holds for all $f\in V^*$. This on the other hand requires
  $\ker A=\{0\}$, or $\overline{\range{A^\transpose}} = W^*$. Then, on the
  other hand, we see that $\range{A^\transpose}$ is closed since $\range{A}$ is
  closed and the closed range theorem holds. Therefore, we obtain the
  following theorem for the case that we require a unique solution for
  all right hand sides.
\end{remark}

\begin{Theorem}{infsup-well-posedness2}
  Let $a(\cdot,\cdot)$ on $V\times W$ be a bounded bilinear form. % such that
%  \begin{gather}
%    a(v,w) \le M \norm{v}_V \norm{w}_W.
%  \end{gather}
  Let for some $\ellipa>0$ the inf-sup-conditions
  \begin{align}
    \inf_{w\in W}\sup_{v\in V}
    \frac{a(v,w)}{\norm{v}_V\norm{w}_W}
    &\ge \ellipa,\\
    \inf_{v\in V}\sup_{w\in W}
    \frac{a(v,w)}{\norm{v}_V\norm{w}_W}
    &\ge \ellipa
  \end{align}
  hold.  Then, the problem finding $v\in V$ such that
  \begin{gather}
    a(v,w) = f(w) \qquad\forall w\in W,
  \end{gather}
  has a unique solution for $f\in W^*$ and
  \begin{gather}
    \norm{v}_V \le \frac1\ellipa \norm{f}_{W^*}.
  \end{gather}
\end{Theorem}

\begin{Problem}{closed-range}
  Assume for $A\colon V\to W^*$ that $\range A$ is closed. Show
  \begin{enumerate}
  \item $A:V\to W^*$ is surjective if and only if $A^\transpose$ is
    injective.
  \item Show that with all other assumptions unchanged, the second
    inf-sup condition in \slideref{Theorem}{infsup-well-posedness2} is
    not only sufficient, but equivalent to $A$ injective.
  \item \slideref{Theorem}{infsup-well-posedness2} seems excessive if
    we only want to establish one of the statements
    \begin{xalignat}3
      \forall f&\in W^*& \exists u&\in V& a(u,w) &= f(w)\\
      \forall g&\in V^*& \exists u&\in W& a(v,u) &= g(v)
    \end{xalignat}
    Is this true or is there equivalence of both inf-sup conditions
    with only one of these statements?
  \end{enumerate}
\end{Problem}

\begin{remark}
  If we compare \slideref{Theorem}{infsup-well-posedness2} with
  \slideref{Corollary}{infsup-well-posedness1}, we see that the only
  difference lies in the fact that the second inf-sup condition
  ensures surjectivity of $A$ by injectivity of $A^\transpose$. In some cases
  it may be impossible difficult to prove both inf-sup conditions. Then, it is
  sufficient to prove one inf-sup condition, say the first, and then
  only injectivity of $A^\transpose$. Although we verify less than the
  assumptions of \slideref{Theorem}{infsup-well-posedness2}, the
  closed range theorem saves us from the additional work. We further
  note that this notion is symmetric between $A$ and $A^\transpose$, that is,
  it is sufficient to prove inf-sup for either operator and
  injectivity for the other.
\end{remark}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{The inf-sup condition for mixed problems}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{intro}
  In the previous section, we have developed a framework for
  well-posedness of problems which are not $V$-elliptic. In principle,
  this theory can be applied to the bilinear form
  $\mathcal A((u,p),(v,q))$ as a whole. On the other hand, we can
  formally split the solution of a constrained minimization problem
  into the reduced problem and then computing the Lagrange multiplier,
  which more clearly exhibits the relation of the two spaces $V$ and
  $Q$ involved in the mixed formulation. Here are the resulting
  theorems.
\end{intro}

\begin{Definition}{mixed-weak}
  The abstract saddle-point problem in weak form reads:
  find $(u,p)\in V\times Q$ such that
  \begin{gather}
    \label{eq:saddle-point-weak}
    \arraycolsep.1em
    \begin{matrix}
      a(u,v) &+& b(v,p) &=& f(v) &\quad&\forall v\in V, \\
      b(u,q) && &=& g(q) &&\forall q\in Q.
    \end{matrix}
  \end{gather}
  Here, $V$ and $Q$ are Hilbert spaces chosen, such that $a(.,.)$ and
  $b(.,.)$ are bounded bilinear forms on $V\times V$ and $V\times Q$,
  respectively. The bounds $\bounda$ and $\boundb$ are such that
  \begin{gather}
    \label{eq:saddle-point-bounds}
    \sup_{u,v\in V}\frac{a(u,v)}{\norm{u}_V\norm{v}_V} = \bounda,
    \qquad
    \sup_{\substack{v\in V\\q\in Q}}\frac{b(v,q)}{\norm{v}_V\norm{q}_Q} = \boundb.
  \end{gather}
\end{Definition}

\begin{Theorem}{infsup-mixed1}
  Let $V$ and $Q$ be Hilbert spaces. Let $a(\cdot,\cdot)$ and
  $b(\cdot,\cdot)$ be bounded bilinear forms. Then, the weak
  formulation find $u\in V$ and $p\in Q$ such that
  \begin{gather}
    a(u,v) + b(v,p) + b(u,q)
    = f(v)+g(q)
    \qquad\forall v\in V, q\in Q,
  \end{gather}
  has a unique solution for any $f\in V^*$ and any $g\in Q^*$ if and
  only if there exists $\ellipa>0$ such that
  \begin{gather}
    \label{eq:infsup:system-infsup}
    \inf_{\substack{u\in V\\p\in Q}}
    \sup_{\substack{v\in V\\q\in Q}}
    \quad \frac{a(u,v) + b(v,p) + b(u,q)}{\norm{(u,p)}_{V\times
        Q}\norm{(v,q)}_{V\times Q}} \ge \ellipa.
  \end{gather}
\end{Theorem}

\begin{proof}
  Straight application of \slideref{Theorem}{infsup-well-posedness2}.
\end{proof}

Often, like in the case of Stokes' equations, the properties of the
form $a(\cdot,\cdot)$ or the form $b(\cdot,\cdot)$ are already known
and you want to combine them to a mixed formulation. Thus, the theorem
above is unwieldy, since it requires to do the analysis from
scratch. We thus provide an equivalence theorem, which allows us to
separate the properties of the bilinear forms.

\begin{Theorem}{infsup-mixed2}
  Let $V$ and $Q$ be Hilbert spaces and let
  \begin{gather}
    \begin{split}
      \ker B &= \bigl\{v\in V \big| b(v,q) = 0 \;\forall q\in Q\bigr\}.
    \end{split}
  \end{gather}
  Then, the mixed problem finding $(u,p)\in V\times Q$ such that
  \begin{gather}
    a(u,v) + b(v,p) + b(u,q) = f(v) \quad\forall v\in V, q\in Q,
  \end{gather}
  is well-posed if and only if the reduced problem
  finding $u\in \ker B$ such that
    \begin{gather}
      a(u,v) = f(v) \quad\forall v\in \ker B
    \end{gather}
    is well-posed for any $f\in V^*$ and there is a positive constant
    $\infsupc$ such that
    \begin{gather}
      \inf_{q\in Q}\sup_{v\in V} \frac{b(v,q)}{\norm{v}_V\norm{q}_Q} \ge \infsupc.
    \end{gather}
\end{Theorem}

\begin{proof}
  In order to show the ``if'', we note that by well-posedness of the
  reduced problem, $u\in V$ is well-determined and bounded by the data
  $f\in V^*$ without knowledge of the Lagrange
  multiplier. Furthermore, there holds $b(u,q) = 0$.

  Entering this into the mixed formulation, the Lagrange multiplier
  $p$ is determined by
  \begin{gather}
    b(v,p) = f(v) - a(u,v), \qquad\forall v\in V.
  \end{gather}
  Applying \slideref{Corollary}{infsup-well-posedness1} to the
  bilinear form $b(.,.)$, we deduce that this equation has a unique
  solution $p\in Q$ if and only if $f-a(u,\cdot) \in \polar{\ker B}$, which
  is true due to the statement of the reduced problem.

  For the only if, we note that choosing $v=0$ in the mixed problem,
  we see that there holds
  \begin{gather}
    b(u,q) = 0 \qquad \forall q\in Q,
  \end{gather}
  and thus $u\in \ker B$. For such $u$ holds in the mixed formulation
  \begin{gather}
    f(v) = a(u,v) + b(v,p) = a(u,v) \qquad \forall v \in \ker B.
  \end{gather}
  In order to deduce the inf-sup condition, we note that
  well-posedness of the mixed problem implies the inf-sup
  condition~\eqref{eq:infsup:system-infsup}. Since the infimum will
  not decrease by taking a subset, we confine the set to $u=0$ and
  obtain
  \begin{align}
    &\inf_{p\in Q}\sup_{v\in V} \frac{b(v,p)}{\norm{v}_V\norm{q}_Q}
    \\ = &
           \inf_{p\in Q}
           \sup_{\substack{v\in V\\q\in Q}}
    \frac{a(0,v) + b(v,p) + b(0,q)}{\norm{(0,p)}_{V\times Q} \norm{(v,q)}_{V\times Q}}
    \\ \ge&
            \inf_{\substack{u\in U\\p\in Q}}
    \sup_{\substack{v\in V\\q\in Q}}
    \frac{a(u,v) + b(v,p) + b(u,q)}{\norm{(u,p)}_{V\times Q} \norm{(v,q)}_{V\times Q}} \\\ge& \ellipa > 0
  \end{align}
\end{proof}

\begin{Problem}{inhomogeneous-continuity}
  Show that \slideref{Theorem}{infsup-mixed2} can be extended to the
  case with right hand side $f(v)+g(q)$ with $g\in Q^*$.

\begin{solution}
  We want to solve the problem
  \begin{align}
    a(u,v) + b(v,p) + b(u,q) = f(v)+g(q) \quad\forall v\in V, q\in Q,
  \end{align}
  where $b(v,p)$ fulfills a inf-sup condition.

  \begin{enumerate}
  \item Due to \slideref{Theorem}{infsup-well-equivalence}, the
    operator $B: V\to Q^*$ is surjective and thus, there exists
    $u_g\in V$ such that
    \begin{gather}
      b(u_g,q) = q(q) \quad\forall q\in Q.
    \end{gather}
  \item Now consider the function $u_0 = u-u_g$. For $u$ to solve the
    original problem $u_0$ has to solve
    \begin{align}
      a(u_0+u_g,v) + b(v,p) + b(u_0+u_g,q) = f(v)+g(q) \quad\forall v\in V, q\in Q\\
      \Leftrightarrow a(u_0,v) + b(v,p) + b(u_0,q) = f(v)-a(u_g,v) \quad\forall v\in V, q\in Q
    \end{align}
  \item Due to $a(u_g,v) \leq \bounda \norm{u_g}_V\norm{v}_V$,
    the right-hand side $f(\cdot)-a(u_g,\cdot)$ is in $V^*$
    and we are in the setting of
    \slideref{Theorem}{infsup-mixed2}.
  \end{enumerate}
\end{solution}
\end{Problem}

\begin{remark}
  Since $V$ is a Hilbert space, the decomposition
  $V = \ker B \oplus \ker B^\perp$ is uniquely determined and there is
  a corresponding decomposition $V^* = \polar{(\ker B^\perp)} \oplus \polar{\ker B} = I_\flat\ker B \oplus I_\flat \ortho{\ker B} $,
  such that $f = f^0+f^\perp$ above. The way we solve the reduced
  problem first and then compute the Lagrange multiplier implies that
  the solution $u$ only depends on $f^\perp$ only.
\end{remark}

\begin{remark}
  We have imposed well-posedness of the reduced problem only in an
  abstract way. Depending on $a(.,.)$ we can formulate two conditions:
  ellipticity on $\ker B$ or inf-sup stability on $\ker B$. Indeed,
  most problems considered in this class will have symmetric bilinear
  forms $a(.,.)$, such that ellipticity serves as our usual
  assumption.  In these cases, note that $V$-ellipticity already
  implies the well-posedness on $\ker B$.
\end{remark}

\begin{Notation}{v0-kernel}
  Since the kernels of the bilinear form $b(\cdot,\cdot)$ play an
  important role, we abbreviate
  \begin{gather}
    \begin{split}
      V^0 = \ker B &= \bigl\{v\in V \;\big|\; b(v,q) = 0 \quad\forall q\in Q\bigr\},\\
      Q^0 = \ker{B^\transpose} &= \bigl\{q\in Q \;\big|\; b(v,q) = 0 \quad\forall v\in V\bigr\}.
    \end{split}
  \end{gather}
  We als define for $g\in Q^*$ and for $f\in V^*$ the affine spaces
  \begin{gather}
    \begin{split}
      V^g &= \bigl\{v\in V \;\big|\; b(v,q) = g(q) \quad\forall q\in Q\bigr\},\\
      Q^f &= \bigl\{q\in Q \;\big|\; b(v,q) = f(v) \quad\forall v\in V\bigr\}.
    \end{split}
  \end{gather}
\end{Notation}

\begin{intro}
  We summarize the results of this section in a theorem for
  well-posedness of the mixed formulation in
  \slideref{Definition}{mixed-weak} with simplified assumptions.  It
  will be the basis for further results in this course. We know from
  the discussion above that this assumption is only sufficient and
  weaker conditions may be imposed on $a(.,.)$. But indeed, it helps
  us through a lot of problems and is a good compromise between
  generality and ease of use.
\end{intro}

\begin{Theorem}{mixed-elliptic}
  Let the bilinear form $a(.,.)$ be positive semi-definite on $V$ and
  elliptic on $V^0 = \ker B$. Let the bilinear form $b(\cdot,\cdot)$
  be inf-sup stable. Thus, there are constants $\ellipa>0$ and
  $\infsupc>0$
  \begin{gather}
    \inf_{v\in V^0} \frac{a(v,v)}{\norm{u}_V^2} = \ellipa,  \qquad
    \inf_{q\in Q}\sup_{v\in V} \frac{b(v,q)}{\norm{v}_V\norm{q}_Q} = \infsupc.
  \end{gather}
  Then, the abstract mixed problem in \slideref{Definition}{mixed-weak}
  has a unique solution.
\end{Theorem}

\begin{Problem}{mixed-elliptic}
  Derive bounds for the solutions $u\in V$ and $p\in Q$ using the
  assumptions of the previous theorem.
\end{Problem}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Galerkin approximation of mixed problems}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\input{../mixed/galerkin}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


\section{Discriminant Analysis I}

\begin{frame}
  \frametitle{Discriminant Analysis}

  \structure{Discriminant analysis} methods are \emph{discriminative modeling} methods that model the posterior through its factorization

  \begin{displaymath}
    p(y|\vec x) = \frac{p(y)\cdot p(\vec x|y)}{\sum_y p(y)\cdot p(\vec x|y)}
  \end{displaymath}
\end{frame}

\subsection{Gaussian Classifier}

\begin{frame}
  \frametitle{Gaussian Classifier}

  We call the Bayesian classifier \structure{Gaussian}, if the class conditional density $p(\vec x|y)$ is Gaussian, i.\,e.

  \begin{eqnarray*}
     p(\vec x | y) &=& \mathcal {N} (\vec x ;\vec {\mu}_y, \mat\Sigma_y) \\
                   &=& \frac{1}{\sqrt{\det 2\pi  \mat\Sigma_y}}
                       e^{-\frac{1}{2}(\vec x -\vec{\mu}_y)^T \mat\Sigma_y^{-1}(\vec x - \vec{\mu}_y)}
  \end{eqnarray*}
%
  where
%
  \begin{center}
    \begin{minipage}{0.6\textwidth}
      \begin{itemize}
        \item[$\vec x \in \real^d$:] $d$-dimensional feature vector\\
        \item[$\vec {\mu}_y \in \real^d $:] mean vector of class y\\
        \item[$ \mat\Sigma_y \in \real^{d\times d}$:] positive definite covariance matrix.
      \end{itemize}
    \end{minipage}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Gaussian Classifier \cont}

  \structure{Facts about Gaussian classifiers:}

  \begin{itemize}
    \item In general the decision boundary is \structure{quadratic} in the components $x_i$ \\
      of the feature vector $\vec x$. \\[.5cm]
    \item If all classes share the same covariance, the decision boundary is \structure{linear} \\
      in the components $x_i$ of the feature vector $\vec x$. \\[.5cm]
    \item If all covariance matrices are diagonal matrices, \\
      then we get a \structure{Na{\"i}ve Bayes} classifier.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Gaussian Classifier \cont}

  \structure{Facts about Gaussian classifiers (cont.):}

  \begin{itemize}
    \item If the joint covariance matrix is $\mat\Sigma$ and priors are identical, \\
      classification requires the minimization of the \structure{Mahalanobis distance}
      \begin{eqnarray*}
        y^* &=& \argmin_{y} \frac{1}{2} (\vec x-\vec\mu_y)^T\mat\Sigma^{-1}(\vec x-\vec\mu_y)
      \end{eqnarray*}
      \pause
    \item If all covariance matrices are the identity matrix, we get the \\
      \structure{Nearest Neighbor} classifier based on the $L_2$-norm:
      \begin{eqnarray*}
        y^* &=& \argmin_{y} \frac{1}{2} (\vec x-\vec\mu_y)^T(\vec x-\vec\mu_y)
      \end{eqnarray*}
      The prototype vectors are the mean vectors.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Gaussian Classifier \cont}

  \structure{From linear to quadratic decision boundaries:} \\[.5cm]

  A compromise between linear and quadratic decision boundaries \\
  can be achieved by using \structure{regularized covariance matrices}:
%
  \begin{eqnarray*}
    \mat\Sigma_y (\alpha)= \alpha \mat\Sigma_y + (1-\alpha)\mat\Sigma
  \end{eqnarray*}
%
  where $\alpha\in [0,1]$ and $\mat\Sigma$ denotes the joint covariance.
  \spread

  Obviously we have the extremes:
  \begin{itemize}
    \item Linear decision boundary: $\quad ~~\alpha = 0$
    \item Quadratic decision boundary: $\alpha =1$
  \end{itemize}
\end{frame}

\subsection{Feature Transform}

\begin{frame}
  \frametitle{Feature Transform}

  Can we find a feature transform
%
  \begin{eqnarray*}
    \phi: \real^d &\rightarrow &\real^d
   \end{eqnarray*}
%
  to generate features $\phi(\vec x)$ that share the same covariance matrix?
\end{frame}

\begin{frame}
  \frametitle{Feature Transform \cont}

  The symmetric positive semidefinite covariance matrix $\mat\Sigma\in \real^{d\times d}$ \\
  can be decomposed using SVD:
%
  \begin{eqnarray*}
    \mat\Sigma&=& \pause \mat{U}\mat{D}\mat{U}^T \pause = (\mat{U}\mat{D}^{\frac{1}{2}})(\mat{U}\mat{D}^{\frac{1}{2}})^T \pause =
    (\mat{U}\mat{D}^{\frac{1}{2}})\cdot \mat{I} \cdot(\mat{U}\mat{D}^{\frac{1}{2}})^T
  \end{eqnarray*}
%
  where $\mat{I}\in \real^{d\times d}$ is the identity matrix.
  \pause

  \begin{itemize}
    \item \structure{Determinant:}
      \begin{displaymath}
        \det \mat\Sigma = \prod_{i=1}^d d_{i,i},
      \end{displaymath}
      where $d_{i,i}$ are the diagonal elements of $\mat D$, i.\,e.\ the \structure{singular values}. \pause
    \item \structure{Inverse:}
      \begin{displaymath}
        \mat\Sigma^{-1} = \mat{U}\mat{D}^{-1}\mat{U}^T
                          = (\mat{U}\mat{D}^{-\frac{1}{2}})\cdot \mat{I} \cdot(\mat{U}\mat{D}^{-\frac{1}{2}})^T
      \end{displaymath}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Feature Transform \cont}

   Now we incorporate this:

  \begin{eqnarray*}
    \mathcal {N} (\vec x ;\vec {\mu}, \mat\Sigma)
      &=& \frac{1}{\sqrt{\det 2\pi  \mat\Sigma}}
          e^{-\frac{1}{2}(\vec x -\vec{\mu})^T \, \mat\Sigma^{-1} \, (\vec x - \vec{\mu})} \\[.3cm] \pause
      &=& \frac{1}{\sqrt{\det 2\pi  \mat\Sigma}}
          e^{-\frac{1}{2}(\vec x -\vec{\mu})^T \, (\mat{U}\mat{D}^{-\frac{1}{2}})\cdot \mat{I} \cdot(\mat{U}\mat{D}^{-\frac{1}{2}})^T \, (\vec x - \vec{\mu})} \\[.3cm] \pause
      &=& \frac{1}{\sqrt{\det 2\pi  \mat\Sigma}}
          e^{-\frac{1}{2} \big( (\mat{D}^{-\frac{1}{2}}\mat{U}^T)\vec x - (\mat{D}^{-\frac{1}{2}}\mat{U}^T)\vec\mu \big)^T \,\mat{I} ~
          \big( (\mat{D}^{-\frac{1}{2}}\mat{U}^T)\vec x- (\mat{D}^{-\frac{1}{2}}\mat{U}^T)\vec\mu \big) }
  \end{eqnarray*}
\end{frame}

\begin{frame}
  \frametitle{Feature Transform \cont}

  The classwise transform $\phi_y$ is even a linear function:
%
  \begin{displaymath}
    \vec x' \quad = \quad \phi_y(\vec x) \quad = \quad \mat{D}_y^{-\frac{1}{2}}\mat{U}^T_y\vec x
  \end{displaymath}
  \pause

  It is straight forward  to show that $\vec x'$ is normally distributed
  \begin{displaymath}
     p(\vec x' | y) \quad = \quad \mathcal {N} (\vec x' ; \vec {\mu}'_y, \mat\Sigma'_y)
                    \quad = \quad \mathcal {N} (\vec x' ; \mat{D}_y^{-\frac{1}{2}}\mat{U}^T_y\vec {\mu}_y, \mat{I})
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Feature Transform \cont}

  \structure{Conclusions:}

  \begin{itemize}
    \item All classes $y$ share the same covariance matrix that is the identity matrix. \\[.3cm]
    \item The decision boundary is linear. \\[.5cm] \pause
    \item \vorsicht \structure{Huge disadvantage:} \\
      feature transform depends on class number $y$! \\[.3cm]
    \item If we have a classified training set, we can compute a transform \\
      for each class such that all covariance matrices are the identity matrix. \\[.3cm]
    \item Classification requires the application of different transforms.
  \end{itemize}
\end{frame}

\input{nextTime.tex}

\subsection{Linear Discriminant Analysis}

\begin{frame}
  \frametitle{Linear Discriminant Analysis}

  \begin{algorithmic}
    \STATE \structure{Input:} training data:  $S = \{ (\vec x_1, y_1), (\vec x_2, y_2), (\vec x_3, y_3), \dots, (\vec x_m, y_m) \}$
      \pause
    \STATE 1. ML estimation of the \structure{joint covariance matrix}:
      \begin{displaymath}
        \widehat{\mat\Sigma}= \frac{1}{m} \sum_{i=1}^m (\vec x_i - \vec \mu_{y_i})(\vec x_i - \vec \mu_{y_i})^T
      \end{displaymath}
      \pause
     \STATE 2. Compute SVD of covariance matrix: $\widehat{\mat \Sigma}= \mat{U}\mat{D}\mat{U}^T$ \pause
     \STATE 3. Assign transform:
       \begin{displaymath}
         \phi= \mat D^{-\frac{1}{2}} \mat{U}^T
       \end{displaymath}
       \pause
     \STATE 4. Compute mean vectors for all $y$
       \begin{displaymath}
          \vec{\mu}'_y = \phi(\vec\mu_y)= \mat{D}^{-\frac{1}{2}} \mat{U}^T \vec{\mu}_y
       \end{displaymath}
       \pause \vspace{-.5cm}
     \STATE \structure{Output:} feature transform $\phi$, transformed mean vectors $\vec{\mu}'_y$
  \end{algorithmic}
\end{frame}

\begin{frame}
  \frametitle{Linear Discriminant Analysis \cont}

  Decision rule using \structure{sphered data $\phi(\vec x)$}:

  \begin{eqnarray*}
    y^* &=& \pause \argmax_y p(y|\vec \phi(\vec x))\\ \pause
        &=& \argmax_y
              \left\{
                \log p(y) - \frac{1}{2} \big( \phi(\vec x) - \phi(\vec\mu_y) \big)^T \big( \phi(\vec x)-\phi(\vec\mu_y) \big)
              \right\} \\ \pause
        &=& \argmin_y \left\{ \frac{1}{2}\left\|\phi(\vec x)-\phi(\vec\mu_y) \right\|_2^2 - \log p(y) \right\}
  \end{eqnarray*}

  where $\|.\|_2$ denotes the $L_2$ norm.
\end{frame}

\begin{frame}
  \frametitle{Linear Discriminant Analysis \cont}

  \structure{Conclusions:}

  \begin{itemize}
    \item If all classes share the \structure{same prior}, \\ the decision rule is the
      \structure{Nearest Neighbor} decision rule, \\
      where transformed mean vectors serve as prototypes. \\[.5cm]
    \item The feature transform $\phi$ does not change the dimension of features.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Linear Discriminant Analysis \cont}

  \begin{figure}
    \resizebox{0.35\linewidth}{!}{
      \input{\texfigdir/lda_nearest_neighbor.pstex_t}
    }
    \caption{Nearest Neighbor classification for two classes}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Linear Discriminant Analysis \cont}

  \structure{2 classes:} insights from geometrical analysis of sphered data \\[.3cm]

  \begin{itemize}
    \item Angle between $\phi(\vec x)$ and $(\phi(\vec \mu_1)-\phi(\vec \mu_0))$ can be used \\
      for decision making.\\[.5cm]
    \item Decision rule:
      {\small
      \begin{displaymath}
        y^* = \left\{
                \begin{array}{cl}
                  0, & \quad \mbox{if} \quad
                       \phi(\vec x)^T \big( \phi(\vec\mu_1) - \phi(\vec\mu_0) \big) <
                       \frac{1}{2} \big(
                         \phi(\vec{\mu}_1)^T \phi(\vec{\mu}_1) -
                         \phi(\vec{\mu}_0)^T \phi(\vec{\mu}_0)
                        \big) \\[.3cm]
%                       \big( \phi(\vec{\mu}_1) + \phi(\vec{\mu}_0) \big)^T
%                       \big( \phi(\vec{\mu}_1) - \phi(\vec{\mu}_0) \big) \\[.3cm]
                  1, & \quad \mbox{otherwise}.
                \end{array}
              \right.
      \end{displaymath}
      }
    \item Coordinate orthogonal to the 1-D subspace spanned by $(\phi(\vec \mu_1)-\phi(\vec \mu_0))$ does not affect relative distances.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Linear Discriminant Analysis \cont}

  \structure{$K$ classes:} insights from geometrical analysis of sphered data \\[.3cm]

  \begin{itemize}
    \item Class centroids span $(K-1)$-dimensional subspace. \\[.5cm]
    \item Relative differences are not affected by coordinates in the $(d-K+1)$-dimensional
          subspace that is orthogonal to the  $(K-1)$-dimensional subspace spanned
          by class centroids.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Linear Discriminant Analysis \cont}

  \structure{Objective:} \\[.5cm]

  Will we gain an advantage if we transform features by $$\phi: \real^d\rightarrow \real^k$$ in higher $(k>d)$ or lower dimensional $(k<d)$ spaces?
\end{frame}

\subsection{Lessons Learned}

\begin{frame}
  \frametitle{Lessons Learned}

 \begin{itemize}
    \item Relationship between Bayesian classifier, Gaussian classifier, and \\
      Nearest Neighbor classifier. \\[.5cm]
    \item Mahalanobis distance \\[.5cm]
    \item Linear Discriminant Analysis is a regularized Nearest Neighbor classifier \\[.5cm]
    \item Class centroids span $(K-1)$-dimensional subspace
  \end{itemize}
\end{frame}

\input{nextTime.tex}

\subsection{Further Readings}

\begin{frame}
  \frametitle{Further Readings}

  You are required to be familiar with \structure{linear algebra} and \structure{matrix calculus}:

  \begin{itemize}
    \item SIAMS best selling book in the last decade:\\[.15cm]
      Lloyd N. Trefethen, David Bau III: \\
      \structure{Numerical Linear Algebra}, \\
      SIAM, Philadelphia, 1997. \\[0.15cm]
    \item All about matrix derivatives and related problems is described in the Matrix Cookbook:
      \structure{\url{http://www.matrixcookbook.com}} \\[.3cm]
  \end{itemize}

  Basics on \structure{discriminant analysis} can be found in

  \begin{itemize}
    \item T. Hastie, R. Tibshirani, and J. Friedman: \\
      \structure{The Elements of Statistical Learning --}\\
      \structure{ Data Mining, Inference, and Prediction},\\
      2nd edition, Springer, New York, 2009.
   \end{itemize}
\end{frame}

\subsection{Comprehensive Questions}

\begin{frame}
  \frametitle{Comprehensive Questions}

  \begin{itemize}
    \item What is a Gaussian classifier? \\[1cm]
    \item What is the idea behind the feature transform for the LDA? \\[1cm]
    \item Formulate the LDA for normally distributed classes. \\[1cm]
    \item What is the dimensionality of the LDA subspace for $K$ classes?
  \end{itemize}
\end{frame}


\section{Math equations}

\subsection{Inline equations (in text)}
Have you ever heard of Pythagorean theorem: \(a^2 + b^2 = c^2\) ? It's pretty neat.

\subsection{Display equations (n separate lines)}
In physics, the mass-energy equivalence is stated by the equation:
\begin{center}
    $E=mc^2$
\end{center}
Albert Einstein discovered it in 1905!

\subsection{Fractions, summations, products, roots, powers}
    \textbf{Fractions}\\
        This is 0.5 presented as a fraction: \( \frac{1}{2} \). Neat right?\\
        \vspace{0.5cm}

    \textbf{Summation can be inserted as shown on Eqn. \ref{eq:sum}}\\
        \begin{equation}
            \sum_{x=1}^{n}x^2=1
            \label{eq:sum}
        \end{equation}
        \vspace{0.5cm}

    \textbf{Product sequence can be inserted as shown in Eqn. \ref{eq:prod}}\\
        \begin{equation}
            \prod_{x=1}^{n}x^2
            \label{eq:prod}
        \end{equation}
        \vspace{0.5cm}

    \textbf{Square Root}\\
        The square root of 100 is \(\sqrt{100}=10\).
        \vspace{0.5cm}

    \textbf{Power}\\
        Two to the power of five is \(2^5 = 32 \).
        \vspace{0.5cm}


\subsection{2016 Free-Response Answers}

\begin{enumerate}
	\item \begin{enumerate}
		\item We can approximate $R^\prime(2)$ as the tangent slope between 1 and 3.
			\begin{equation*}
				R^\prime(2) \approx \frac{R(3)-R(1)}{3-1} = \frac{950-1190}{2} = -120.
			\end{equation*}
			So, we estimate $R^\prime(3)$ to be -120 liters/hour$^2$.
		\item Using a left Riemann sum using the values in the table,
			\begin{equation*}
				\int_{0}^{8}{R(t)\d{t}} \approx (1-0)1340 + (3-1)1190 + (6-3)950 + (8-6)740 = 1340 + 2380 + 2850 + 1480 = 8050.
			\end{equation*}
			So, the left Riemann sum approximates the total water removed to be 8050 liters.
		\item Using our answer from part (b) and integrating, we can find the total amount of water added or removed.
			\begin{equation*}
				\Delta \text{Water} = -8050 + \int_{0}{8}{2000e^{-t^2/20}\d{t}} \approx -214.
			\end{equation*}
			Since we know there is 50000 liters of water at $t=0$, we can add the net change to find the amount of water at $t=8$.
			\begin{equation*}
				\text{Water}_8 = \text{Water}_0 + \Delta \text{Water} = 50000 - 214 = 49768.
			\end{equation*}
			So, to the nearest liter, we approximate the amount of water in the tank at $t=8$ to be 49768 liters.
		\item At $t=0$, $W(0) = 2000 > R(0) = 1340$, so $W(0)-R(0) > 0$.
			At $t=8$, $W(8) = 81.52 < R(8) = 700$, so $W(8)-R(8) < 0$.
			Since both $W$ and $R$ are continuous, so is $W - R$.
			So, by the Intermediate Value Theorem, there is some $0 < t < 8$ such that $W(t)-R(t)=0$, or $W(t)=R(t)$.
	\end{enumerate}

	\item \begin{enumerate}
		\item We can apply the Fundamental Theorem of Calculus to find $x(3)$.
			\begin{equation*}
				\int_{0}^{3}{\dd{x}{t}\d{t}} = x(3) - x(0) = x(3) - 5 = 9.377 \implies x(3) = 14.377.
			\end{equation*}
			Looking at the graph, we see that $y(3)=-\frac{1}{2}$.
			So, at $t=3$, the particle's position is $\left(14.377, -0.5\right)$.
		\item Looking at the graph, we see that $y^\prime(3) = \frac{1}{2}$.
			Evaluating $\dd{x}{t}$ at $t=3$, we see that $x^\prime(3) = 9+\sin{27}$.
			So,
			\begin{equation*}
				\dd{y}{x} = \frac{\dd{y}{t}}{\dd{x}{t}} = \frac{\frac{1}{2}}{9+\sin{27}} \approx 0.0502.
			\end{equation*}
		\item Using our formula for parametric speed,
			\begin{equation*}
				s = \sqrt{(x^\prime(t))^2 + (y^\prime(t))^2} = \sqrt{\left(\frac{1}{2}\right)^2 + \left(9+\sin{27}\right)^2} \approx 9.969.
			\end{equation*}
		\item Starting with the formula for parametric arc length and splitting the integral into two peices,
			\begin{align*}
				\int_{0}^{2}{\sqrt{\left(y^\prime(t)\right)^2+\left(x^\prime(t)\right)^2}\d{t}} &= \int_{0}^{1}{\sqrt{(-2)^2+(t^2+\sin{(3t^2)})^2}\d{t}} + \int_{1}^{2}{\sqrt{(0)^2+(t^2+\sin{(3t^2)})^2}\d{t}} \\
				&\approx 2.237 + 2.112 \\
				&= 4.439.
			\end{align*}
	\end{enumerate}

	\item \begin{enumerate}
		\item Since we know by the Fundamental Theorem of Calculus that $f$ is the derivative of $g$, $g$ has critical points where $f$ is 0.
			$x=10$ is one such critical point.
			Both to the left and right of $x=10$, $f$ is negative.
			So, although $x=10$ is a critical point, it is neither a relative minimum or relative maximum of $g$ because $g$ is decreasing both left and right of $x=10$.
		\item Inflection points occur when the second derivative changes sign.
			Since $f$ is the derivative of $g$, inflection points of $g$ occur when the derivative of $f$ changes sign.
			To the left of $x=4$, the derivative of $f$ is positive.
			To the right of $x=4$, the derivative of $f$ is negative.
			So, $x=4$ is indeed an inflection point for $g$.
		\item $g$ has critical points where $f$ is 0, which is at $x=-2$, $x=2$, $x=6$, and $x=10$.
			Of these, $x=2$ and $x=10$ do not change sign, so they cannot be absolute extrema.
			Evaluating the remaining critical points and the endpoints using the geometry of the graph of $f$,
			\begin{align*}
				g(-4) &= -4 \\
				g(-2) &= -8 \\
				g(6) &= 8 \\
				g(12) &= -4.
			\end{align*}
			So, the absolute minimum of $g$ is at $x=-2$, and the absolute maximum of $g$ is at $x=6$.
		\item To the left $x=2$, $g(x)$ is negative whenever there is more area between $f$ and the $x$-axis above the $x$-axis than below.
			So, all points $[-4,2]$ have $g(x) \leq 0$.
			To the left of $x=2$, the opposite is true.
			So, all points $[10,12]$ have $g(x) \leq 0$.
			Putting these two results together, $g(x) \leq 0$ on $[-4,2] \cup [10,12]$.
	\end{enumerate}

	\item \begin{enumerate}
		\item Implicitly differentiating,
			\begin{equation*}
				\d{^2y}{x^2} = 2x - \frac{1}{2}\dd{y}{x} = 2x - \frac{1}{2}\left(x^2 - \frac{1}{2}y\right).
			\end{equation*}
		\item Since we have expression for the first and second derivatives, it makes the most sense to apply a second derivative test.
			First, checking that $(-2,8)$ is indeed a critical point,
			\begin{equation*}
				\dd{y}{x}_{(-2,8)} = (-2)^2 - \frac{1}{2}(8) = 0.
			\end{equation*}
			Next, evaluating the second derivative,
			\begin{equation*}
				\dd{^2y}{x^2}_{(-2,8)} = 2(-2) - \frac{1}{2}\left((-2)^2 - \frac{1}{2}(8)\right) = -4 - 0 = -4.
			\end{equation*}
			Since the second derivative is negative at this critical point, the second derivative test tells us that this is a relative maximum.
		\item Since we know that $g(-1)=2$, both the numerator and denominator of the limit are in an indeterminate form of 0/0.
			So, we can apply L'H\^{o}pital's Rule.
			\begin{equation*}
				\lim_{x\to -1}{\left(\frac{g(x)-2}{3(x+1)^2}\right)} = \lim_{x\to-1}\left(\frac{g^\prime(x)}{6(x+1)}\right).
			\end{equation*}
			Using the given differential equation, we can see that at $(-1,2)$, $g^\prime(-1)=0$.
			So again we have an indeterminate form of 0/0 and can apply L'H\^{o}pital's Rule.
			\begin{equation*}
				\lim_{x\to-1}\left(\frac{g^\prime(x)}{6(x+1)}\right) = \lim_{x\to -1}{\frac{g^{\prime\prime}(x)}{6}}.
			\end{equation*}
			Using our answer from part (b), we know that at $(-1,2)$, $g^{\prime\prime}(-1) = -2$.
			So,
			\begin{equation*}
				\lim_{x\to -1}{\frac{g^{\prime\prime}(x)}{6}} = \frac{-2}{6} = -\frac{1}{3}.
			\end{equation*}
		\item Applying two iterations of Euler's method starting at $(0,2)$ with $\Delta x = \frac{1}{2}$,
			\begin{table}[H]
				\begin{center}
					\begin{tabular}{|c|c|c|c|c|}
						\hline
						$(x,y)$ & $\dd{y}{x}$ & $\Delta x$ & $\Delta y = \Delta x\dd{y}{x}$ & $(x+\Delta x, y+\Delta y)$ \\
						\hline
						$(0,2)$ & $-1$ & $\frac{1}{2}$ & $-\frac{1}{2}$ & $(\frac{1}{2},\frac{3}{2})$ \\
						\hline
						$(\frac{1}{2},\frac{3}{2})$ & $-\frac{1}{2}$ & $\frac{1}{2}$ & $-\frac{1}{4}$ & $(1,\frac{5}{4})$ \\
						\hline
					\end{tabular}
				\end{center}
			\end{table}
			So, our application of Euler's method approximates $h(1)$ to be 5/4.
	\end{enumerate}

	\item \begin{enumerate}
		\item Applying the formula for average value,
			\begin{equation*}
				\frac{1}{10-0}\int_{0}^{10}{\frac{1}{20}\left(3+h^2\right)\d{h}} = \frac{1}{200}\left(3h+\frac{h^3}{3}\right) = \frac{109}{60}.
			\end{equation*}
			So, the average radius of the funnel is $\frac{109}{60}$ inches.
		\item Applying the volume formula for a solid of revolution,
			\begin{align*}
				V &= \pi\int_{0}^{10}{\left(\frac{1}{20}\left(3+h^2\right)\right)\d{h}} \\
				&= \frac{\pi}{400}\left(\frac{h^5}{5}+2h^3+9h\right)\biggr\rvert_0^{10} \\
				&= \frac{\pi}{400}\left(20000+2000+90\right) \\
				&= \frac{2209\pi}{40}.
			\end{align*}
		\item Applying the chain rule,
			\begin{align*}
				\dd{r}{t} &= \dd{r}{h}\dd{h}{t} \\
				&= \frac{1}{10}h\dd{h}{t}.
			\end{align*}
			Solving with the information given,
			\begin{equation*}
				-\frac{1}{5} = \frac{1}{10}(3)\dd{h}{t} \implies \dd{h}{t} = -\frac{2}{3}.
			\end{equation*}
			So, at this instant, the height of the liquid is decreasing at 2/3 inches per second.
	\end{enumerate}

	\item \begin{enumerate}
		\item Finding several derivatives at $x=1$,
			\begin{align*}
				f(1) &= 1 \\
				f^\prime(1) &= -\frac{1}{2} \\
				f^{\prime\prime}(1) &= \frac{1}{4} \\
				f^{(3)}(1) &= -\frac{1}{4}.
			\end{align*}
			Applying the Taylor Series formula centered at $x=1$,
			\begin{equation*}
				f(x) = 1 - \frac{1}{2}(x-1) + \frac{1}{8}(x-1)^2 - \frac{1}{24}(x-1)^3 + \ldots + (-1)^n\frac{1}{n2^n}(x-1)^n.
			\end{equation*}
		\item Since we know that the series is centered at $x=1$ and the radius of convergence is 2, we know the series converges on $(-1,3)$ and need to check the endpoints.
			When $x=-1$,
			\begin{equation*}
				\sum_{n=1}^{\infty}{(-1)^n\frac{1}{n2^n}(-1-1)^n} = \sum_{n=1}^{\infty}{\frac{1}{n}}
			\end{equation*}
			diverges by the P-Test.
			When $x=3$,
			\begin{equation*}
				\sum_{n=1}^{\infty}{(-1)^n\frac{1}{n2^n}(3-1)^n} = \sum_{n=1}^{\infty}{(-1)^n\frac{1}{n}}
			\end{equation*}
			converges by the Alternating Series Test.
			So, the invterval of convergence is $(-1,3]$.
		\item Using the first three terms of our series from part (a) with $x=1.2$,
			\begin{equation*}
				f(1.2) \approx 1 - \frac{1}{2}(1.2-1) + \frac{1}{8}(1.2-1)^2 = 1 - \frac{1}{10} + \frac{1}{200} = 0.905.
			\end{equation*}
		\item Since this series is alternating, we can apply the Alternating Series Estimation Theorem.
			\begin{equation*}
				\abs{f(1.2)-P_2(1.2)} \leq \abs{\frac{-1}{2^3\cdot 3} (.2)^3 } = \frac{1}{3000} \leq 0.001.
			\end{equation*}
			So, by the Alternating Series Estimation Theorem, the error is certainly at most 0.001.
	\end{enumerate}
\end{enumerate}