%!TEX root = RBM.tex

\subsubsection{Rao-Blackwellized Tempered Sampling\protect\footnote{Available at \protect\url{https://github.com/lzhbrian/MCMC/blob/master/src/partition/RTS.m} in Matlab}}

\para{Algorithm}
Similar to AIS, Rao-Blackwellized Tempered (RTS)\cite{carlson2016partition} Sampling also has a set of inverse temperatures $\{0= \beta_{1} < \beta_{2} < ... < \beta_{K} =1\}$, which can define a sequence of
\begin{equation}
f_{k}(\mathbf x) \propto f(\mathbf x)^{\beta_{k}} p_{1}(\mathbf x)^{1-\beta_{k}}
\end{equation}

Different from AIS, we do not traverse $\mathbf \beta$. Instead we sample a $\beta^{*}$ every loop, from the $\mathbf \beta$ set with the distribution $(\beta|x)$.

Subsequently, we sample from $x_{k}$ to $x_{k+1}$ by the probability of $q(x|\beta^{*})$ just like what we did in AIS, shown in Figure~\ref{fig:xkxk1}. However, what also different from AIS is that, we have to iterate from $x_{k}$ to $x_{k+1}$ many times(i.e. 50 times in\cite{carlson2016partition} ) for the sake of getting a better $x_{k+1}$.

At the last of every loop, we update the lower variance estimator $\hat{\mathbf c}$ by
\begin{equation}
\hat{c}_{k} = \hat{c}_{k} + \frac{1}{N}q(\beta_{k}|x)
\end{equation}

Finally, we get $Z_{k}$ by
\begin{equation}
\hat{Z}_{k}^{RTS} = \hat{Z}_{k}\frac{r_{1}\hat{c}_{k}}{r_{k}\hat{c}_{1}},~~k=2,...,K
\end{equation}
in which what we do care is $Z_{B} \approx \hat{Z}_{K}^{RTS}$.

The posterier distribution $q(\beta_{k}|x)$ in the above equations is defined by:
\begin{equation}
q(\beta_{k}|x) = \frac{f_{k}(x)r_{k}/\hat{Z}_{k}}{\sum_{k'=1}^{K} f_{k'}(x)r_{k'}/\hat{Z}_{k'}}
\end{equation}

\para{Practice}
In the paper\cite{carlson2016partition}, Carlson et al. note an initializing method to initialize $Z_{k}$, whose procedure is just like the above process. The only difference is that they sampled $\beta_{k}$ by uniform distribution in every loop, not by the distribution $(\beta|x)$. They claim that after doing such initializing work, then we conduct the algorithms above would acquire a better result.

In our real practice, we directly use the initializing method mentioned above by selecting $\beta_{k}$ with a uniform distribution in every loop. We also initialize the value of $Z_{A}$ by the method we have mentioned in the AIS section using the dataset. And we have found that the result is already satisfying, there is no need to conduct more loops with $\beta_{k}$ sampled by $(\beta|x)$.

Also, we found that we have to conduct the procedure above for several times s.t. we can acquire our desired partition function value.(i.e. We did it for 100 times, that is to say we update $\mathbf Z$ for 100 times).

	\begin{algorithm}
        \caption{Rao-Blackwellized Tempered Sampling}
        \begin{algorithmic}
        	\Require $\{\beta_{k},r_{k}\}_{k=1,...,K}$
            \State Initialize $b_{A}$ by dataset
        	\State Initialize $log \hat{Z}_{k}, k=2,...,K$
            \For{$n = 1 \to Runtime$}
            	\State Initialize $\beta \in \{\beta_{1}...\beta_{K}\}$
            	\State Initialize $\hat{c}_{k}=0, k=1,...,K$
                \For{$t = 1 \to N$}
                    \For{$t = 1 \to Transition time$}
                        \State Sample $\mathbf x_{k+1}$ given $\mathbf x_{k}$ using $q(x|\beta^{*})$
                    \EndFor
                    \State $\mathbf x^{*} \gets \mathbf x_{k+1}$
    	            \State Sample $\beta^{*} \sim (\beta|\mathbf x^{*})$ or $\beta^{*} \in \{\beta_{1}...\beta_{K}\}$
    	            \State Update $\hat{c}_{k} \gets \hat{c}_{k}+\frac{1}{N} q(\beta_{k}|\mathbf x^{*})$
    			\EndFor
                \State Update $\hat{Z}^{RTS}_{k} \gets \hat{Z}_{k}\frac{r_{1}\hat{c}_{k}}{r_{k}\hat{c}_{1}}, k=2,...,K$
            \EndFor
        \end{algorithmic}
    \end{algorithm}


\section{Probability}

    \subsection{What is probability?}
    Probability is the measurement of the chance of an event occurring.
    If you've ever checked the weather and seen that it's a "20\% chance of rain", you've seen probability in action.
    To start, we'll need to define some terms.
    \begin{definition}
        Outcomes: A possible result, typically referencing all of the possible results from the situation.
    \end{definition}
    \begin{definition}
        Events: These typically refer to desired outcomes
    \end{definition}
    Typically when finding probability, we'll divide all possible outcomes from our desired outcomes.
    Thinking in terms of set notation, we'll refer to all of our desired outcomes as n(A), where A is our desired outcome.
    n(S) will refer to all of the possible outcomes, and P(A) referring to the likelihood of A occurring.
    To generalize, we'll use the equation below.
    \begin{equation*}
        P(A) = \frac{n(A)}{n(S)}
    \end{equation*}
    Since the number of desired outcomes will never be greater than the number of total outcomes, we'll always have a decimal between 0 (0\%) and 1 (100\%), which we can convert to a percentage (ex. $\frac{1}{4}=0.25=25\%$), though typically we'll simply leave it as a reduced version of the fraction.

    \subsection{Mutually Exclusive Events and Non-Mutually Exclusive Events}
    Mutually Exclusive events are events that have zero overlap with other events.
    Recall the Venn Diagram of Sets in the Combinations section, and assume A and B are both events.
    To find the probability of either of them happening, we'll have to remember the equation:
    \begin{equation*}
        n(A\cup B) = n(A) + n(B)
    \end{equation*}
    This equation finds all the ways A or B can happen, with zero overlap.
    The problem with this equation is that it only finds the number of wanted outcomes.
    Remember, to find the probability of something, we find the wanted outcomes, and divide it by all of the possible outcomes.
    After this, our equation will look like this:
    \begin{equation*}
        \frac{n(A \cup B)}{n(S)} = \frac{n(A) + n(B)}{n(S)}
    \end{equation*}
    Which reduces to:
    \begin{equation*}
        P(A\cup B) = P(A) + P(B)
    \end{equation*}
    For non-mutually exclusive events, we must instead use a different equation, more accurate to the one we've discussed in the Sets Unit.
    Following the same logic we used earlier, our equation works out to:
    \begin{equation*}
        P(A\cup B) = P(A) + P(B) - P(A\cap B)
    \end{equation*}
    Where $P(A\cap B)$ equals the probability of both of those events occurring.

    \subsection{Independent Probability}
    When two events occur at the same time, or one after another with no effect on each other, they are called Independent Events.
    If you roll a die, then select a card, this would be independent, as the outcomes don't affect each other.
    So solve an Independent Probability question we'll have to think in terms of sets, so the probability of A and B, can be solved with the equation below.
    \begin{equation*}
        P(A\cap B) = P(A) \cap(B)
    \end{equation*}
        But what happens if we need to find the probability of $(A \mbox{ or } B)$?
        To find that we'll need to use another equation.
    \begin{equation*}
        P(A\cup B) = P(A) + P(B) - P(A\cap B)
    \end{equation*}

    \subsection{Markov Chains}
    Markov Chains are ways of predicting probability after successive iterations.
    Suppose that Product A has a 70\% chance of repurchasing, and Product B has a 70\% chance of repurchasing, how would we model this? We'd use a Markov Chain. Shown below is our probability matrix.
    \begin{equation*}
        P =
        \begin{bmatrix}
            0.7 & 0.3\\
            0.3 & 0.7
        \end{bmatrix}
    \end{equation*}
    Now, we have to find the initial probability of something, which we call a \textbf{transition matrix}. We'll represent this in the form of how many people started buying A or B, (shown below). Since this is the initial time, we'll refer to it as $S_{0}$, though you may also hear it called a \textbf{probability vector}.
    \begin{equation*}
    S_{0} =
        \begin{bmatrix}
            0.6 & 0.4
        \end{bmatrix}
    \end{equation*}
    To find the next probability vector, or $S_{1}$, we'll have to multiply the transition matrix by the probability vector.
    \begin{equation*}
        \begin{bmatrix}
            0.6 & 0.4
        \end{bmatrix}
        \cdot
        \begin{bmatrix}
            0.7 & 0.3\\
            0.3 & 0.7
        \end{bmatrix}
        =
        \begin{bmatrix}
            0.54 & 0.46
        \end{bmatrix}
        = S_{1}
    \end{equation*}
    Much like rolling a die, the more times you repeat this process, the closer you are to the actual results. Though what if there was an easier way to do this?

    \subsection{Steady State Vectors}
    In Markov Chains, the probability vectors will eventually stop changing.
    A probability vector that remains unchanged upon multiplication is called the \textbf{steady state vector}.
    This vector will represent the long term trend of the event.
    To find this without repeating the Markov Chain multiplication, we must think of the problem in terms of a system of equations.
    \begin{equation*}
        \begin{bmatrix}
            a & b
        \end{bmatrix}
        \cdot
        \begin{bmatrix}
            0.7 & 0.3\\
            0.3 & 0.7
        \end{bmatrix}
        =
        \begin{bmatrix}
            a & b
        \end{bmatrix}
    \end{equation*}
    To have a systems of equations from this, we must first assert that $a + b = 1$, and find the other by multiplying for one of the values.
    In this case, we can see that $0.7a + 0.3b = a$.
    Now solve this systems of equations like you have done in Grade 9.

    \subsection{Odds}
    Have you ever heard the (seemingly wrong) Roll Up The Rim odds?
    They claim that 1 in 5 people are winners!
    But what does this mean?
    Odds are the degree of confidence that someone has that an event will occur.
    To find the odds of something occurring, we must use the formula below.
    \begin{equation*}
        P(A) : P(A)'
    \end{equation*}
    This equation represents a ratio of the probability of A happening, against the probability of A \textbf{not} happening. Likewise with the odds of something \textbf{not occurring}, we can use the reverse of this formula.
    \begin{equation*}
        P(A)' : P(A)
    \end{equation*}
    What happens if we want to win a bet though? You've probably heard of betting odds before. To do that, we'll use the equation below, where X will equal the amount won.
    \begin{equation*}
        \frac{P(A)}{P(A)'} = \frac{\emph{Amount Bet}}{x}
    \end{equation*}
    After this, isolate X, which will give you the following equation.
    \begin{equation*}
        X = \emph{Amount Bet}\cdot\frac{P(A)'}{P(A)}
    \end{equation*}


\chapter{Neural Networks}\label{sec:neural_network}
Neural networks (NN) are machine learning algorithms inspired by the biological neural network that constitute the brain. A neural network is a collection of nodes called neurons that transmits and processes signals to and from other neurons connected to it, much like the biological neural network. The goal of a neural network is to approximate some function by learning parameters that results in the best approximation. They are popular as they can usually be constructed to do this fairly well, several people (\cite{Cybenko1989}, \cite{Funahashi1989}) have shown that a neural network with one hidden layer can approximate any function on a compact domain arbitrarily closed, if provided with a sufficient number of neurons.
\\
\\
In section \ref{sec:feedforward_nn}, we introduce the most common neural network, the feedforward neural network, which is the type of network we refer to throughout the dissertation, when we use the term neural network. In section \ref{sec:backprop} we examine a common method for training a neural network called backpropagation and in section \ref{sec:early_stopping} we introduce a popular method of regularizing neural networks to avoid overfitting.

\section{Feedforward Neural Networks} \label{sec:feedforward_nn}
The most simple neural networks are called feedforward neural networks or multilayer perceptrons (MLPs). They are called feedforward as information flows in one direction through the neurons. The neurons are typically arranged in layers to indicate which neurons receive and sends data to which neurons. These layers are typically vector-valued with each element of the vector playing the role as a neuron. As such one can describe a neural network as a chain of functions. Say we have three layers, then the neural network is $f(\mathbf{x}) = f^{(3)}\lr{f^{(2)}\lr{f^{(1)}\lr{\mathbf{x}}}}$, with $f^{(1)}$ being the first layer to pass the data, $f^{(2)}$ the second layer and so on. The final layer is called the output layer, while the intermediate layers, that do not directly show their output to the user, are called hidden layers. \\
\\
A feedfoward neural network is shown in figure \ref{fig:mlp} where the data flows from the input layer through two hidden layers and finally to the output layer. While the number of neurons in the input- and output layer depends on the example and the label dimensions, the number of neurons in the hidden layers as well as the number of hidden layers are design decisions chosen by the architect. \\
\\
The data is sent through these layers by multiplying the inputs with weights $\boldsymbol{w}$, adding a bias $\boldsymbol{b}$ and finally applying an activation function $g$ to the result. An activation function is a non-linear function that ensures that we do not just pass linear transformations of the examples through the network, which is necessary to learn non-linear functions. Activation functions come in many different forms, some of the most popular can be seen in figure \ref{fig:act_funcs}. For a network with two hidden layers the result of the first hidden layer is $f^{(1)}\lr{\boldsymbol{X}} = g_1 \lr{\boldsymbol{X} \boldsymbol{w}_1 + \boldsymbol{b}_1}$ and the result of the second hidden layer is $f^{(2)} \lr{f^{(1)}\lr{\boldsymbol{X}}} = g_2 \lr{f^{(1)}\lr{\boldsymbol{X}} \boldsymbol{w}_2 + \boldsymbol{b}_2}$. The subscripts indicate that the activation functions, weights and biases are unique for each layer. At last the result of the output layer is $f^{\text{out}} \lr{f^{(2)} \lr{f^{(1)}\lr{\boldsymbol{X}}}} = g_3 \lr{f^{(2)} \lr{f^{(1)}\lr{\boldsymbol{X}}} \boldsymbol{w}_3 + \boldsymbol{b}_3}$. Sometimes the biases are omitted, as they can be represented in the weight vector for an example-matrix that are padded with a column of ones. \\
\\
While the choice of activation function are chosen as a design decision by the architect for the hidden layers, the activation function for the output layer are defined by the learning task. If the neural network is performing regression then we typically do not have an activation function for the output layer, as the label space for regression is the real numbers and need not be mapped to a specific set of values. However for binary classification a sigmoid-function is used, while a softmax-function is used for multiclass classification, as these map real values to the interval $[0,1]$ and thus provides valid probabilities for belonging to a specific class. Some popular choices of activation functions are shown in figure \ref{fig:act_funcs}. Taking an activation value $a$ as input, these are defined as follows:\\
\\
Logistic sigmoid:
\begin{equation} \label{eq:sigmoid}
    g(a) = \sigma(a) \equiv \frac{1}{1 + e^{-a}}
\end{equation}
Softmax for $K$ classes:
\begin{equation*}
    g(a)_i = \frac{e^{a_i}}{\sum_{j=1}^K e^{a_j}} \quad \text{for} \quad i=1 ,\dots, K
\end{equation*}
Hyperbolic tangens:
\begin{equation} \label{eq:tanh}
    g(a) = \tanh \lr{a} \equiv \frac{e^a - e^{-a}}{e^a + e^{-a}}
\end{equation}
Rectified linear unit (ReLU):
\begin{equation} \label{eq:relu}
    g(a) = \max \lr{0, a}
\end{equation}
Exponential linear unit (ELU):
\begin{equation*}
    g(a) = \begin{cases}
    a & a \geq 0 \\
    \alpha \lr{e^a - 1} & a < 0
    \end{cases} \quad \text{for } \alpha > 0
\end{equation*}
Step:
\begin{equation*}
    g(a) = \mathbf{1}_{[a > 0]}
\end{equation*}

\begin{figure}
    \centering
    \begin{neuralnetwork}[height = 4]
        \newcommand{\x}[2]{$x_#2$}
        \newcommand{\y}[2]{$\hat{y}_#2$}
        \newcommand{\hfirst}[2]{\small $h^{(1)}_#2$}
        \newcommand{\hsecond}[2]{\small $h^{(2)}_#2$}
        \inputlayer[count=3, bias=false, title=Input\\layer, text=\x]
        \hiddenlayer[count=4, bias=false, title=Hidden\\layer 1, text=\hfirst] \linklayers
        \hiddenlayer[count=3, bias=false, title=Hidden\\layer 2, text=\hsecond] \linklayers
        \outputlayer[count=2, title=Output\\layer, text=\y] \linklayers
    \end{neuralnetwork}
    \caption{Feedforward neural network with 2 hidden layers and arrows indicating where neurons feed their data.}
    \label{fig:mlp}
\end{figure}

\begin{figure}
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{pics/act_func_fig.pdf}
    \caption{Plot of some of the most used activation functions. The Python code for producing this figure can be seen in appendix \ref{app:act_funcs}}
    \label{fig:act_funcs}
\end{figure}

\section{Backpropagation} \label{sec:backprop}
As mentioned in section \ref{sec:train_val} we seek to find the hypothesis $\hat{h}^*_S$ that minimizes our empirical loss. For notational convenience we will collect the bias and weights in a parameter vector $\boldsymbol{\theta}$, which can be interpreted as using examples padded with a column of ones, so that $\boldsymbol{x}\boldsymbol{w}+\boldsymbol{b} = \boldsymbol{x}_{\text{padded}} \boldsymbol{\theta}$. In the case of neural networks $\hat{h}^*_S$ determine the weights used in each neuron as described in section \ref{sec:feedforward_nn}. To obtain these optimal weights we train the neural network by minimizing the gradient of the (perhaps regularized) loss function $J$ with respect to the weights. To learn these weights we use a minimization algorithm like stochastic gradient descent described in section \ref{sec:sgd}. However the gradient of the loss function is not readily available in neural networks as we have to trace the information back through the network that produced $\hat{y}$ for which we calculate the loss. The solution is an algorithm proposed by \cite{Rumelhart:1986a} called back-propagation or backprop for short.\\
\\
Back-propagation uses the chain rule of calculus, which states that
\begin{equation*}
    \frac{d \, f\lr{g\lr{x}}}{d \, x} = \frac{d \, f\lr{g\lr{x}}}{d \, g \lr{x}} \frac{d \, g\lr{x}}{d \, x}
\end{equation*}
for a real number $x$ and functions $g$ and $f$. This rule can be generalized for a vector $\boldsymbol{x} \in \mathbb{R}^m$
\begin{equation*}
    \frac{\partial f\lr{g\lr{\boldsymbol{x}}}}{\partial x_i} = \sum_j \frac{\partial f\lr{g\lr{\boldsymbol{x}}}}{\partial g(x_j)} \frac{\partial g(x_j)}{\partial x_i}
\end{equation*}
where $g: \mathbb{R}^m \rightarrow \mathbb{R}^n$ and $f: \mathbb{R}^n \rightarrow \mathbb{R}$. In a neural network with one hidden layer we find the gradient of the loss function $J(\boldsymbol{\theta})$ (with $\boldsymbol{X}$ and $\boldsymbol{y}$ as implicit inputs to keep notation uncluttered) by
\begin{equation} \label{eq:loss_grad}
    \nabla J\lr{\boldsymbol{\theta}} = \begin{pmatrix}
    \frac{\partial J\lr{f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}}{\partial \theta_1} \\
    \vdots\\
    \frac{\partial J\lr{f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}}{\partial \theta_m}
    \end{pmatrix}
\end{equation}
where each element is found by
\begin{equation} \label{eq:loss_grad_element}
    \frac{\partial J\lr{f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}}{\partial \theta_i} = \frac{\partial J\lr{f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}}{\partial f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}
    \frac{\partial f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}{\partial f^{(1)}\lr{\boldsymbol{\theta}}}
    \frac{\partial f^{(1)}\lr{\boldsymbol{\theta}}}{\partial \theta_i}
\end{equation}
which can easily be generalized to networks with more than one hidden layer, by using the chain rule in a similar fashion. A computational graph for calculating this in a neural network is shown in figure \ref{fig:backprop_compgraph}. This provides the fundamentals to understand how the gradient is computed in practice. A vector of weights (perhaps chosen by an update of stochastic gradient descent) is sent through the network as shown by algorithm \ref{alg:forward_prop} to provide a loss value. This is called forward-propagation. Then the gradient in equation \ref{eq:loss_grad} is computed using the relation in equation \ref{eq:loss_grad_element} by back-propagation as shown in algorithm \ref{alg:back_prop}. The result of the backpropagation algorithm is gradients on the weights that can be directly used for learning the weights that minimizes the gradient on the loss in a stochastic gradient descent algorithm as the one shown in algorithm \ref{alg:sgd}.\\
\\
Some activation functions like the ReLU-function $g(a) = \max \lr{0, a}$ are not differential, which might sound problematic for backpropagation and gradient based learning. However, as mentioned by \cite{Goodfellow-et-al-2016}, gradient descent still performs well enough for these models, partly because training algorithms usually do not arrive at a local minimum of the loss function. This means that we do not expect to get a gradient of exactly $\boldsymbol{0}$, and therefore it is not a problem that the minimum of the loss function corresponds to points with an undefined gradient. Furthermore \cite{Goodfellow-et-al-2016} mentions that most software return one of the one-sided derivatives, which can be heuristically justified since a computer is subject to numerical error anyway.

\begin{figure}
    \centering
    \begin{tikzpicture}[minimum size=1.5cm, node distance={45mm}, thick, main/.style = {draw, circle}]
        \node[main] (0) {$\boldsymbol{\theta}$};
        \node[main] (1) [above of=0]{$f^{(1)} \lr{\boldsymbol{\theta}}$};
        \node[main] (2) [right of=1] {$\frac{\partial f^{(1)}\lr{\boldsymbol{\theta}}}{\partial \theta_i}$};
        \node[main] (3) [right of=2] {$\frac{\partial J\lr{f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}}{d \theta_i}$};
        \node[main] (4) [above of=1] {$f^{\text{out}} \lr{ f^{(1)}\lr{\boldsymbol{\theta}}}$};
        \node[main] (5) [right of=4] {$\frac{\partial f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}{\partial f^{(1)}\lr{\boldsymbol{\theta}}} $};
        \node[main] (6) [right of=5] {$\frac{\partial J\lr{f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}}{\partial f^{(1)}\lr{\boldsymbol{\theta}}}$};
        \node[main] (7) [above of=4] {$J \lr{f^{\text{out}} \lr{ f^{(1)}\lr{\boldsymbol{\theta}}}}$};
        \node[main] (8) [right of=7] {$\frac{\partial J\lr{f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}}}{\partial f^{\text{out}} \lr{f^{(1)}\lr{\boldsymbol{\theta}}}} $};

        \draw[->] (0) -- node[midway, left] {$f^{(1)}$} (1);
        \draw[->] (1) -- node[midway, above] {$f'$} (2);
        \draw[->] (2) -- node[midway, above, pos=0.9] {$\times$} (3);
        \draw[->] (1) -- node[midway, left] {$f^{\text{out}}$} (4);
        \draw[->] (4) -- node[midway, above] {$f'$} (5);
        \draw[->] (5) -- node[midway, above, pos=0.8] {$\times$} (6);
        \draw[->] (6) -- (3);
        \draw[->] (4) -- node[midway, left] {$J$} (7);
        \draw[->] (7) -- node[midway, above] {$f'$} (8);
        \draw[->] (8) -- (6);
    \end{tikzpicture}
    \caption{Computational graph of calculating the elements of the gradient in equation \ref{eq:loss_grad_element} for the network with one hidden layer. $f'$ indicates that the function of the node is derived with regard to its input and $\times$ indicates that the node is a product of the nodes pointing to it. }
    \label{fig:backprop_compgraph}
\end{figure}

\begin{algorithm}\label{alg:forward_prop}
\SetAlgoLined
\KwInput{Number of layers, $l$}
\KwInput{The weight vectors of each layer $\boldsymbol{w}^{(i)}, i \in \lrc{1, \dots, l}$}
\KwInput{The activation functions of each layer, $g^{(i)}, i \in \lrc{1, \dots, l}$}
\KwInput{The example to process, $\boldsymbol{x}$}
\KwInput{The target label for the example, $\boldsymbol{y}$}
\KwInput{The (regularized) loss function $J$}
\KwOutput{The (regularized) loss on the example, $J\lr{\boldsymbol{\hat{y}}, \boldsymbol{y}, \boldsymbol{\theta}}$}
 initialize $\boldsymbol{h}^{(0)} \leftarrow \boldsymbol{x}$ \\
\For{$k = 1, \dots, l$}{
    $\boldsymbol{a}^{(k)} \leftarrow \boldsymbol{w}^{(k)} \boldsymbol{h}^{(k-1)}$ \\
    $\boldsymbol{h}^{(k)} \leftarrow g^{(k)} \lr{\boldsymbol{a}^{(k)}}$ \\
}
$\hat{\boldsymbol{y}} \leftarrow \boldsymbol{h}^{(l)}$ \\
$J\lr{\boldsymbol{\hat{y}}, \boldsymbol{y}, \boldsymbol{\theta}} \leftarrow L\lr{\boldsymbol{\hat{y}}, \boldsymbol{y}} + \alpha \Omega \lr{\boldsymbol{\theta}}$ \\
 \caption{Forwardpropagation through a neural network and the computation of the cost function. For simplicity this demonstration uses only a single input example $\boldsymbol{x}$, in practice one typically uses a minibatch of examples. We have also omitted the bias terms for simplicity, as these can be part of the weights $\boldsymbol{w}^{(i)}$ with an example $\boldsymbol{x}$ padded with a column of 1's. The collection of weights are denoted by $\boldsymbol{\theta}$.}
\end{algorithm}

\begin{algorithm}\label{alg:back_prop}
\SetAlgoLined
\KwInput{Same variables as forwardpropagation in algorithm \ref{alg:forward_prop}}
\KwOutput{The gradient of the regularized loss function on the weights, $ \boldsymbol{d} = \nabla_{\boldsymbol{w}}J$}
Perform forward propagation as per algorithm \ref{alg:forward_prop}\\
Computation of the loss-gradient on the output layer:\\
$\boldsymbol{d} \leftarrow \nabla_{\hat{\boldsymbol{y}}} J =  \nabla_{\hat{\boldsymbol{y}}} L\lr{\hat{\boldsymbol{y}}, \boldsymbol{y}}$ \\
\For{$k = l, l-1, \dots, 1$}{
    $\boldsymbol{d} \leftarrow \nabla_{\boldsymbol{a}^{(k)}}J = \boldsymbol{d} \odot g^{(k)'} \lr{\boldsymbol{a}^{(k)}}$\\
    $\nabla_{\boldsymbol{\theta}^{(k)}} J \leftarrow \boldsymbol{d} \boldsymbol{h}^{(k-1)^\top} + \lambda \nabla_{\boldsymbol{\theta}^{(k)}} \Omega \lr{\boldsymbol{\theta}}$\\
    $\boldsymbol{d} \leftarrow \nabla_{\boldsymbol{h}^{(k-1)}} J = \boldsymbol{\theta}^{(k)^\top} \boldsymbol{d}$\\
}
\caption{Back-propagation}
\end{algorithm}

\clearpage
\section{Early Stopping} \label{sec:early_stopping}
For this regularization methods we introduce a validation set, which is a third split of the data that is separate from the training set, that trains our neural networks, and separate from the test set, that evaluates our neural network. This validation set is used for selecting a non-overfitting set of parameters proposed by the training of the algorithm. This means that the validation set contributes to the selection of the optimal hypothesis, which is why we must keep it separate from the test set, which we use to evaluate our algorithm.
\\
\\
\cite{Goodfellow-et-al-2016} argues that when we have large models with sufficient capacity to overfit, we often observe that the training error decreases steadily over time, while the validation error begins to rise again at some point before ending the training. This means that we obtain a better model if we stop training and return the parameters at the point with the lowest validation error. This is the concept of regularizing using early stopping. The algorithm terminates training when no parameters have improved the validation error over some pre-specified number of iterations called the patience, $\phi$. The measure of improvement can be controlled by a hyperparameter $\delta_{\text{min}}$, making sure we only constitute an improvement if the decrease in validation error is more than $\delta_{\text{min}}$. The parameters $\boldsymbol{\theta}$ used for calculating the loss are provided by a training algorithm like ADAM from section \ref{alg:adam} using back-propagation from section \ref{sec:backprop}, while the loss is provided by the network output for these parameters and the chosen loss function. This procedure is specified more formally in algorithm \ref{alg:es}.\\
\\
One can think of early stopping as a hyperparameter selection algorithm that chooses the number of training steps for the training algorithm. The only significant cost of choosing this parameter automatically with early stopping is running the evaluation of the validation error periodically during training. However this can be done in parallel to the training process, as in Tensorflow where one can run parallel on the GPU. An additional cost is the need to maintain a copy of the optimal parameters, but this is negligible according to \cite{Goodfellow-et-al-2016} since they are written to infrequently and never during the training-algorithm, and since they can preferably be stored in a slower and larger form of memory like in host memory or disk drive.
\\
\\
\cite{bishop1995} and \cite{sjoberg_ljung1995} argues that early stopping has the effect of restricting the optimization procedure to a small volume of parameter space in the neighborhood of the initial parameter value. \cite{Goodfellow-et-al-2016} shows with a simple linear model with a quadratic error function and gradient descent how early stopping is equivalent to L2 regularization in equation \ref{eq:L2_reg}, but that early stopping has the advantage of automatically determining the correct amount of regularization while L2 regularization requires many training experiments with different values of its hyperparameter $\alpha$.

\begin{algorithm}\label{alg:es}
\SetAlgoLined
\KwInput{The number of steps between evaluations, $n$}
\KwInput{The patience parameter i.e. the number of times of observing worsening validation error before terminating, $\phi$}
\KwInput{The measure of improvement, $\delta_{\text{min}}$}
\KwOutput{Parameters when terminating $\boldsymbol{\theta}^{*}$, training step when terminating $i^{*}$}
    initialize $i, j \leftarrow 0$ \\
    initialize $v \leftarrow \infty$\\
    Let $\boldsymbol{\theta}_0$ be the initial parameters\\
    $\boldsymbol{\theta}^* \leftarrow \boldsymbol{\theta}_0$\\
    $i^* \leftarrow i$\\
    \While{$j < \phi$}{
        Update $\boldsymbol{\theta}$ by running the training algorithm for $n$ steps\\
        $i \leftarrow i + n$\\
        $v' \leftarrow \hat{L}\lr{S_{\text{val}}, \boldsymbol{\theta}}$\\
        \If{$v' < v - \delta_{\text{min}}$}{
            $j \leftarrow 0$\\
             $\boldsymbol{\theta}^* \leftarrow \boldsymbol{\theta}$\\
            $i^* \leftarrow i$\\
            $v \leftarrow v'$\\
            \Else{
                $j \leftarrow j +1$
            }
        }
    }
 \caption{Early stopping}
\end{algorithm}


\chapter{Dual space and trace}
You may have learned in high school that given a matrix
\[
	\begin{bmatrix}
		a & c \\
		b & d
	\end{bmatrix}
\]
the trace is the sum along the diagonals $a+d$
and the determinant is $ad-bc$.
But we know that a matrix is somehow
just encoding a linear map using a choice of basis.
Why would these random formulas somehow not
depend on the choice of a basis?

In this chapter, we are going to
give an intrinsic definition of $\Tr T$,
where $T \colon V \to V$ and $\dim V < \infty$.
This will give a coordinate-free definition
which will in particular imply the trace $a+d$
doesn't change if we take a different basis.

In doing so, we will introduce two new constructions:
the \emph{tensor product} $V \otimes W$
(which is a sort of product of two spaces,
with dimension $\dim V \cdot \dim W$)
and the \emph{dual space} $V^\vee$,
which is the set of linear maps $V \to k$ (a $k$-vector space).
Later on, when we upgrade from a vector space $V$
to an inner product space $V^\vee$,
we will see that the dual space gives a nice
interpretation of the ``transpose'' of a matrix.
You'll already see some of that come through here.

The trace is only defined for finite-dimensional
vector spaces, so if you want you can restrict
your attention to finite-dimensional vector spaces for this chapter.
(On the other hand we do not need the
ground field to be algebraically closed.)

The next chapter will then do the same for the determinant.

\section{Tensor product}
\prototype{$\RR[x] \otimes \RR[y] = \RR[x,y]$.}
We know that $\dim (V \oplus W) = \dim V + \dim W$,
even though as sets $V \oplus W$ looks like $V \times W$.
What if we wanted a real ``product'' of spaces,
with multiplication of dimensions?

For example, let's pull out
my favorite example of a real vector space, namely
\[ V = \left\{ ax^2 + bx + c \mid a,b,c \in \RR \right\}. \]
Here's another space, a little smaller:
\[ W = \left\{ dy + e \mid d,e \in \RR \right\}. \]
If we take the direct sum, then we would get some rather unnatural
vector space of dimension five
(whose elements can be thought of as pairs $(ax^2+bx+c,dy+e)$).
But suppose we want a vector space
whose elements are \emph{products} of polynomials in $V$ and $W$;
it would contain elements like $4x^2y + 5xy + y + 3$.
In particular, the basis would be
\[ \left\{ x^2y, x^2, xy, x, y, 1 \right\} \]
and thus have dimension six.

For this we resort to the \emph{tensor product}.
It does exactly this, except that the ``multiplication''
is done by a scary\footnote{%
	Seriously, $\otimes$ looks \emph{terrifying} to non-mathematicians,
	and even to many math undergraduates.}
symbol $\otimes$:
think of it as a ``wall'' that separates the elements
between the two vector spaces.
For example, the above example might be written as
\[ 4x^2 \otimes y + 5x \otimes y + 1 \otimes y + 3 \otimes 1. \]
(This should be read as $(4x^2 \otimes y) + (5x \otimes y) + \dots$;
addition comes after $\otimes$.)
Of course there should be no distinction
between writing $4x^2 \otimes y$ and $x^2 \otimes 4y$
or even $2x^2 \otimes 2y$.
While we want to keep the $x$ and $y$ separate,
the scalars should be free to float around.

Of course, there's no need to do everything
in terms of just the monomials.
We are free to write
\[ (x + 1) \otimes (y + 1). \]
If you like, you can expand this as
\[ x \otimes y + 1 \otimes y + x \otimes 1 + 1 \otimes 1. \]
Same thing.
The point is that we can take any two of our polynomials
and artificially ``tensor'' them together.

The definition of the tensor product does exactly this,
and nothing else.\footnote{I'll only define this
	for vector spaces for simplicity.
	The definition for modules over a commutative ring $R$ is exactly the same.}
\begin{definition}
	Let $V$ and $W$ be vector spaces over the same field $k$.
	The \vocab{tensor product} $V \otimes_k W$ is the abelian group
	generated by elements of the form $v \otimes w$, subject to relations
	\begin{align*}
		(v_1 + v_2) \otimes w &= v_1 \otimes w + v_2 \otimes w \\
		v \otimes (w_1 + w_2) &= v \otimes w_1 + v \otimes w_2 \\
		(c \cdot v) \otimes w &= v \otimes (c \cdot w).
	\end{align*}
	As a vector space,
	its action is given by
	$c \cdot (v \otimes w) = (c \cdot v) \otimes w = v \otimes (c \cdot w)$.
\end{definition}
Here's another way to phrase the same idea.
We define a \vocab{pure tensor} as an
element of the form $v \otimes w$ for $v \in V$ and $w \in W$.
But we let the $\otimes$ wall be ``permeable'' in the sense that
\[ (c \cdot v) \otimes w = v \otimes (c \cdot w) = c \cdot (v \otimes w) \]
and we let multiplication and addition distribute as we expect.
Then $V \otimes W$ consists of sums of pure tensors.

\begin{example}
	[Infinite-dimensional example of tensor product: two-variable polynomials]
	Although it's not relevant to this chapter,
	this definition works equally well with infinite-dimensional
	vector spaces.
	The best example might be
	\[ \RR[x] \otimes_\RR \RR[y] = \RR[x,y]. \]
	That is, the tensor product of polynomials in $x$
	with real polynomials in $y$
	turns out to just be two-variable polynomials $\RR[x,y]$.
\end{example}

\begin{remark}
	[Warning on sums of pure tensors]
	Remember the elements of $V \otimes_k W$
	really are \emph{sums} of these pure tensors!
	If you liked the previous example,
	this fact has a nice interpretation ---
	not every polynomial in $\RR[x,y] = \RR[x] \otimes_\RR \RR[y]$ factors
	as a polynomial in $x$ times a polynomial in $y$
	(i.e.\ as pure tensors $f(x) \otimes g(y)$).
	But they all can be written as sums of pure tensors $x^a \otimes y^b$.
\end{remark}

As the example we gave suggested,
the basis of $V \otimes_k W$ is literally the ``product''
of the bases of $V$ and $W$.
In particular, this fulfills our desire that
$\dim (V \otimes_k W) = \dim V \cdot \dim W$.
\begin{proposition}[Basis of $V \otimes W$]
	Let $V$ and $W$ be finite-dimensional $k$-vector spaces.
	If $e_1, \dots, e_m$ is a basis of $V$ and $f_1, \dots, f_n$ is a basis of $W$,
	then the basis of $V \otimes_k W$
	is precisely $e_i \otimes f_j$, where $i=1,\dots,m$ and $j=1,\dots,n$.
\end{proposition}
\begin{proof}
	Omitted; it's easy at least to see that this basis is spanning.
\end{proof}

\begin{example}[Explicit computation]
	Let $V$ have basis $e_1$, $e_2$ and $W$ have basis $f_1, f_2$.
	Let $v = 3e_1 + 4e_2 \in V$ and $w = 5f_1 + 6f_2 \in W$.
	Let's write $v \otimes w$ in this basis for $V \otimes_k W$:
	\begin{align*}
		v \otimes w &= (3e_1+4e_2) \otimes (5f_1+6f_2) \\
		&= (3e_1) \otimes (5f_1) +  (4e_2) \otimes (5f_1)
		+ (3e_1) \otimes (6f_2) + (4e_2) \otimes (6f_2) \\
		&= 15 (e_1 \otimes f_1) + 20(e_2 \otimes f_1)
		+ 18 (e_1 \otimes f_2) + 24(e_2 \otimes f_2).
	\end{align*}
	So you can see why tensor products are a nice ``product'' to
	consider if we're really interested in $V \times W$
	in a way that's more intimate than just a direct sum.
\end{example}

\begin{abuse}
	Moving forward, we'll almost always abbreviate $\otimes_k$ to just $\otimes$,
	since $k$ is usually clear.
\end{abuse}
\begin{remark}
	Observe that to define a linear map $V \otimes W \to X$,
	I only have to say what happens to each pure tensor $v \otimes w$,
	since the pure tensors \emph{generate} $V \otimes W$.
	But again, keep in mind that
	$V \otimes W$ consists of \emph{sums} of these pure tensors!
	In other words, $V \otimes W$ is generated by pure tensors.
\end{remark}

\begin{remark}
	Much like the Cartesian product $A \times B$ of sets,
	you can tensor together any two vector spaces $V$ and $W$ over the same field $k$;
	the relationship between $V$ and $W$ is completely irrelevant.
	One can think of the $\otimes$ as a ``wall'' through which one can pass
	scalars in $k$, but otherwise keeps the elements of $V$ and $W$ separated.
	Thus, $\otimes$ is \textbf{content-agnostic}.

	This also means that even if $V$ and $W$ have some relation to each other,
	the tensor product doesn't remember this.
	So for example $v \otimes 1 \neq 1 \otimes v$,
	just like $(g,1_G) \neq (1_G,g)$ in the group $G \times G$.
\end{remark}

\section{Dual space}
\prototype{Rotate a column matrix by $90$ degrees.}

Consider the following vector space:
\begin{example}
	[Functions from $\RR^3 \to \RR$]
	The set of real functions $f(x,y,z)$ is an
	infinite-dimensional real vector space.
	Indeed, we can add two functions to get $f+g$,
	and we can think of functions like $2f$.
\end{example}
This is a terrifyingly large vector space,
but you can do some reasonable reductions.
For example, you can restrict your attention to just
the \emph{linear maps} from $\RR^3$ to $\RR$.

That's exactly what we're about to do.
This definition might seem strange at first, but bear with me.

\begin{definition}
	Let $V$ be a $k$-vector space.
	Then $V^\vee$, the \vocab{dual space} of $V$, is defined
	as the vector space whose elements are \emph{linear maps from $V$ to $k$}.
\end{definition}
The addition and multiplication are pointwise:
it's the same notation we use when we write $cf+g$ to mean $c \cdot f(x) + g(x)$.
The dual space itself is less easy to think about.

Let's try to find a basis for $V^\vee$.
First, here is a very concrete interpretation of the vector space.
Suppose for example $V = \RR^3$.
We can think of elements of $V$ as column matrices, like
\[ v = \begin{bmatrix}
		2 \\ 5 \\ 9
	\end{bmatrix}
	\in V. \]
Then a linear map $f \colon V \to k$ can be interpreted as a \emph{row matrix}:
\[
	f = \begin{bmatrix}
		3 & 4 & 5
	\end{bmatrix}
	\in V^\vee. \]
Then
\[
	f(v) = \begin{bmatrix}
		3 & 4 & 5
	\end{bmatrix}
	\begin{bmatrix}
		2 \\ 5 \\ 9
	\end{bmatrix}
	= 71. \]

More precisely: \textbf{to specify a linear map $V \to k$,
I only have to tell you where each basis element of $V$ goes}.
In the above example, $f$ sends $e_1$ to $3$, $e_2$ to $4$, and $e_3$ to $5$.
So $f$ sends \[ 2e_1 + 5e_2 + 9e_3 \mapsto 2 \cdot 3 + 5 \cdot 4 + 9 \cdot 5 = 71. \]

Let's make all this precise.
\begin{proposition}[The dual basis for $V^\vee$]
	Let $V$ be a finite-dimensional vector space with basis $e_1, \dots, e_n$.
	For each $i$ consider the function $e_i^\vee \colon V \to k$
	defined by
	\[
		e_i^\vee(e_j)
		= \begin{cases}
			1 & i=j \\
			0 & i \neq j.
		\end{cases}
	\]
	In more humane terms, $e_i^\vee(v)$
	gives the coefficient of $e_i$ in $v$.

	Then $e_1^\vee$, $e_2^\vee$, \dots, $e_n^\vee$ is a basis of $V^\vee$.
\end{proposition}

\begin{example}[Explicit example of element in $V^\vee$]
	In this notation, $f = 3e_1^\vee + 4e_2^\vee + 5e_3^\vee$.
	Do you see why the ``sum'' notation works as expected here?
	Indeed
	\begin{align*}
		f(e_1) &= (3e_1^\vee + 4e_2^\vee + 5e_3^\vee)(e_1) \\
		&= 3e_1^\vee(e_1) + 4e_2^\vee(e_1) + 5e_3^\vee(e_1) \\
		&= 3 \cdot 1 + 4 \cdot 0 + 5 \cdot 0 = 3.
	\end{align*}
	That's exactly what we wanted.
\end{example}

You might be inclined to point out that $V \cong V^\vee$ at this point,
since there's an obvious isomorphism $e_i \mapsto e_i^\vee$.
You might call it ``rotating the column matrix by $90\dg$''.
The issue is that this isomorphism
depends very much on which basis you choose:
if I pick a different basis, then the isomorphism
will be intrinsically different.

It \emph{is} true that $V$ and $V^\vee$ are isomorphic
for finite-dimensional $V$,
but you should already know that \emph{any} two $k$-vector spaces
of the same dimension are isomorphic.
In light of this, the fact that $V \cong V^\vee$ is not especially impressive.

\section{$V^\vee \otimes W$ gives matrices from $V$ to $W$}
Goal of this section:
\begin{moral}
	If $V$ and $W$ are finite-dimensional $k$-vector spaces
	then $V^\vee \otimes W$ represents linear maps $V \to W$.
\end{moral}

Here's the intuition.
If $V$ is three-dimensional and $W$ is five-dimensional, then we can think
of the maps $V \to W$ as a $5 \times 3$ array of numbers.
We want to think of these maps as a vector space:
(since one can add or scale matrices).
So it had better be a vector space with dimension $15$,
but just saying ``$k^{\oplus 15}$'' is not really that satisfying
(what is the basis?).

To do better, we consider the tensor product
\[ V^\vee \otimes W \]
which somehow is a product of maps out of $V$ and the target space $W$.
We claim that this is in fact the space we want:
i.e.\ \textbf{there is a natural bijection between elements of $V^\vee \otimes W$
and linear maps from $V$ to $W$}.

First, how do we interpret an element of $V^\vee \otimes W$ as a map $V \to W$?
For concreteness, suppose $V$ has a basis $e_1$, $e_2$, $e_3$,
and $W$ has a basis $f_1$, $f_2$, $f_3$, $f_4$, $f_5$.
Consider an element of $V^\vee \otimes W$, say
\[ e_1^\vee \otimes (f_2 + 2f_4) + 4e_2^\vee \otimes f_5. \]
We want to interpret this element as a function $V \to W$:
so given a $v \in V$,
we want to output an element of $W$.
There's really only one way to do this:
feed in $v \in V$ into the $V^\vee$ guys on the left.
That is, take the map
\[ v \mapsto e_1^\vee(v) \cdot (f_2 + 2f_4) + 4e_2^\vee(v) \cdot f_5 \in W. \]
So, there's a natural way to interpret any element
$\xi_1 \otimes w_1 + \dots + \xi_m \otimes w_m \in V^\vee \otimes W$
as a linear map $V \to W$.
The claim is that in fact, every linear map $V \to W$ has
such an interpretation.

First, for notational convenience,
\begin{definition}
	Let $\Hom(V,W)$ denote the set of linear maps from $V$ to $W$
	(which one can interpret as matrices which send $V$ to $W$),
	viewed as a vector space over $k$.
	(The ``$\Hom$'' stands for homomorphism.)
\end{definition}
\begin{ques}
	Identify $\Hom(V,k)$ by name.
\end{ques}

We can now write down something that's more true generally.
\begin{theorem}[$V^\vee \otimes W$ $\iff$ linear maps $V \to W$]
	\label{thm:vect_hom_dualization}
	Let $V$ and $W$ be finite-dimensional vector spaces.
	We described a map
	\[ \Psi \colon V^\vee \otimes W \to \Hom(V,W) \]
	by sending $\xi_1 \otimes w_1 + \dots + \xi_m \otimes w_m$ to the linear map
	\[ v \mapsto \xi_1(v) w_1 + \dots + \xi_m(v) w_m. \]
	Then $\Psi$ is an isomorphism of vector spaces, i.e.\ every linear map $V \to W$
	can be uniquely represented as an element of $V^\vee \otimes W$ in this way.
\end{theorem}

The above is perhaps a bit dense, so here is a concrete example.
\begin{example}[Explicit example]
	Let $V = \RR^2$ and take a basis $e_1$, $e_2$ of $V$.
	Then define $T : V \to V$ by
	\[ T = \begin{bmatrix}
			1 & 2 \\ 3 & 4
		\end{bmatrix}. \]
	Then we have
	\[ \Psi(e_1^\vee \otimes e_1 + 2e_2^\vee \otimes e_1
		+ 3e_1^\vee \otimes e_2 + 4e_2^\vee \otimes e_2) = T. \]
	The beauty is that the $\Psi$ definition is basis-free;
	thus even if we change the basis,
	although the above expression will look completely different,
	the \emph{actual element} in $V^\vee \otimes V$ doesn't change.
\end{example}

Despite this, we'll indulge ourselves in using coordinates for the proof.
\begin{proof}
	[Proof of \Cref{thm:vect_hom_dualization}]
	This looks intimidating, but it's actually not difficult.
	We proceed in two steps:
	\begin{enumerate}
		\ii First, we check that $\Psi$ is \emph{surjective};
		every linear map has at least one representation in $V^\vee \otimes W$.
		To see this, take any $T : V \to W$.
		Suppose $V$ has basis $e_1$, $e_2$, $e_3$ and that
		$T(e_1) = w_1$, $T(e_2) = w_2$ and $T(e_3) = w_3$.
		Then the element
		\[ e_1^\vee \otimes w_1 + e_2^\vee \otimes w_2 + e_3^\vee \otimes w_3 \]
		works, as it is contrived to agree with $T$ on the basis elements $e_i$.
		\ii So it suffices to check now that $\dim V^\vee \otimes W = \dim \Hom(V,W)$.
		Certainly, $V^\vee \otimes W$ has dimension $\dim V \cdot \dim W$.
		But by viewing $\Hom(V,W)$ as $\dim V \cdot \dim W$ matrices, we see that
		it too has dimension $\dim V \cdot \dim W$. \qedhere
	\end{enumerate}
\end{proof}
So there is a \textbf{natural isomorphism} $V^\vee \otimes W \cong \Hom(V,W)$.
While we did use a basis liberally in the
\emph{proof that it works}, this doesn't change the
fact that the isomorphism is ``God-given'',
depending only on the spirit of $V$ and $W$ itself
and not which basis we choose to express the vector spaces in.

\section{The trace}
We are now ready to give the definition of a trace.
Recall that a square matrix $T$ can be thought of as a map $T \colon V \to V$.
According to the above theorem,
\[ \Hom(V, V) \cong V^\vee \otimes V \]
so every map $V \to V$ can be thought of as an element of $V^\vee \otimes V$.
But we can also define an
\emph{evaluation map} $\opname{ev} : V^\vee \otimes V \to k$
by ``collapsing'' each pure tensor: $f \otimes v \mapsto f(v)$.
So this gives us a composed map
\begin{diagram}
	\Hom(V, V) & \rTo^\cong & V^\vee \otimes V & \rTo^{\opname{ev}} & k.
\end{diagram}
This result is called the \vocab{trace} of a matrix $T$.

\begin{example}[Example of a trace]
	Continuing the previous example,
	\[ \Tr T = e_1^\vee(e_1) + 2e_2^\vee(e_1)
		+ 3e_1^\vee(e_2) + 4e_2^\vee(e_2)
		= 1 + 0 + 0 + 4 = 5. \]
	And that is why the trace is the sum of the diagonal entries.
\end{example}

\section{\problemhead}

\begin{problem}
	[Trace is sum of eigenvalues]
	Let $V$ be an $n$-dimensional vector space
	over an algebraically closed field $k$.
	Let $T \colon V \to V$ be a linear map with
	eigenvalues $\lambda_1$, $\lambda_2$, \dots, $\lambda_n$
	(counted with algebraic multiplicity).
	Show that $\Tr T = \lambda_1 + \dots + \lambda_n$.
	\begin{hint}
		Follows by writing $T$ in an eigenbasis:
		then the diagonal entries are the eigenvalues.
	\end{hint}
\end{problem}

\begin{dproblem}
	[Product of traces]
	Let $T \colon V \to V$ and $S \colon W \to W$ be linear maps
	of finite-dimensional vector spaces $V$ and $W$.
	Define $T \otimes S \colon V \otimes W \to V \otimes W$
	by $v \otimes w \mapsto T(v) \otimes S(w)$.
	Prove that \[ \Tr(T \otimes S) = \Tr(T) \Tr(S). \]
	\begin{hint}
		Again one can just take a basis.
	\end{hint}
\end{dproblem}

\begin{dproblem}
	[Traces kind of commute]
	\gim
	Let $T \colon V \to W$ and $S \colon W \to V$ be linear maps
	between finite-dimensional vector spaces $V$ and $W$.
	Show that \[ \Tr(T \circ S) = \Tr(S \circ T). \]
	\begin{hint}
		One solution is to just take a basis.
		Otherwise, interpret $T \otimes S \mapsto \Tr(T \circ S)$ as a
		linear map $(V^\vee \otimes W) \otimes (W^\vee \otimes V) \to k$,
		and verify that it is commutative.
	\end{hint}
	\begin{sol}
		Although we could give a coordinate calculation,
		we instead opt to give a cleaner proof.
		This amounts to drawing the diagram
		\begin{diagram}
			&& (W^\vee \otimes V) \otimes (V^\vee \otimes W)
			&\rIsom& (V^\vee \otimes W) \otimes (W^\vee \otimes V) && \\
			& \ldTo^{\text{compose}} & \dTo && \dTo & \rdTo^{\text{compose}} & \\
			\Hom(W, W) & \lIsom & W^\vee \otimes W & & V^\vee \otimes V & \rIsom & \Hom(V, V) \\
			& \rdTo_{\Tr} & \dTo_{\opname{ev}} && \dTo_{\opname{ev}} & \ldTo_{\Tr} & \\
			&& k & \rIsom{\id} & k &&
		\end{diagram}
		It is easy to check that the center rectangle commutes,
		by checking it on pure tensors $\xi_W \otimes v \otimes \xi_V \otimes w$.
		So the outer hexagon commutes and we're done.
		This is really the same as the proof with bases;
		what it amounts to is checking the assertion is true for
		matrices that have a $1$ somewhere and $0$ elsewhere,
		then extending by linearity.
	\end{sol}
\end{dproblem}

\begin{problem}
	[Putnam 1988]
	\gim
	Let $V$ be an $n$-dimensional vector space.
	Let $T : V \to V$ be a linear map and suppose there exists $n+1$ eigenvectors,
	any $n$ of which are linearly independent.
	Does it follow that $T$ is a scalar multiple of the identity?
	\begin{hint}
		Look at the trace of $T$.
	\end{hint}
	\begin{sol}
		See \url{https://mks.mff.cuni.cz/kalva/putnam/psoln/psol886.html}.
	\end{sol}
\end{problem}


\section{Branching Bisimulation relation between LTSs}
\label{app:A}
\begin{definition}
 \label{branching_bisimulation}
For two LTSs $L_1 = (S_1, \Lambda\cup\{\tau\}, \rightarrow_1, i_1)$, $L_2 = (S_2, \Lambda, \rightarrow_2, i_2)$ a  relation $R \subseteq S_1 \times S_2$ is called a branching   bisimulation relation if for all $s \in S_1$ and $t \in S_2$ such that $R(s,t)$, the following conditions are met, for $a\in \Lambda\cup\{\tau\}$:
  \begin{itemize}
  \item[1.] if $s \arrow{}{a} s'$ in~$L_1$, then either
    \begin{itemize}
    \item [--] $a = \tau$ and $R(s',t)$, or
    \item [--] for some $n \geq 0$, there exist $t_1, \ldots, t_n$ and~$t'$ in~$S_2$ such that \\
     $t \arrow{}{\tau} t_1 \arrow{}{\tau}\ldots \arrow{}{\tau} t_{n}
      \arrow{}{a} t'$ in~$L_2$, $R(s,t_n)$ and~$R(s',t')$;
    \end{itemize}
  \item[2.] if $t \arrow{}{a} t'$ in~$L_2$, then either
    \begin{itemize}
    \item [--] $a = \tau$ and $R(s,t')$, or
    \item [--] for some $n \geq 0$, there exist $s_1, \ldots, s_n$
      and~$s'$ in~$S_2$ such that \\
      $s \arrow{}{\tau} s_1
      \arrow{}{\tau} \ldots \arrow{}{\tau} s_{n}
      \arrow{}{a} s'$ in~$L_1$, $R(s_n,t)$ and~$R(s',t')$;
    \end{itemize}
%  \item[3.] if $s\downarrow$ in~$Z$, then, for some $n\geq 0$, %there
%    exist $t_1, \ldots, t_n$ in~$\ST'$ such that $t %\xrightarrow{\tau}
%    t_1 \xrightarrow{\tau} \ldots \xrightarrow{\tau} t_{n}$ %in~$Z'$,
%    $R(s,t_n)$ and $t_n \downarrow$;
%  \item[4.] if $t\downarrow$ in~$Z'$, then, for some $n\geq 0$, %there
%    exist $s_1, \ldots, s_n$ in~$\ST$ such that $s %\xrightarrow{\tau}
%    s_1 \xrightarrow{\tau} \ldots \xrightarrow{\tau} s_{n}$ %in~$Z$,
%    $R(s_n,t)$ and $s_n \downarrow$.
  \end{itemize}
$L_1$ and~$L_2$
%two states $s$ and~$t$
are branching bisimilar,
%notation $L_1 \bisim_{\!b\ } L_2$,
if there exists a   branching bisimulation relation~$R$ for $L_1$ and~$L_2$ such that~$R(i_1,i_2)$.
\end{definition}


\documentclass[./Thesis.tex]{subfiles}
\begin{document}

\chapter{Agda Briefly}
\label{chap:agda-briefly}

\epigraph{
  The world of mathematics is becoming very large, the complexity of
  mathematics is becoming very high, and there is a danger of an
  accumulation of mistakes.
}{Vladimir Voevodsky \cite{voevodsky-quote}}

The goal of this chapter is two-fold. The main goal is to introduce the reader
to Agda and its style of dependently typed proofs. The secondary goal is to
introduce the reader to the syntax and proof styles used in this thesis. Luckily,
these goals are complementary so we will tackle them simultaneously.

\section{Functions and Datatypes}
\label{sec:functions-and-datatypes}

The code snippet below defines the standard boolean datatype ($\mathbb{Z}/[2]$).
The keyword \AgdaKeyword{data} allows users to define their own types.
You can think about the type $\AgdaPrimitiveType{Set}$ as the type of all types\footnotemark.
The following lines introduce the common names for boolean values while
expressing that they are of type boolean.
This example also highlights that names in Agda need not be ASCII identifiers.
They can be arbitrary unicode strings namely $\AgdaDatatype{𝔹}$.
\footnotetext{
  This allows for a type theoretic analog of Russell's
  Paradox. In actuality
  $\AgdaPrimitiveType{Set}_{n} : \AgdaPrimitiveType{Set}_{n + 1}$
  but as everything in this thesis lives in
  $\AgdaPrimitiveType{Set}_0 = \AgdaPrimitiveType{Set}$
  we can ignore this complexity.
}
\begin{code}
  data 𝔹 : Set where
    false : 𝔹
    true  : 𝔹

  not : 𝔹 → 𝔹
  not false = true
  not true  = false

  not-example : 𝔹
  not-example = not false
\end{code}
The function $\AgdaFunction{not}$ is defined by case analysis and follows
the standard mathematical definition closely. Functions in \Agda{} must be
total. In other words, they must cover every possible value in their domain.
The function defined above clearly satisfies this condition. Note that function
application, as seen in $\AgdaFunction{not-example}$ is simply an empty space
separating the function and its argument. This convention first appeared in
Alonzo Church's \textit{lambda calculus} and as \Agda{} is an
extension of the lambda calculus the convention carries over. Investigating the
lambda calculus is a thesis worthy exercise and in the past many Reed theses
have attempted this challenge. In our case, we refer the interested reader to the
following sources \cite{harper} \cite{hott-book}. This being said we consider the
lambda calculus to be a formalism of the abstract mapping notation common when
describing functions in mathematics. Note the following similarities between the
mathematical notation and the lambda notation described in \ref{eqn:mapping}.
\begin{align}
  \label{eqn:mapping}
  x \mapsto x^2 \text{ is the same as } λ x \, → \, x^2
\end{align}
\Agda{} supports arbitrary lambda expressions and we can use this fact to define
the identity function for booleans in lambda notation.
\begin{code}[hide]
  module Identity₁ where
\end{code}
\begin{code}
    id : 𝔹 → 𝔹
    id = λ x → x
\end{code}
The following identity function is definitionally equal to the lambda function
described above as every function in \Agda{} is desugared to the lambda calculus.
\begin{code}[hide]
  module Identity₂ where
\end{code}
\begin{code}
    id : 𝔹 → 𝔹
    id x = x
\end{code}
The astute reader may find the definitions above suspicious. Why is the identity
function defined with a domain and codomain of the booleans? Does \Agda{}
require a programmer to create a new identity function for every type?
Thankfully, \Agda{} adopts a solution pervasive to mathematics, universal
quantification.
\begin{code}[hide]
  module Identity₃ where
\end{code}
\begin{code}
    id : ∀ (A : Set) → A → A
    id A x = x

    id-𝔹 : 𝔹 → 𝔹
    id-𝔹 = id 𝔹
\end{code}
The general identity function depicted above introduces two new concepts.
It introduces universal quantification, or type theoretically, dependent types.
A dependent type comes in two parts. The type of the second half $A \, → \, A$ depends
on the input of the first $∀ (A : \AgdaPrimitiveType{Set}) →$. This is powerful
innovation and we will come to see that it drastically increases language expressivity.
It also introduces function currying. This concept allows multi-argument
functions to be encoded as functions that return functions.
\begin{align}
  \label{eqn:currying}
  (x, y) \mapsto x^2 + y^2 \equiv λ x \, → λ y \, → \, x^2 + y^2
\end{align}
Every function in \Agda{} is automatically curried so the identity function
described above is definitionally equal to the following function.
\begin{code}[hide]
  module Identity₄ where
\end{code}
\begin{code}
    id : ∀ (A : Set) → A → A
    id = λ A → λ x → x
\end{code}
Notice how the dependent type behaves similarly to normal function types.
This is because all function types in \Agda{} are actually dependently typed.
The syntax $A \, → \, B$ is just \textit{syntactic sugar} for a dependent type that ignores
the type of the input $(\_ : A) \, → \, B$. Lastly we make specifying the type
of the input optional as \Agda{} can almost always infer that type. Note that we
can still specify the type as seen in $\AgdaFunction{id-example₂}$ and
$\AgdaFunction{id-example₃}$
\begin{code}[hide]
  module Identity₅ where
\end{code}
\begin{code}
    id : ∀ {A : Set} → A → A
    id {A} x = x

    id-example₁ : 𝔹
    id-example₁ = id true

    id-example₂ : 𝔹
    id-example₂ = id {𝔹} true

    id-example₃ : 𝔹
    id-example₃ = id {A = 𝔹} true
\end{code}
\section{Baby's First Proofs}
\label{sec:first-proofs}
In order to prove interesting mathematical statements we first need a notion of
equality. The following code defines a datatype that represents equality. The
first line informs \Agda{} that equality is a binary relation over an arbitrary
type. The datatype only has a single constructor which requires that the left
hand side is the ``same'' as the right hand side. Thus the constructor is named
$\AgdaInductiveConstructor{≡-refl}$ as these constraints make the constructor
similar to the reflexivity axiom of equivalence classes.
\begin{code}[hide]
  module Equality where
\end{code}
\begin{code}
    data _≡_ {A : Set} : A → A → Set where
      ≡-refl : ∀ {x : A} → x ≡ x
\end{code}
In practice the left hand side and right hand side need not be the exact same
code. They do need to simplify to the same value. In the following code
$\AgdaFunction{not} \, \AgdaInductiveConstructor{true}$ simplifies to
$\AgdaInductiveConstructor{false}$ so the reflexivity constructor is allowed.
\begin{code}
    simple : not true ≡ false
    simple = ≡-refl
\end{code}
These concepts guide the following proof where we prove that $\AgdaFunction{not}$ is an
involution. The proof proceeds by cases, when the input is
$\AgdaInductiveConstructor{false}$ the expression
$\AgdaFunction{not} \,
(\AgdaFunction{not} \, \AgdaInductiveConstructor{false})$
simplifies to $\AgdaInductiveConstructor{false}$ via repeated applications of
the definition of $\AgdaFunction{not}$. The process proceeds symmetrically when the input is
$\AgdaInductiveConstructor{true}$. Note that the case split is required as
$\AgdaFunction{not}$ does not simplify for arbitrary booleans. This is unlike
the identity function which would simplify for any input.
\begin{code}
    not-involution : ∀ (b : 𝔹) → not (not b) ≡ b
    not-involution false = ≡-refl
    not-involution true = ≡-refl
\end{code}
\section{Natural Numbers}
\label{sec:natural-numbers}
The set of mathematical statements provable with only finite constructions is
significantly smaller than the set of all provable mathematical statements.
Thankfully \Agda{} can express many infinite constructions\footnotemark{}.
Infinite constructions are defined by an \textit{inductively defined datatype}
\cite{agda}.
In the following code we define the simplest inductively defined datatype, the Peano
natural numbers. This definition is similar to our definition of the booleans
but the $\AgdaInductiveConstructor{suc}$ constructor recursively
contains another natural.
\footnotetext{It can easily construct any ordinal less than
  $\varepsilon_0$ and with effort can express larger ordinals.
  \cite{ordinals}
}
\begin{code}[hide]
  module Naturals where
    open import Relation.Binary.PropositionalEquality
      using (_≡_; _≢_; module ≡-Reasoning)
      renaming (refl to ≡-refl; sym to ≡-sym; cong to ≡-cong; cong₂ to ≡-cong₂)
    open ≡-Reasoning
\end{code}
\begin{code}
    data ℕ : Set where
      zero : ℕ
      suc  : ℕ → ℕ

    three : ℕ
    three = suc (suc (suc zero))
\end{code}
We can consume these naturals by \textit{pattern matching} \cite{agda}. In the
following example the variable $m$ is bound by pattern matching and is equal to
the input $n$ minus one. This is because we are ``pealing'' off one successor
constructor.
\begin{code}
    isZero : ℕ → 𝔹
    isZero zero = true
    isZero (suc m) = false
\end{code}
\begin{code}[hide]
    infixl 7 _+_
\end{code}
The example above is not a compelling use of natural numbers as it does not
require any properties unique to the natural numbers. The code sample
below defines addition on Peano naturals a more compelling use. As our definition was inductively
defined any non trivial operation on naturals must be inductively defined as
well.
\begin{code}
    _+_ : ℕ → ℕ → ℕ
    zero + n = n
    suc n + m = suc (n + m)
\end{code}
The previous function name was defined with mixfix syntax. This is a syntax unique
to \Agda{} that allows for the construction of complex mathematical operators. A
valid name may be interspersed with underscores. At every underscore \Agda{}
expects a user to supply one argument. Addition is a binary operator so we
supply one underscore on either side of the addition symbol. \\

This is a proof assistant so lets prove a simple property of the naturals,
namely that addition is associative. In order to do this we need a definition of
associativity. The following definition is a function that takes
an arbitrary binary operation and returns a specification of associativity for
that binary operation. Note that even arguments can be defined with mixfix.
\begin{code}
    Associative : ∀ {A : Set} → (A → A → A) → Set
    Associative {A} _∙_ = ∀ (x y z : A) → (x ∙ (y ∙ z)) ≡ ((x ∙ y) ∙ z)
\end{code}
The actual proof begins by cases. When the
first argument is zero, $\AgdaInductiveConstructor{zero} + (y + z)$ and
$(\AgdaInductiveConstructor{zero} + y) + z$ both simplify to $y + z$
definitionally. Thus we can invoke reflexivity. The second case is more
complicated and is proved using equational reasoning combinators. The first
combinator $\AgdaFunction{\_≡⟨⟩\_}$ can only be used when the two lines are
definitionally equal to each other. This is the reflexivity axiom in disguise.
The next combinator we use is $\AgdaFunction{\_≡⟨\_⟩\_}$ and it allows us to
supply a proof that the first line equals the second.
We invoke the inductive hypothesis and wrap a
$\AgdaInductiveConstructor{suc}$ around the result. This is allowable as our
equality type is congruent over any function.
Lastly we push the $\AgdaInductiveConstructor{suc}$
inside the term. This is the second line in the definition of addition applied
in reverse.
\begin{code}
    +-assoc : Associative _+_
    +-assoc zero y z = ≡-refl
    +-assoc (suc x) y z = begin
      suc x + (y + z)   ≡⟨⟩
      suc (x + (y + z)) ≡⟨ ≡-cong suc (+-assoc x y z) ⟩
      suc ((x + y) + z) ≡⟨⟩
      (suc (x + y) + z) ≡⟨⟩
      (suc x + y) + z   ∎
\end{code}
\section{Record Types}
\label{sec:record-types}
We introduce one last critical construct, the record construction. A record is
very similar to a datatype but there are some differences. Most importantly
records have only one constructor so they can not be used to define any of the
datatype given above. Unlike datatypes, records contain a list of named fields.
We define a record type that bundles together all the proofs required for a
binary operator to be a semigroup. In this case there is only one constraint
namely associativity. \\
\begin{code}
    record IsSemigroup {A : Set} (_∙_ : A → A → A) : Set where
      constructor IsSemigroup✓
      field
        ∙-assoc : Associative _∙_
\end{code}
We prove that $\AgdaFunction{\_+\_}$ is a semigroup but supplying every
field of the $\AgdaDatatype{IsSemigroup}$ record.
\begin{code}
    +-isSemigroup : IsSemigroup _+_
    +-isSemigroup = IsSemigroup✓ +-assoc
\end{code}
Importantly the type of fields can depend on the value of fields previously in
the list. This allows us to define the less than or equal to relation $n \leq m$
as $\exists k. \, n + k = m$. The reason fields act similarly to exists
that when we construct a record we ``forget'' everything specific to the value
except the requirements specified by the fields.
\begin{code}
    record _≤_ (n : ℕ) (m : ℕ) : Set where
      constructor lte
      field
        k : ℕ
        pf : n + k ≡ m
\end{code}

% \begin{code}
%   Reflexive : ∀ {A : Set} → (A → A → Set) → Set
%   Reflexive _R_ = ∀ {x} → x R x

%   ≡-reflexive : ∀ {A : Set} → Reflexive {A = A} _≡_
%   ≡-reflexive = ≡-refl
% \end{code}
% \begin{code}
%   Symmetric : ∀ {A : Set} → (A → A → Set) → Set
%   Symmetric _R_ = ∀ {x y} → x R y → y R x

%   ≡-sym : ∀ {A : Set} → Symmetric {A = A} _≡_
%   ≡-sym {A} {x} {.x} ≡-refl = ≡-refl
% \end{code}
% Agda supports mixfix syntax for defining names (datatypes, functions, and arguments).
% As shown below anywhere an underscore is given an argument should be supplied.
% \begin{code}
%   Transitive : ∀ {A : Set} → (A → A → Set) → Set
%   Transitive _R_ = ∀ {x y z} → x R y → y R z → x R z

%   ≡-trans : ∀ {A : Set} → Transitive {A = A} _≡_
%   ≡-trans {A} {x} {.x} {.x} ≡-refl ≡-refl = ≡-refl
% \end{code}
% \begin{code}
%   ≡-cong : ∀ {A B : Set} (f : A → B) {x y : A} → x ≡ y → f x ≡ f y
%   ≡-cong {A} {B} f {x} {.x} ≡-refl = ≡-refl
% \end{code}
% \begin{code}
%   infix  1 begin_
%   begin_ : ∀ {A : Set} {x y : A} → x ≡ y → x ≡ y
%   begin x≡y = x≡y

%   infix  3 _∎
%   _∎ : ∀ {A : Set} (x : A) → x ≡ x
%   x ∎ = ≡-refl

%   infixr 2 _≡⟨_⟩_
%   _≡⟨_⟩_ : ∀ {A : Set} (x : A) {y z : A} → x ≡ y → y ≡ z → x ≡ z
%   x ≡⟨ x≡y ⟩ y≡z = ≡-trans x≡y y≡z

%   infixr 2 _≡⟨⟩_
%   _≡⟨⟩_ : ∀ {A : Set} {y : A} (x : A) → x ≡ y → x ≡ y
%   x ≡⟨⟩ ≡-refl = ≡-refl
% \end{code}

\end{document}


\documentclass[../../main.tex]{subfiles}
\begin{document}
\subsubsection{Map?}
The operation $map?$ will return a boolean which indicates if the passed in argument is a $KV$
\begin{schema}{Map?[V]}
  m? : V \\
  bol! : Boolean \\
  map?~\_ : V \fun Boolean
  \where
  bol! = map?(m?) @ bol! = true \iff m? : KV \implies V \hide (Scalar, Collection)
\end{schema}
where $V \hide (Scalar, Collection)$ is used to indicate that $m?$ is of type $V$
\begin{zed}
  V ::= Scalar ~| ~Collection ~| ~KV
\end{zed}
but in order for $bol! = true$, $m?$ must not be of type $Scalar ~\lor Collection$ such that
\begin{argue}
  X = \ldata x_{0}, x_{1}, x_{2}, x_{3}, x_{4} \rdata \\
  \t1 x_{0} = 0 \\
  \t1 x_{1} = foo \\
  \t1 x_{2} = \langle baz, \ qux \rangle \\
  \t1 x_{3} = \ldata abc \mapsto 123, \ def \mapsto 456 \rdata \\
  \t1 x_{4} = \langle \ldata ghi \mapsto 789, \ jkl \mapsto 101112 \rdata, \ \ldata ghi \mapsto 131415, \ jkl \mapsto 161718 \rdata \rangle \\
  map?(X) = true & KV by definition\\
  map?(x_{3}) = true & KV \\
  map?(x_{2}) = false & Collection \\
  map?(x_{4}) = false & Collection of maps\\
  map?(x_{0}) = false & Scalar \\
  map?(x_{1}) = false & String
\end{argue}
\end{document}


\paragraph{Opt.:}
$\argmin_x \operatorname{score}(x) - \sum \lambda_i \operatorname{Regularizer}_i(x)$, $\operatorname{score}(x) := \operatorname{mean}(\operatorname{layer}_l[x])$ (using GD)

\paragraph{Gradient Feature Attribution}
$\nabla_x \operatorname{logit}_t(x)$

\paragraph{Shapley Values}
$\displaystyle C_i := \sum_{S \subseteq P \setminus \{i\}} \frac{\lvert S \rvert! (\lvert P\rvert - \lvert S \rvert - 1) !}{\lvert P \rvert !} \left(f(S \cup \{i\}) - f(S) \right)$

Need to define $f(S)$ (e.g., set pixels not in $S$ to 0)

\paragraph{Prop.} $\sum_i C_i = f(P)$

\paragraph{Robustness} Robust NN rely on more robust features, more aligned with human perception


\section{Theories and Signatures}
\label{sec:theories-signatures}

\begin{figure*}[t]
\[
\footnotesize
\renewcommand{\arraystretch}{1.0}
\begin{array}[t]{l}
\mbox{\textbf{Theory Elements}}\\
\SET\ s\ [\EQUALS\ \mathit{set}\ ]\\
\CONST\ c\ [\COLON \mathit{set}\ ]\ [\EQUALS\ \mathit{term}\ ]\\
{}[\STABLE{}]\ \RELATION\ r\ [\COLON \mathit{set}\ ]\ [\EQUALS \mathit{prop}\ ]\\
\EQUIVALENCE \COLON \mathit{set}\\
\MODEL\ M\COLON\mathit{theory}\\
\AXIOM\ a\ [\ M\COLON\mathit{theory}\ ]^{*}\ [x\COLON\mathit{set}\ ]^*\EQUALS\mathit{prop}\\
\\
\mbox{\textbf{Propositions}}\\
\TRUE\\
\FALSE\\
\NOT\ \Prop\\
\Prop \AAND \Prop\\
\Prop \OOR \Prop\\
\Prop \IIMPLY \Prop\\
\Prop \IIFF \Prop\\
r [\ \Term\ ]^*\\
\Term\EQUALS\Term\\ %[\IN\ \Set]\\
\ALL\ [x\COLON \Set] \PERIOD \Prop\\
\SOME\ [x\COLON \Set] \PERIOD \Prop\\
\UNIQUE\ [x\COLON \Set] \PERIOD \Prop\\
\end{array}
\qquad
\begin{array}[t]{l}
\mbox{\textbf{Sets}}\\
\ZERO\\
\ONE\\
\BOOL\\
s\\
\mathit{Model}\PERIOD \mathit{name}\\
\Set \TIMES \cdots \TIMES \Set\\
\Setexp \ARROW \Setexp\\
\Label\ [\COLON \Setexp\ ]\ \PLUS \cdots \PLUS \Label\ [\COLON \Setexp\ ]\\
\LBRACE \Ident [\ \COLON \Setexp\ ]\ \BAR \Proposition \RBRACE\\
\Setexp \PERCENT \metav{relation}\\
%\RZ\ \Setexp\\
\\
\mbox{\textbf{Terms}}\\
x\\
\LPAREN \Term\COMMA \cdots \COMMA \Term \RPAREN\\
\Term\PERIOD \metav{n}\\
\Label\ [\ \Term\ ]\\
\MATCH\ \Term\ \WITH\ \mbox{\textit{pattern-matches}}\\
\LAMBDA\ x\COLON\Set\ \PERIOD\ \Term\\
\Term\ \Term\\
\Term \PERCENT \metav{relation}\\
\LET\ x \PERCENT \metav{relation}\ \IN\ \Term\EQUALS\Term\\
\Term \SUBIN \Set\\
\Term \SUBOUT \Set\\
\THE\ x\ [\COLON\Set\ ] \PERIOD \Prop\\
\LET\ x\ [\COLON\Set\ ] \EQUALS\Term\ \IN\ \Term
\end{array}
\]
\vspace{-0.6truecm}
\caption{Input Language Summary}
\label{fig:input}
\end{figure*}

In this section we describe first-order theories and signatures.
Our system translates the former into the later.

\subsection{Theories}
\label{sec:theories}

A \emph{theory} is a description of a mathematical structure, such as
a group, a vector space, a directed graph, etc. A theory consists of
%
\begin{itemize}
\item a list of \emph{basic sets},
\item a list of \emph{basic constants} belonging to specified sets,
\item a list of \emph{basic relations} on specified sets,
\item a list of axioms.
\end{itemize}
%
To take a simple example, consider the theory of a semigroup in which
every element has a (possibly non-unique) square root; recall that a
semigroup is a set with an associative binary operation and a neutral
element.\footnote{An example of a semigroup with square roots is the
  complex numbers with multiplication as the binary operation.} In our
system it could be written as follows:
\vspace{-0.2truecm}
{
\small
\VerbatimInput{semigroup.thy}
}
\goodbreak\goodbreak
\noindent The theory is enclosed by \Verb|thy|\ldots\Verb|end|. This theory
defines one basic set \Verb|s|, and two basic constants: an element
\Verb|e| of \Verb|s| and a (curried) binary infix operator \Verb|*| on
the set \Verb|s|. The \Verb|implicit| operator is not part of the
theory proper, but signals to the type checker that bound
variables named \Verb|x| or \Verb|y| or \Verb|z| should be assumed to
range over \Verb|s| unless otherwise specified. Finally,
we have three axioms. Axiom arguments, e.g., \Verb|x|, \Verb|y|, and
\Verb|z| in the associativity axiom, name the free variables occuring
in the axiom. It is not too big a mistake to think of them as being
universally quantified.

It is important to note that theories do not include proofs, but
rather just the statements of the axioms (and theorems) specified to
hold. Thus although axioms can be defined, one cannot actually refer
to them within the theory.

There are several features of theories that our system supports other
than those shown in this example above; the input language is
summarized in Figure~\ref{fig:input}, where brackets imply optional
elements.

Theories may declare or define relations. They may be \Verb|stable|,
i.e., their computational interpretation is trivial (see
Section~\ref{sec:implementation} for further discussion of this
point). Axioms can universally quantify over all models of a theory.
This is useful for describing universality properties, such as
initiality of an algebra or finality of a coalgebra.

The propositions are the familiar ones from first-order logic;
$\UNIQUE$ is unique existence ($\exists!$). In addition to the basic
empty ($\ZERO$) and unit ($\ONE$) sets, one can form cartesian
products, function spaces, tagged disjoint unions, subsets, and
quotients by stable equivalence relations. The corresponding
introduction and elimination forms appear in the language of terms.
For example, $\Term \PERCENT \metav{relation}$ is the equivalence
class under $\metav{relation}$ containing $\Term$, while $\LET\ x
\PERCENT \metav{relation} \mbox{\Verb| = |} \Term_1\ \IN\ \Term_2$
binds $x$ to a representative of the equivalence class $\Term_1$ to be
used in $\Term_2$. The expression $\Term \SUBIN \Set$ injects $\Term$
into a given subset (recording a proof obligation of the term actually
being a member of the subset), while $\Term \SUBOUT \Set$ projects
$\Term$ from a subset out into its superset $\Set$. The value of the
description operator $\THE\ x \,.\, \Prop$ is the unique $x$
satisfying $\Prop$; using it incurs the obligation of proving that
there is exactly one such~$x$.

\subsection{Signatures}
\label{sec:signatures}

On the logical side, we have models described by theories.  Thus on
the programming side we should have implementations being described by
specifications.  Our tool thus translates theories into
\emph{signatures}, which are ML's module interfaces.

Signatures allow us to require the existence of certain types, as well
as values of given type.  This allows decidable typechecking, but we
need more expressiveness in order to faithfully translate the content
of a theory.  We therefore generate signatures augmented by assertion
comments, which specify constraints on the values and functions an
implementation beyond their type.  It is the responsibility of the
programmer to check that the implementation satisfies these
assertions, as \RZ does not attempt to do any theorem proving.

Assertions are written in ordinary classical first-order logic. Since
programmers typically are not trained in constructive logic, this may
make it easier to verify the assertions.
\goodbreak
The output for the theory \Verb|SQGROUP| above is then:
{\small \VerbatimInput{semigroup.mli}}

At the ML level we have required a type \Verb|s|, and three values
\Verb|e|, \Verb|*|, and \Verb|sqrt|, of types \Verb|s|,
\Verb|s->s->s|, and \Verb|s->s|, respectively. The third value was
generated from the square root axiom, which has a non-trivial
computational content, cf.\ Subsection~\ref{subsec:real-transl}.

Comments contain other requirements, not expressible in ML, that
further contstrain the allowed implementations. The assertion
\Verb|PER(=s=)| abbreviates the requirement that \Verb|=s=| be a
partial equivalence relation on~\Verb|s|; its domain \Verb+||s||+ is
the subset of terms of type \Verb|s| that realize semigroup
elements, and the relation \Verb|=s=| identifies (possibly different)
terms realizing the same abstract semigroup element. These data
together determine a modest set. The assertion following the
declarations of \Verb|e| asserts that \Verb|e| realizes a valid
semigroup element, and the one following \Verb|*| asserts that
\Verb|*| must not be affected by the choice of realizers. Both
\Verb|e| and \Verb|*| must of course still satisfy the \Verb|unit| and
\Verb|assoc| axioms. Finally, the new function \Verb|sqrt| derived
from the logic must compute square roots. Since the theory requires
existence but not uniqueness of square roots, there is no requirement
that \Verb|sqrt| be invariant with respect to the partial equivalence
relation on \Verb|s|; different realizers of the same semigroup
element are allowed to produce (realizers of) different square roots.

\subsection{Parameterized Theories}
\label{sec:param-theor-funct}

A theory may be parameterized by one or more models of other theories.
For example, a theory \Verb|Real| of the reals may be parameterized in
terms of a model \Verb|N| of the naturals.  A theory of free groups may be
parameterized in terms of the generating set.

Parameterized theories serve two purposes.  A model of a
parameterized theory is a generic implementation that, given any
implementation of the parameters, returns an implementation of the
resulting theory.  At the level of ML, this would be a function from
modules to modules, a so-called \emph{functor}, and so a
parameterized theory can be translated into the signature of a
functor.

Alternatively, once we have described a parameterized theory
\Verb|Real|, we may wish to use it to describe a single specific
implementation of real numbers based on a specific model \Verb|N1|
(implementation) of the natural numbers; this can be described as
an implementation satisfying the theory \Verb|Real(N1)|.

The dual nature of parameterized theories as being both a description
of a parameterized model (a $\Pi$ type) and something which can be
applied to a model to produce a specialized theory (a $\lambda$) is
very reminiscent the type inclusion of Automath~\cite{automath}.  ML
does not permit applications of functor signatures, however, so we
beta-reduce all theory applications before generating signatures;
\Verb|Real(N1)| would produce a signature for a real-number
implementation that refers directly to \Verb|N1| rather than to a
generic parameter \Verb|N|.

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "case"
%%% End:


\section{Basic elements of  Lie groups and Lie algebras theory.}
Let us recall the definitions of the  Lie group Theory taken from \cite{Iserles.ea_AN2000} and \cite{Varadarajan_book1984}.

\subsection{Differential equation (evolving) on a manifold $\mathcal M$}
\begin{definition}
 A $d$-dimensional manifold $\mathcal M$ is a $d$-dimensional smooth surface $ M\subset \RR^n$ for some $n\geq d$.
\end{definition}
\begin{definition}
  Let $\mathcal M$ be a $d$-dimensional manifold and suppose that $\rho(t) \in\mathcal M$  is a smooth curve such that $\rho(0) = p$. A tangent vector at $p$ is defined as
  \begin{equation}
    \label{eq:12}
    a = \left. \frac{d}{dt} (\rho(t)) \right|_{t=0}.
  \end{equation}
The set of all tangents at $p$ is called the tangent space at $p$ and denoted by $T\mathcal M|_p$. It has the structure of a linear space.
\end{definition}
\begin{definition}
   A (tangent) vector field on $\mathcal M$ is a smooth function $F : \mathcal M \rightarrow T\mathcal M$ such that $F (p) \in T\mathcal M|_p$ for all $p \in \mathcal M$. The collection of all vector fields on $\mathcal M$ is denoted by $\mathcal X(\mathcal M)$.
 \end{definition}

 \begin{definition}[Differential equation (evolving) on $\mathcal M$]
   Let $F$ be a tangent vector field on $\mathcal M$. By a differential equation (evolving) on $\mathcal M$ we mean a differential equation of the form
   \begin{equation}
     \dot y =F(y), t\geq  0, y(0)\in \mathcal M\label{eq:13}
   \end{equation}
   where $F \in \mathcal X(\mathcal M)$. Whenever convenient, we allow $F$ in~\eqref{eq:13} to be a function of time, $F = F(t,y)$. The flow of $F$ is the solution operator $\Psi_{t,F} : \mathcal M \rightarrow  \mathcal M$ such that
   \begin{equation}
     y(t) = \Psi_{t,F} (y0).\label{eq:14}
   \end{equation}
 \end{definition}

 \subsection{Lie algebra and Lie group}
 \begin{definition}[commutator]
   Given two vector fields $F, G$ on $\RR^n$ , the commutator $H = [F, G]$ can
   be computed componentwise at a given point $y ∈ \RR^n$ as
   \begin{equation}
     H_i(y)= \sum_{j=1}^n  G_j(y)\frac{\partial F_i(y)}{\partial y_j}   −F_j(y) \frac{\partial G_i(y)}{\partial y_j} .\label{eq:15}
   \end{equation}
 \end{definition}

 \begin{lemma}\label{lemma:LieBracket}
The commutator of vector fields satisfies the identities
\begin{equation}
  \label{eq:16}
  \begin{array}[lclr]{lclr}
    \protect{[}F, G]&=& −\protect{[}G, F ] & (skew symmetry), \\
    \protect{[} \alpha F,G] &=& \alpha \protect{[}F,G], \alpha \in \RR &  \\
    \protect{[}F + G, H]&=& \protect{[}F, H] + \protect{[}G, H] & (bilinearity),\\
    0 &=&  \protect{[}F,\protect{[}G,H]]+\protect{[}G,\protect{[}H,F]]+\protect{[}H,\protect{[}F,G]] &(Jacobi’s identity).
  \end{array}
\end{equation}
\end{lemma}
\begin{definition}
  A Lie algebra of vector fields is a collection of vector fields which is closed under linear combination and commutation. In other words, letting $\mathfrak g$ denote the Lie algebra,

  \begin{equation}
    \begin{array}[lclr]{l}
    B \in \mathfrak g \implies \alpha B \in \mathfrak  g \text{ for all } \alpha ∈ R .\\
    B_1,B_2 \in\mathfrak g \implies B_1 +B_2, [B_1,B_2]\in\mathfrak g\label{eq:17}
    \end{array}
\end{equation}

Given a collection of vector fields $B = {B_1 , B_2 , \ldots}$, the least Lie algebra of vector fields containing $B$ is called the Lie algebra generated by $B$
\end{definition}

\begin{definition}
  A Lie algebra is a linear space $V$ equipped with a Lie bracket, a bilinear, skew-symmetric mapping
  \begin{equation}
    \label{eq:18}
    [ \cdot , \cdot ] : V \times V \rightarrow V
  \end{equation}
that obeys identities \eqref{eq:16} from Lemma~\ref{lemma:LieBracket}
\end{definition}

\begin{definition}[(General) Lie algebra]
  A Lie algebra homomorphism is a linear map between two Lie algebras, $\varphi : \mathfrak g \rightarrow \mathfrak h$, satisfying the identity
  \begin{equation}
\varphi ([v, w]_{\mathfrak g}) = [\varphi(v), \varphi(w)]_{\mathfrak h}, v, w in \mathfrak g\label{eq:19}.
\end{equation}
An invertible homomorphism is called an isomorphism.
\end{definition}

\begin{definition}
  A Lie group is a differential manifold $\mathcal G$ equipped with a product $\glaw : \mathcal G\times \mathcal G →\rightarrow \mathcal  G$ satisfying
  \begin{equation}
    \label{eq:20}
    \begin{array}[lclr]{lr}
      p \glaw(q \glaw r) = (p\glaw q)\glaw r, \forall  p, q, r ∈ \mathcal G &\text{(associativity)}\\
      \exists I \in \mathcal G \text{ such that } I\glaw p = p \glaw I = p,  \forall p \in \mathcal G&\text{(identity element)}\ \\
      \forall p \in \mathcal G, \exists  p^{-1}  \in \mathcal G \text{ such that }  p^{-1}\glaw p = I&\text{(inverse) }\ \\
      \text{ The maps}  (p, r)  \rightarrow p\glaw r \text{ and }  p  \rightarrow p^{-1} \text{are smooth functions }&\text{(smoothness)}\
    \end{array}
\end{equation}
\end{definition}

\begin{definition}[Lie algebra $\mathfrak g $ of a Lie group $\mathcal G$]
  The Lie algebra $\mathfrak g$ of a Lie group $\mathcal G$ is defined as the linear space of all tangents to $G$ at the identity $I$. The Lie bracket in $\mathfrak g$ is defined as
  \begin{equation}
    [a,b]= \left.\frac{\partial^2 }{\partial s\partial t} \rho(s)\sigma(t)\rho(-s)\right|_{s=t=0}\label{eq:21}
\end{equation}
where $\rho(s)$ and $\sigma(t)$ are two smooth curves on $\mathcal G$ such that $\rho(0) = \sigma(0) = I$, and
$\dot \rho(0) = a$ and $\dot \sigma(0) = b$.
\end{definition}

\subsection{Actions of a group $\mathcal G$ on  manifold $\mathcal M$}
\begin{definition}
   A left  action of Lie Group $\mathcal G$ on a manifold $\mathcal M$ is a smooth map $\Lambda^l: \mathcal G \times  \mathcal M \rightarrow \mathcal M$ satisfying
\begin{equation}
  \label{eq:22}
  \begin{array}[lcl]{rcl}
    \Lambda^l(I,y) &=& y, \quad \forall y \in \mathcal M \\
    \Lambda^l(p,\Lambda(r,y)) &=& \Lambda^l(p\glaw r, y) , \quad \forall p,r \in \mathcal G,\quad  \forall y \in \mathcal M .
  \end{array}
\end{equation}
\end{definition}

\begin{definition}
   A  right  action of Lie Group $\mathcal G$ on a manifold $\mathcal M$ is a smooth map $\Lambda^r: \mathcal M \times \mathcal G   \rightarrow \mathcal M$ satisfying
\begin{equation}
  \label{eq:23}
  \begin{array}[lcl]{rcl}
    \Lambda^r(y,I) &=& y, \quad \forall y \in \mathcal M \\
    \Lambda^r(\Lambda(y,r), p) &=& \Lambda^r(y,  r\glaw p) , \quad \forall p,r \in \mathcal G,\quad  \forall y \in \mathcal M .
  \end{array}
\end{equation}
\end{definition}

A given smooth curve  $S(\cdot) : t\in \RR \mapsto S(t)\in \mathcal G$ in $\mathcal G$ such that $S(0)= I$ produces a flow $\Lambda^l(S(t),\cdot)$ (resp. $\Lambda^r(\cdot, S(t))$) on $\mathcal M$ and by differentiation we find a tangent vector field
\begin{equation}
  \label{eq:24}
  F(y) = \left. \frac{d}{dt} (\Lambda^l(S(t),y) \right|_{t=0}\quad( \text{resp.  }  F(y) = \left. \frac{d}{dt} (\Lambda^r(y,S(t)) \right|_{t=0} )
\end{equation}
that defines a ordinary differential equation on a Lie Group
\begin{equation}
  \label{eq:25}
  \dot y(t) = F(y(t)) = \left. \frac{d}{dt} (\Lambda^l(S(t),y) \right|_{t=0}  \quad( \text{resp.  }\dot y(t) = F(y(t)) = \left. \frac{d}{dt} (\Lambda^r(y,S(t)) \right|_{t=0})
\end{equation}

\begin{lemma}
  Let $\lambda^l_{*} : \mathfrak g \rightarrow \mathcal X(\mathcal M) $ (resp. $\lambda^r_{*} : \mathfrak g \rightarrow \mathcal X(\mathcal M) $ be defined as
  \begin{equation}
  \lambda^l_{*}(a)(y) = \left.\frac{d}{ds}{ \Lambda^l (\rho(s), y)}\right|_{s=0} \quad (\text{ resp. }  \lambda^r_{*}(a)(y) = \left.\frac{d}{ds}{ \Lambda^r (y, \rho(s))}\right|_{s=0})\label{eq:26}
\end{equation}
 where $\rho(s)$ is a curve in $\mathcal G$ such that $\rho(0)=I$ and $\dot\rho (0)=a$. Then $\lambda^l_{8}$ is a linear
map between Lie algebras such that
\begin{equation}
  [a, b]_{\mathfrak g} = [\lambda^l_{*}(a), \lambda^l_{*}(b)]_{\mathcal X(\mathcal M)}.\label{eq:27}
\end{equation}
\end{lemma}

The following product between an element of an algebra $a \in \mathfrak g$ with an element of a group $\sigma  \in \mathcal G$
 can be defined. This will served as a basis for defining the exponential map.
\begin{definition}
  We define the left product $(\cdot, \cdot)^l : \mathfrak g \times \mathcal G \rightarrow  \mathcal G$ of an element of an algebra $a \in \mathfrak g$ with an element of a group $\sigma  \in \mathcal G$ as
  \begin{equation}
 (a, \sigma)^l = a \cdot \sigma = \left.\frac{d}{ds} \rho(s) \glaw \sigma \right|_{s=0}\label{eq:28}
\end{equation}
where $\rho(s)$ is a smooth curve such that $\dot\rho(0)=a$ and $\rho(0)=I$. In the same way, we can define the right product $(\cdot, \cdot)^r : \mathcal G \times \mathfrak g  \rightarrow   \mathcal G$
\begin{equation}
  \label{eq:29}
  (\sigma,a)^r = \sigma \cdot a  = \left.\frac{d}{ds} \sigma \glaw \rho(s)   \right|_{s=0}
\end{equation}
\end{definition}

\subsection{Exponential map}
\begin{definition}
  Let $\mathcal G$ be a Lie group and $\mathfrak g$ its Lie algebra. The exponential mapping $exp : \mathfrak g \rightarrow \mathcal G$ is defined as $\exp(a) = \sigma(1)$ where $\sigma (t)$ satisfies the  differential equation
\begin{equation}
\dot \sigma(t) = a \cdot \sigma(t), \quad \sigma (0) = I.\label{eq:30}
\end{equation}
\end{definition}

Let us define $a^k$ as
\begin{equation}
  \label{eq:31}
  \left\{\begin{array}[l]{l}
    a^k = \underbrace{a\glaw a \glaw \ldots a\glaw a}_{k \text{ times}} \text{ for } k \geq 1 \\
    a^0  = I
  \end{array}\right.
\end{equation}
The exponential map can be expressed as
\begin{equation}
  \label{eq:32}
  \exp(at) = \sum_{k=0}^\infty \frac{(ta)^k}{k!}
\end{equation}
since it is  a solution of \eqref{eq:30}. A simple computation allows to check this claim:
\begin{equation}
  \label{eq:33}
   \frac{d}{dt}\exp(at) = \sum_{k=1}^\infty  k t^{k-1} \frac{a^k}{k!} = a \glaw \sum_{k=0}^\infty  t^{k} \frac{a^k}{k!} = a \glaw \exp(at).
\end{equation}
A similar computation gives
\begin{equation}
  \label{eq:34}
  \frac{d}{dt}\exp(at)  = \sum_{k=0}^\infty  t^{k} \frac{a^k}{k!} \glaw a = \exp(at) \glaw a.
\end{equation}
The exponential mapping $exp : \mathfrak g \rightarrow \mathcal G$ can also be defined as $\exp(a) = \sigma(1)$ where $\sigma (t)$ satisfies the  differential equation
\begin{equation}
  \label{eq:35}
  \dot \sigma(t) = \sigma(t) \cdot a, \quad \sigma (0) = I.
\end{equation}

\begin{theorem}
  \label{Theorem:solutionofLieODE}
  Let $\Lambda^l:\mathcal G\times\mathcal M \rightarrow \mathcal M$ be a left  group action and $\lambda^l_{∗} : \mathfrak g\rightarrow \mathcal X(\mathcal M)$ the corresponding Lie algebra homomorphism. For any $a \in \mathfrak g$ the flow of the vector field $F = \lambda^l_{a}(a)$, i.e. the solution of the equation
  \begin{equation}
    \dot y(t) = F(y(t)) = \lambda^l_{*}(a)(y(t)),\quad  t \geq 0, y(0) = y_0 \in \mathcal M,\label{eq:36}
\end{equation}
  is given as
  \begin{equation}
y(t) = \Lambda^l(\exp(ta), y_0).\label{eq:37}
\end{equation}
Let $\Lambda^r:\mathcal M\times\mathcal G \rightarrow \mathcal M$ be a right group action and $\lambda^r_{∗} : \mathfrak g\rightarrow \mathcal X(\mathcal M)$ the corresponding Lie algebra homomorphism. For any $a \in \mathfrak g$ the flow of the vector field $F = \lambda^r_{*}(a)$, i.e. the solution of the equation
  \begin{equation}
    \dot y(t) = F(y(t)) = \lambda^r_{*}(a)(y(t)),\quad  t \geq 0, y(0) = y_0 \in \mathcal M,\label{eq:38}
\end{equation}
  is given as
  \begin{equation}
y(t) = \Lambda^r(y_0,\exp(ta)).\label{eq:39}
\end{equation}

\end{theorem}

\subsection{Translation (Trivialization) maps}
The left and right translation maps defined by
\begin{equation}
  \label{eq:148}
  \begin{array}{rcl}
    L_z  : \mathcal G \times \mathcal G &\rightarrow& \mathcal G \quad \text{ (left translation map )} \\
    y &\mapsto&  z \glaw y
  \end{array}
\end{equation}
and
\begin{equation}
  \label{eq:149}
  \begin{array}{rcl}
    R_z(y)  :  \mathcal G \times  \mathcal G  & \rightarrow& \mathcal G \quad \text{ (right translation map )} \\
    y  &\mapsto&  y \glaw z
  \end{array}
\end{equation}

If we identify the manifold $\mathcal M$ with the group $\mathcal G$, The left and right translations can be interpreted as the simplest example of group action on the manifold. Note that the left translation map can be viewed as a left or right action on the group.

If we consider $L_z(y)$ as a right group action $ L_z(y) = \Lambda^r( z, y) =z \glaw y $, by differentiation we get a $L'_z : T \mathfrak g \cong  \mathfrak g \rightarrow T_z\mathcal G$ with $\dot\rho (0)=a$ such that
\begin{equation}
  \label{eq:150}
  \lambda^r_{*}(a)(z) = L'_z(a) = \left.\frac{d}{ds}{ \Lambda^r (z, \rho(s))}\right|_{s=0} = z \glaw a
\end{equation}
The map
\begin{equation}
  \label{eq:152}
  \begin{array}{rcl}
  L'_z  : \mathfrak g &\rightarrow& T_z\mathcal G  \\
         a &\mapsto&  z \glaw a
  \end{array}
\end{equation}
determines an isomorphism of $\mathfrak g$ with the tangent space  $T_z\mathcal G$. In other words, the  tangent space can be identified to $\mathfrak g$ as
\begin{equation}
  \label{eq:153}
  T_z\mathcal G =\{L'_z(a) = z \glaw a \mid a \in \mathfrak g  \}
\end{equation}

Respectively, if we consider $R_z(y)$ as a left group action $ R_z(y) = \Lambda^l( y, z) =y \glaw z $, by differentiation we get a $R'_z : T \mathfrak g \cong  \mathfrak g \rightarrow T_z\mathcal G$ with $\dot\rho (0)=a$ such that
\begin{equation}
  \label{eq:150}
  \lambda^l_{*}(a)(z) = R'_z(a) = \left.\frac{d}{ds}{ \Lambda^l (\rho(s),z)}\right|_{s=0} = a \glaw z
\end{equation}
The map
\begin{equation}
  \label{eq:152}
  \begin{array}{rcl}
  R'_z  : \mathfrak g &\rightarrow& T_z\mathcal G  \\
         a &\mapsto&  a \glaw z
  \end{array}
\end{equation}
determines an isomorphism of $\mathfrak g$ with the tangent space  $T_z\mathcal G$. In other words, the  tangent space can be identified to $\mathfrak g$ as
\begin{equation}
  \label{eq:153}
  T_z\mathcal G =\{R'_z(a) = a \glaw z \mid a \in \mathfrak g  \}
\end{equation}
Any tangent vector $F : \mathcal G \rightarrow T_z\mathcal G$ can be written in either of the forms
\begin{equation}
  \label{eq:155}
  F(z) = L'_z(f(a)) = R'_z(g(z))
\end{equation}
where $f,g \mathcal G \rightarrow \mathfrak g$.
\subsection{Adjoint representation}
\begin{definition}
Let $p \in \mathcal G$ and let $\sigma (t)$ be a smooth curve on $\mathcal G$ such that $\sigma (0)$ = I and $\dot \sigma(0) = b \in \mathfrak g$. The adjoint representation is defined as
\begin{equation}
\Ad_p(b) =\left. \frac{d}{dt} p\sigma(t)p^{-1}\right|_{t=0}\label{eq:40}
\end{equation}
The derivative of $\Ad$ with respect to the first argument is denoted $\ad$. Let $\rho(s)$ be a smooth curve on $\mathcal G$ such that $\rho(0) = I$  and $\dot \rho(0) = a$, it  yields:
\begin{equation}
  \label{eq:41}
    \ad_a(b) = \left.\frac{d}{ds} \Ad_{\rho(s)}(b)\right|_{s=0}  = [a, b]
\end{equation}
\end{definition}
The adjoint representation can also be expressed with the map
\begin{equation}
  \label{eq:154}
  \Ad_p(b)  = (L_p \glaw R_{p^{-1}})' (b) = (L'_p \glaw R'_{p^{-1}}) (b) = p \glaw b \glaw p^{-1}
\end{equation}

For a tangent vector given in~\eqref{eq:155}, we have
\begin{equation}
  \label{eq:151}
  g(z) = Ad_z(f(z))
\end{equation}
Another important relation relating $\Ad$, $\ad$ and $\exp$ is
\begin{equation}
  \label{eq:164}
  \Ad_{\exp(a)} =\exp{\ad_a}
\end{equation}

\subsection{Differential of the exponential map} There are multiple ways to represent the differential of $\exp(\cdot)$ at a point $a\in \mathfrak g$. Let us start by the following definition of the differential map at $a\in\mathfrak g$
\begin{equation}
  \label{eq:147}
  \begin{array}{lcl}
    \exp_a' & : & \mathfrak g \rightarrow  T_{exp(a)}\mathcal G\\
            & &  v \mapsto \exp'_a(v)  = \left.\frac{d}{dt} \exp(a+tv)\right|_{t=0}
  \end{array}
\end{equation}
The definition is very similar to the definition of the directional derivative of $\exp$ in the direction $v \in \mathfrak g$ at a point $a\in\mathfrak g$. Using the expression \eqref{eq:153} of the tangent space at $\exp(a)$, we can defined another expression of the differential map denoted as $\dlexp_a : \mathfrak g  \rightarrow \mathfrak g$ such that
\begin{equation}
  \label{eq:156}
  \dlexp_a = L'_{\exp^{-1}(a)} \glaw \exp_a' = L'_{\exp(-a)} \glaw \exp_a'
\end{equation}
This expression appears as a trivialization of the differential map $\exp'_a$. Using the expression of $L'_z$ in \eqref{eq:152}.
In~\cite[Theorem 2.14.13]{Varadarajan_book1984}, an explicit formula relates $\dlexp_{a}$ to the iteration of the adjoint operator:
\begin{equation}
  \label{eq:43}
  \dlexp_a(b) = \sum_{k=0}^\infty \frac{(-1)^k}{(k+1)!} (\ad_a(b))^k \coloneqq \frac{e - \exp\glaw\ad_a}{\ad_a}(b)
\end{equation}
where $(\ad_a)^k$ is the kth iteration of the adjoint operator:
\begin{equation}
  \label{eq:44}
  \left\{\begin{array}[l]{l}
    (\ad_a)^k(b) = \underbrace{[a, [ a, [ \ldots, a, [ a, b]]]}_{k \text{ times}} \text{ for } k \geq 1 \\
    (\ad_a)^0(b)  = b
  \end{array}\right.
\end{equation}
It is also possible to define the right trivialized differential of the exponential map
\begin{equation}
  \label{eq:162}
  \drexp_a = R'_{\exp^{-1}(a)} \glaw \exp_a' = R'_{\exp(-a)} \glaw \exp_a'
\end{equation}
that is
\begin{equation}
  \label{eq:163}
  \drexp_a(b) = \exp'_a(b) \glaw \exp(-a)
\end{equation}
With these expression, we have equivalently for
\begin{equation}
  \label{eq:157}
   \exp_a'(b)  = \exp_a \glaw \dlexp_a(b)\quad \text{ and } \exp_a'(b)  = \drexp_a(b) \glaw   \exp(a)
\end{equation}

To avoid to burden to much the notation, we introduced the unified definition of the differential map  that corresponds to $\dexp=\drexp$
\begin{definition}
The differential of the exponential mapping, denoted by $\dexp_a : \mathfrak g \times \mathfrak g \rightarrow \mathfrak g$ is defined as the ``right trivialized'' tangent of the exponential map
\begin{equation}
  \label{eq:42}
  \frac{d}{dt} (\exp(a(t))) = \dexp_{a(t)}(a'(t)) \exp(a(t))
\end{equation}
\end{definition}
An explicit formula relates $\dexp_{a}$ to the iteration of the adjoint operator:
\begin{equation}
  \label{eq:43}
  \dexp_a(b) = \sum_{k=0}^\infty \frac{1}{(k+1)!} (\ad_a(b))^k \coloneqq \frac{\exp\glaw\ad_a-e}{\ad_a}(b)
\end{equation}

\begin{ndrva}
  Say what is not the Jacobian in $\RR^4$
\end{ndrva}

As for $\Ad_a$ and $\ad_a$, the mapping $\dexp_{a}(b)$ is a linear mapping in its second argument for a fixed $a$. Using the relation~\eqref{eq:164}, we can also relate the right and the lest trivialization tangent
\begin{equation}
  \label{eq:165}
\dlexp_a (b) =   (\Ad_{\exp(a)} \glaw \dexp(a))(b) = (\exp(\ad_{-a}) \glaw \frac{e - \exp\glaw\ad_a}{\ad_a})(b) = \frac{e - \exp\glaw\ad_{-a}}{\ad_a}(b) = \dexp_{-a}(b)
\end{equation}
It is also possible to define the  the ``left trivialized'' tangent of the exponential map
\begin{equation}
  \label{eq:46}
   \frac{d}{dt} (\exp(a(t))) =  \exp(a(t)) \dlexp_{a(t)}(a'(t)) = \exp(a(t)) \dexp_{-a(t)}(a'(t))
\end{equation}

\begin{ndrva}
  other notation and Lie derivative
  \begin{equation}
    \label{eq:178}
      Df \cdot \widehat \Omega (p) = (\widehat \Omega^r f )(p)
  \end{equation}
\end{ndrva}

\paragraph{Inverse of the exponential map}

The function $\dexp_{a}$ is an analytical function so it possible to invert it to get
\begin{equation}
  \label{eq:45}
  \dexp^{-1}_{a} = \sum_{k=0}^\infty \frac{B_k}{(k)!} (\ad_a)^k(b)
\end{equation}
where $B_k$ are the Bernouilli number.

\subsection{Differential of a map $f : \mathcal G \rightarrow \mathfrak g$}

We follow the notation developed in~\cite{Owren.Welfert_BIT2000}. Let us first define the differential of the map $f : \mathcal G \rightarrow \mathfrak g$ as
\begin{equation}
  \label{eq:166}
  \begin{array}[rcl]{rcl}
    f'_z : T_z\mathcal G &\rightarrow&T_{f(z)}\mathfrak g \cong  \mathfrak g\\
    b &\mapsto& \left.\frac{d}{dt} f(z\glaw \exp(t L'_{z^{-1}}(b))) \right|_{t=0}
  \end{array}
\end{equation}
The image of $b$ by $f'_z$   is obtained by first identifying $b$ with an element of $v \in \mathfrak g$ thanks to the left representation of $T_{f(z)}\mathfrak g$ view the left translation map $v= t L'_z(b)$. The exponential mapping transforms $v$ an element $y$ of the Lie Group $\mathcal G$. Then $f'_z$ is obtained by
\begin{equation}
  \label{eq:167}
  f'_z(b) = \lim_{t\rightarrow 0} \frac{f(z\glaw y) - f(z)}{t}
\end{equation}
As we have done for the exponential mapping, it is possible to get a left trivialization of
\begin{equation}
  \label{eq:169}
  \dd f_z = (f\glaw L_z)' = f'_z \glaw L'_z
\end{equation}
thus
\begin{equation}
  \label{eq:170}
  \dd f_z (a) =  f'_z \glaw L'_z(a) = f'_z(L'_z(a)) =  \left.\frac{d}{dt} f(z\glaw \exp(t a )) \right|_{t=0}
\end{equation}

\paragraph{Newton Method}
Let us imagine that we want to solve $f(y) = 0 $ for $y \in \mathcal G$. A newton method can be written as
%%% Local Variables:
%%% mode: latex
%%% TeX-master: "DevNotes"
%%% End: