% !TEX root = ../root.tex
\section{Theoretical principles}\label{sec:theory}
This section will present the theoretical aspects of the technologies employed in the project, and provides a knowledge base for better understanding the problem and the employed solution.

\subsection{Ultra-wideband}
Ultra-wideband (UWB in short) is a radio technology that use a very low energy level for short-range, high-bandwidth communications over a large portion of the radio spectrum.

Conventional narrowband systems employs the modulation of a continuous-waveform signal with a specific carrier frequency to transmit and receive information.
This continuous waveform has a well-defined signal energy in a narrow frequency band that exposes it to detection and interception.
On the other hand, UWB transmitters generate extremely thin pulse (duty cycle of 0.5\%) whose average energy content is very low, in fact barely above the background noise level.

Short pulses like the ones described above can yield a very wide bandwidth, often in the order of several \si{\giga\hertz}.
On a theoretical standpoint, this means that UWB may allow for high channel capacities \emph{C}, as predicted by Shannon–Hartley, shown below (where \emph{B} is the channel bandwidth and \emph{SNR} is the signal-to-noise ratio of the signal):
\begin{equation}
    \textit{C} = \textit{B}\log_2(1 + \textit{SNR})
\end{equation}

The UWB technology finds easy applications in \emph{ToF}-based positioning systems, due to its ability of avoiding reflection overlapping (an inherited benefit of the brevity of the pulses).
Moreover, the wide spectrum ensures that at least a portion of the signal follows a line-of-sight path.

Other applications may include short-range, high-speed communication systems capable of replacing current wireless technologies while having a smaller footprint on the overall level of emission.

\subsection{Ranging algorithm}
An overview of different ranging techniques is hereby presented and discussed.
The techniques analyzed here mainly strive to calculate the most accurate possible \emph{time of flight} (TOF), which can then be converted into distance with the following trivial equation:
\begin{equation}
    \textit{distance} = \textit{ToF} \times c
\end{equation}
The devices involved in the process will be referred to as initiator (\(\mathbf{I}\)) and responder (\(\mathbf{R}\)).
Furthermore, only solutions that can cope with unsynchronized (i.e. not sharing the same base clock) devices are taken into consideration.

\fig{5cm}{twr.png}{Two-way ranging: different approaches}{twr}

\subsubsection{Single-sided 2WR}
This is the most straightforward approach:
\begin{enumerate}
    \item \(\mathbf{I}\) sends poll message and notes time \(T_{poll_{TX}}\)
    \item \(\mathbf{R}\) receives poll message and notes time \(T_{poll_{RX}}\)
    \item After an arbitrary time, \(\mathbf{R}\) sends response message and notes time \(T_{resp_{TX}}\)
    \item \(\mathbf{I}\) receives response message and notes time \(T_{resp_{RX}}\)
\end{enumerate}
Assuming that \(\mathbf{R}\) includes all its recorded data into the response message, \(\mathbf{I}\) can assume that:
\begin{equation}
    \textit{ToF} = \frac{(T_{resp_{RX}} - T_{poll_{TX}}) - (T_{resp_{TX}} - T_{poll_{RX}})}{2}
\end{equation}
One of the major shortcomings of this scheme is that clock frequency differences between the two devices will cause vast drifts over the tiniest delay.

\subsubsection{Double-sided 2WR}
This method improves upon the previous ones by trying to compensate for the introduced drift:
\begin{enumerate}
    \item \(\mathbf{I}\) sends poll message and notes time \(T_{poll_{TX}}\)
    \item \(\mathbf{R}\) receives poll message and notes time \(T_{poll_{RX}}\)
    \item After an arbitrary time, \(\mathbf{R}\) sends response message and notes time \(T_{resp_{TX}}\) as well as \(T_{reply1} = T_{resp_{TX}} - T_{poll_{RX}}\)
    \item \(\mathbf{I}\) receives response message and notes time \(T_{resp_{RX}}\) as well as \(T_{round1} = T_{resp_{RX}} - T_{poll_{TX}}\)
    \item After an arbitrary time, \(\mathbf{I}\) sends final message and notes time \(T_{final_{TX}}\) as well as \(T_{reply2} = T_{final_{TX}} - T_{resp_{RX}}\)
    \item \(\mathbf{R}\) receives response message and notes time \(T_{final_{RX}}\) as well as \(T_{round2} = T_{final_{RX}} - T_{resp_{TX}}\)
\end{enumerate}
Assuming that \(\mathbf{R}\) includes the calculated time intervals in the response message, \(\mathbf{I}\) can assume that:
\begin{equation}
    \textit{ToF} = \frac{T_{round1} \times T_{round2} - T_{reply1} \times T_{reply2}}{T_{round1} + T_{round2} + T_{reply1} + T_{reply2}}
\end{equation}
Although this method is known to be more reliable than the previous, it requires that ranging measurements are transmitted through the radio, which depending on the application may not be desirable.

\subsubsection{Single-sided 2WR with drift compensation}
This method provides drift correction whilst maintaining the range computation on the initializer:
\begin{enumerate}
    \item \(\mathbf{I}\) sends poll message and notes time \(T_{poll_{TX}}\)
    \item \(\mathbf{R}\) receives poll message
    \item After an arbitrary time, \(\mathbf{R}\) sends response message
    \item \(\mathbf{I}\) receives response message and notes time \(T_{resp1_{RX}}\)
    \item After the same amount of time, \(\mathbf{R}\) sends another response message
    \item \(\mathbf{I}\) receives response message and notes time \(T_{resp2_{RX}}\)
\end{enumerate}

In this way \(\mathbf{I}\) need not use any data based on other clocks.
The equation becomes:
\begin{equation}
    \textit{ToF} = \frac{(T_{resp2_{RX}} - T_{poll_{TX}}) - 2 \times (T_{resp2_{RX}} - T_{resp1_{TX}})}{2}
\end{equation}

\subsection{Trilateration}
\emph{Trilateration} consists in determining the location of an object by measuring its distance from reference points.
With the advent of electronic means of measuring distances, this technique found widespread usage in the fields of geolocalization, navigation, and surveying.
In particular, GPS technology leverages on trilateration for providing its services.

The process of trilateration can be easily visualized in two dimensions by considering the circle projected by the set of points equidistant from a given reference.
The radius of such circle is the distance between an anchor whose location is known (the center of the circle) and a tag whose location is to be found.

\fig{5cm}{trilat.png}{Trilateration: example case in two dimensions}{trilat}

Adding another anchor and projecting its distance from the tag brings down the possible location to two points.
In the ideal case, a third anchor would project a circle intercepting precisely in one of these points.
However, this is rarely the case and may simply be unfeasible in case of a moving object (as the application of this project implies). \nameref{fig:trilat} shows however that an educated estimate can still be achieved.

\subsection{CAN Bus}
A \emph{Controller Area Network} (CAN) bus is a standard designed to allow microcontrollers, sensors, and other devices to communicate with each other without a host computer.
It was designed in 1983 by \emph{Robert Bosch GmbH} for automotive usage, but eventually found applications in many other contexts.

A CAN network is made up of nodes connected to each other through a two-wire bus.
The wires form a differential pair of \SI{120}{\ohm} nominal impedance.
Each node is capable of sending or receiving messages, properly called called frames.
A frame consists of an ID which can be \SI{11}{\bit} or \SI{29}{\bit} long, and up to eight data bytes.

CAN uses a lossless arbitration method in case of multiple nodes simultaneously attempting to transmit data.
Transmitted bit can be recessive or dominant.
If two such bits are beind transmitted at the same time, the node transmitting the recessive bits suspends its operation and reattempts the transfer after a given number of clock cycles, effectively implementing a prioritized communication system.
For this strategy to work, each node on a CAN network must agree on a bit rate and is required to sample every bit of data at the same time as the others.

Due to the lack of a shared clock signal, a synchronization mechanism is employed.
\emph{Hard synchronization} occurs after a recessive-to-dominant bit transition, and simply restart the timing from there.
Each bit is divided into time intervals called \emph{quanta}, and the first quantum is assigned to the so-called synchronization segment.
Whenever a recessive-to-dominant bit transition occurs outside of the synchronization segment, the sampling time is moved accordingly.
To ensure enough transitions to perform the above procedure, a bit of opposite polarity is inserted after five consecutive equal bits, through a process called \emph{bit stuffing}.

\fig{2cm}{can_smpl.png}{CAN Bus: bit timing example }{can_smpl}

Most CAN interfaces support automatic filtering of undesired frames through acceptance filter.
The process consists in performing a bitwise \emph{and} operation between the incoming frame ID and an acceptance bit-mask, and then comparing the result with the content of a filter register: if these match, the frame is added to the reception queue.
In this way, a receiving node can choose to accept frames of a specific type, or sent by a particular other node.


\section{Qualitative Sensitivity Measures: The Morris Method} \label{comp_morris}

This section introduces the Morris method for input screening. After the usefulness of input screening is discussed, the Morris method is applied to the Rust model.

\subsection{Input Screening} \label{screening}

Screening methods aim at identifying uninfluential inputs and ranking them according to their importance. These qualitative sensitivity methods, as opposed to quantitative ones, do not allow us to exactly quantify each input's effect on the model outcome. Nevertheless, there are several settings in which screening methods can be successfully applied. These are the FF and the FP settings discussed in \cref{var_based_sa}.

Since screening methods come at a lower computational cost than other sensitivity measures, they are well adapted to models with a large number of inputs. In such settings, other sensitivity methods may not be tractable anymore. Input screening can then guide research by identifying uninfluential inputs that can be left out of subsequent analyses. Such a subsequent analysis can be a more thorough, expensive sensitivity method which has then come into reach by having to consider only a subset of inputs. Screening can also foster scientific discovery by telling us which inputs are most important for the model output. Future models can then build on these results by particularly focusing on these important inputs \citep{R21}.

To illustrate the usefulness of input screening, consider the following examples of their application found in \citet{GPWMS17} and \citet{MMA18}. \citet{GPWMS17} conduct input screening in hydrology. They apply seven different screening methods to a lake model and learn that a certain input has major influence on their model's output. By identifying the main driver of uncertainty in the model, they are able to make an informed recommendation about which inputs should be estimated more accurately. \citet{MMA18} use input screening to identify uninfluential inputs that could then be ignored in the subsequent development of a meta-model, thus facilitating and guiding modelling.

While applying quantitative sensitivity methods, \citet{HMSW19} use their sensitivity indices for input ranking as well to ease the interpretation of model mechanics.

With these examples of the usefulness of input screening in mind, the following section discusses the Morris method which is then applied to the Rust model.

\subsection{Morris Method for Independent Inputs} \label{classic_morris}

The Morris method was introduced to identify the uninfluential input variables of a model, especially in cases where there are many inputs and/or the evaluation of a model is time-consuming \citep{M91}.

Consider the same setup as employed in the preceding sections. Let $x = \{x_1, \dots, x_k\}$ denote a sample of values assigned to the $X_i$'s. $f(x)$ is then the model output obtained for the values in $x$. Now consider a second sample $x_{\Delta_i} = \{x_1, \dots, x_{i-1}, x_i + \Delta, x_{i+1}, \dots, x_k\}$ that is identical to $x$ up to input $x_i$ which is varied by $\Delta$. Then, one elementary effect for input $i$ is derived by
\begin{equation*}
EE_i = \frac{f(x_{\Delta_i}) - f(x)}{\Delta}.
\end{equation*}

The above elementary effect is computed $N$ times, each for a varying $\Delta$ \citep{GM17}. The actual sensitivity measures resulting from the Morris method are the mean, denoted by $\mu^\ast_i$, and the standard deviation, denoted by $\sigma_i$, taken from all $N$ different elementary effects per input $i$.
\begin{align}
\mu_i^\ast& = \frac{1}{N} \sum_{r=1}^N \vert EE_{i, r} \vert, \label{mu}\\
\sigma_i& = \sqrt{\frac{1}{N-1} \sum_{r=1}^N (EE_{i, r} - \mu_i)^2}, \label{sigma}
\end{align}

\noindent with $EE_{i, r}$ denoting the $r$-th elementary effect of input $i$, $r = 1,\ \dots, N$, and $\vert \cdot \vert$ the absolute value. Note that in \citet{M91} the absolute value was absent and elementary effects could potentially cancel each other out \citep{CCS07}. Therefore, \citet{CCS07} proposed the version presented above, thus making the screening method more robust. A total of $2 k N$ model evaluations is needed to compute the full set of sensitivity measures using the Morris method.

$\mu_i^\ast$ and $\sigma_i$ can now be used to identify non-influential model inputs. Uninfluential inputs exhibit a $\mu_i^\ast$ close to zero. If $\mu_i^\ast$ is large, it depends on $\sigma_i$ whether there exist substantial non-linear or interaction effects. A low $\sigma_i$ indicates that non-linear effects are non-existent, whereas a high $\sigma_i$ suggests large interaction or non-linear effects \citep{GM17}.

The Morris method exhibits some drawbacks, though. Firstly, as they stand, the sensitivity indices derived by the Morris method are not suited for screening inputs under dependence. To see why consider two inputs $X_i$ and $X_j$ which are dependent, i.e. $G(x_i, x_j) \neq G(x_i)G(x_j)$, where $G(\cdot)$ again denotes the cumulative distribution function. If $x_i$ changes, $x_j$ should change as well due to the dependence between the two inputs. The sensitivity indices presented above are derived using a one-at-a-time approach that does not allow for the screening of dependent inputs \citep{GM17}.

Secondly, similar to the Sobol' indices, the researcher or practitioner conducting sensitivity analysis has to take two indices per input into account. Compare to the arguments made in \cref{var_based_sa}.

Thirdly, there exists no clear interpretation of the absolute values of the sensitivity indices. They only provide a ranking of inputs and give a hint of which inputs are the least influential ones \citep{GM17}.

On the advantages, Morris indices are easily computed, with a much lower computational burden than the Shapley effects as presented in \cref{comp_shap}. Recall that Shapley effects as computed by using the algorithm in \citet{SNS16} came at a cost of $N_V+m \cdot N_I \cdot N_O \cdot (k-1)$ model evaluations. Even the more efficient approach by \citet{PRB20} needed $2^k$ model runs. See \cref{comparison} for a discussion of the respective computational costs.

\citet{M91} points out that his method does not rely on simplifying assumptions, e.g. monotonicity of the model or input sparsity. He argues that if those assumptions hold, one could apply other, more effective and economical procedures, e.g. based on Latin hypercube designs. However, the Morris method does not rely on such assumptions and will work well if these assumptions are justifiable or not \citep{M91}.

Considering the interpretation of Morris indices, $\mu_i^\ast$ and $\sigma_i$, it is apparent that not only input ranking is feasible but we can also learn something about the underlying model structure, i.e. whether interaction or non-linear effects are present.

\citet{BP16} group the Morris method to the family of local sensitivity measures, but they acknowledge that screening methods like the Morris sensitivity measures stand apart from other local sensitivity measures. While the elementary effects themselves consider only local changes, the actual measures for input importance, $\mu_i^\ast$ and $\sigma_i$, average over these $N$ elementary effects. Thus, they take $N$ local changes per input $i$ into account \citep{M91}. Indeed, \citet{CCS11} make a case for the Morris method to be seen a global sensitivity method.

\subsection{Algorithm for Extended Morris Method}

\begin{figure}[t]
	\caption{Uncertainty in Morris Indices - 100 Replicates}
    \label{morris_replicates}
    \begin{centering}
	\vspace*{-4mm}
	\begin{centering}
	\includegraphics[scale=0.9]{../figures/boxplot_morris_replicates_100_replicates_27_draws.png}
    \end{centering}
    \end{centering}

    \small
    \textit{Notes:} Boxplots of the four Morris indices for the model inputs $RC$ and $\theta_{11}$. 100 Morris indices were estimated with $N=27$. The left panel shows \textit{independent}, the right one \textit{full} Morris indices.
\end{figure}

\begin{figure}[t]
	\caption{Convergence of Morris Indices}
    \label{morris_convergence}
    \begin{centering}
	\vspace*{-4mm}
	\begin{centering}
	\includegraphics[scale=0.9]{../figures/morris_convergence.png}
    \end{centering}
    \end{centering}

    \small
    \textit{Notes:} Estimated Morris indices for different sample sizes $N$. The left panel shows \textit{independent} Morris indices, the right one \textit{full} Morris indices.
\end{figure}

In this section I introduce the extended Morris method for dependent samples as proposed by \citet{GM17}.

To grasp the computational procedure of the extended elementary effects, some more notation is needed. Following \citet{GM17}, let $X' = \{X_1',\ X_2',\ \dots,\ X_k'\}$ be $k$ dependent random inputs, following the joint probability density function $g(X)$. Thus, $X'$ is just a set of inputs independently drawn from the set $X$. Input subsets denoted by $\bar{X}$ are conditionally drawn inputs. Hence, let $\bar{X_{-i}'}$ follow the conditional probability density function $g(\bar{X_{-i}'} \mid X_{-i})$. That is $\bar{X_{-i}'}$ is drawn conditionally on the inputs in the first set $X_{-i}$. The input denoted by $\bar{X_{-i}}$ is conditionally drawn following the probability density function $g(\bar{X_{-i}} \mid X_i')$. % Hence, $\bar{X_{-i}'}$ and $\bar{X_{-i}}$ differ by [].

Analogously to the independent and full Sobol' indices \citep{MTA15}, \citet{GM17} developed the following elementary effects for dependent inputs.
\begin{align}
EE_i^{ind} = \frac{f(\bar{x_i}',\ x_{-i}) - f(x_i,\ x_{-i})}{\Delta},\\
EE_i^{full} = \frac{f(x_i',\ \bar{x_{-i}}) - f(x_i,\ x_{-i})}{\Delta},
\end{align}

\noindent where
\begin{itemize}
\item $EE_i^{ind}$ denotes \textit{independent} elementary effects for input $i$, effects that exclude the contributions attributable to the dependence between input $X_i$ and $X_j$ for $i \neq j$, and
\item $EE_i^{full}$ denotes \textit{full} elementary effects for input $i$, that include the effects due to correlation with other inputs.
\end{itemize}

As in the case of the classic Morris method, the sensitivity measures for input $i$ are derived by considering $N$ random samples yielding $2 N$ elementary effects, once for the independent and once for the full elementary effects. \citet{GM17} compute the corresponding sensitivity measures as shown in \cref{mu,sigma}. Since two sets of elementary effects are computed, they end up with a set of four sensitivity measures, $(\mu^{\ast ind}_i,\ \sigma_i^{ind})$ and $(\mu^{\ast full}_i,\ \sigma_i^{full})$.

Interpretation-wise, $X_i$ is an unimportant input if all sensitivity measures are essentially zero. If $\mu^{\ast ind}_i$ is strongly larger than zero and $\sigma_i^{ind}$ is close to zero, input $X_i$ is an important input due to its own, isolated contribution. When all sensitivity measures except $\mu^{\ast full}_i$ are close to zero, $X_i$'s contribution is due to the dependence with other important inputs. Strong interaction effects are present if either or both of the two $\sigma$'s are larger than zero \citep{GM17}.

The computation of the extended Morris indices requires the generation of dependent samples. According to \citet{GM17}, the computation involves the following steps:

\begin{enumerate}
    \item Create independent, uniformly distributed samples.
    \item Transform these uniformly distributed samples into dependent samples of a target distribution.
    \item Use the above methods for the computation of the extended elementary effects.
    \item As in the case of the classic Morris method, average over all $N$ elementary effects per input $i$ and compute their standard deviation. Do so for the \textit{independent} and \textit{full} elementary effects.
\end{enumerate}

In what follows I stick to the version of the algorithm as implemented in the \textit{econsa} Python-package \citep{OSE21}. There, the radial design is used for obtaining independent samples. To derive dependent samples, the inverse Nataf transformation is applied. The total computational cost amounts to $3kN$ model runs.

\subsection{Morris Indices for the Rust Model} \label{morris_rust_model}

\begin{table}[t]
    \caption{Relative Difference Morris Indices}
    \label{rel_diff_morris}
    \centering

    \begin{threeparttable}
        \begin{centering}
            \input{../figures/morris_convergence_relative_difference.tex}
            \begin{tablenotes}
                \small
                \item \textit{Notes:} The percentage difference between Morris indices for different values of $N$. For example, for $\mu^{\ast,\ full}$ Percentage changes are calculated by $\frac{\mu^{\ast,\ full,\ n-1}_i - \mu^{\ast,\ full,\ n}_i}{\mu^{\ast,\ full,\ n}_i}$, for $i \in \{RC,\ \theta_{11}\}$ and $n$ denoting the different values of $N$.
            \end{tablenotes}
        \end{centering}

        \end{threeparttable}
\end{table}

Similar to the procedure described in \cref{comp_shap}, I estimate Morris indices in two different ways: Firstly, I estimate 100 replicates for a relatively low sample size to get a sense of the variability in estimates, Then, I consecutively increase the sample size to investigate
the estimation procedure further. Uniformly distributed samples are drawn from a Sobol'
sequence \citep{S76}.\footnote{The Sobol' sequence is a quasi-random sequence used for Quasi-Monte Carlo methods. \citet{KTA12} argue that Quasi-Monte Carlo estimators converge faster than estimators based on pseudo-random numbers.}

Firstly, I estimate 100 sets of Morris indices. In order to ensure comparability to the Shapley effects, I use the same total computational cost as for the data used for \cref{boxplot_shapley}. I set $N = 27$, which amounts
to a total cost of 162 model evaluations, which differs slightly from the computational
cost of the Shapley effects (160) because of rounding.\footnote{The same computational cost as for Shapley effects is achieved by solving $6 \cdot N = 160 \Leftrightarrow N = \frac{160}{6} \approx 26.67$.} As stated above, there are four different
sensitivity indices estimated: $(\mu^{\ast ind}_i,\ \sigma_i^{ind})$ and $(\mu^{\ast full}_i,\ \sigma_i^{full})$. The distribution of the
estimated Morris indices is given in \cref{morris_replicates}.

Analysing \cref{morris_replicates}, we can get a sense of the uncertainty inherent in the estimation procedure,
given the admittedly small sample size. For the independent Morris indices, there exists a
very clear ranking: both values indicate that $RC$ is uninfluential. For $\theta_{11}$ the situation is
different: both independent indices, $(\mu^{\ast ind}_{\theta_{11}},\ \sigma_{\theta_{11}}^{ind})$, are nonzero. Surprisingly, for the full Morris indices,
we fail to observe a clear input ranking. Estimates of $(\mu^{\ast full}_i,\ \sigma_i^{full})$ are very volatile, but
clearly different from zero. Seemingly, no
clear input ranking is achieved. However, Morris indices were primarily developed to
identify non-influential inputs. Both inputs are important if the full Morris indices are
considered.

To investigate whether the individual pairs of full Morris indices, $(\mu^{\ast full}_i,\ \sigma_i^{full})$, for the
two inputs $i \in \{RC,\ \theta_{11}\}$ yield the correct input ranking, I simply compare their values
per replicate. Details are given in \cref{accuracy}. Surprisingly, only about 60 of the 100 replicates yield the correct importance ranking
for $\mu^{\ast full}_i$. I conclude that ranking based on full Morris indices in the case of the Rust model can be misleading given the small sample size.
The influence of dependence seems to be so large that indices cannot be ranked according to their full variance contribution.

\begin{table}
	\centering
	\caption{Accuracy of Morris Indices}
	\label{accuracy}
	\begin{threeparttable}
	\centering
	\input{../figures/correct_rankings.tex}
	\begin{tablenotes}
	\small
	\item \textit{Notes:} Share of the correct input rankings for $100$ Morris indices based on $\mu^{\ast,\ full}$ and $\mu^{\ast,\ ind}$. As a benchmark, the accuracy of Shapley effects is presented. For Morris indices the accuracy for different sample sizes is reported.
	\end{tablenotes}
	\end{threeparttable}
\end{table}

Secondly, I compute Morris indices for a set of number of draws. I consider $N \in \{100,\ 500,\ 1000,\ 1500,\ 2000,\ 2500,\ 3000\}$. The corresponding results are shown in \cref{morris_convergence}. The obtained Morris indices clearly confirm the input ranking implied by the Shapley effects for both, $\mu^{\ast ind}_i$ and $\mu^{\ast full}_i$. Further, Morris indices
seem to converge as $N$ increases. In \Cref{rel_diff_morris} specific information on the convergence behaviour is provided. \citet{ASAV19} show for the classical (independent) Morris indices that they do indeed converge with increasing sample size. Since in the end, the full Morris indices are estimated in a similar fashion, similar behaviour for full indices is expected. However, \textit{full} Morris indices do not converge for all replicates as seen in \cref{morris_replicates} and supported further by the results in \cref{accuracy}. In fact, the share of correct rankings steadily declines as $N$ increases. Intrigued by this result, I investigate whether this behaviour can be observed for a larger sample size as well. To that end, I consider the replicate of Morris indices with $N=500$ which shows the largest difference $\mu^{\ast,\ full}_{RC} - \mu^{\ast,\ full}_{\theta_{11}}$ and re-estimate the full indices for larger sample sizes, i.e. for $N \in \{3\,000,\ 4\,000,\ 6\,000,\ 8\,000\}$. Details are shown in \cref{appendix_morris_seed_67} in Appendix A. There, Morris indices seem to converge, but still yield the wrong input ranking. Unfortunately, due to the computational limitations I face, I cannot investigate this further.
% Although Morris indices are volatile, \citet{GM17} point out that a high estimation precision is not necessarily needed as long as the identification of uninfluential inputs is achieved.

Let us inspect the Morris indices for $N = 3\,000$ in more detail. Consult \cref{morris_3000} for the
resulting sensitivity indices. The computational cost for this value of $N$ amounts to $18\,000$
model evaluations. As in the other trials, \cref{morris_3000} shows $(\mu^{\ast ind}_{RC},\ \sigma_{RC}^{ind})$ equal to zero.
If only the independent Morris indices are considered, we erroneously
conclude that input $RC$ is a non-influential input by neglecting the variance contribution due to dependence. Since none of the inputs exhibits $(\mu^{\ast ind}_i,\ \sigma_i^{ind})$ and $(\mu^{\ast full}_i,\ \sigma_i^{full})$ close to zero at the same time, both inputs have to be considered as influential ones and thus cannot be fixed without influencing the output.
Since both independent indices for $\theta_{11}$, $(\mu^{\ast ind}_{\theta_{11}},\ \sigma_{\theta_{11}}^{ind})$, are high, there are either strong interaction and/or non-linear effects present.
% Since independent sensitivity indices for $RC$ are both zero, the reason for the high $\sigma_{\theta_{11}}^{ind}$ must be driven by the presence of non-linear effects in $\theta_{11}$. Regarding the model structure we can conclude that there are no interaction effects between $RC$ and $\theta_{11}$.

\begin{table}[t]
    \centering
    \caption{Morris Indices for $N=3\,000$}
    \label{morris_3000}
    \begin{threeparttable}
    \begin{centering}
        \input{../figures/table_morris_conv_3000}
        \begin{tablenotes}
            \small
            \item \textit{Notes:} \textit{Independent} and \textit{full} Morris indices for $RC$ and $\theta_{11}$ using $N=3\,000$.
        \end{tablenotes}
    \end{centering}

    \end{threeparttable}

\end{table}


\documentclass[a4paper]{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{physics}
%\usepackage[round]{natbib}
%\usepackage{bibtex}
\usepackage{fancyvrb}
\usepackage{hyperref}
\usepackage[linesnumbered,boxed]{algorithm2e}
\usepackage{subfigure}

\numberwithin{equation}{section} % Remove this line for global equation numbering

\title{Stefano's awesome MATH1058 coursework report}
\author{Stefano Coniglio}
\date{\today}

\begin{document}

% This produces the title: to modify contents, change the \title, \author, and
% \date in the preamble
\maketitle

\begin{abstract}
  This document reports on something really interesting, hopefully.
\end{abstract}

\section{Introduction}
\label{sec:intro}

We consider, in this work, a definitely relevant problem which, in formal terms, can be expressed as follows:
%
\begin{quote}
  {\bf Very Interesting Problem (VIP):} Given a quite relevant set of givens $A \in \mathbb{R}^{m \times n}, b \in \mathbb{R}^n, c \in \mathbb{R}^n$, the VIP problem calls for a very, very relevant solution $x \in \mathbb{R}^n$ which satisfies the following very, very, very (!) relevant property:
  \begin{equation}\label{equation:theRelevantOne}
    z = \max_{x \in \mathbb{R}^n_+} \{cx: Ax \leq b\}
  \end{equation}
\end{quote}

Relying on Equation~\eqref{equation:theRelevantOne}, the algorithm computes $x$ proceeding according to the following pseudocode:

\begin{algorithm}[H]\label{alg:theGoodOne}
 \KwData{this text}
 \KwResult{how to write algorithm with \LaTeX2e }
 initialization\;
 \While{not at end of this document}{
  read current\;
  \eIf{understand}{
   go to next section\;
   current section becomes this one\;
   }{
   go back to the beginning of current section\;
  }
 }
 \caption{How to write algorithms.}
\end{algorithm}

The pseudocode reported in Algorithm~\ref{alg:theGoodOne} was taken from~\cite{coniglio}.\footnote{Actually, it was not, but I thought that citing myself was a good idea.}

\section{Solution approach and implementation}

We had a great solution idea, which we implemented in the following beautiful piece of Python code:

\begin{Verbatim}[numbers=left]
def beautiful_function(myUnusedInput, mySecondUnusedInput):
    if do_something_awesome() == True:
        return youWillNeverBugOutOnMe
    else:
        return myVeryUndefinedOutput
\end{Verbatim}

\section{Experimental results and analysis}

We have run a set of experiments on this very costly piece of hardware: {\em xxx}. Our findings are summarized in the quite colorful Figure~\ref{fig:whatANiceFigure}.

\begin{figure}[h]
  \subfigure[]{\includegraphics[scale=0.4]{./template_placeholder.png}}
  \subfigure[]{\includegraphics[scale=0.4]{./template_placeholder.png}}
  \caption{One very nice chart entirely unrelated to this coursework (a) and an apparently very similar chart stil unrelated to this coursework (b).}
  \label{fig:whatANiceFigure}
\end{figure}

We should not forget to explain what conclusions the figure allows us to draw, nor why.

Also, including a table may be a good idea. Take a look at Table~\ref{tab:1}:

\begin{table}[htbp]
  \caption{A nice table. Notice that captions preceed tables, whereas they follow figures.}
  \begin{center}
\begin{tabular}{l|lll}
  header    & header2 & header3 \\
  \hline
  name1     & 12      & 12\\
  $s = |S|$ & 5       & 42
\end{tabular}
  \end{center}
\label{tab:1}
\end{table}

\section{Conclusions}

We have reported on a set of quite interesting findings.

% At the end of the document, include the bibliography.
% You can use any citation style: this one is simple.
\bibliographystyle{plain}
% This assumes you have a single BibTeX file called references.bib: change to
% match your file name.
\bibliography{template_references}

\end{document}


\section{Test Description and Success Criteria}
In order to test the model, the boresight vector on the spacecraft, and the orientation of the spacecraft were varied to be in each quadrant of the body frame. There were a total of 16 different tests performed in this manner. Eight different boresight vector placements (holding the spacecraft orientation constant), and eight different spacecraft orientations (holding the boresight vector constant). There was a 17th test performed in order to check the azimuth angle when the miss angle is zero. For every combination of parameters, the pointing reference frame was calculated and compared to the simulated pointing frame, and using that, the miss angle and azimuth angle were calculated.

The boresight vector was calculated in the same manner that it was calculated in the model using numpy libraries to do so. It was then compared to the model's boresight vector. The test was deemed a success if the boresight vector was withing 1E-10 of the model's boresight vector using the unitTestSupport script. It should be noted that the boresight vector that the user passes to the model doesn't need to be a unit vector because the model will normalize the vector.

The miss angle was calculated in two separate ways. The first method mirrored the module's method. Just as in the module, a direction cosine martix was created to represent the pointing frame from the inertial frame. Then the boresight vector was projected into this frame, and the miss angle was calculated using standard trigonometry. The key difference in the first method of validation is that the validation used the python numpy library primarily rather than the RigidBodyKinematics.py file.

The second method used the existing inertial reference frame in order to calculate the miss angle of the boresight. In this method, the baseline vector was projected into the inertial frame. Then just as the first, the miss angle was calculated using standard trigonometry.

The second method relied on the direction cosine matrices created in the first method. That being said, the first operation in the second method was to calculate the position vector of the spacecraft in the inertial reference frame. Then the baseline vector was projected into the inertial frame using the direction cosine matrix from the inertial frame to the pointing frame. Then the position vector in the inertial frame was dotted with the baseline projection. Just as in the first method, the product of the two vectors was passed through the arccosine function in order to calculate the miss angle of the boresight.

Once the miss angle calculations were complete, the calculated miss angles were compared to the miss angle pulled from the mission simulation. If the calculated miss angles were withing 1E-10 of the simulation miss angle, the test passed.

Then the azimuth angle was calculated using the same method as in the model. Again, it should be noted that the standard python numpy library was used in the validation calculation. The test passed when the calculated azimuth angle was within 1E-10 of the simulation azimuth angle.

\begin{table}[htbp]
	\caption{Tolerance Table (Absolute)}
	\label{tab:label}
	\centering \fontsize{10}{10}\selectfont
	\begin{tabular}{ c | c } % Column formatting,
		\hline
		Parameter & Tolerance\\
		\hline
		Boresight Vector & 1E-10 \\
		Miss Angle & 1E-10 \\
		Azimuth Angle & 1E-10 \\
		\hline
	\end{tabular}
\end{table}

\section{Test Parameters}

\begin{table}[htbp]
	\caption{Table of test parameters}
	\label{tab:label}
	\centering \fontsize{10}{10}\selectfont
	\begin{tabular}{ c | c | c | c | c | c } % Column formatting,
		\hline
		Test    & Boresight Vector &  $\sigma_{B/N}$  & Test    & Boresight Vector &  $\sigma_{B/N}$ \\
		\hline
		1 & $\frac{\sqrt[]{3}}{3}$, $\frac{\sqrt[]{3}}{3}$, $\frac{\sqrt[]{3}}{3}$ & 0.0, 0.0, 0.0  & 9 & 0.0, 0.0, 1.0 & -0.079, 0.191, 0.191  \\
		2 & -$\frac{\sqrt[]{3}}{3}$, $\frac{\sqrt[]{3}}{3}$, $\frac{\sqrt[]{3}}{3}$ & 0.0, 0.0, 0.0  & 10 & 0.0, 0.0, 1.0 & -0.261, 0.108, 0.631 \\
		3 & $\frac{\sqrt[]{3}}{3}$, -$\frac{\sqrt[]{3}}{3}$, $\frac{\sqrt[]{3}}{3}$ & 0.0, 0.0, 0.0  & 11 & 0.0, 0.0, 1.0 & 0.261, 0.108, -0.631  \\
		4 & -$\frac{\sqrt[]{3}}{3}$, -$\frac{\sqrt[]{3}}{3}$, $\frac{\sqrt[]{3}}{3}$ & 0.0, 0.0, 0.0  & 12 & 0.0, 0.0, 1.0 & 0.079, 0.191, -0.191  \\
		5 & $\frac{\sqrt[]{3}}{3}$, $\frac{\sqrt[]{3}}{3}$, -$\frac{\sqrt[]{3}}{3}$ & 0.0, 0.0, 0.0 & 13 & 0.0, 0.0, 1.0 & 0.079, -0.191, 0.191 \\
		6 & -$\frac{\sqrt[]{3}}{3}$, $\frac{\sqrt[]{3}}{3}$, -$\frac{\sqrt[]{3}}{3}$ & 0.0, 0.0, 0.0 & 14 & 0.0, 0.0, 1.0 & 0.261, -0.108, 0.631 \\
		7 & $\frac{\sqrt[]{3}}{3}$, -$\frac{\sqrt[]{3}}{3}$, -$\frac{\sqrt[]{3}}{3}$ & 0.0, 0.0, 0.0  & 15 & 0.0, 0.0, 1.0 & -0.261, -0.108, -0.631  \\
		8 & -$\frac{\sqrt[]{3}}{3}$, -$\frac{\sqrt[]{3}}{3}$, -$\frac{\sqrt[]{3}}{3}$ & 0.0, 0.0, 0.0  & 16 & 0.0, 0.0, 1.0 & -0.079, -0.191, -0.191  \\
		&  &  & 17 & 1.0, 0.0, 0.0 & 0.0, 0.0, 0.0\\
		\hline
	\end{tabular}
\end{table}
It should be noted that the boresight vector passed to the model does not need to be a unit vector because the model will normalize the vector.

\section{Test Results}

\begin{table}[H]
	\caption{Pass or Fail Table}
	\label{tab:label}
	\centering \fontsize{10}{10}\selectfont
	\begin{tabular}{ c | c | c | c } % Column formatting,
		\hline
		Test & Pass/Fail & Test    & Pass/Fail\\
		\hline
		1 & \input{AutoTex/Result0} & 9  & \input{AutoTex/Result8} \\
		2 & \input{AutoTex/Result1} & 10 & \input{AutoTex/Result9} \\
		3 & \input{AutoTex/Result2} & 11 & \input{AutoTex/Result10} \\
		4 & \input{AutoTex/Result3} & 12 & \input{AutoTex/Result11} \\
		5 & \input{AutoTex/Result4} & 13 & \input{AutoTex/Result12} \\
		6 & \input{AutoTex/Result5} & 14 & \input{AutoTex/Result13} \\
		7 & \input{AutoTex/Result6} & 15 & \input{AutoTex/Result14} \\
		8 & \input{AutoTex/Result7} & 16 & \input{AutoTex/Result15} \\
		&                         & 17 & \input{AutoTex/Result16} \\
		\hline
	\end{tabular}
\end{table}


\documentclass[oneside]{memoir}

\usepackage{notestemplate}
\usepackage{import}
\usepackage{standalone}
\usepackage{fancyhdr} % Required for custom headers
\usepackage{lastpage} % Required to determine the last page for the footer
\usepackage{extramarks} % Required for headers and footers
\usepackage{lipsum} % Used for inserting dummy 'Lorem ipsum' text into the template
\usepackage{hyperref}
\hypersetup{pdftex,colorlinks=true,allcolors=blue}
\usepackage{hypcap}
\usetikzlibrary{matrix,arrows.meta}

\logo{~/LibreMath/Auxiliary Resources/resources/png/logo.png}
\institute{Rice University}
%\faculty{Faculty of Whatever Sciences}
\department{Department of Mathematics}
\title{Introduction to Groups, Rings, and Fields}
\subtitle{A first exposure to core concepts in abstract algebra}
\author{\textit{Author}\\Gabriel \textsc{Gress}}
%\supervisor{Linus \textsc{Torvalds}}
%\context{Well, I was bored...}
\date{\today}

%\makeindex

\begin{document}
% \section{}

\maketitle

\tableofcontents
\input{2020-02-04-NumThryPrelim}
\input{2020-03-24-IntroGroups}
\input{2020-03-25-Subgroups}
\input{2020-03-31-NormalSubgroups}
\input{2020-04-01-QuotientGroup+Homo}
\input{2020-04-07-GroupProd}
\input{2020-03-26-CyclicGroups}
\input{2020-04-08-PermGroup}
%\input{GroupRepresentations}
%\input{AlternatingGroup}
\input{2020-04-28-GroupAction}
%\input{Automorphisms}
%\input{GroupRepresentations2}
%\input{Sylow}
\input{2020-04-09-ClassificationThms}
%\input{NilpotentSolvableGroups}
%\input{ProofFundThmFinAbelGrps}

\chapter{Rings}
\label{cha:rings}

\input{2020-02-05-IntroRings}
\input{2020-02-12-Subrings}

\section{Ring Homomorphisms and Quotient Rings}

\input{2020-02-18-QuotientRing}
\input{2020-02-19-RingHomo}
\input{2020-02-25-RingProd}
\input{2020-02-26-DivisioninRings}
\input{2020-03-03-GaussianInt}
\input{2020-03-04-GaussianInt2}
\input{2020-03-10-GaussInt3}
\input{2020-03-11-FiniteFields}
%\input{ModuleTheory}

%\printindex

\end{document}

Abstract Algebra - Introductory/Algebra I/Notes/source/AlgebraI.tex


\chapter{Integrals and ratios}
\label{ch:spectral_preprocessing}

\begin{wrapfigure}{o}{0.72\textwidth}
    \centering
    \vspace{-3cm}
    \includegraphics[width=0.85\textwidth]{integrals-ratios-fig1.png}
    \label{fig:integrals-ratios-fig1}
\end{wrapfigure}

Peak integration is an essential element of spectroscopy for measuring concentrations, spectral contributions, etc.

\begin{wrapfigure}{o}{0.95\textwidth}
    \centering
    % \vspace{-1cm}
    \includegraphics[width=\textwidth]{integrals-ratios-fig2.png}
    \caption{To display a preview, select a spectrum and enable preview of individual integrals with their play buttons.}
    \label{fig:integrals-ratios-fig2}
\end{wrapfigure}

To compute integrals, we use the \textit{Integrate Spectra} widget. Let’s compute the ratios of two integrals to establish an internal standard in our dataset. Add two integrals and then feed them into the \textit{Feature Constructor} (\textit{Edit Domain} is optional—we use it to simplify column names). In \textit{Feature Constructor}, we can create new numeric features with Python expressions.
To work with Feature Constructor more easily, uncheck “Output as metas”, which will replace the original spectra with their integrals (and reduce the number of columns in your data).
We can use \textit{Feature Constructor} whenever we would like to create a new column from existing data.

\begin{figure}[h]
\vspace{-0.5cm}

\stackinset{r}{-0.65\linewidth}{t}{+0.15\linewidth}
% {r} is left-right
% {t} is top-bottom
  {\includegraphics[scale=0.45]{integrals-ratios-fig4.png}}
  {\includegraphics[scale=0.4]{integrals-ratios-fig3.png}}
  \caption{We added a Numeric feature, the ratio of I1 and I2 called R1.}
  \label{fig:spectral-PCA-fig2}
\end{figure}

\noindent The produced data can be inspected by connecting other widgets.


\documentclass[12pt, letterpaper]{article}
\usepackage[english]{babel}
\usepackage{amsmath}
\usepackage[margin=0.7in]{geometry}

  %%%%%%%%%%%%
  % PREAMBLE %
  %%%%%%%%%%%%
\begin{document}
\selectlanguage{english}

\section{The crystal state and associated parameter gradients}
  The purpose of this document is to discuss the code in \texttt{g\_gradients.py} and \texttt{a\_g\_conversion.h}, which is presumably used for parameter refinement in \textit{DIALS}, per the Waterman (2016) paper, section B3.  Code such as this has been in the package at least since 2006.  Unfortunately, the following notes predate the current code, and I notice that slightly different conventions are used, \textit{e.g.}, the notes give the fractionalization matrix as upper triangular, but the comments in the code indicate lower triangular.  Therefore at some point these notes will have to be reconciled with the actual code.

Let's begin by defining notation.  All formulae will be given here in terms of the reciprocal lattice, therefore the vectors $\mathbf{a}$, $\mathbf{b}$, and $\mathbf{c}$ will be understood to refer to the reciprocal cell basis vectors, even though these vectors may be denoted with superscript asterisks in most literature.  In addition we have the reciprocal space $\mathbf{A}$ matrix,

  \begin{equation}
    \mathbf{A} =
    \left[
    \begin{array}{c c c}
       a_{x}  & b_{x} & c_{x} \\
       a_{y}  & b_{y} & c_{y} \\
       a_{z}  & b_{z} & c_{z} \\
    \end{array}
    \right]
    \text{,}
    \label{eqn:expansion}
  \end{equation}

representing the state variables of the crystal (cell parameters and orientation), in terms of column vectors denoting the basis vectors expressed as laboratory $x,y,z$ components.

In practice we express $\mathbf{A}$ as

  \begin{equation}
      \mathbf{A} = \mathbf{UF}
    \text{,}
    \label{eqn:sabbrev}
  \end{equation}

with unitary matrix $\mathbf{U}$ expressing the crystal rotation ($e.g.$, as a composite resulting from Euler rotations $\phi$, $\theta$ and $\psi$), and a fractionalization matrix $\mathbf{F}$ given by

  \begin{equation}
    \mathbf{F} =
    \left[
    \begin{array}{c c c}
       a_{x}  & b_{x} & c_{x} \\
       0  & b_{y} & c_{y} \\
       0  & 0 & c_{z} \\
    \end{array}
    \right]
    \text{.}
    \label{eqn:expansionF}
  \end{equation}

Here it is understood that $\mathbf{F}$ represents the standard orientation of the crystal, which by convention is taken to mean that the reciprocal $\mathbf{a}$ vector lies along laboratory $x$, with $\mathbf{b}$ in the $xy$ plane.

In contrast to the state variables, the parameters most useful for model fitting turn out to be the elements of the reciprocal space symmetric metric tensor,

  \begin{equation}
    \mathbf{G} =
    \left[
    \begin{array}{c}
       g_{11}  \\ g_{22} \\ g_{33} \\
       g_{12}  \\ g_{13} \\ g_{23}
       \\
    \end{array}
    \right] =
    \left[
    \begin{array}{c}
       g_{0}  \\ g_{1} \\ g_{2} \\
       g_{3}  \\ g_{4} \\ g_{5}
       \\
    \end{array}
    \right] =
     \left[
    \begin{array}{c}
       \mathbf{a}\cdot\mathbf{a}  \\ \mathbf{b}\cdot\mathbf{b} \\ \mathbf{c}\cdot\mathbf{c} \\
       \mathbf{a}\cdot\mathbf{b}  \\ \mathbf{a}\cdot\mathbf{c} \\ \mathbf{b}\cdot\mathbf{c}
       \\
    \end{array}
    \right]
   \text{.}
    \label{eqn:expansionG}
  \end{equation}

 The reason for prefering the metrical tensor components is that high-symmetry Bravais cells will only refine a subset of the six components, with monoclinic refining four, orthorhombic three, tetragonal and hexagonal two, and cubic only one.

 A key strategy in parameter refinement is to use the chain rule.  Suppose we are performing iterative non-linear least-squares parameter refinement with a refinement target $\chi$ that is a function of $\mathbf{A}$.  Suppose that at an intermediate step of the calculation we have the gradient of the target with respect to each element $a_{ij}$,

   \begin{equation}
    \frac {\partial \chi} {\partial a_{ij}}
       \text{,}
    \label{eqn:frac}
  \end{equation}

 where $i$ denotes the row and $j$ the column of the matrix $\mathbf{A}$.

Then in order to calculate the gradient of the target with respect to the free parameters $g_{ij}$, one would employ the chain rule.  For the specfic example of $g_{11}$ this would be performed as follows over a total of nine terms:

   \begin{equation}
    \frac {\partial \chi} {\partial g_{11}} =
    \frac {\partial \chi} {\partial a_{11}} \frac {\partial a_{11}}{\partial g_{11}}+
    \frac {\partial \chi} {\partial a_{12}} \frac {\partial a_{12}}{\partial g_{11}}+
    \frac {\partial \chi} {\partial a_{13}} \frac {\partial a_{13}}{\partial g_{11}}+
    \ldots+
    \frac {\partial \chi} {\partial a_{31}} \frac {\partial a_{31}}{\partial g_{11}}+
    \frac {\partial \chi} {\partial a_{32}} \frac {\partial a_{32}}{\partial g_{11}}+
    \frac {\partial \chi} {\partial a_{33}} \frac {\partial a_{33}}{\partial g_{11}}
       \text{.}
    \label{eqn:fracC}
  \end{equation}

It is important to note that \textit{DIALS} employs a related chain rule formula applicable to the special case where we start with the derivatives of the target with respect to the six non-zero elements of the fractionalization matrix,

     \begin{equation}
    \frac {\partial \chi} {\partial f_{ij}}
       \text{.}
    \label{eqn:fracF}
  \end{equation}

  However, I do not have the specific formulae documented here apart from the code itself.

  In any case, the critical piece will then be the relationship between $\mathbf{F}$ and $\mathbf{G}$.  Here are the full equations to convert between the two, $\mathbf{F}$ to $\mathbf{G}$:
      \begin{flalign*}
    &g_{0} = a_{x}^{2} &\\
    &g_{1} = b_{x}^{2} + b_{y}^{2} &\\
    &g_{2} = c_{x}^{2} + c_{y}^{2} + c_{z}^{2} &\\
    &g_{3} = a_{x} b_{x} &\\
    &g_{4} = a_{x} c_{x} &\\
    &g_{5} = b_{x} c_{x} + b_{y} c_{y} &\\
    \end{flalign*}
  Note the use of two differing nomenclatures for the subscript of $g$, which are nevertheless equivalent.  In the $\mathbf{G}$ to $\mathbf{F}$ direction:
      \begin{flalign*}
    &a_{x} = g_{0}^{1/2} &\\
    &b_{x} = g_{3} / a_{x} &\\
    &c_{x} = g_{4} / a_{x} &\\
    &b_{y} = (g_{1} - b_{x}^2)^{1/2} &\\
    &c_{y} = \dfrac{g_{5} - b_{x}c_{x}}{b_{y}} &\\
    &c_{z} = (g_{2} - c_{x}^2 - c_y^2)^{1/2} &\\
    \end{flalign*}
Notice the presence of the square root in three of the $\mathbf{G}$ to $\mathbf{F}$ formulae.  A negative value inside the radical should never occur, and if it did would represent an illegal trial value for a free parameter.

      \end{document}


\section{Lecture 4 - The Lazy Approach}

\begin{frame}
  \frametitle{The Lazy Approach}

  \scriptsize

  Notice that
  \begin{itemize}
    \item Assignments $\mu$ of $\varphi$ are many (potentially $\infty$),
          infeasible to check if any of them is a model {\bf systematically}
    \item Models $\babst{\mu}$ of $\babst{\varphi}$ are finite in number,
          and easy to enumerate with a SAT-solver
    \item A model $\babst{\mu}$ is nothing but a {\bf conjunction of \tatoms},
          can be checked efficiently with a \tsolver
  \end{itemize}
  \vfill
  These observations suggest us a methodology
  to tackle the SMT(\T) problem
  \begin{itemize}
    \item Enumerate a Boolean model $\babst{\mu}$ of $\babst{\varphi}$ (abstraction). If no model
	  exist we are done ($\varphi$ is unsatisfiable)
    \item Check if $\babst{\mu}$ is satisfiable using the \tsolver. If so $\babst{\mu}$ can be extended
          to a model $\mu$ of $\varphi$, and so we are done ! ($\varphi$ is satisfiable)
    \item It not, we tell the SAT-solver not to enumerate $\babst{\mu}$ again,
          thus {\bf cutting away systematically an infinite number}
	  of assignments for $\varphi$ (refinement)
    \item It can be blocked by adding a clause $\neg \babst{\mu}$. Go up
    \item It terminates because there are finite Boolean models
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{The Lazy Approach}

  \scriptsize

  The lazy approach falls into the so-called {\bf abstraction-refinement}
  paradigm
  \vfill
  \begin{center}
  \scalebox{.5}{\input{ar.pdf_t}}
  \end{center}

\end{frame}

\begin{frame}
  \frametitle{The Lazy Approach}

  \scriptsize

  The interaction described naturally falls within the
  CDCL style, enriched with a \tsolver
      $$\varphi \equiv
      (x=3 \vee \neg (x<3))\ \swedge\
      (x=3 \vee \neg (x>3))\ \swedge\
      (x>3 \vee \neg (x<3))\ \swedge\
      (x>3 \vee \neg (x=3))$$
  \vfill
  \begin{columns}

    \begin{column}{.5cm}
      \vspace{-165pt}
      $\babst{\varphi} \equiv$
    \end{column}

    \begin{column}{5cm}
      $(\coltwoat{\coloneat{a_1}{8-13}}{2-6} \vee \coltwoat{\neg a_2}{9-13})$ \\
      $(\coltwoat{\coloneat{a_1}{8-13}}{2-6} \vee \coltwoat{\coloneat{\neg a_3}{3-6}}{10-13})$ \\
      $(\coloneat{\coltwoat{a_3}{3-6}}{10-13} \vee \coltwoat{\neg a_2}{9-13})$ \\
      $(\coloneat{\coltwoat{a_3}{3-6}}{10-13} \vee \coloneat{\coltwoat{\neg a_1}{8-13}}{2-6})$ \\
      \onslide<6->{$(\coloneat{\coltwoat{\neg a_1}{10-13}}{6} \vee \coltwoat{\coloneat{\neg a_3}{6}}{10-13})$ \\}
      \onslide<7->{$(\coltwoat{\neg a_1}{8-13})$ \\}
      \onslide<13->{$(\coloneat{a_1}{13} \vee \coloneat{a_2}{13} \vee \coloneat{a_3}{13})$ \\}
      \onslide<14->{$(\ )$ \\}
      \bigskip
      $a_1 \equiv x=3$ \\
      $a_2 \equiv x<3$ \\
      $a_3 \equiv x>3$ \\
      \bigskip
      $\babst{\mu}$: $\{\ \only<2-6|handout:0>{a_1}\only<3-6|handout:0>{, a_3}\only<8-13|handout:0>{\neg a_1}\only<9-13|handout:0>{, \neg a_2}\only<10-13|handout:0>{, \neg a_3}\ \}$ \\
      \bigskip
      \begin{tabular}{rl}
      SAT-solver: & \only<1,4-5,11-12|handout:0>{Idle}\only<2|handout:0>{Decision}\only<3,8-10|handout:0>{BCP}\only<6,13|handout:0>{Learn}\only<7,14|handout:0>{Conf. Analysis, Backtrack}\only<15>{UNS} \\
        \tsolver: & \only<1,2,3,6-10,13->{Idle}\only<4,5,11,12|handout:0>{Is $\babst{\mu}$ \T-satisfiable ?}\only<5,12|handout:0>{ NO}
      \end{tabular}
    \end{column}

    \begin{column}{5cm}
      \begin{overlayarea}{5cm}{5cm}
	\only<1|handout:0>{\scalebox{.6}{\input{search_0.pdf_t}}}
	\only<2|handout:0>{\scalebox{.6}{\input{search_1.pdf_t}}}
	\only<3,4,5|handout:0>{\scalebox{.6}{\input{search_2.pdf_t}}}
	\only<6,7|handout:0>{\scalebox{.6}{\input{search_3.pdf_t}}}
	\only<8|handout:0>{\scalebox{.6}{\input{search_4.pdf_t}}}
	\only<9|handout:0>{\scalebox{.6}{\input{search_5.pdf_t}}}
	\only<10-12|handout:0>{\scalebox{.6}{\input{search_6.pdf_t}}}
	\only<13->{\scalebox{.6}{\input{search_7.pdf_t}}}
      \end{overlayarea}
    \end{column}

  \end{columns}

\end{frame}

\begin{frame}
  \frametitle{Lecture 4 - Exercize 1}

  \scriptsize

  \begin{tabular}{ccc}
    \begin{minipage}{.4\textwidth}
     $$
     \begin{array}{l}
     (a_1 \vee \neg a_2) \\
     (a_1 \vee \neg a_3) \\
     (a_3 \vee \neg a_2) \\
     (a_3 \vee \neg a_1) \\
     (\neg a_1 \vee \neg a_3)
     \end{array}
     $$
    \end{minipage}
    & ~~~~~~ &
    \begin{minipage}{.4\textwidth}
      \begin{tabular}{ccl}
	\hline
	Trail & dl & Reason \\
	\hline
	$a_1$ & 1 & Decision \\
	$a_3$ & 1 & $(a_3 \vee \neg a_1)$ \\
	\hline
      \end{tabular}
      \bigskip \\
      $\{ \dec{a_1}{1}, \dec{a_3}{1} \}$
    \end{minipage}
  \end{tabular}

  \vfill
  \pause

  \begin{minipage}{\textwidth}
    \begin{prooftree}
    \AxiomC{$(\neg a_1 \vee \neg a_3)$}
    \AxiomC{$( a_3 \vee \neg a_1 )$}
    \BinaryInfC{$(\neg a_1)$}
    \end{prooftree}
  \end{minipage}

  \vfill
  \pause
  Conflict clause: $(\neg a_1)$ \pause \\
  Backtracking level: 0
\end{frame}

\begin{frame}
  \frametitle{Lecture 4 - Exercize 1}

  \scriptsize

  \begin{tabular}{ccc}
    \begin{minipage}{.4\textwidth}
     $$
     \begin{array}{l}
     (a_1 \vee \neg a_2) \\
     (a_1 \vee \neg a_3) \\
     (a_3 \vee \neg a_2) \\
     (a_3 \vee \neg a_1) \\
     (\neg a_1 \vee \neg a_3) \\
     (\neg a_1) \\
     (a_1 \vee a_2 \vee a_3)
     \end{array}
     $$
    \end{minipage}
    & ~~~~~~ &
    \begin{minipage}{.4\textwidth}
      \begin{tabular}{ccl}
	\hline
	Trail & dl & Reason \\
	\hline
	$\neg a_1$ & 0 & $(\neg a_1)$ \\
	$\neg a_2$ & 0 & $(a_1 \vee \neg a_2)$ \\
	$\neg a_3$ & 0 & $(a_1 \vee \neg a_3)$ \\
	\hline
      \end{tabular}
      \bigskip \\
      $\{ \neg \dec{a_1}{0}, \neg \dec{a_2}{0}, \neg \dec{a_3}{0} \}$
    \end{minipage}
  \end{tabular}

  \vfill
  \pause

  \begin{minipage}{\textwidth}
    \begin{prooftree}
    \AxiomC{$(a_1 \vee a_2 \vee a_3)$}
    \AxiomC{$(a_3 \vee \neg a_1)$}
    \BinaryInfC{$(a_1 \vee a_2)$}
    \AxiomC{$(a_1 \vee \neg a_2)$}
    \BinaryInfC{$(a_1)$}
    \AxiomC{$(\neg a_1)$}
    \BinaryInfC{$\bot$}
    \end{prooftree}
  \end{minipage}

  \vfill
  \pause
  Conflict clause: $\bot$ \pause \\
  Backtracking level: 0
\end{frame}


\documentclass[aspectratio=149]{beamer}

\input{../../shared_slides.tex}

% reference: https://yuxinchen2020.github.io/ele522_optimization/lectures/variance_reduction.pdf
% or
% https://ieeexplore-ieee-org.uaccess.univie.ac.at/stamp/stamp.jsp?tp=&arnumber=9226504

\usepackage{booktabs}

\title{Variance reduction for stochastic gradient methods}
\date{\today}

\begin{document}
\maketitle
\frame{\tableofcontents}

\section{Introduction}%

\begin{frame}
  \frametitle{The finite sum problem}
  A common Task in (supervised) machine learning:
  \begin{equation}
    \min_{x\in \R^d} f(x) := \frac{1}{n} \sum_{i=1}^{n} \underbrace{f_i(x)}_{\textcolor{blue}{\text{loss for $i$-th sample}}} + \underbrace{\psi(x)}_{\textcolor{blue}{\text{regularizer}}}
  \end{equation}
  where the $i$-th sample is $(a_i, y_i)$.

  \textbf{Examples:}
  \begin{itemize}
    \item linear regression: $f_i(x) = {(a_i^T x -y_i)}^2$, and $\psi=0$
    \item logistic regression: $f_i(x) = \log(1+e^{-y_i a_i^T x})$, and $\psi=0$
          ``sigmoid function'' and logistic loss.
    \item Lasso: $f_i$ as for linear regression but $\psi(x) = \Vert x \Vert_1$
    \item SVM: $f_i(x) = \max \{0 , 1 - y_i a_i^T x\}$ and $\psi(x)= \Vert x \Vert^2$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Gradient descent}
  \begin{algorithm}[H]
    \caption{(batch) GD}
    \begin{algorithmic}[1]
      \For{$k = 1,2, \dots$}
      \State{ $x_{k+1} = x_k  - \alpha_k \nabla f(x_k) $}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}

  \begin{itemize}
    \item gradient can be computed via
          \begin{equation}
            \nabla f(x) = \nabla \left(\sum_{i=1}^{n}f_i(x)\right) = \sum_{i=1}^{n} \nabla f_i(x_k)
          \end{equation}
    \item good convergence properties
    \item can be \textbf{expensive} if $n$ is large!
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Stochastic gradient descent}
  \begin{algorithm}[H]
    \caption{SGD}\label{sgd}
    \begin{algorithmic}[1]
      \For{$k = 1,2, \dots$}
      \State{pick $i_k$ uniform at random in $[n]$}
      \State{$x_{k+1} = x_k  - \alpha_k \nabla f_{i_k}(x_k)$}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}

  We already noticed that:
  \begin{itemize}
    \item unbiased: $\E[\nabla f_{i_k}(x)] = \sum_{i=1}^{n} \P[i=i_k] \nabla f_i(x) = \sum_{i=1}^{n} \frac{1}{n} \nabla f_i(x)$
    \item large stepsizes fail to suppress noise in the stoch.\ gradients \\
          $\rightarrow$ leads to oscillations
    \item decreasing stepsizes mitigate this problem but \textbf{slows down} convergence (too \emph{conservative})
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Recall SGD}
  \begin{block}{template}
    \begin{equation}
      x_{k+1} = x_k - \alpha_k g_k
    \end{equation}
  \end{block}
  \begin{itemize}
    \item $g_k$ is an unbiased estimator of the true gradient $\nabla F(x_k)$
    \item convergence depends on \textbf{variance} $\E [ \Vert g_k - \nabla F(x_k) \Vert ] \le \sigma_g$ \\
          (not strictly necessary)
    \item vanilla SGD uses $g_k = \nabla f_{i_k}(x_k)$ \\
          \textbf{issue:} $\sigma_g$ is non-negligible even close to the solution
    \item \textbf{Q:} can we choose $g_k$ in a different way to reduce variability?
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Minibatching}
  \begin{algorithm}[H]
    \caption{minibatch SGD}
    \begin{algorithmic}[1]
      \For{$k = 1,2, \dots$}
      \State{pick $I_k$ random subset of $[n]$ with $\vert I_k \vert = b $}
      \State{$x_{k+1} = x_k  - \alpha_k \sum_{i \in I_k} \nabla f_{i}(x_k)$}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}
  \begin{itemize}
    \item typically we make a (uniform) \emph{random} choice $i_k \in [n] = \{1, \dots, n\}$\\
          (or random reshuffling)
    \item by increasing the size to a \textbf{random subset} $I_k \subset [n]$ of size $b \ll n$ we can
          \begin{itemize}
            \item decrease variance
            \item increase cost only moderately,
            \item no improvement in the rate
          \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{A simple idea}
  Consider
  \begin{itemize}
    \item estimator $X$ for parameter $\mu$ ($\E[X]=\mu$ and $\Var[X]=\sigma^2$)
    \item want to \textcolor{blue}{keep unbiased} but \textbf{\textcolor{blue}{reduce variance}}
    \item find $Y$ such that $\E[Y] = 0$ but $\Cov (X,Y)$ \textbf{is large} and define
          \begin{equation}
            \tilde{X} := X-Y
          \end{equation}
    \item remains unbiased
    \item $\Var[\tilde{X}]$ can be much smaller than $\Var[X]$ if $X,Y$ are highly correlated
          \begin{equation}
            \Var[\tilde{X}] = \Var[X]+\Var[Y]-2\Cov[X,Y]
          \end{equation}
  \end{itemize}
\end{frame}

\section{SAG}%
\label{sec:}

\begin{frame}
  \frametitle{Stochastic average gradient (SAG), 2013}
  \begin{itemize}
    \item \textbf{maintain table} containing gradients $g_i$ of $f_i$
    \item pick random $i_k \in [n]$ and
          \begin{equation}
            g_{i_k}^k := \nabla f_{i_k}(x^{k})
          \end{equation}
          set $g_{i}^k = g_i^{k-1}$ for all $i\neq i_k$ (remain the same)
    \item Update
          \begin{equation}
            x^{k+1} = x^k - \alpha_k \frac{1}{n} \sum_{i=1}^{n} g_i^k.
          \end{equation}
    % \item assuming gradients do not change too much along trajectory
    \item gradient estimator \textcolor{red}{no longer unbiased}
    \item Isn't it expensive to average these gradients?
          \begin{equation}
            x^{k+1} = x^k - \alpha_k  \Big( \frac{g_{i_k}^k}{n} - \frac{g_{i_k}^{k-1}}{n} + \underbrace{\frac{1}{n}\sum_{i=1}^{n} g_i^{k-1}}_{\text{old table average}} \Big)
          \end{equation}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SAG variance reduction}
  Gradient estimator in SAG:
  \begin{equation}
    x^{k+1} = x^k - \alpha_k \frac{1}{n} \Big( \underbrace{g_{i_k}^k}_X - \underbrace{g_{i_k}^{k-1} - \sum_{i=1}^{n} g_i^{k-1}}_{Y} \Big)
  \end{equation}
  \begin{itemize}
    \item Indeed $\E[X] = \nabla f(x^k)$, but $\E[Y]\neq 0$ $\rightarrow $ is \textbf{biased estimator}
    \item $X$ and $Y$ are correlated as $X-Y \to 0$:
          \begin{itemize}
            \item $x^k$ and $x^{k-1}$ both converge to $x^*$ $\Rightarrow$ $ \nabla f_i(x^k) - \nabla f_i(x^{k-1}) \to 0$
            \item the last term converges to $\nabla f(x^*)= 0$
          \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Convergence}
  As always, initialization plays a role: $D^2 := \Vert x^0 -x^* \Vert^2$.

  \begin{equation}
    \begin{aligned}
      \text{SAG:}& \quad \frac{n}{k}(f(x^0)-f^*) + \frac{L}{k}D^2 \\
      \text{GD:}& \quad \frac{L}{k}D^2 \\
      \text{SGD:}& \quad \frac{L}{\sqrt{k}}D^2 \\
    \end{aligned}
  \end{equation}
  \begin{itemize}
    \item Achieves \textcolor{blue}{linear convergence} in the \textbf{strongly} convex setting.
    \item proofs are difficult (and computer-aided)
  \end{itemize}

  \begin{center}
    Same gradient oracle cost as SGD, but same converge rate as GD.
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Experiments from the original paper}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{SAG-experiments}
    \caption{ Solving $\ell_2$-regularized logistic regression.}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{More ``naive'' implementation}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=0.9\textheight,keepaspectratio]{SAG}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{SAG experiments}
  \begin{itemize}
    \item does not work well out of the box
    \item needs a \textcolor{blue}{warm up} to get good $(g_1^0, g_2^0, \dots, g_m^0)$
          \begin{itemize}
            \item achieved by running one full epoch of SGD
          \end{itemize}
    \item reguires hand tuned stepsize or line search
  \end{itemize}

\end{frame}

\section{SAGA}%
\label{sec:}

\begin{frame}
  \frametitle{SAGA, 2014}
  Very similar to SAG:
  \begin{itemize}
    \item maintain table containing gradients $g_i$ of $f_i$
    \item pick random $i_k \in [n]$ and
          \begin{equation}
            g_{i_k}^k := \nabla f_{i_k}(x^{k})
          \end{equation}
          set $g_{i}^k = g_i^{k-1}$ for all $i\neq i_k$ (remain the same)
    \item Update
          \begin{equation}
            x^{k+1} = x^k - \alpha_k  \Big( g_{i_k}^k - g_{i_k}^{k-1} + \frac{1}{n}\sum_{i=1}^{n} g_i^k \Big)
          \end{equation}
    \item estimator now \textbf{unbiased}!
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{For Comparison}
  SAGA gradient estimate:
  \begin{equation}
    g_{i_k}^k - g_{i_k}^{k-1} + \frac{1}{n}\sum_{i=1}^{n} g_i^k.
  \end{equation}

  SAG gradient estimate:
  \begin{equation}
    \frac{1}{n}g_{i_k}^k - \frac{1}{n}g_{i_k}^{k-1} + \frac{1}{n}\sum_{i=1}^{n} g_i^k.
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Experiments from the original paper}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{SAGA-experiments}
    \caption{Solving regularized logistic regression. First row is $\ell_2$-regularized; second row is $\ell_1$.}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{More ``naive'' implementation}

  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=0.9\textheight,keepaspectratio]{SAGA.png}
    \caption{Highlights the low variance of SAG.}
  \end{figure}
\end{frame}

\section{SVRG}%

\begin{frame}
  \frametitle{Stochastic Variance Reduced Gradient (SVRG), 2013}

  \begin{algorithm}[H]
    \caption{SVRG}\label{}
    \begin{algorithmic}[1]
      \For{$k=1,2, \dots$}
      \State{Set $x^1 = \tilde{x} = \tilde{x}^{k}$}
      \State{Compute $\tilde{\mu} := \nabla f(\tilde{x})$ \hfill \textcolor{blue}{//update snapshot}}
      \For{$l=1,2, \dots, m$ }\hfill \textcolor{blue}{//$m$ iterations per epoch}
      \State{pick $i_l$ uniform at random in $[n]$}
      \State{Set $x^{l+1} = x^l - \alpha (\nabla f_{i_l}(x^l) - \nabla f_{i_l}(\tilde{x}) + \tilde{\mu})$}
      \EndFor{}
      \State{$\tilde{x}^{k+1}= x^{m+1}$}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}
  \begin{itemize}
    \item Does \textbf{\textcolor{blue}{not need to store}} full table of gradients.
    \item requires \emph{batch} gradient computation every \emph{epoch}
    \item per iteration cost is comparable to that of SGD if $m \ge n$
    \item convergence rates similar to SAGA, but simpler analysis.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SVRG}
  \textbf{key idea:} by storing old point we can
  \begin{equation}
    \underbrace{\nabla f_{i_k}(x^k) - \nabla f_{i_k}(x^{\text{old}})}_{\text{$\to 0$ if $x\approx x^{\text{old}}$}} + \underbrace{\nabla f(x^{\text{old}})}_{\text{$\to 0$ if $x^{\text{old}}\approx x^*$}}
  \end{equation}
  \begin{itemize}
    \item is an unbiased estimate of $\nabla f(x^k)$
    \item converges to $0$ (meaning reduced variability) if $x^k\approx x^{\text{old}} \approx x^*$
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{SVRG: Theorem}

  \textcolor{gray}{Each $f_i$ is convex and $L$-smooth, and sum is $\mu$-strongly convex.}

  \begin{theorem}
    Choose $m$ large enough s.t. $\rho = \frac{1}{\mu \alpha (1-2L \alpha)m} + \frac{2L \alpha}{1-2L \alpha}< 1$, then
    \begin{equation}
      \E[F(x_s^{old})- F(x^*)] \le \rho^s [F(x_0^{old})-F(x^*)]
    \end{equation}
  \end{theorem}
  Computational cost:
  \begin{itemize}
    \item per epoch: $(m+n)$
    % \item complexity: $(n+ \kappa) \log(1/\epsilon)$
    \item inner loop is annoying (has to choose $m$) $\rightarrow$ loopless variant
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SVRG: Convergence Proof}
  Denote $g_s^k = \nabla f_{i_l}(x_s^k) - \nabla f_{i_l}(x_s^{old}) + \nabla F(x_s^{old})$. Conditioning on everything prior to $x_s^{k+1}$ we get
  \begin{equation}
    \begin{aligned}
      \MoveEqLeft \E [\Vert x_s^{k+1} - x^* \Vert^2] = \E [\Vert x_s^{k}- \alpha g_s^k - x^* \Vert^2] \\
      &= \Vert x_s^k-x^* \Vert^2 - 2 \alpha {(x_s^k-x^* )}^T \E[g_s^t] + \alpha^2 \E[\Vert g_s^k \Vert^2] \\
      &= \Vert x_s^k-x^* \Vert^2 - 2 \alpha {(x_s^k-x^* )}^T \textcolor{blue}{\nabla F(x_s^t)} + \alpha^2 \E[\Vert g_s^k \Vert^2] \\
      &\le \Vert x_s^k-x^* \Vert^2 - 2 \alpha (F(x_s^k) - F(x^*))+ \alpha^2 \E[\Vert g_s^k \Vert^2]
    \end{aligned}
  \end{equation}

  \begin{itemize}
     \item \textbf{key step:} control $\E[\Vert g_s^k \Vert^2]$
   \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SVRG: convergence Proof}
  \begin{lemma}%
    \begin{equation}
      \E[\Vert g_s^k \Vert^2] \le 4L [ F(x_s^k)- F(x^*) + F(x_s^{old}-F(x^*)) ]
    \end{equation}

  \end{lemma}

\end{frame}

\begin{frame}
  \frametitle{Comparison}

  % \begin{tabular}{l | c}
  %   SVRG & $(n + \kappa) \log(1/\epsilon)$\\
  %   GD   & $n \kappa \log(1/\epsilon)$ \\
  %   SGD  & $\kappa^2 / \epsilon$
  % \end{tabular}

  % \textbf{\textcolor{blue}{Strongly convex}} problems: number gradient calls to compute:
  % \begin{equation}
  %   \E [f(x_k)] - f^* \le \epsilon
  % \end{equation}
  % is given by

  \textcolor{gray}{$f$ is $L$-smooth and $\mu$-strongly convex. Condition number: $\kappa = L/\mu$.}\\
  \textbf{\textcolor{blue}{Strongly convex}} problems: number gradient calls to compute $\E [f(x_k)] - f^* \le \epsilon$ is given by
  \begin{center}
    \begin{tabular}{c c c}
      SVRG / SAGA  & GD & SGD \\
      \midrule
      $(n + \kappa) \log \frac{1}{\epsilon}$ & $n \kappa \log \frac{1}{\epsilon}$ & $\kappa^2 / \epsilon$
    \end{tabular}
  \end{center}
  \vspace{1cm}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{SAGA-table-comparison}
    \caption{Summary of other relevant properties.}
  \end{figure}

  % \begin{itemize}
  %   \item SVRG:\ $(n + \kappa) \log(1/\epsilon)$
  %   \item GD:\ $n \kappa \log(1/\epsilon)$
  %   \item SGD:\ $\kappa^2 / \epsilon$
  % \end{itemize}

\end{frame}

\section{Katyusha}%
\label{sec:}

\begin{frame}
  \frametitle{Variance reduction + momentum/acceleration}
  \begin{block}{}
    \centering
    \textbf{Katyusha}
  \end{block}

  \textbf{\textcolor{blue}{Strongly convex}} problems: number gradient calls to compute $\E [f(x_k)] - f^* \le \epsilon$ is given by
  \begin{center}
  \begin{tabular}{c c c c c}
    Katyusha & SVRG / SAGA  & GD & NAG & SGD \\
    \midrule
    $(n + \sqrt{n \kappa}) \log \frac{1}{\epsilon}$ &$(n + \kappa) \log \frac{1}{\epsilon}$ & $n \kappa \log \frac{1}{\epsilon}$ & $n \sqrt{\kappa} \log \frac{1}{\epsilon}$ & $\kappa^2 / \epsilon$
  \end{tabular}
  \end{center}
  \vspace{1cm}
  \begin{itemize}
    \item Improvement critical for \textbf{\textcolor{blue}{ill conditioned}} ($\kappa \gg n$) problems.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Katyusha in the non-strongly convex setting}
  \textbf{\textcolor{blue}{just convex}} problems: number gradient calls to compute $\E [f(x_k)] - f^* \le \epsilon$ is given by
  \begin{center}
  \begin{tabular}{c c c c c c}
    lower bound & Katyusha & SVRG & GD & NAG & SGD \\
    \midrule
    $n + \sqrt{\frac{n L}{\epsilon}}$ & $n \log \frac{1}{\epsilon} + \sqrt{\frac{n L}{\epsilon}}$ &$n + \sqrt{n}\frac{L}{\epsilon}$ & $n \frac{L}{\epsilon}$ & $n \sqrt{\frac{L}{\epsilon}}$ & $ \frac{1}{\epsilon^2}$
  \end{tabular}
  \end{center}
  \vspace{1cm}
  \textcolor{gray}{Almost matches lower bound.}

\end{frame}

\begin{frame}
  \frametitle{Summary}

  \begin{itemize}
    \item Variance reduction recovers the rates of batch (deterministic methods)
    \item but (more or less) keeps number of gradient calls of SGD
    \item still require batch gradient computations sometimes
    \item requires offline setting (multiple passes through data)
    \item requires knowledge of (multiple) parameters to get good stepsize
  \end{itemize}

  \textcolor{gray}{
  Check out this fantastic review paper:
  https://ieeexplore-ieee-org.uaccess.univie.ac.at/stamp/stamp.jsp?tp=&arnumber=9226504}

\end{frame}

\end{document}