% !TEX root = ../root.tex
\section{Theoretical principles}\label{sec:theory}
This section will present the theoretical aspects of the technologies employed in the project, and provides a knowledge base for better understanding the problem and the employed solution.

\subsection{Ultra-wideband}
Ultra-wideband (UWB in short) is a radio technology that use a very low energy level for short-range, high-bandwidth communications over a large portion of the radio spectrum.

Conventional narrowband systems employs the modulation of a continuous-waveform signal with a specific carrier frequency to transmit and receive information.
This continuous waveform has a well-defined signal energy in a narrow frequency band that exposes it to detection and interception.
On the other hand, UWB transmitters generate extremely thin pulse (duty cycle of 0.5\%) whose average energy content is very low, in fact barely above the background noise level.

Short pulses like the ones described above can yield a very wide bandwidth, often in the order of several \si{\giga\hertz}.
On a theoretical standpoint, this means that UWB may allow for high channel capacities \emph{C}, as predicted by Shannon–Hartley, shown below (where \emph{B} is the channel bandwidth and \emph{SNR} is the signal-to-noise ratio of the signal):
\begin{equation}
    \textit{C} = \textit{B}\log_2(1 + \textit{SNR})
\end{equation}

The UWB technology finds easy applications in \emph{ToF}-based positioning systems, due to its ability of avoiding reflection overlapping (an inherited benefit of the brevity of the pulses).
Moreover, the wide spectrum ensures that at least a portion of the signal follows a line-of-sight path.

Other applications may include short-range, high-speed communication systems capable of replacing current wireless technologies while having a smaller footprint on the overall level of emission.

\subsection{Ranging algorithm}
An overview of different ranging techniques is hereby presented and discussed.
The techniques analyzed here mainly strive to calculate the most accurate possible \emph{time of flight} (TOF), which can then be converted into distance with the following trivial equation:
\begin{equation}
    \textit{distance} = \textit{ToF} \times c
\end{equation}
The devices involved in the process will be referred to as initiator (\(\mathbf{I}\)) and responder (\(\mathbf{R}\)).
Furthermore, only solutions that can cope with unsynchronized (i.e. not sharing the same base clock) devices are taken into consideration.

\fig{5cm}{twr.png}{Two-way ranging: different approaches}{twr}

\subsubsection{Single-sided 2WR}
This is the most straightforward approach:
\begin{enumerate}
    \item \(\mathbf{I}\) sends poll message and notes time \(T_{poll_{TX}}\)
    \item \(\mathbf{R}\) receives poll message and notes time \(T_{poll_{RX}}\)
    \item After an arbitrary time, \(\mathbf{R}\) sends response message and notes time \(T_{resp_{TX}}\)
    \item \(\mathbf{I}\) receives response message and notes time \(T_{resp_{RX}}\)
\end{enumerate}
Assuming that \(\mathbf{R}\) includes all its recorded data into the response message, \(\mathbf{I}\) can assume that:
\begin{equation}
    \textit{ToF} = \frac{(T_{resp_{RX}} - T_{poll_{TX}}) - (T_{resp_{TX}} - T_{poll_{RX}})}{2}
\end{equation}
One of the major shortcomings of this scheme is that clock frequency differences between the two devices will cause vast drifts over the tiniest delay.

\subsubsection{Double-sided 2WR}
This method improves upon the previous ones by trying to compensate for the introduced drift:
\begin{enumerate}
    \item \(\mathbf{I}\) sends poll message and notes time \(T_{poll_{TX}}\)
    \item \(\mathbf{R}\) receives poll message and notes time \(T_{poll_{RX}}\)
    \item After an arbitrary time, \(\mathbf{R}\) sends response message and notes time \(T_{resp_{TX}}\) as well as \(T_{reply1} = T_{resp_{TX}} - T_{poll_{RX}}\)
    \item \(\mathbf{I}\) receives response message and notes time \(T_{resp_{RX}}\) as well as \(T_{round1} = T_{resp_{RX}} - T_{poll_{TX}}\)
    \item After an arbitrary time, \(\mathbf{I}\) sends final message and notes time \(T_{final_{TX}}\) as well as \(T_{reply2} = T_{final_{TX}} - T_{resp_{RX}}\)
    \item \(\mathbf{R}\) receives response message and notes time \(T_{final_{RX}}\) as well as \(T_{round2} = T_{final_{RX}} - T_{resp_{TX}}\)
\end{enumerate}
Assuming that \(\mathbf{R}\) includes the calculated time intervals in the response message, \(\mathbf{I}\) can assume that:
\begin{equation}
    \textit{ToF} = \frac{T_{round1} \times T_{round2} - T_{reply1} \times T_{reply2}}{T_{round1} + T_{round2} + T_{reply1} + T_{reply2}}
\end{equation}
Although this method is known to be more reliable than the previous, it requires that ranging measurements are transmitted through the radio, which depending on the application may not be desirable.

\subsubsection{Single-sided 2WR with drift compensation}
This method provides drift correction whilst maintaining the range computation on the initializer:
\begin{enumerate}
    \item \(\mathbf{I}\) sends poll message and notes time \(T_{poll_{TX}}\)
    \item \(\mathbf{R}\) receives poll message
    \item After an arbitrary time, \(\mathbf{R}\) sends response message
    \item \(\mathbf{I}\) receives response message and notes time \(T_{resp1_{RX}}\)
    \item After the same amount of time, \(\mathbf{R}\) sends another response message
    \item \(\mathbf{I}\) receives response message and notes time \(T_{resp2_{RX}}\)
\end{enumerate}

In this way \(\mathbf{I}\) need not use any data based on other clocks.
The equation becomes:
\begin{equation}
    \textit{ToF} = \frac{(T_{resp2_{RX}} - T_{poll_{TX}}) - 2 \times (T_{resp2_{RX}} - T_{resp1_{TX}})}{2}
\end{equation}

\subsection{Trilateration}
\emph{Trilateration} consists in determining the location of an object by measuring its distance from reference points.
With the advent of electronic means of measuring distances, this technique found widespread usage in the fields of geolocalization, navigation, and surveying.
In particular, GPS technology leverages on trilateration for providing its services.

The process of trilateration can be easily visualized in two dimensions by considering the circle projected by the set of points equidistant from a given reference.
The radius of such circle is the distance between an anchor whose location is known (the center of the circle) and a tag whose location is to be found.

\fig{5cm}{trilat.png}{Trilateration: example case in two dimensions}{trilat}

Adding another anchor and projecting its distance from the tag brings down the possible location to two points.
In the ideal case, a third anchor would project a circle intercepting precisely in one of these points.
However, this is rarely the case and may simply be unfeasible in case of a moving object (as the application of this project implies). \nameref{fig:trilat} shows however that an educated estimate can still be achieved.

\subsection{CAN Bus}
A \emph{Controller Area Network} (CAN) bus is a standard designed to allow microcontrollers, sensors, and other devices to communicate with each other without a host computer.
It was designed in 1983 by \emph{Robert Bosch GmbH} for automotive usage, but eventually found applications in many other contexts.

A CAN network is made up of nodes connected to each other through a two-wire bus.
The wires form a differential pair of \SI{120}{\ohm} nominal impedance.
Each node is capable of sending or receiving messages, properly called called frames.
A frame consists of an ID which can be \SI{11}{\bit} or \SI{29}{\bit} long, and up to eight data bytes.

CAN uses a lossless arbitration method in case of multiple nodes simultaneously attempting to transmit data.
Transmitted bit can be recessive or dominant.
If two such bits are beind transmitted at the same time, the node transmitting the recessive bits suspends its operation and reattempts the transfer after a given number of clock cycles, effectively implementing a prioritized communication system.
For this strategy to work, each node on a CAN network must agree on a bit rate and is required to sample every bit of data at the same time as the others.

Due to the lack of a shared clock signal, a synchronization mechanism is employed.
\emph{Hard synchronization} occurs after a recessive-to-dominant bit transition, and simply restart the timing from there.
Each bit is divided into time intervals called \emph{quanta}, and the first quantum is assigned to the so-called synchronization segment.
Whenever a recessive-to-dominant bit transition occurs outside of the synchronization segment, the sampling time is moved accordingly.
To ensure enough transitions to perform the above procedure, a bit of opposite polarity is inserted after five consecutive equal bits, through a process called \emph{bit stuffing}.

\fig{2cm}{can_smpl.png}{CAN Bus: bit timing example }{can_smpl}

Most CAN interfaces support automatic filtering of undesired frames through acceptance filter.
The process consists in performing a bitwise \emph{and} operation between the incoming frame ID and an acceptance bit-mask, and then comparing the result with the content of a filter register: if these match, the frame is added to the reception queue.
In this way, a receiving node can choose to accept frames of a specific type, or sent by a particular other node.


\section{Qualitative Sensitivity Measures: The Morris Method} \label{comp_morris}

This section introduces the Morris method for input screening. After the usefulness of input screening is discussed, the Morris method is applied to the Rust model.

\subsection{Input Screening} \label{screening}

Screening methods aim at identifying uninfluential inputs and ranking them according to their importance. These qualitative sensitivity methods, as opposed to quantitative ones, do not allow us to exactly quantify each input's effect on the model outcome. Nevertheless, there are several settings in which screening methods can be successfully applied. These are the FF and the FP settings discussed in \cref{var_based_sa}.

Since screening methods come at a lower computational cost than other sensitivity measures, they are well adapted to models with a large number of inputs. In such settings, other sensitivity methods may not be tractable anymore. Input screening can then guide research by identifying uninfluential inputs that can be left out of subsequent analyses. Such a subsequent analysis can be a more thorough, expensive sensitivity method which has then come into reach by having to consider only a subset of inputs. Screening can also foster scientific discovery by telling us which inputs are most important for the model output. Future models can then build on these results by particularly focusing on these important inputs \citep{R21}.

To illustrate the usefulness of input screening, consider the following examples of their application found in \citet{GPWMS17} and \citet{MMA18}. \citet{GPWMS17} conduct input screening in hydrology. They apply seven different screening methods to a lake model and learn that a certain input has major influence on their model's output. By identifying the main driver of uncertainty in the model, they are able to make an informed recommendation about which inputs should be estimated more accurately. \citet{MMA18} use input screening to identify uninfluential inputs that could then be ignored in the subsequent development of a meta-model, thus facilitating and guiding modelling.

While applying quantitative sensitivity methods, \citet{HMSW19} use their sensitivity indices for input ranking as well to ease the interpretation of model mechanics.

With these examples of the usefulness of input screening in mind, the following section discusses the Morris method which is then applied to the Rust model.

\subsection{Morris Method for Independent Inputs} \label{classic_morris}

The Morris method was introduced to identify the uninfluential input variables of a model, especially in cases where there are many inputs and/or the evaluation of a model is time-consuming \citep{M91}.

Consider the same setup as employed in the preceding sections. Let $x = \{x_1, \dots, x_k\}$ denote a sample of values assigned to the $X_i$'s. $f(x)$ is then the model output obtained for the values in $x$. Now consider a second sample $x_{\Delta_i} = \{x_1, \dots, x_{i-1}, x_i + \Delta, x_{i+1}, \dots, x_k\}$ that is identical to $x$ up to input $x_i$ which is varied by $\Delta$. Then, one elementary effect for input $i$ is derived by
\begin{equation*}
EE_i = \frac{f(x_{\Delta_i}) - f(x)}{\Delta}.
\end{equation*}

The above elementary effect is computed $N$ times, each for a varying $\Delta$ \citep{GM17}. The actual sensitivity measures resulting from the Morris method are the mean, denoted by $\mu^\ast_i$, and the standard deviation, denoted by $\sigma_i$, taken from all $N$ different elementary effects per input $i$.
\begin{align}
\mu_i^\ast& = \frac{1}{N} \sum_{r=1}^N \vert EE_{i, r} \vert, \label{mu}\\
\sigma_i& = \sqrt{\frac{1}{N-1} \sum_{r=1}^N (EE_{i, r} - \mu_i)^2}, \label{sigma}
\end{align}

\noindent with $EE_{i, r}$ denoting the $r$-th elementary effect of input $i$, $r = 1,\ \dots, N$, and $\vert \cdot \vert$ the absolute value. Note that in \citet{M91} the absolute value was absent and elementary effects could potentially cancel each other out \citep{CCS07}. Therefore, \citet{CCS07} proposed the version presented above, thus making the screening method more robust. A total of $2 k N$ model evaluations is needed to compute the full set of sensitivity measures using the Morris method.

$\mu_i^\ast$ and $\sigma_i$ can now be used to identify non-influential model inputs. Uninfluential inputs exhibit a $\mu_i^\ast$ close to zero. If $\mu_i^\ast$ is large, it depends on $\sigma_i$ whether there exist substantial non-linear or interaction effects. A low $\sigma_i$ indicates that non-linear effects are non-existent, whereas a high $\sigma_i$ suggests large interaction or non-linear effects \citep{GM17}.

The Morris method exhibits some drawbacks, though. Firstly, as they stand, the sensitivity indices derived by the Morris method are not suited for screening inputs under dependence. To see why consider two inputs $X_i$ and $X_j$ which are dependent, i.e. $G(x_i, x_j) \neq G(x_i)G(x_j)$, where $G(\cdot)$ again denotes the cumulative distribution function. If $x_i$ changes, $x_j$ should change as well due to the dependence between the two inputs. The sensitivity indices presented above are derived using a one-at-a-time approach that does not allow for the screening of dependent inputs \citep{GM17}.

Secondly, similar to the Sobol' indices, the researcher or practitioner conducting sensitivity analysis has to take two indices per input into account. Compare to the arguments made in \cref{var_based_sa}.

Thirdly, there exists no clear interpretation of the absolute values of the sensitivity indices. They only provide a ranking of inputs and give a hint of which inputs are the least influential ones \citep{GM17}.

On the advantages, Morris indices are easily computed, with a much lower computational burden than the Shapley effects as presented in \cref{comp_shap}. Recall that Shapley effects as computed by using the algorithm in \citet{SNS16} came at a cost of $N_V+m \cdot N_I \cdot N_O \cdot (k-1)$ model evaluations. Even the more efficient approach by \citet{PRB20} needed $2^k$ model runs. See \cref{comparison} for a discussion of the respective computational costs.

\citet{M91} points out that his method does not rely on simplifying assumptions, e.g. monotonicity of the model or input sparsity. He argues that if those assumptions hold, one could apply other, more effective and economical procedures, e.g. based on Latin hypercube designs. However, the Morris method does not rely on such assumptions and will work well if these assumptions are justifiable or not \citep{M91}.

Considering the interpretation of Morris indices, $\mu_i^\ast$ and $\sigma_i$, it is apparent that not only input ranking is feasible but we can also learn something about the underlying model structure, i.e. whether interaction or non-linear effects are present.

\citet{BP16} group the Morris method to the family of local sensitivity measures, but they acknowledge that screening methods like the Morris sensitivity measures stand apart from other local sensitivity measures. While the elementary effects themselves consider only local changes, the actual measures for input importance, $\mu_i^\ast$ and $\sigma_i$, average over these $N$ elementary effects. Thus, they take $N$ local changes per input $i$ into account \citep{M91}. Indeed, \citet{CCS11} make a case for the Morris method to be seen a global sensitivity method.

\subsection{Algorithm for Extended Morris Method}

\begin{figure}[t]
	\caption{Uncertainty in Morris Indices - 100 Replicates}
    \label{morris_replicates}
    \begin{centering}
	\vspace*{-4mm}
	\begin{centering}
	\includegraphics[scale=0.9]{../figures/boxplot_morris_replicates_100_replicates_27_draws.png}
    \end{centering}
    \end{centering}

    \small
    \textit{Notes:} Boxplots of the four Morris indices for the model inputs $RC$ and $\theta_{11}$. 100 Morris indices were estimated with $N=27$. The left panel shows \textit{independent}, the right one \textit{full} Morris indices.
\end{figure}

\begin{figure}[t]
	\caption{Convergence of Morris Indices}
    \label{morris_convergence}
    \begin{centering}
	\vspace*{-4mm}
	\begin{centering}
	\includegraphics[scale=0.9]{../figures/morris_convergence.png}
    \end{centering}
    \end{centering}

    \small
    \textit{Notes:} Estimated Morris indices for different sample sizes $N$. The left panel shows \textit{independent} Morris indices, the right one \textit{full} Morris indices.
\end{figure}

In this section I introduce the extended Morris method for dependent samples as proposed by \citet{GM17}.

To grasp the computational procedure of the extended elementary effects, some more notation is needed. Following \citet{GM17}, let $X' = \{X_1',\ X_2',\ \dots,\ X_k'\}$ be $k$ dependent random inputs, following the joint probability density function $g(X)$. Thus, $X'$ is just a set of inputs independently drawn from the set $X$. Input subsets denoted by $\bar{X}$ are conditionally drawn inputs. Hence, let $\bar{X_{-i}'}$ follow the conditional probability density function $g(\bar{X_{-i}'} \mid X_{-i})$. That is $\bar{X_{-i}'}$ is drawn conditionally on the inputs in the first set $X_{-i}$. The input denoted by $\bar{X_{-i}}$ is conditionally drawn following the probability density function $g(\bar{X_{-i}} \mid X_i')$. % Hence, $\bar{X_{-i}'}$ and $\bar{X_{-i}}$ differ by [].

Analogously to the independent and full Sobol' indices \citep{MTA15}, \citet{GM17} developed the following elementary effects for dependent inputs.
\begin{align}
EE_i^{ind} = \frac{f(\bar{x_i}',\ x_{-i}) - f(x_i,\ x_{-i})}{\Delta},\\
EE_i^{full} = \frac{f(x_i',\ \bar{x_{-i}}) - f(x_i,\ x_{-i})}{\Delta},
\end{align}

\noindent where
\begin{itemize}
\item $EE_i^{ind}$ denotes \textit{independent} elementary effects for input $i$, effects that exclude the contributions attributable to the dependence between input $X_i$ and $X_j$ for $i \neq j$, and
\item $EE_i^{full}$ denotes \textit{full} elementary effects for input $i$, that include the effects due to correlation with other inputs.
\end{itemize}

As in the case of the classic Morris method, the sensitivity measures for input $i$ are derived by considering $N$ random samples yielding $2 N$ elementary effects, once for the independent and once for the full elementary effects. \citet{GM17} compute the corresponding sensitivity measures as shown in \cref{mu,sigma}. Since two sets of elementary effects are computed, they end up with a set of four sensitivity measures, $(\mu^{\ast ind}_i,\ \sigma_i^{ind})$ and $(\mu^{\ast full}_i,\ \sigma_i^{full})$.

Interpretation-wise, $X_i$ is an unimportant input if all sensitivity measures are essentially zero. If $\mu^{\ast ind}_i$ is strongly larger than zero and $\sigma_i^{ind}$ is close to zero, input $X_i$ is an important input due to its own, isolated contribution. When all sensitivity measures except $\mu^{\ast full}_i$ are close to zero, $X_i$'s contribution is due to the dependence with other important inputs. Strong interaction effects are present if either or both of the two $\sigma$'s are larger than zero \citep{GM17}.

The computation of the extended Morris indices requires the generation of dependent samples. According to \citet{GM17}, the computation involves the following steps:

\begin{enumerate}
    \item Create independent, uniformly distributed samples.
    \item Transform these uniformly distributed samples into dependent samples of a target distribution.
    \item Use the above methods for the computation of the extended elementary effects.
    \item As in the case of the classic Morris method, average over all $N$ elementary effects per input $i$ and compute their standard deviation. Do so for the \textit{independent} and \textit{full} elementary effects.
\end{enumerate}

In what follows I stick to the version of the algorithm as implemented in the \textit{econsa} Python-package \citep{OSE21}. There, the radial design is used for obtaining independent samples. To derive dependent samples, the inverse Nataf transformation is applied. The total computational cost amounts to $3kN$ model runs.

\subsection{Morris Indices for the Rust Model} \label{morris_rust_model}

\begin{table}[t]
    \caption{Relative Difference Morris Indices}
    \label{rel_diff_morris}
    \centering

    \begin{threeparttable}
        \begin{centering}
            \input{../figures/morris_convergence_relative_difference.tex}
            \begin{tablenotes}
                \small
                \item \textit{Notes:} The percentage difference between Morris indices for different values of $N$. For example, for $\mu^{\ast,\ full}$ Percentage changes are calculated by $\frac{\mu^{\ast,\ full,\ n-1}_i - \mu^{\ast,\ full,\ n}_i}{\mu^{\ast,\ full,\ n}_i}$, for $i \in \{RC,\ \theta_{11}\}$ and $n$ denoting the different values of $N$.
            \end{tablenotes}
        \end{centering}

        \end{threeparttable}
\end{table}

Similar to the procedure described in \cref{comp_shap}, I estimate Morris indices in two different ways: Firstly, I estimate 100 replicates for a relatively low sample size to get a sense of the variability in estimates, Then, I consecutively increase the sample size to investigate
the estimation procedure further. Uniformly distributed samples are drawn from a Sobol'
sequence \citep{S76}.\footnote{The Sobol' sequence is a quasi-random sequence used for Quasi-Monte Carlo methods. \citet{KTA12} argue that Quasi-Monte Carlo estimators converge faster than estimators based on pseudo-random numbers.}

Firstly, I estimate 100 sets of Morris indices. In order to ensure comparability to the Shapley effects, I use the same total computational cost as for the data used for \cref{boxplot_shapley}. I set $N = 27$, which amounts
to a total cost of 162 model evaluations, which differs slightly from the computational
cost of the Shapley effects (160) because of rounding.\footnote{The same computational cost as for Shapley effects is achieved by solving $6 \cdot N = 160 \Leftrightarrow N = \frac{160}{6} \approx 26.67$.} As stated above, there are four different
sensitivity indices estimated: $(\mu^{\ast ind}_i,\ \sigma_i^{ind})$ and $(\mu^{\ast full}_i,\ \sigma_i^{full})$. The distribution of the
estimated Morris indices is given in \cref{morris_replicates}.

Analysing \cref{morris_replicates}, we can get a sense of the uncertainty inherent in the estimation procedure,
given the admittedly small sample size. For the independent Morris indices, there exists a
very clear ranking: both values indicate that $RC$ is uninfluential. For $\theta_{11}$ the situation is
different: both independent indices, $(\mu^{\ast ind}_{\theta_{11}},\ \sigma_{\theta_{11}}^{ind})$, are nonzero. Surprisingly, for the full Morris indices,
we fail to observe a clear input ranking. Estimates of $(\mu^{\ast full}_i,\ \sigma_i^{full})$ are very volatile, but
clearly different from zero. Seemingly, no
clear input ranking is achieved. However, Morris indices were primarily developed to
identify non-influential inputs. Both inputs are important if the full Morris indices are
considered.

To investigate whether the individual pairs of full Morris indices, $(\mu^{\ast full}_i,\ \sigma_i^{full})$, for the
two inputs $i \in \{RC,\ \theta_{11}\}$ yield the correct input ranking, I simply compare their values
per replicate. Details are given in \cref{accuracy}. Surprisingly, only about 60 of the 100 replicates yield the correct importance ranking
for $\mu^{\ast full}_i$. I conclude that ranking based on full Morris indices in the case of the Rust model can be misleading given the small sample size.
The influence of dependence seems to be so large that indices cannot be ranked according to their full variance contribution.

\begin{table}
	\centering
	\caption{Accuracy of Morris Indices}
	\label{accuracy}
	\begin{threeparttable}
	\centering
	\input{../figures/correct_rankings.tex}
	\begin{tablenotes}
	\small
	\item \textit{Notes:} Share of the correct input rankings for $100$ Morris indices based on $\mu^{\ast,\ full}$ and $\mu^{\ast,\ ind}$. As a benchmark, the accuracy of Shapley effects is presented. For Morris indices the accuracy for different sample sizes is reported.
	\end{tablenotes}
	\end{threeparttable}
\end{table}

Secondly, I compute Morris indices for a set of number of draws. I consider $N \in \{100,\ 500,\ 1000,\ 1500,\ 2000,\ 2500,\ 3000\}$. The corresponding results are shown in \cref{morris_convergence}. The obtained Morris indices clearly confirm the input ranking implied by the Shapley effects for both, $\mu^{\ast ind}_i$ and $\mu^{\ast full}_i$. Further, Morris indices
seem to converge as $N$ increases. In \Cref{rel_diff_morris} specific information on the convergence behaviour is provided. \citet{ASAV19} show for the classical (independent) Morris indices that they do indeed converge with increasing sample size. Since in the end, the full Morris indices are estimated in a similar fashion, similar behaviour for full indices is expected. However, \textit{full} Morris indices do not converge for all replicates as seen in \cref{morris_replicates} and supported further by the results in \cref{accuracy}. In fact, the share of correct rankings steadily declines as $N$ increases. Intrigued by this result, I investigate whether this behaviour can be observed for a larger sample size as well. To that end, I consider the replicate of Morris indices with $N=500$ which shows the largest difference $\mu^{\ast,\ full}_{RC} - \mu^{\ast,\ full}_{\theta_{11}}$ and re-estimate the full indices for larger sample sizes, i.e. for $N \in \{3\,000,\ 4\,000,\ 6\,000,\ 8\,000\}$. Details are shown in \cref{appendix_morris_seed_67} in Appendix A. There, Morris indices seem to converge, but still yield the wrong input ranking. Unfortunately, due to the computational limitations I face, I cannot investigate this further.
% Although Morris indices are volatile, \citet{GM17} point out that a high estimation precision is not necessarily needed as long as the identification of uninfluential inputs is achieved.

Let us inspect the Morris indices for $N = 3\,000$ in more detail. Consult \cref{morris_3000} for the
resulting sensitivity indices. The computational cost for this value of $N$ amounts to $18\,000$
model evaluations. As in the other trials, \cref{morris_3000} shows $(\mu^{\ast ind}_{RC},\ \sigma_{RC}^{ind})$ equal to zero.
If only the independent Morris indices are considered, we erroneously
conclude that input $RC$ is a non-influential input by neglecting the variance contribution due to dependence. Since none of the inputs exhibits $(\mu^{\ast ind}_i,\ \sigma_i^{ind})$ and $(\mu^{\ast full}_i,\ \sigma_i^{full})$ close to zero at the same time, both inputs have to be considered as influential ones and thus cannot be fixed without influencing the output.
Since both independent indices for $\theta_{11}$, $(\mu^{\ast ind}_{\theta_{11}},\ \sigma_{\theta_{11}}^{ind})$, are high, there are either strong interaction and/or non-linear effects present.
% Since independent sensitivity indices for $RC$ are both zero, the reason for the high $\sigma_{\theta_{11}}^{ind}$ must be driven by the presence of non-linear effects in $\theta_{11}$. Regarding the model structure we can conclude that there are no interaction effects between $RC$ and $\theta_{11}$.

\begin{table}[t]
    \centering
    \caption{Morris Indices for $N=3\,000$}
    \label{morris_3000}
    \begin{threeparttable}
    \begin{centering}
        \input{../figures/table_morris_conv_3000}
        \begin{tablenotes}
            \small
            \item \textit{Notes:} \textit{Independent} and \textit{full} Morris indices for $RC$ and $\theta_{11}$ using $N=3\,000$.
        \end{tablenotes}
    \end{centering}

    \end{threeparttable}

\end{table}


\documentclass[a4paper]{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{physics}
%\usepackage[round]{natbib}
%\usepackage{bibtex}
\usepackage{fancyvrb}
\usepackage{hyperref}
\usepackage[linesnumbered,boxed]{algorithm2e}
\usepackage{subfigure}

\numberwithin{equation}{section} % Remove this line for global equation numbering

\title{Stefano's awesome MATH1058 coursework report}
\author{Stefano Coniglio}
\date{\today}

\begin{document}

% This produces the title: to modify contents, change the \title, \author, and
% \date in the preamble
\maketitle

\begin{abstract}
  This document reports on something really interesting, hopefully.
\end{abstract}

\section{Introduction}
\label{sec:intro}

We consider, in this work, a definitely relevant problem which, in formal terms, can be expressed as follows:
%
\begin{quote}
  {\bf Very Interesting Problem (VIP):} Given a quite relevant set of givens $A \in \mathbb{R}^{m \times n}, b \in \mathbb{R}^n, c \in \mathbb{R}^n$, the VIP problem calls for a very, very relevant solution $x \in \mathbb{R}^n$ which satisfies the following very, very, very (!) relevant property:
  \begin{equation}\label{equation:theRelevantOne}
    z = \max_{x \in \mathbb{R}^n_+} \{cx: Ax \leq b\}
  \end{equation}
\end{quote}

Relying on Equation~\eqref{equation:theRelevantOne}, the algorithm computes $x$ proceeding according to the following pseudocode:

\begin{algorithm}[H]\label{alg:theGoodOne}
 \KwData{this text}
 \KwResult{how to write algorithm with \LaTeX2e }
 initialization\;
 \While{not at end of this document}{
  read current\;
  \eIf{understand}{
   go to next section\;
   current section becomes this one\;
   }{
   go back to the beginning of current section\;
  }
 }
 \caption{How to write algorithms.}
\end{algorithm}

The pseudocode reported in Algorithm~\ref{alg:theGoodOne} was taken from~\cite{coniglio}.\footnote{Actually, it was not, but I thought that citing myself was a good idea.}

\section{Solution approach and implementation}

We had a great solution idea, which we implemented in the following beautiful piece of Python code:

\begin{Verbatim}[numbers=left]
def beautiful_function(myUnusedInput, mySecondUnusedInput):
    if do_something_awesome() == True:
        return youWillNeverBugOutOnMe
    else:
        return myVeryUndefinedOutput
\end{Verbatim}

\section{Experimental results and analysis}

We have run a set of experiments on this very costly piece of hardware: {\em xxx}. Our findings are summarized in the quite colorful Figure~\ref{fig:whatANiceFigure}.

\begin{figure}[h]
  \subfigure[]{\includegraphics[scale=0.4]{./template_placeholder.png}}
  \subfigure[]{\includegraphics[scale=0.4]{./template_placeholder.png}}
  \caption{One very nice chart entirely unrelated to this coursework (a) and an apparently very similar chart stil unrelated to this coursework (b).}
  \label{fig:whatANiceFigure}
\end{figure}

We should not forget to explain what conclusions the figure allows us to draw, nor why.

Also, including a table may be a good idea. Take a look at Table~\ref{tab:1}:

\begin{table}[htbp]
  \caption{A nice table. Notice that captions preceed tables, whereas they follow figures.}
  \begin{center}
\begin{tabular}{l|lll}
  header    & header2 & header3 \\
  \hline
  name1     & 12      & 12\\
  $s = |S|$ & 5       & 42
\end{tabular}
  \end{center}
\label{tab:1}
\end{table}

\section{Conclusions}

We have reported on a set of quite interesting findings.

% At the end of the document, include the bibliography.
% You can use any citation style: this one is simple.
\bibliographystyle{plain}
% This assumes you have a single BibTeX file called references.bib: change to
% match your file name.
\bibliography{template_references}

\end{document}


\documentclass[oneside]{memoir}

\usepackage{notestemplate}
\usepackage{import}
\usepackage{standalone}
\usepackage{fancyhdr} % Required for custom headers
\usepackage{lastpage} % Required to determine the last page for the footer
\usepackage{extramarks} % Required for headers and footers
\usepackage{lipsum} % Used for inserting dummy 'Lorem ipsum' text into the template
\usepackage{hyperref}
\hypersetup{pdftex,colorlinks=true,allcolors=blue}
\usepackage{hypcap}
\usetikzlibrary{matrix,arrows.meta}

\logo{~/LibreMath/Auxiliary Resources/resources/png/logo.png}
\institute{Rice University}
%\faculty{Faculty of Whatever Sciences}
\department{Department of Mathematics}
\title{Introduction to Groups, Rings, and Fields}
\subtitle{A first exposure to core concepts in abstract algebra}
\author{\textit{Author}\\Gabriel \textsc{Gress}}
%\supervisor{Linus \textsc{Torvalds}}
%\context{Well, I was bored...}
\date{\today}

%\makeindex

\begin{document}
% \section{}

\maketitle

\tableofcontents
\input{2020-02-04-NumThryPrelim}
\input{2020-03-24-IntroGroups}
\input{2020-03-25-Subgroups}
\input{2020-03-31-NormalSubgroups}
\input{2020-04-01-QuotientGroup+Homo}
\input{2020-04-07-GroupProd}
\input{2020-03-26-CyclicGroups}
\input{2020-04-08-PermGroup}
%\input{GroupRepresentations}
%\input{AlternatingGroup}
\input{2020-04-28-GroupAction}
%\input{Automorphisms}
%\input{GroupRepresentations2}
%\input{Sylow}
\input{2020-04-09-ClassificationThms}
%\input{NilpotentSolvableGroups}
%\input{ProofFundThmFinAbelGrps}

\chapter{Rings}
\label{cha:rings}

\input{2020-02-05-IntroRings}
\input{2020-02-12-Subrings}

\section{Ring Homomorphisms and Quotient Rings}

\input{2020-02-18-QuotientRing}
\input{2020-02-19-RingHomo}
\input{2020-02-25-RingProd}
\input{2020-02-26-DivisioninRings}
\input{2020-03-03-GaussianInt}
\input{2020-03-04-GaussianInt2}
\input{2020-03-10-GaussInt3}
\input{2020-03-11-FiniteFields}
%\input{ModuleTheory}

%\printindex

\end{document}

Abstract Algebra - Introductory/Algebra I/Notes/source/AlgebraI.tex


\documentclass[12pt, letterpaper]{article}
\usepackage[english]{babel}
\usepackage{amsmath}
\usepackage[margin=0.7in]{geometry}

  %%%%%%%%%%%%
  % PREAMBLE %
  %%%%%%%%%%%%
\begin{document}
\selectlanguage{english}

\section{The crystal state and associated parameter gradients}
  The purpose of this document is to discuss the code in \texttt{g\_gradients.py} and \texttt{a\_g\_conversion.h}, which is presumably used for parameter refinement in \textit{DIALS}, per the Waterman (2016) paper, section B3.  Code such as this has been in the package at least since 2006.  Unfortunately, the following notes predate the current code, and I notice that slightly different conventions are used, \textit{e.g.}, the notes give the fractionalization matrix as upper triangular, but the comments in the code indicate lower triangular.  Therefore at some point these notes will have to be reconciled with the actual code.

Let's begin by defining notation.  All formulae will be given here in terms of the reciprocal lattice, therefore the vectors $\mathbf{a}$, $\mathbf{b}$, and $\mathbf{c}$ will be understood to refer to the reciprocal cell basis vectors, even though these vectors may be denoted with superscript asterisks in most literature.  In addition we have the reciprocal space $\mathbf{A}$ matrix,

  \begin{equation}
    \mathbf{A} =
    \left[
    \begin{array}{c c c}
       a_{x}  & b_{x} & c_{x} \\
       a_{y}  & b_{y} & c_{y} \\
       a_{z}  & b_{z} & c_{z} \\
    \end{array}
    \right]
    \text{,}
    \label{eqn:expansion}
  \end{equation}

representing the state variables of the crystal (cell parameters and orientation), in terms of column vectors denoting the basis vectors expressed as laboratory $x,y,z$ components.

In practice we express $\mathbf{A}$ as

  \begin{equation}
      \mathbf{A} = \mathbf{UF}
    \text{,}
    \label{eqn:sabbrev}
  \end{equation}

with unitary matrix $\mathbf{U}$ expressing the crystal rotation ($e.g.$, as a composite resulting from Euler rotations $\phi$, $\theta$ and $\psi$), and a fractionalization matrix $\mathbf{F}$ given by

  \begin{equation}
    \mathbf{F} =
    \left[
    \begin{array}{c c c}
       a_{x}  & b_{x} & c_{x} \\
       0  & b_{y} & c_{y} \\
       0  & 0 & c_{z} \\
    \end{array}
    \right]
    \text{.}
    \label{eqn:expansionF}
  \end{equation}

Here it is understood that $\mathbf{F}$ represents the standard orientation of the crystal, which by convention is taken to mean that the reciprocal $\mathbf{a}$ vector lies along laboratory $x$, with $\mathbf{b}$ in the $xy$ plane.

In contrast to the state variables, the parameters most useful for model fitting turn out to be the elements of the reciprocal space symmetric metric tensor,

  \begin{equation}
    \mathbf{G} =
    \left[
    \begin{array}{c}
       g_{11}  \\ g_{22} \\ g_{33} \\
       g_{12}  \\ g_{13} \\ g_{23}
       \\
    \end{array}
    \right] =
    \left[
    \begin{array}{c}
       g_{0}  \\ g_{1} \\ g_{2} \\
       g_{3}  \\ g_{4} \\ g_{5}
       \\
    \end{array}
    \right] =
     \left[
    \begin{array}{c}
       \mathbf{a}\cdot\mathbf{a}  \\ \mathbf{b}\cdot\mathbf{b} \\ \mathbf{c}\cdot\mathbf{c} \\
       \mathbf{a}\cdot\mathbf{b}  \\ \mathbf{a}\cdot\mathbf{c} \\ \mathbf{b}\cdot\mathbf{c}
       \\
    \end{array}
    \right]
   \text{.}
    \label{eqn:expansionG}
  \end{equation}

 The reason for prefering the metrical tensor components is that high-symmetry Bravais cells will only refine a subset of the six components, with monoclinic refining four, orthorhombic three, tetragonal and hexagonal two, and cubic only one.

 A key strategy in parameter refinement is to use the chain rule.  Suppose we are performing iterative non-linear least-squares parameter refinement with a refinement target $\chi$ that is a function of $\mathbf{A}$.  Suppose that at an intermediate step of the calculation we have the gradient of the target with respect to each element $a_{ij}$,

   \begin{equation}
    \frac {\partial \chi} {\partial a_{ij}}
       \text{,}
    \label{eqn:frac}
  \end{equation}

 where $i$ denotes the row and $j$ the column of the matrix $\mathbf{A}$.

Then in order to calculate the gradient of the target with respect to the free parameters $g_{ij}$, one would employ the chain rule.  For the specfic example of $g_{11}$ this would be performed as follows over a total of nine terms:

   \begin{equation}
    \frac {\partial \chi} {\partial g_{11}} =
    \frac {\partial \chi} {\partial a_{11}} \frac {\partial a_{11}}{\partial g_{11}}+
    \frac {\partial \chi} {\partial a_{12}} \frac {\partial a_{12}}{\partial g_{11}}+
    \frac {\partial \chi} {\partial a_{13}} \frac {\partial a_{13}}{\partial g_{11}}+
    \ldots+
    \frac {\partial \chi} {\partial a_{31}} \frac {\partial a_{31}}{\partial g_{11}}+
    \frac {\partial \chi} {\partial a_{32}} \frac {\partial a_{32}}{\partial g_{11}}+
    \frac {\partial \chi} {\partial a_{33}} \frac {\partial a_{33}}{\partial g_{11}}
       \text{.}
    \label{eqn:fracC}
  \end{equation}

It is important to note that \textit{DIALS} employs a related chain rule formula applicable to the special case where we start with the derivatives of the target with respect to the six non-zero elements of the fractionalization matrix,

     \begin{equation}
    \frac {\partial \chi} {\partial f_{ij}}
       \text{.}
    \label{eqn:fracF}
  \end{equation}

  However, I do not have the specific formulae documented here apart from the code itself.

  In any case, the critical piece will then be the relationship between $\mathbf{F}$ and $\mathbf{G}$.  Here are the full equations to convert between the two, $\mathbf{F}$ to $\mathbf{G}$:
      \begin{flalign*}
    &g_{0} = a_{x}^{2} &\\
    &g_{1} = b_{x}^{2} + b_{y}^{2} &\\
    &g_{2} = c_{x}^{2} + c_{y}^{2} + c_{z}^{2} &\\
    &g_{3} = a_{x} b_{x} &\\
    &g_{4} = a_{x} c_{x} &\\
    &g_{5} = b_{x} c_{x} + b_{y} c_{y} &\\
    \end{flalign*}
  Note the use of two differing nomenclatures for the subscript of $g$, which are nevertheless equivalent.  In the $\mathbf{G}$ to $\mathbf{F}$ direction:
      \begin{flalign*}
    &a_{x} = g_{0}^{1/2} &\\
    &b_{x} = g_{3} / a_{x} &\\
    &c_{x} = g_{4} / a_{x} &\\
    &b_{y} = (g_{1} - b_{x}^2)^{1/2} &\\
    &c_{y} = \dfrac{g_{5} - b_{x}c_{x}}{b_{y}} &\\
    &c_{z} = (g_{2} - c_{x}^2 - c_y^2)^{1/2} &\\
    \end{flalign*}
Notice the presence of the square root in three of the $\mathbf{G}$ to $\mathbf{F}$ formulae.  A negative value inside the radical should never occur, and if it did would represent an illegal trial value for a free parameter.

      \end{document}


\section{Lecture 4 - The Lazy Approach}

\begin{frame}
  \frametitle{The Lazy Approach}

  \scriptsize

  Notice that
  \begin{itemize}
    \item Assignments $\mu$ of $\varphi$ are many (potentially $\infty$),
          infeasible to check if any of them is a model {\bf systematically}
    \item Models $\babst{\mu}$ of $\babst{\varphi}$ are finite in number,
          and easy to enumerate with a SAT-solver
    \item A model $\babst{\mu}$ is nothing but a {\bf conjunction of \tatoms},
          can be checked efficiently with a \tsolver
  \end{itemize}
  \vfill
  These observations suggest us a methodology
  to tackle the SMT(\T) problem
  \begin{itemize}
    \item Enumerate a Boolean model $\babst{\mu}$ of $\babst{\varphi}$ (abstraction). If no model
	  exist we are done ($\varphi$ is unsatisfiable)
    \item Check if $\babst{\mu}$ is satisfiable using the \tsolver. If so $\babst{\mu}$ can be extended
          to a model $\mu$ of $\varphi$, and so we are done ! ($\varphi$ is satisfiable)
    \item It not, we tell the SAT-solver not to enumerate $\babst{\mu}$ again,
          thus {\bf cutting away systematically an infinite number}
	  of assignments for $\varphi$ (refinement)
    \item It can be blocked by adding a clause $\neg \babst{\mu}$. Go up
    \item It terminates because there are finite Boolean models
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{The Lazy Approach}

  \scriptsize

  The lazy approach falls into the so-called {\bf abstraction-refinement}
  paradigm
  \vfill
  \begin{center}
  \scalebox{.5}{\input{ar.pdf_t}}
  \end{center}

\end{frame}

\begin{frame}
  \frametitle{The Lazy Approach}

  \scriptsize

  The interaction described naturally falls within the
  CDCL style, enriched with a \tsolver
      $$\varphi \equiv
      (x=3 \vee \neg (x<3))\ \swedge\
      (x=3 \vee \neg (x>3))\ \swedge\
      (x>3 \vee \neg (x<3))\ \swedge\
      (x>3 \vee \neg (x=3))$$
  \vfill
  \begin{columns}

    \begin{column}{.5cm}
      \vspace{-165pt}
      $\babst{\varphi} \equiv$
    \end{column}

    \begin{column}{5cm}
      $(\coltwoat{\coloneat{a_1}{8-13}}{2-6} \vee \coltwoat{\neg a_2}{9-13})$ \\
      $(\coltwoat{\coloneat{a_1}{8-13}}{2-6} \vee \coltwoat{\coloneat{\neg a_3}{3-6}}{10-13})$ \\
      $(\coloneat{\coltwoat{a_3}{3-6}}{10-13} \vee \coltwoat{\neg a_2}{9-13})$ \\
      $(\coloneat{\coltwoat{a_3}{3-6}}{10-13} \vee \coloneat{\coltwoat{\neg a_1}{8-13}}{2-6})$ \\
      \onslide<6->{$(\coloneat{\coltwoat{\neg a_1}{10-13}}{6} \vee \coltwoat{\coloneat{\neg a_3}{6}}{10-13})$ \\}
      \onslide<7->{$(\coltwoat{\neg a_1}{8-13})$ \\}
      \onslide<13->{$(\coloneat{a_1}{13} \vee \coloneat{a_2}{13} \vee \coloneat{a_3}{13})$ \\}
      \onslide<14->{$(\ )$ \\}
      \bigskip
      $a_1 \equiv x=3$ \\
      $a_2 \equiv x<3$ \\
      $a_3 \equiv x>3$ \\
      \bigskip
      $\babst{\mu}$: $\{\ \only<2-6|handout:0>{a_1}\only<3-6|handout:0>{, a_3}\only<8-13|handout:0>{\neg a_1}\only<9-13|handout:0>{, \neg a_2}\only<10-13|handout:0>{, \neg a_3}\ \}$ \\
      \bigskip
      \begin{tabular}{rl}
      SAT-solver: & \only<1,4-5,11-12|handout:0>{Idle}\only<2|handout:0>{Decision}\only<3,8-10|handout:0>{BCP}\only<6,13|handout:0>{Learn}\only<7,14|handout:0>{Conf. Analysis, Backtrack}\only<15>{UNS} \\
        \tsolver: & \only<1,2,3,6-10,13->{Idle}\only<4,5,11,12|handout:0>{Is $\babst{\mu}$ \T-satisfiable ?}\only<5,12|handout:0>{ NO}
      \end{tabular}
    \end{column}

    \begin{column}{5cm}
      \begin{overlayarea}{5cm}{5cm}
	\only<1|handout:0>{\scalebox{.6}{\input{search_0.pdf_t}}}
	\only<2|handout:0>{\scalebox{.6}{\input{search_1.pdf_t}}}
	\only<3,4,5|handout:0>{\scalebox{.6}{\input{search_2.pdf_t}}}
	\only<6,7|handout:0>{\scalebox{.6}{\input{search_3.pdf_t}}}
	\only<8|handout:0>{\scalebox{.6}{\input{search_4.pdf_t}}}
	\only<9|handout:0>{\scalebox{.6}{\input{search_5.pdf_t}}}
	\only<10-12|handout:0>{\scalebox{.6}{\input{search_6.pdf_t}}}
	\only<13->{\scalebox{.6}{\input{search_7.pdf_t}}}
      \end{overlayarea}
    \end{column}

  \end{columns}

\end{frame}

\begin{frame}
  \frametitle{Lecture 4 - Exercize 1}

  \scriptsize

  \begin{tabular}{ccc}
    \begin{minipage}{.4\textwidth}
     $$
     \begin{array}{l}
     (a_1 \vee \neg a_2) \\
     (a_1 \vee \neg a_3) \\
     (a_3 \vee \neg a_2) \\
     (a_3 \vee \neg a_1) \\
     (\neg a_1 \vee \neg a_3)
     \end{array}
     $$
    \end{minipage}
    & ~~~~~~ &
    \begin{minipage}{.4\textwidth}
      \begin{tabular}{ccl}
	\hline
	Trail & dl & Reason \\
	\hline
	$a_1$ & 1 & Decision \\
	$a_3$ & 1 & $(a_3 \vee \neg a_1)$ \\
	\hline
      \end{tabular}
      \bigskip \\
      $\{ \dec{a_1}{1}, \dec{a_3}{1} \}$
    \end{minipage}
  \end{tabular}

  \vfill
  \pause

  \begin{minipage}{\textwidth}
    \begin{prooftree}
    \AxiomC{$(\neg a_1 \vee \neg a_3)$}
    \AxiomC{$( a_3 \vee \neg a_1 )$}
    \BinaryInfC{$(\neg a_1)$}
    \end{prooftree}
  \end{minipage}

  \vfill
  \pause
  Conflict clause: $(\neg a_1)$ \pause \\
  Backtracking level: 0
\end{frame}

\begin{frame}
  \frametitle{Lecture 4 - Exercize 1}

  \scriptsize

  \begin{tabular}{ccc}
    \begin{minipage}{.4\textwidth}
     $$
     \begin{array}{l}
     (a_1 \vee \neg a_2) \\
     (a_1 \vee \neg a_3) \\
     (a_3 \vee \neg a_2) \\
     (a_3 \vee \neg a_1) \\
     (\neg a_1 \vee \neg a_3) \\
     (\neg a_1) \\
     (a_1 \vee a_2 \vee a_3)
     \end{array}
     $$
    \end{minipage}
    & ~~~~~~ &
    \begin{minipage}{.4\textwidth}
      \begin{tabular}{ccl}
	\hline
	Trail & dl & Reason \\
	\hline
	$\neg a_1$ & 0 & $(\neg a_1)$ \\
	$\neg a_2$ & 0 & $(a_1 \vee \neg a_2)$ \\
	$\neg a_3$ & 0 & $(a_1 \vee \neg a_3)$ \\
	\hline
      \end{tabular}
      \bigskip \\
      $\{ \neg \dec{a_1}{0}, \neg \dec{a_2}{0}, \neg \dec{a_3}{0} \}$
    \end{minipage}
  \end{tabular}

  \vfill
  \pause

  \begin{minipage}{\textwidth}
    \begin{prooftree}
    \AxiomC{$(a_1 \vee a_2 \vee a_3)$}
    \AxiomC{$(a_3 \vee \neg a_1)$}
    \BinaryInfC{$(a_1 \vee a_2)$}
    \AxiomC{$(a_1 \vee \neg a_2)$}
    \BinaryInfC{$(a_1)$}
    \AxiomC{$(\neg a_1)$}
    \BinaryInfC{$\bot$}
    \end{prooftree}
  \end{minipage}

  \vfill
  \pause
  Conflict clause: $\bot$ \pause \\
  Backtracking level: 0
\end{frame}


\documentclass[aspectratio=149]{beamer}

\input{../../shared_slides.tex}

% reference: https://yuxinchen2020.github.io/ele522_optimization/lectures/variance_reduction.pdf
% or
% https://ieeexplore-ieee-org.uaccess.univie.ac.at/stamp/stamp.jsp?tp=&arnumber=9226504

\usepackage{booktabs}

\title{Variance reduction for stochastic gradient methods}
\date{\today}

\begin{document}
\maketitle
\frame{\tableofcontents}

\section{Introduction}%

\begin{frame}
  \frametitle{The finite sum problem}
  A common Task in (supervised) machine learning:
  \begin{equation}
    \min_{x\in \R^d} f(x) := \frac{1}{n} \sum_{i=1}^{n} \underbrace{f_i(x)}_{\textcolor{blue}{\text{loss for $i$-th sample}}} + \underbrace{\psi(x)}_{\textcolor{blue}{\text{regularizer}}}
  \end{equation}
  where the $i$-th sample is $(a_i, y_i)$.

  \textbf{Examples:}
  \begin{itemize}
    \item linear regression: $f_i(x) = {(a_i^T x -y_i)}^2$, and $\psi=0$
    \item logistic regression: $f_i(x) = \log(1+e^{-y_i a_i^T x})$, and $\psi=0$
          ``sigmoid function'' and logistic loss.
    \item Lasso: $f_i$ as for linear regression but $\psi(x) = \Vert x \Vert_1$
    \item SVM: $f_i(x) = \max \{0 , 1 - y_i a_i^T x\}$ and $\psi(x)= \Vert x \Vert^2$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Gradient descent}
  \begin{algorithm}[H]
    \caption{(batch) GD}
    \begin{algorithmic}[1]
      \For{$k = 1,2, \dots$}
      \State{ $x_{k+1} = x_k  - \alpha_k \nabla f(x_k) $}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}

  \begin{itemize}
    \item gradient can be computed via
          \begin{equation}
            \nabla f(x) = \nabla \left(\sum_{i=1}^{n}f_i(x)\right) = \sum_{i=1}^{n} \nabla f_i(x_k)
          \end{equation}
    \item good convergence properties
    \item can be \textbf{expensive} if $n$ is large!
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Stochastic gradient descent}
  \begin{algorithm}[H]
    \caption{SGD}\label{sgd}
    \begin{algorithmic}[1]
      \For{$k = 1,2, \dots$}
      \State{pick $i_k$ uniform at random in $[n]$}
      \State{$x_{k+1} = x_k  - \alpha_k \nabla f_{i_k}(x_k)$}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}

  We already noticed that:
  \begin{itemize}
    \item unbiased: $\E[\nabla f_{i_k}(x)] = \sum_{i=1}^{n} \P[i=i_k] \nabla f_i(x) = \sum_{i=1}^{n} \frac{1}{n} \nabla f_i(x)$
    \item large stepsizes fail to suppress noise in the stoch.\ gradients \\
          $\rightarrow$ leads to oscillations
    \item decreasing stepsizes mitigate this problem but \textbf{slows down} convergence (too \emph{conservative})
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Recall SGD}
  \begin{block}{template}
    \begin{equation}
      x_{k+1} = x_k - \alpha_k g_k
    \end{equation}
  \end{block}
  \begin{itemize}
    \item $g_k$ is an unbiased estimator of the true gradient $\nabla F(x_k)$
    \item convergence depends on \textbf{variance} $\E [ \Vert g_k - \nabla F(x_k) \Vert ] \le \sigma_g$ \\
          (not strictly necessary)
    \item vanilla SGD uses $g_k = \nabla f_{i_k}(x_k)$ \\
          \textbf{issue:} $\sigma_g$ is non-negligible even close to the solution
    \item \textbf{Q:} can we choose $g_k$ in a different way to reduce variability?
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Minibatching}
  \begin{algorithm}[H]
    \caption{minibatch SGD}
    \begin{algorithmic}[1]
      \For{$k = 1,2, \dots$}
      \State{pick $I_k$ random subset of $[n]$ with $\vert I_k \vert = b $}
      \State{$x_{k+1} = x_k  - \alpha_k \sum_{i \in I_k} \nabla f_{i}(x_k)$}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}
  \begin{itemize}
    \item typically we make a (uniform) \emph{random} choice $i_k \in [n] = \{1, \dots, n\}$\\
          (or random reshuffling)
    \item by increasing the size to a \textbf{random subset} $I_k \subset [n]$ of size $b \ll n$ we can
          \begin{itemize}
            \item decrease variance
            \item increase cost only moderately,
            \item no improvement in the rate
          \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{A simple idea}
  Consider
  \begin{itemize}
    \item estimator $X$ for parameter $\mu$ ($\E[X]=\mu$ and $\Var[X]=\sigma^2$)
    \item want to \textcolor{blue}{keep unbiased} but \textbf{\textcolor{blue}{reduce variance}}
    \item find $Y$ such that $\E[Y] = 0$ but $\Cov (X,Y)$ \textbf{is large} and define
          \begin{equation}
            \tilde{X} := X-Y
          \end{equation}
    \item remains unbiased
    \item $\Var[\tilde{X}]$ can be much smaller than $\Var[X]$ if $X,Y$ are highly correlated
          \begin{equation}
            \Var[\tilde{X}] = \Var[X]+\Var[Y]-2\Cov[X,Y]
          \end{equation}
  \end{itemize}
\end{frame}

\section{SAG}%
\label{sec:}

\begin{frame}
  \frametitle{Stochastic average gradient (SAG), 2013}
  \begin{itemize}
    \item \textbf{maintain table} containing gradients $g_i$ of $f_i$
    \item pick random $i_k \in [n]$ and
          \begin{equation}
            g_{i_k}^k := \nabla f_{i_k}(x^{k})
          \end{equation}
          set $g_{i}^k = g_i^{k-1}$ for all $i\neq i_k$ (remain the same)
    \item Update
          \begin{equation}
            x^{k+1} = x^k - \alpha_k \frac{1}{n} \sum_{i=1}^{n} g_i^k.
          \end{equation}
    % \item assuming gradients do not change too much along trajectory
    \item gradient estimator \textcolor{red}{no longer unbiased}
    \item Isn't it expensive to average these gradients?
          \begin{equation}
            x^{k+1} = x^k - \alpha_k  \Big( \frac{g_{i_k}^k}{n} - \frac{g_{i_k}^{k-1}}{n} + \underbrace{\frac{1}{n}\sum_{i=1}^{n} g_i^{k-1}}_{\text{old table average}} \Big)
          \end{equation}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SAG variance reduction}
  Gradient estimator in SAG:
  \begin{equation}
    x^{k+1} = x^k - \alpha_k \frac{1}{n} \Big( \underbrace{g_{i_k}^k}_X - \underbrace{g_{i_k}^{k-1} - \sum_{i=1}^{n} g_i^{k-1}}_{Y} \Big)
  \end{equation}
  \begin{itemize}
    \item Indeed $\E[X] = \nabla f(x^k)$, but $\E[Y]\neq 0$ $\rightarrow $ is \textbf{biased estimator}
    \item $X$ and $Y$ are correlated as $X-Y \to 0$:
          \begin{itemize}
            \item $x^k$ and $x^{k-1}$ both converge to $x^*$ $\Rightarrow$ $ \nabla f_i(x^k) - \nabla f_i(x^{k-1}) \to 0$
            \item the last term converges to $\nabla f(x^*)= 0$
          \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Convergence}
  As always, initialization plays a role: $D^2 := \Vert x^0 -x^* \Vert^2$.

  \begin{equation}
    \begin{aligned}
      \text{SAG:}& \quad \frac{n}{k}(f(x^0)-f^*) + \frac{L}{k}D^2 \\
      \text{GD:}& \quad \frac{L}{k}D^2 \\
      \text{SGD:}& \quad \frac{L}{\sqrt{k}}D^2 \\
    \end{aligned}
  \end{equation}
  \begin{itemize}
    \item Achieves \textcolor{blue}{linear convergence} in the \textbf{strongly} convex setting.
    \item proofs are difficult (and computer-aided)
  \end{itemize}

  \begin{center}
    Same gradient oracle cost as SGD, but same converge rate as GD.
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Experiments from the original paper}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{SAG-experiments}
    \caption{ Solving $\ell_2$-regularized logistic regression.}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{More ``naive'' implementation}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=0.9\textheight,keepaspectratio]{SAG}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{SAG experiments}
  \begin{itemize}
    \item does not work well out of the box
    \item needs a \textcolor{blue}{warm up} to get good $(g_1^0, g_2^0, \dots, g_m^0)$
          \begin{itemize}
            \item achieved by running one full epoch of SGD
          \end{itemize}
    \item reguires hand tuned stepsize or line search
  \end{itemize}

\end{frame}

\section{SAGA}%
\label{sec:}

\begin{frame}
  \frametitle{SAGA, 2014}
  Very similar to SAG:
  \begin{itemize}
    \item maintain table containing gradients $g_i$ of $f_i$
    \item pick random $i_k \in [n]$ and
          \begin{equation}
            g_{i_k}^k := \nabla f_{i_k}(x^{k})
          \end{equation}
          set $g_{i}^k = g_i^{k-1}$ for all $i\neq i_k$ (remain the same)
    \item Update
          \begin{equation}
            x^{k+1} = x^k - \alpha_k  \Big( g_{i_k}^k - g_{i_k}^{k-1} + \frac{1}{n}\sum_{i=1}^{n} g_i^k \Big)
          \end{equation}
    \item estimator now \textbf{unbiased}!
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{For Comparison}
  SAGA gradient estimate:
  \begin{equation}
    g_{i_k}^k - g_{i_k}^{k-1} + \frac{1}{n}\sum_{i=1}^{n} g_i^k.
  \end{equation}

  SAG gradient estimate:
  \begin{equation}
    \frac{1}{n}g_{i_k}^k - \frac{1}{n}g_{i_k}^{k-1} + \frac{1}{n}\sum_{i=1}^{n} g_i^k.
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Experiments from the original paper}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{SAGA-experiments}
    \caption{Solving regularized logistic regression. First row is $\ell_2$-regularized; second row is $\ell_1$.}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{More ``naive'' implementation}

  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=0.9\textheight,keepaspectratio]{SAGA.png}
    \caption{Highlights the low variance of SAG.}
  \end{figure}
\end{frame}

\section{SVRG}%

\begin{frame}
  \frametitle{Stochastic Variance Reduced Gradient (SVRG), 2013}

  \begin{algorithm}[H]
    \caption{SVRG}\label{}
    \begin{algorithmic}[1]
      \For{$k=1,2, \dots$}
      \State{Set $x^1 = \tilde{x} = \tilde{x}^{k}$}
      \State{Compute $\tilde{\mu} := \nabla f(\tilde{x})$ \hfill \textcolor{blue}{//update snapshot}}
      \For{$l=1,2, \dots, m$ }\hfill \textcolor{blue}{//$m$ iterations per epoch}
      \State{pick $i_l$ uniform at random in $[n]$}
      \State{Set $x^{l+1} = x^l - \alpha (\nabla f_{i_l}(x^l) - \nabla f_{i_l}(\tilde{x}) + \tilde{\mu})$}
      \EndFor{}
      \State{$\tilde{x}^{k+1}= x^{m+1}$}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}
  \begin{itemize}
    \item Does \textbf{\textcolor{blue}{not need to store}} full table of gradients.
    \item requires \emph{batch} gradient computation every \emph{epoch}
    \item per iteration cost is comparable to that of SGD if $m \ge n$
    \item convergence rates similar to SAGA, but simpler analysis.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SVRG}
  \textbf{key idea:} by storing old point we can
  \begin{equation}
    \underbrace{\nabla f_{i_k}(x^k) - \nabla f_{i_k}(x^{\text{old}})}_{\text{$\to 0$ if $x\approx x^{\text{old}}$}} + \underbrace{\nabla f(x^{\text{old}})}_{\text{$\to 0$ if $x^{\text{old}}\approx x^*$}}
  \end{equation}
  \begin{itemize}
    \item is an unbiased estimate of $\nabla f(x^k)$
    \item converges to $0$ (meaning reduced variability) if $x^k\approx x^{\text{old}} \approx x^*$
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{SVRG: Theorem}

  \textcolor{gray}{Each $f_i$ is convex and $L$-smooth, and sum is $\mu$-strongly convex.}

  \begin{theorem}
    Choose $m$ large enough s.t. $\rho = \frac{1}{\mu \alpha (1-2L \alpha)m} + \frac{2L \alpha}{1-2L \alpha}< 1$, then
    \begin{equation}
      \E[F(x_s^{old})- F(x^*)] \le \rho^s [F(x_0^{old})-F(x^*)]
    \end{equation}
  \end{theorem}
  Computational cost:
  \begin{itemize}
    \item per epoch: $(m+n)$
    % \item complexity: $(n+ \kappa) \log(1/\epsilon)$
    \item inner loop is annoying (has to choose $m$) $\rightarrow$ loopless variant
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SVRG: Convergence Proof}
  Denote $g_s^k = \nabla f_{i_l}(x_s^k) - \nabla f_{i_l}(x_s^{old}) + \nabla F(x_s^{old})$. Conditioning on everything prior to $x_s^{k+1}$ we get
  \begin{equation}
    \begin{aligned}
      \MoveEqLeft \E [\Vert x_s^{k+1} - x^* \Vert^2] = \E [\Vert x_s^{k}- \alpha g_s^k - x^* \Vert^2] \\
      &= \Vert x_s^k-x^* \Vert^2 - 2 \alpha {(x_s^k-x^* )}^T \E[g_s^t] + \alpha^2 \E[\Vert g_s^k \Vert^2] \\
      &= \Vert x_s^k-x^* \Vert^2 - 2 \alpha {(x_s^k-x^* )}^T \textcolor{blue}{\nabla F(x_s^t)} + \alpha^2 \E[\Vert g_s^k \Vert^2] \\
      &\le \Vert x_s^k-x^* \Vert^2 - 2 \alpha (F(x_s^k) - F(x^*))+ \alpha^2 \E[\Vert g_s^k \Vert^2]
    \end{aligned}
  \end{equation}

  \begin{itemize}
     \item \textbf{key step:} control $\E[\Vert g_s^k \Vert^2]$
   \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SVRG: convergence Proof}
  \begin{lemma}%
    \begin{equation}
      \E[\Vert g_s^k \Vert^2] \le 4L [ F(x_s^k)- F(x^*) + F(x_s^{old}-F(x^*)) ]
    \end{equation}

  \end{lemma}

\end{frame}

\begin{frame}
  \frametitle{Comparison}

  % \begin{tabular}{l | c}
  %   SVRG & $(n + \kappa) \log(1/\epsilon)$\\
  %   GD   & $n \kappa \log(1/\epsilon)$ \\
  %   SGD  & $\kappa^2 / \epsilon$
  % \end{tabular}

  % \textbf{\textcolor{blue}{Strongly convex}} problems: number gradient calls to compute:
  % \begin{equation}
  %   \E [f(x_k)] - f^* \le \epsilon
  % \end{equation}
  % is given by

  \textcolor{gray}{$f$ is $L$-smooth and $\mu$-strongly convex. Condition number: $\kappa = L/\mu$.}\\
  \textbf{\textcolor{blue}{Strongly convex}} problems: number gradient calls to compute $\E [f(x_k)] - f^* \le \epsilon$ is given by
  \begin{center}
    \begin{tabular}{c c c}
      SVRG / SAGA  & GD & SGD \\
      \midrule
      $(n + \kappa) \log \frac{1}{\epsilon}$ & $n \kappa \log \frac{1}{\epsilon}$ & $\kappa^2 / \epsilon$
    \end{tabular}
  \end{center}
  \vspace{1cm}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{SAGA-table-comparison}
    \caption{Summary of other relevant properties.}
  \end{figure}

  % \begin{itemize}
  %   \item SVRG:\ $(n + \kappa) \log(1/\epsilon)$
  %   \item GD:\ $n \kappa \log(1/\epsilon)$
  %   \item SGD:\ $\kappa^2 / \epsilon$
  % \end{itemize}

\end{frame}

\section{Katyusha}%
\label{sec:}

\begin{frame}
  \frametitle{Variance reduction + momentum/acceleration}
  \begin{block}{}
    \centering
    \textbf{Katyusha}
  \end{block}

  \textbf{\textcolor{blue}{Strongly convex}} problems: number gradient calls to compute $\E [f(x_k)] - f^* \le \epsilon$ is given by
  \begin{center}
  \begin{tabular}{c c c c c}
    Katyusha & SVRG / SAGA  & GD & NAG & SGD \\
    \midrule
    $(n + \sqrt{n \kappa}) \log \frac{1}{\epsilon}$ &$(n + \kappa) \log \frac{1}{\epsilon}$ & $n \kappa \log \frac{1}{\epsilon}$ & $n \sqrt{\kappa} \log \frac{1}{\epsilon}$ & $\kappa^2 / \epsilon$
  \end{tabular}
  \end{center}
  \vspace{1cm}
  \begin{itemize}
    \item Improvement critical for \textbf{\textcolor{blue}{ill conditioned}} ($\kappa \gg n$) problems.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Katyusha in the non-strongly convex setting}
  \textbf{\textcolor{blue}{just convex}} problems: number gradient calls to compute $\E [f(x_k)] - f^* \le \epsilon$ is given by
  \begin{center}
  \begin{tabular}{c c c c c c}
    lower bound & Katyusha & SVRG & GD & NAG & SGD \\
    \midrule
    $n + \sqrt{\frac{n L}{\epsilon}}$ & $n \log \frac{1}{\epsilon} + \sqrt{\frac{n L}{\epsilon}}$ &$n + \sqrt{n}\frac{L}{\epsilon}$ & $n \frac{L}{\epsilon}$ & $n \sqrt{\frac{L}{\epsilon}}$ & $ \frac{1}{\epsilon^2}$
  \end{tabular}
  \end{center}
  \vspace{1cm}
  \textcolor{gray}{Almost matches lower bound.}

\end{frame}

\begin{frame}
  \frametitle{Summary}

  \begin{itemize}
    \item Variance reduction recovers the rates of batch (deterministic methods)
    \item but (more or less) keeps number of gradient calls of SGD
    \item still require batch gradient computations sometimes
    \item requires offline setting (multiple passes through data)
    \item requires knowledge of (multiple) parameters to get good stepsize
  \end{itemize}

  \textcolor{gray}{
  Check out this fantastic review paper:
  https://ieeexplore-ieee-org.uaccess.univie.ac.at/stamp/stamp.jsp?tp=&arnumber=9226504}

\end{frame}

\end{document}


% !TEX root = main.tex
% !TEX spellcheck = en-US

\section{Non-malleability of $\sonicprotfs$}
\label{sec:sonic}

\markulfdone{22.04}{I prefer the notation $Pr[\chi \gets \FF_q; \chi' \gets \adv( ...): \chi=\chi']$ as one can start reading at the left. But I will let that one go and will admit that it's a question of taste and it might be an aquired taste. }

\subsection{Preliminaries}
\begin{definition}[$(q_1, q_2)\mhyph\ldlog$ assumption]\label{def:ldlog}
  Let $\adv$ be a $\ppt$ adversary that gets as input
  $\gone{\chi^{-q_1}, \ldots, 1, \ldots, \chi^{q_1}}, \gtwo{\chi^{-q_2},
    \ldots, 1, \ldots, \chi^{q_2}}$, for some randomly picked
  $\chi \in \FF_p$, the assumption requires that $\adv$ cannot compute $\chi$. That is
	\[
    \condprob{\chi = \adv(\gone{\chi^{-q_1}, \ldots, 1, \ldots,
        \chi^{q_1}}, \gtwo{\chi^{-q_2}, \ldots, 1, \ldots, \chi^{q_2}
      })}{\chi \sample \FF_p} \leq \negl.
	\]
\end{definition}

\begin{definition}[$(q_1, q_2)\mhyph\uldlog$ assumption]\label{def:uldlog}
	Let $\adv$ be a $\ppt$ adversary that gets oracle access to $\initU$ with internal algorithms $(\kgen_{\ldlog}, \upd_{\ldlog}, \verifyCRS)$, where $\kgen_{\ldlog}$ and $\upd_{\ldlog}$ are defined as follows:
	\begin{itemize}
		\item $\kgen_\ldlog(\secpar)$ samples $\chi \sample \FF_p$ and defines
		$\Ch:=(\gone{\chi^{-q_1}, \ldots, 1, \chi, \ldots,
			\chi^{q_1}}, \gtwo{\chi^{-q_2}, \ldots, 1, \chi, \ldots, \chi^{q_2}
		})$.
		\item $\upd_\ldlog(\Ch, \{\rho_j \}_{j=1}^n)$
		parses $\Ch$ as $\left( \gone{\smallset{A_i}_{i = -q_1}^{q_1}},
		\gtwo{\smallset{B_i}_{i = -q_2}^{q_2}} \right)$, samples
		$\widetilde{\chi} \sample \FF_p$, and defines
		$\widetilde{\Ch} :=
		\left( \gone{\smallset{\widetilde{\chi}^i A_i}_{i = -q_1}^{q_1}},
		\gtwo{\smallset{\widetilde{\chi}^i B_i}_{i = -q_2}^{q_2}} \right)$.
	\end{itemize}
	Then
	\[
	\prob{\bar{\chi} \gets \adv^{\initU}(\secpar)} \leq \negl,
	\]
	where $\left( \gone{\smallset{\bar{\chi}^i}_{i = -q_1}^{q_1}},
	\gtwo{\smallset{\bar{\chi}^i}_{i = -q_2}^{q_2}} \right)$ is the finalized challenge.
\end{definition}

\subsection{\sonic{} Protocol Rolled-out}
In this section we present $\sonic$'s constraint system and algorithms. Reader
familiar with them may jump directly to the next section.

 \begin{figure}[h!]
 \centering
 	\begin{pcvstack}[center,boxed]
 		\begin{pchstack}
 			\procedure{$\kgen(\secparam, \maxdeg)$} {
 				\alpha, \chi \sample \FF^2_p \\ [\myskip]
 				\pcreturn \gone{\smallset{\chi^i}_{i = -\multconstr}^{\multconstr},
           \smallset{\alpha \chi^i}_{i = -\multconstr, i \neq
             0}^{\multconstr}},\\
         \pcind \gtwo{\smallset{\chi^i, \alpha \chi^i}_{i =
             -\multconstr}^{\multconstr}}, \gtar{\alpha}\\
 				\hphantom{\hspace*{5.5cm}}
 		}

 			\pchspace

 			\procedure{$\com(\srs, \maxconst, \p{f}(X))$} {
 				\p{c}(X) \gets \alpha \cdot X^{\dconst - \maxconst} \p{f}(X) \\ [\myskip]
 				\pcreturn \gone{c} = \gone{\p{c}(\chi)}\\ [\myskip]
 				\hphantom{\pcind \pcif \sum_{i = 1}^{\abs{\vec{z}}} r_i \cdot
           \gone{\sum_{j = 1}^{t_j} \gamma_i^{j - 1} c_{i, j} - \sum_{j = 1}^{t_j}
             s_{i, j}} \bullet \gtwo{1} + } }
 		\end{pchstack}
 		% \pcvspace

 		\begin{pchstack}
 			\procedure{$\open(\srs, z, s, f(X))$}
 			{
 				\p{o}(X) \gets \frac{\p{f}(X) - \p{f}(z)}{X - z}\\ [\myskip]
 				\pcreturn \gone{\p{o}(\chi)}\\ [\myskip]
 				\hphantom{\hspace*{5.5cm}}
 			}

 			\pchspace

 			\procedure{$\verify(\srs, \maxconst, \gone{c}, z, s, \gone{\p{o}(\chi)})$}
       {
         \pcif \gone{\p{o}(\chi)} \bullet \gtwo{\alpha \chi} + \gone{s - z
         \p{o}(\chi)} \bullet \gtwo{\alpha} = \\ [\myskip] \pcind \gone{c}
         \bullet \gtwo{\chi^{- \dconst + \maxconst}} \pcthen  \pcreturn 1\\
         [\myskip]
         \rlap{\pcelse \pcreturn 0.} \hphantom{\pcind \pcif \sum_{i =
             1}^{\abs{\vec{z}}} r_i \cdot \gone{\sum_{j = 1}^{t_j} \gamma_i^{j -
               1} c_{i, j} - \sum{j = 1}^{t_j} s_{i, j}} \bullet \gtwo{1} + } }
 		\end{pchstack}
 	\end{pcvstack}

 	\caption{$\PCOMs$ polynomial commitment scheme.}
 	\label{fig:pcoms}
 \end{figure}

\oursubsub{The Constraint System}
\label{sec:sonic_constraint_system}
\cref{fig:pcoms} presents a variant of KZG~\cite{AC:KatZavGol10} polynomial commitment schemes used in \sonic{}. \sonic's system of constraints composes of three $\multconstr$-long vectors
$\va, \vb, \vc$ which corresponds to left and right inputs to multiplication
gates and their outputs. It hence holds $\va \cdot \vb = \vc$.

There is also $\linconstr$ linear constraints of the form
\[
  \va \vec{u_q} + \vb \vec{v_q} + \vc \vec{w_q} = k_q,
\]
where $\vec{u_q}, \vec{v_q}, \vec{w_q}$ are vectors for the $q$-th linear
constraint with instance value $k_q \in \FF_p$. Furthermore define polynomials
\begin{equation}
  \begin{split}
    \p{u_i}(Y) & = \sum_{q = 1}^\linconstr Y^{q + \multconstr} u_{q, i}\,,\\
    \p{v_i}(Y) & = \sum_{q = 1}^\linconstr Y^{q + \multconstr} v_{q, i}\,,\\
  \end{split}
  \qquad
  \begin{split}
    \p{w_i}(Y) & = -Y^i - Y^{-i} + \sum_{q = 1}^\linconstr Y^{q +
      \multconstr} w_{q, i}\,,\\
    \p{k}(Y) & = \sum_{q = 1}^\linconstr Y^{q + \multconstr} k_{q}.
  \end{split}
\end{equation}

$\sonic$ constraint system requires that
\begin{align}
  \label{eq:sonic_constraint}
  \vec{a}^\top \cdot \vec{\p{u}} (Y) + \vec{b}^\top \cdot \vec{\p{v}} (Y) +
  \vec{c}^\top \cdot \vec{\p{w}} (Y) + \sum_{i = 1}^{\multconstr} a_i b_i (Y^i +
  Y^{-i}) - \p{k} (Y) = 0.
\end{align}

In \sonic{} we will use commitments to the following polynomials.
\begin{align*}
  \pr(X, Y) & = \sum_{i = 1}^{\multconstr} \left(a_i X^i Y^i + b_i X^{-i} Y^{-i}
              + c_i X^{-i - \multconstr} Y^{-i - \multconstr}\right) \\
  \p{s}(X, Y) & = \sum_{i = 1}^{\multconstr} \left( u_i (Y) X^{-i} +
                v_i(Y) X^i + w_i(Y) X^{i + \multconstr}\right)\\
  \pt(X, Y) & = \pr(X, 1) (\pr(X, Y) + \p{s}(X, Y)) - \p{k}(Y)\,.
\end{align*}

Polynomials $\p{r} (X, Y), \p{s} (X, Y), \p{t} (X, Y)$ are designed such that
$\p{t} (0, Y) = \vec{a}^\top \cdot \vec{\p{u}} (Y) + \vec{b}^\top \cdot
\vec{\p{v}} (Y) + \vec{c}^\top \cdot \vec{\p{w}} (Y) + \sum_{i =
  1}^{\multconstr} a_i b_i (Y^i + Y^{-i}) - \p{k} (Y) $. That is, the prover is
asked to show that $\p{t} (0, Y) = 0$, cf.~\cref{eq:sonic_constraint}.

Furthermore, the commitment system in $\sonic$ is designed such that it is
infeasible for a $\ppt$ algorithm to commit to a polynomial with non-zero
constant term.

\oursubsub{Algorithms Rolled out}
\ourpar{$\sonic$ SRS generation $\kgen(\REL)$.} The SRS generating algorithm picks
randomly $\alpha, \chi \sample \FF_p$ and outputs
	\[
      \srs = \left( \gone{\smallset{\chi^i}_{i = -\dconst}^{\dconst},
          \smallset{\alpha \chi^i}_{i = -\dconst, i \neq 0}^{\dconst}},
        \gtwo{\smallset{\chi^i, \alpha \chi^i}_{i = - \dconst}^{\dconst}},
        \gtar{\alpha} \right)
	\]
\ourpar{$\sonic$ prover $\prover(\srs, \inp, \wit=\va, \vb, \vc)$.}
\begin{description}
\item[Message 1] The prover picks randomly randomizers
  $c_{\multconstr + 1}, c_{\multconstr + 2}, c_{\multconstr + 3}, c_{\multconstr
    + 4} \sample \FF_p$. Sets
  $\pr(X, Y) \gets \pr(X, Y) + \sum_{i = 1}^4 c_{\multconstr + i} X^{- 2
    \multconstr - i}$. Commits to $\pr(X, 1)$ and outputs
  $\gone{r} \gets \com(\srs, \multconstr, \pr(X, 1))$.  Then it computes challenge $y = \ro(\tzkproof[0..1])$.
\item[Message 2] $\prover$ commits to $\pt(X, y)$ and outputs
  $\gone{t} \gets \com(\srs, \dconst, \pt(X, y))$. Then it gets a challenge $z = \ro(\tzkproof[0..2])$.
\item[Message 3] The prover computes commitment openings. That is, it outputs
  \begin{align*}
    \gone{o_a} & = \open(\srs, z, \pr(z, 1), \pr(X, 1)) \\
    \gone{o_b} & = \open(\srs, y z, \pr(y z, 1), \pr(X, 1)) \\
    \gone{o_t} & = \open(\srs, z, \pt(z, y), \pt(X, y))
  \end{align*}
  along with evaluations $a' = \pr(z, 1), b' = \pr(y, z), t' = \pt(z, y)$.  Then it
  engages in the signature of correct computation playing the role of the
  helper, i.e.~it commits to $\p{s}(X, y)$ and sends the commitment $\gone{s}$, commitment opening
  \begin{align*}
    \gone{o_s} & = \open(\srs, z, \p{s}(z, y), \p{s}(X, y)), \\
  \end{align*} and $s'=\p{s}(z, y)$.
%
  Then
  it obtains a challenge $u = \ro(\tzkproof[0..3])$.
\item[Message 4] For the next message the prover computes
  $\gone{c} \gets \com(\srs, \dconst, \p{s}(u, Y))$ and
  computes commitments' openings
  \begin{align*}
    \gone{w} & = \open(\srs, u, \p{s}(u, y), \p{s}(X, y)), \\
    \gone{q_y} & = \open(\srs, y,\p{s}(u, y), \p{s}(u, Y)),
  \end{align*}
  and returns $\gone{w}, \gone{q_y}, s = \p{s}(u, y)$. Eventually the prover gets the last challenge
  $z' = \ro(\tzkproof[0..4])$.
\item[Message 5] For the final message, $\prover$ computes opening
  $\gone{q_{z'}} = \open(\srs, z', \p{s}(u, z'), \p{s}(u, X))$ and outputs $\gone{q_{z'}}$.
\end{description}

\ourpar{$\sonic$ verifier $\verifier(\srs, \inp, \zkproof)$.} The verifier
in \sonic{} runs as subroutines the verifier for the polynomial commitment. That
is it sets $t' = a'(b' + s') - \p{k}(y)$ and checks the following:
\begin{equation*}
  \begin{split}
    &\PCOMs.\verifier(\srs, \multconstr, \gone{r}, z, a', \gone{o_a}), \\
    &\PCOMs.\verifier(\srs, \multconstr, \gone{r}, y z, b', \gone{o_b}),\\
    &\PCOMs.\verifier(\srs, \dconst, \gone{t}, z, t', \gone{o_t}),\\
    &\PCOMs.\verifier(\srs, \dconst, \gone{s}, z, s', \gone{o_s}),\\
  \end{split}
  \qquad
  \begin{split}
    &\PCOMs.\verifier(\srs, \dconst, \gone{s}, u, s, \gone{w}),\\
    &\PCOMs.\verifier(\srs, \dconst, \gone{c}, y, s, \gone{q_y}),\\
    &\PCOMs.\verifier(\srs, \dconst, \gone{c}, z', \p{s}(u, z'), \gone{q_{z'}}),
  \end{split}
\end{equation*}
and accepts the proof iff all the checks holds. Note that the value
$\p{s}(u, z')$ that is recomputed by the verifier uses separate challenges $u$
and $z'$. This enables the batching of many proof and outsourcing of this
part of the proof to an untrusted helper.

\subsection{Unique Opening Property of $\PCOMs$}
\begin{lemma}
\label{lem:pcoms_unique_op}
$\PCOMs$ has the unique opening property in the AGM.
\end{lemma}
\begin{proof}
Let
$z \in \FF_p$ be the attribute the polynomial is evaluated at,
$\gone{c} \in \GRP$ be the commitment,
$s \in \FF_p$ the evaluation value, and
$o \in \GRP$ be the commitment opening.
We need to show that for every $\ppt$ adversary $\adv$ probability
\[
  \Pr \left[
    \begin{aligned}
      & \verify(\srs, \gone{c}, z, s, \gone{o}) = 1, \\
      & \verify(\srs, \gone{c}, z, \tilde{s}, \gone{\tilde{o}}) = 1
    \end{aligned}
    \,\left|\, \vphantom{\begin{aligned}
          & \verify(\srs, \gone{c}, z, s, \gone{o}),\\
          & \verify(\srs, \gone{c}, z, s, \gone{\tilde{o}}) \\
          &o \neq \tilde{o})
		\end{aligned}}
      \begin{aligned}
        %& \srs \gets \kgen(\secparam, \maxdeg), \\
        & (\gone{c}, z, s, \gone{o}, \gone{\tilde{o}}) \gets \adv^{\initU}(1^\secpar, \maxdeg)
      \end{aligned}
    \right.\right]
  % \leq \negl.
\]
is at most negligible.

As noted in \cite[Lemma 2.2]{EPRINT:GabWilCio19} it is enough to upper bound the
probability of the adversary succeeding against the ideal verifier, who verifies equality between polynomials.

For a polynomial $f$, its degree upper bound $\maxconst$, evaluation point $z$,
evaluation result $s$, and opening $\gone{o(X)}$ the ideal verifier checks that
\begin{equation}
  \alpha (X^{\dconst - \maxconst}f(X) \cdot X^{-\dconst + \maxconst} -  s) \equiv \alpha \cdot o(X) (X - z)\,,
\end{equation}
what is equivalent to
\begin{equation}
	f(X) -  s \equiv o(X) (X - z)\,.
	\label{eq:pcoms_idealised_check}
\end{equation}
Since $o(X)(X - z) \in \FF_p[X]$ then from the uniqueness of polynomial
composition, there is only one $o(X)$ that fulfills the equation above.
\qed
\end{proof}

\subsection{Unique Response Property}
The unique response property of $\sonicprotfs$ follows from the unique opening
property of the polynomial commitment scheme $\PCOMs$.
\begin{lemma}
\label{lem:sonicprot_ur}
If a polynomial commitment scheme $\PCOMs$ is evaluation binding with security loss $\epsbind (\secpar)$ and has unique openings property with security loss $\epsop(\secpar)$,\COMMENT{ and $(\dconst, \dconst)$-$\uldlog$ problem is $\epsuldlog (\secpar)$-hard,} then $\sonicprotfs$ is $\ur{2}$ against algebraic adversaries with security loss
  \[
    2 \cdot \epsbinding (\secpar) + \epsop (\secpar).
  \]
\end{lemma}

\begin{proof}
  Let $\adv$ be an algebraic adversary tasked to break the $\ur{2}$-ness of
  $\sonicprotfs$. We note that to show $\ur{2}$-ness is is enough to show that the first prover's message determines, along with the verifiers challenges, the rest of it.
  % This is done by game hops. In the games, the adversary outputs two proofs
  We denote by $\zkproof^0$ and $\zkproof^1$ the two proofs for the same statement the adversary outputs.
  To distinguish polynomials and commitments which an honest prover sends in the
  proof from the polynomials and commitments computed by the adversary we write the latter using indices $0$ and $1$ (two indices as we have two transcripts), e.g.~to describe the quotient polynomial provided by the adversary we write $\p{t}^0$ and   $\p{t}^1$ instead of $\p{t}$ as in the description of the protocol.

  We note that since the unique response property requires from $\zkproof^{0}$ and $\zkproof^{1}$ that the first place they possibly differ is the $3$-th prover's message, then the challenge $z$, that is picked by the adversary after the $2$-rd message is the same in both transcripts. This challenge determines the evaluation point of polynomials $\p{r}(X, 1), \p{t}(X, y)$ which commitments are already sent.

  In its third message, the prover provides evaluations of these polynomials along with their openings at $z$ or $yz$. Note that the adversary can output two accepting proofs that differ on their third message only if it  manages to break evaluation binding of one of the opening. Since the commitment scheme is evaluation binding with security loss $\epsbinding (\secpar)$, the adversary can make $\zkproof^{0}$ and $\zkproof^{1}$ differ on the third message with probability at most $\epsbinding (\secpar) $.

  Similarly, in its fourth message, the prover provides an evaluation at $u$ of polynomial $\p{s}(X, y)$, an evaluation at $y$ of $\p{s} (u, Y)$, and the corresponding openings. Note that the adversary can output two accepting proofs that differ on their fourth message only if it manages to break evaluation binding of one of the opening. Since the commitment scheme is evaluation binding with security loss $\epsbinding (\secpar)$, the adversary can make $\zkproof^{0}$ and $\zkproof^{1}$ differ on the fourth message with probability at most $\epsbinding (\secpar) $.

  Next, assume that the transcripts are the same up to the fourth message, but differ at the fifth. In that message, the adversary provides openings of the evaluations. Since the unique opening property, the adversary can open the valid evaluation of a polynomial to two different values with probability at most $\epsop (\secpar)$.%, which is upper-bounded by $\epsuldlog (\secpar)$.

  By the union bound, the adversary is able to break the unique response property with probability upper bounded by $2 \epsbinding (\secpar) + \epsop (\secpar)$.
\iffalse
  \michals{30.4}{Old proof starts here}

  \ncase{Game 0}
  \ngame{0} In this game, the adversary additionally wins if it provides two transcripts that match on all $5$ messages sent by the prover.
  In this game the adversary cannot win.

  \ngame{1} This game is identical to Game $\game{0}$ except that now the
  adversary additionally wins if it provides two transcripts that matches on the first four
  messages of the proof.

  \ncase{Game 0 to Game 1} We show that the probability that $\adv$
  wins in one game but does not in the other is negligible.  Observe that in
  after its $4$-th message, the adversary is given a challenge $z'$ and has to open
  commitment to $\p{s} (u, z')$. Hence, to be able to give two different
  openings in its $5$-th message, $\adv$ has to break the unique opening property of the
  KZG commitment scheme which happens with probability $\epsop (\secpar)$ tops.

  \ngame{2} This game is identical to Game $\game{1}$ except that now the
  adversary additionally wins if it provides two transcripts that matches on the
  first three messages of the proof.

  \ncase{Game 2 to Game 3} In its $4$-th message the adversary computes evaluation
  $s = \p{s} (u, y)$ and the corresponding openings $\gone{w}, \gone{q_y}$. The adversary
  cannot provide two different evaluations for the committed polynomials, since that would
  require breaking the evaluation binding property, which happens (by the union bound)
  with probability at most $2 \cdot \epsbind (\secpar)$. Also, the adversary cannot provide two
  different yet valid opening except probability $2 \cdot \epsop (\secpar)$

  Hence, the probability that adversary wins in one game but does not in the
  other is upper-bounded by $2 \cdot (\epsbind (\secpar) + \epsop (\secpar))$

  \ngame{4} This game is identical to Game $\game{3}$ except that now the
  adversary additionally wins if it provides two transcripts that match on the
  first two messages of the proof.

  \ncase{Game 3 to Game 4} In its $3$-rd message the adversary computes $4$ polynomial
  evaluations and their openings. It also sends commitment to $\p{s} (X, y)$ which is a
  signature of correct computation.

  Probability that the adversary provides different evaluations or polynomial openings is
  upper bounded by $4 \cdot (\epsop (\secpar) + \epsbind (\secpar))$. Since the polynomial
  commitment scheme is deterministic, if commitment to $\p{s^0} (X, y)$ does not equal
  commitment to $\p{s^1} (X, y)$, then at least one of these values has been computed
  incorrectly. Probability that the adversary outputs an accepting proof, where signature
  of correct computation has been computed incorrectly is upper-bounded by $\epss
  (\secpar) + \epsdlog (\secpar)$, cf.~\cref{lem:plonkprot_ur}.

  \ncase{Game 5}  This game is identical to Game $\game{4}$ except that now the
  adversary additionally wins if it provides two transcripts that match on the
  first messages of the proof.

  \ncase{Game 4 to Game 5} In its second message the adversary commits to polynomial
  $\p{t} (X, y)$. Since the commitment scheme is deterministic, probability that adversary
  outputs accepting proofs where commitment to $\p{t^0} (X, y)$ does not equal commitment
  to $\p{t^1} (X)$ is upper-bounded by $\epss (\secpar) + \epsdlog (\secpar)$, cf.~\cref{lem:plonkprot_ur}.

  \ncase{Conclusion} Taking all the games together, the probability that $\adv$ wins
  in Game 5 is upper-bounded by
  \[
    3 \cdot \epsop (\secpar) + 2 \cdot (\epsbind (\secpar) + \epss (\secpar) +
    \epsdlog (\secpar)).
  \]
  \fi
  \qed
\end{proof}

\subsection{Rewinding-Based Knowledge Soundness}
\begin{lemma}
	\label{lem:sonicprot_ss}
  $\sonicprotfs$ is $(2, \noofc + 1)$-rewinding-based knowledge sound against algebraic adversaries who make up to $q$ random oracle queries with security loss
  \[
    \epscss(\secpar,\accProb, q) \leq \left(1 - \frac{\accProb - (q + 1) \left( \frac{\noofc}{p}\right)}{1 - \frac{\noofc}{p}}\right) + (\noofc + 1) \cdot \epsuldlog (\secpar)\,,
  \]
	Here $\accProb$ is a probability that the adversary outputs an accepting proof, and $\epsuldlog(\secpar)$ is the security of $(\maxdeg, \maxdeg)$-$\uldlog$ assumption.
\end{lemma}

% 	$\sonicprot$ is $(2, \noofc + 1)$-computational special sound with security loss $(\epst (\accProb, \secpar), \epss(\secpar))$ against
% 	algebraic adversaries, where
%   \[
%     \epst(\accProb, \secpar) \leq \frac{\accProb - (q + 1) \epsid (\secpar)}{1 - \epsid (\secpar)}\,,
%   \]
%   and
% 	\[
% 	  \epss(\secpar) \leq \epsid(\secpar) + \epsldlog(\secpar) \,.
% 	\]
% 	Here $\accProb$ is a probability that the adversary outputs an accepting proof, $q$ is the upper bound for a number of random oracle queries the adversary makes, $\epsid(\secpar)$ is a soundness error of the idealized verifier, and $\epsldlog(\secpar)$ is security of $(\dconst, \dconst)$-$\ldlog$ assumption.
% \end{lemma}

Let $\adv^{\ro, \initU}(\secparam; r)$ be the adversary who outputs $(\inp, \zkproof)$ such that $\sonicprotfs.\verifier$ accepts the proof. Let $\tdv$ be the tree-building algorithm of \cref{lem:attema} that outputs a tree $\tree$, and let $\extcss$ be an extractor that given the tree output by $\tdv$ reveals the witness for $\inp$. The main idea of the proof is to show that an adversary who breaks rewinding-based knowledge soundness can be used to break a $\uldlog$-problem instance. The proof goes by game hops. Note that since the tree branches after $\adv$'s $2$-nd message, the instance $\inp$, commitments $\gone{\p{r} (\chi, 1), \p{t} (\chi, y)}$, and challenge $y$ are the same in all the transcripts. Also, the tree branches after the second adversary's message where the challenge $z$ is presented, thus tree $\tree$ is built using different values of $z$.	We consider the following games.

  \ncase{Game 0} %
  In this game, the adversary wins if it outputs a valid instance--proof pair $(\inp, \zkproof)$, and the extractor $\extcss$ does not manage to output a witness $\wit$ such that $\REL (\inp, \wit)$ holds.

  \ncase{Game 1} %
  In this game, the environment aborts the game if the tree building algorithm $\tdv$ fails in building a tree of accepting transcripts $\tree$.

  \ncase{Game 0 to Game 1} %
  By \cref{lem:attema} probability that Game 1 is aborted, while Game 0 is not, is, at most
  \[
    1 - \frac{\accProb - (q + 1) \left( \frac{\noofc}{p} \right)} {1 - \frac{\noofc }{p}} \,.
\]

  \ncase{Game 2} %
  In this game the environment additionally aborts if at least one of its proofs in $\tree$ is not accepting by an ideal verifier.

  \ncase{Game 1 to Game 2} %
  As usual, we show a reduction that breaks an instance of a $\uldlog$ assumption when Game 2 is aborted, while Game 1 is not.

  Let $\rdvuldlog$ be a reduction that gets as input an $(\maxdeg, \maxdeg)$-$\uldlog$ instance $\gone{\chi^{-\maxdeg}, \ldots, 1, \ldots, \chi^{\maxdeg}}, \gtwo{\chi^{-\maxdeg}, \ldots, 1, \ldots, \chi^{\maxdeg}}$. Then it can update the instance to another one $\gone{\chi'^{-\maxdeg}, \ldots, 1, \ldots, \chi'^{\maxdeg}}, \gtwo{\chi'^{-\maxdeg}, \ldots, 1, \ldots, \chi'^{\maxdeg}}$. Eventually, the reduction outputs $\chi'$.
	%
	The reduction $\rdvuldlog$ proceeds as follows.
	First, it builds $\adv$'s SRS $\srs$ using the input $\uldlog$ instance. Then it processes the adversary's update query by adding it to the list $\Qsrs$ and passing it to its own update oracle getting instance $\gone{\chi'^{-\maxdeg}, \ldots, 1, \ldots, \chi'^{\maxdeg}}, \gtwo{\chi'^{-\maxdeg}, \ldots, 1, \ldots, \chi'^{\maxdeg}}$. The updated SRS $\srs'$ is then computed and given to $\adv$. $\rdvuldlog$ also takes care of the random oracle queries made by $\adv$. It picks their answers honestly and writes them in $\Qro$. The reduction then starts $\tdv(\srs, \adv, r, \Qro, \Qsrs)$.

  Let $(1, \tree)$ be the output returned by $\tdv$. Let $\inp$ be a relation proven in $\tree$.  Consider a transcript $\zkproof \in \tree$ such that $\vereq_{\inp, \zkproof}(X) \neq 0$, but $\vereq_{\inp, \zkproof}(\chi') = 0$. Since $\adv$ is algebraic, all group elements included in $\tree$ are extended by their representation as a combination of the input $\GRP_1$-elements. Hence, all coefficients of the verification equation polynomial $\vereq_{\inp, \zkproof}(X)$ are known.
  Eventually, the reduction finds $\vereq_{\inp, \zkproof}(X)$ zero points and returns $\chi'$ which is one of them.

  Hence, the probability that the adversary wins in Game 2 but does not win in Game 1 is upper-bounded by $(\noofc + 1) \cdot \epsuldlog (\secpar)$.

  \ncase{Conclusion}
  Note that the adversary can win in Game 2 only if $\tdv$ manages to produce a tree of accepting transcript $\tree$, such that each of the transcripts in $\tree$ is accepting by an ideal verifier. Note that since $\tdv$ produces $(\noofc + 1)$ accepting transcripts for different challenges $z$, it obtains the same number of different evaluations of polynomial $\p{t} (z, y)$ what allows to extract the witness, cf.~\cite{CCS:MBKM19}.

  Hence, the probability that the adversary wins in Game 0 is upper-bounded by
  \[
    \epscss(\secpar,\accProb, q) \leq \left(1 - \frac{\accProb - (q + 1) \left( \frac{\noofc}{p} \right)}{1 - \frac{\noofc}{p}}\right) + (\noofc + 1) \cdot\epsuldlog (\secpar)\,.
  \]

\subsection{Trapdoor-Less Zero-Knowledge of Sonic}
\begin{lemma}
\label{lem:sonic_hvzk}
$\sonicprotfs$ is 2-programmable trapdoor-less zero-knowledge.
\end{lemma}
\begin{proof}
  The simulator proceeds as follows.
  \begin{enumerate}
  \item Pick randomly vectors $\vec{a}$, $\vec{b}$ and set
    \begin{equation}
      \label{eq:ab_eq_c}
      \vec{c} = \vec{a} \cdot \vec{b}.
    \end{equation}
  \item Pick randomizers $c_{\multconstr + 1}, \ldots, c_{\multconstr + 4}$,
    honestly compute polynomials $\p{r}(X, Y), \p{r'}(X, Y), \p{s}(X, Y)$ and
    pick randomly challenges $y$, $z$.
  \item Output commitment $\gone{r} \gets \com(\srs, \multconstr, \p{r} (X,
    1))$ and challenge $y$.
  \item Compute
    \begin{align*}
      & a' = \p{r}(z, 1),\\
      & b' = \p{r}(z, y),\\
      & s' = \p{s}(z, y).
    \end{align*}
  \item Pick polynomial $\p{t}(X, Y)$ such that
    \begin{align*}
      & \p{t} (X, y) = \p{r} (X, 1) (\p{r}(X, y) + \p{s} (X, y)) - \p{k} (Y)\\
      & \p{t} (0, y) = 0
    \end{align*}
  \item Output commitment $\gone{t} = \com (\srs, \dconst, \p{t} (X, y))$ and
    challenge $z$.
  \item Continue following the protocol.
  \end{enumerate}

  We note that the simulation is perfect. This comes since, except polynomial
  $\p{t} (X, Y)$ all polynomials are computed following the protocol. For
  polynomial $\p{t} (X, Y)$ we observe that in a case of both real and simulated
  proof the verifier only learns commitment $\gone{t} = \p{t} (\chi, y)$ and
  evaluation $t' = \p{t} (z, y)$. Since the simulator picks $\p{t} (X, Y)$ such
  that
  \begin{align*}
      \p{t} (X, y) = \p{r} (X, 1) (\p{r}(X, y) + \p{s} (X, y)) - \p{k} (Y)
  \end{align*}
  Values of $\gone{t}$ are equal in both proofs.
  Furthermore, the simulator picks its polynomial such that $\p{t}(0, y) = 0$,
  hence it does not need the trapdoor to commit to it. (Note that the proof
  system's SRS does not allow to commit to polynomials which have non-zero
  constant term). \qed
\end{proof}
\begin{remark}
  As noted in \cite{CCS:MBKM19}, $\sonic$ is statistically subversion zero-knowledge (Sub-ZK). As noted in \cite{AC:ABLZ17short}, one way to achieve
  subversion zero-knowledge is to utilize an extractor that extracts a SRS
  trapdoor from a SRS-generator. Unfortunately, a NIZK made subversion
  zero-knowledge by this approach cannot achieve perfect Sub-ZK as one has to
  count in the probability of extraction failure. However, with the simulation
  presented in \cref{lem:sonic_hvzk}, the trapdoor is not required for the
  simulator as it is able to simulate the execution of the protocol just by
  picking appropriate (honest) verifier's challenges. This result transfers to
  $\sonicprotfs$, where the simulator can program the random oracle to provide
  challenges that fits it.
\end{remark}

\subsection{Simulation Extractability of $\sonicprotfs$}
Since \cref{lem:sonicprot_ur,lem:sonicprot_ss,lem:sonic_hvzk} hold, $\sonicprotfs$ is $\ur{2}$, rewinding-based knowledge sound and trapdoor-less zero-knowledge. We now make use
of \cref{thm:se} and show that $\sonicprotfs$ is simulation-extractable as defined in \cref{def:simext}.

\begin{corollary}[Simulation extractability of $\sonicprotfs$]
  \label{thm:sonicprotfs_se}
  $\sonicprotfs$ is \emph{updatable simulation-extractable} against any $\ppt$ adversary $\advse$ who makes up to $q$ random oracle queries and returns an accepting proof with probability at least $\accProb$ with extraction failure probability
\[
  \epsse(\secpar, \accProb, q) \leq \left(1 - \frac{\accProb - \epsur (\secpar) - (q + 1) \epserr (\secpar)} {1 - \epserr (\secpar)}\right) + (\noofc + 1) \cdot \epsuldlog (\secpar) ,
\]
where $\epserr (\secpar) = \frac{\noofc}{p}$, $p$ is the size of the field, and $\noofc$ is the number of constrains in the circuit.
\end{corollary}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End: