\section{Regression}

The concepts discussed in the following section could also be presented in
random variables instead of sample ones. As the geometry of sample variables
is almost of no difference comparing to the random ones, the logic of
all the theorems is also the same.

\subsection{Geometry of sample variables}

\marginnote{
\begin{multline*}
\sCorr(x,y) = \frac{\sCov(x,y)}{\sqrt{\sVar(x)\sVar(y)}} \\
= \frac{\frac{1}{n-1}\sum_{i=1}^n (x_i - \bar x)(y_i - \bar y)}
{\sqrt{\frac{1}{n-1} \sum_{i=1}^n (x_i - \bar x)\frac{1}{n-1} \sum_{i=1}^n ( y_i -  \hat y)}}
\end{multline*}
}

In the~same manner it was done in the previous section, we define
the~scalar product of two sample variables
$x =
\begin{pmatrix}
x_1 \\
\vdots \\
x_n
\end{pmatrix}$
and
$y =
\begin{pmatrix}
y_1 \\
\vdots \\
y_n
\end{pmatrix}$
as a sample covariation between them:
\[
\langle x, y \rangle = \sCov(x, y).
\]
The main characteristics of a~vector are its length and direction.
Again, we introduce the~length
\[
\sqrt{\sCov(x,x)} = \sqrt{\sVar(x)} = \sigma_x
\]
and the~angle between two sample variables
\[
\cos(x,y) = \frac{\sCov(x,y)}{\sqrt{\sVar(x)\sVar(y)}} = \sCorr(x,y).
\]
Note that from the definition of the angle
it follows that the~sample correlation coefficient can range from $-1$ to $1$.

\begin{marginfigure}
\includegraphics[scale=0.85]{figures/02_basic_projection.pdf}
\caption{Vector $y$ projected onto vector $x$.}
\label{fig:corr_proj}
\end{marginfigure}

Completely analogus to the~case of random variables,
the projection of such a sample variable $y$ onto
$\{cx| c \in \mathbb{R}\}$ is $\hat y = \sCorr(x,y) \cdot y$.

Looking at Figure~\ref{fig:corr_proj}, we can interpret the~square of sample correlation coefficient.
Using the fact that $\cos^2 \varphi$ is the~squared ratio of
the~leg adjacent to $\varphi$ to hypotenuse, we can conclude that
\[
\sCorr^2(x,y) = \frac{\sVar(\hat y)}{\sVar(y)},
\]
as the variance of a vector is associated with the square of its length.
Thus, the~sample correlation coefficient squared shows
the~fraction of variance in $y$ which can be explained
with the~most similar vector proportional to $x$.

\subsection{Sample correlation when a constatnt vector added}

\marginnote{
\begin{align*}
\sCorr(x + \alpha \mathbf{1}, y) &= \frac{\sCov(x + \alpha \mathbf{1}, y)}{\sqrt{\sVar(x + \alpha \mathbf{1}) \sVar(y)}} \\
&= \frac{\sCov(x,y) + \sCov(\alpha \mathbf{1},y)}{\sqrt{\sVar(x)\sVar(y)}} \\
&= \frac{\sCov(x,y)}{\sqrt{\sVar(x)\sVar(y)}} \\
&= \sCorr(x,y)
\end{align*}
}

\begin{theorem}
Adding a~vector of constants does not affect the sample correlation coefficient:
\[
\sCorr(x + \alpha \mathbf{1}, y) = \sCorr(x,y)
\]
where $\alpha \in \mathbb{R}$.
\end{theorem}

\begin{proof}
Firstly, we project vectors $x$ and $y$ onto $\Lin^{\perp}(\mathbf{1})$
in order to get $x^c = x - \bar x$ and $y^c = y - \bar y$ (`c' stands for `centred').
It can be shown that the~matrix corresponding to projecting onto the line spanned by
a~vector of all ones has the~following form
\[
\frac{\mathbf{1} \mathbf{1}^T}{\mathbf{1}^T \mathbf{1}} =
\frac{
\begin{pmatrix}
  1 \\
  \vdots \\
  1
\end{pmatrix}
\begin{pmatrix}
  1 & \ldots & 1
\end{pmatrix}
}{\sum_{i=1}^n 1} =
\begin{pmatrix}
  \frac{1}{n} & \ldots & \frac{1}{n} \\
  \vdots & \ddots & \vdots \\
  \frac{1}{n} & \ldots & \frac{1}{n}
\end{pmatrix}
\]
Thus, projecting onto the~orthogonal subspace is equivalent to
substracting the~projected vector, i.e., the vector of averages, from the original one.

\begin{marginfigure}
\includegraphics[scale=0.65]{figures/02_correlation_constant_centered_variables.pdf}
\caption{Centred vectors $x^c$ and $y^c$.}
\label{fig:corr_xyc}
\end{marginfigure}

Also note that the~angle $\varphi$ between the~original and centred vectors remains the~same.
The~result of this step is shown in Figure~\ref{fig:corr_xyc}.

Then we need to derive a new vector $\tilde x$ with constants added to each component.
Geometrically adding a vector of costants means adding a vector of all ones
scaled by $\alpha \in \mathbb{R}$, i.e., $\alpha \mathbf{1}$.
Then the new vector $\tilde x$ can be broken up into a sum of $\alpha \mathbf{1}$ and
$\beta x$, $\alpha, \beta \in \mathbb{R}$, which can be seen in Figure~\ref{fig:corr_final}.
After that we will project this new vector $\tilde x$ onto $\Lin^{\perp}(\mathbf{1})$.
By the properties of projection it is of no difference whether to project
the whole vector $\tilde x$ or project its parts $\alpha \mathbf{1}$
and $\beta x$ — the result is the same.
So, while $\beta x$ is projected onto the span of $x^c$, the projection of $\alpha \mathbf{1}$
onto the orthohgonal space $\Lin^{\perp}(\mathbf{1})$ yields zero as demonstrated
in Figure~\ref{fig:corr_final}.
Moreover, it follows that the angle between $\tilde x$ and $y$ is still $\varphi$.

\begin{figure}
\begin{center}
\includegraphics[width=0.6\linewidth]{figures/02_correlation_constant_proof.pdf}
\label{fig:corr_final}
\caption{$\sCorr(x + \alpha \mathbf{1}, y) = \sCorr(x,y)$ as the corresponding angles are equal.}
\end{center}
\end{figure}

Finally, putting everything together we finish the proof:
\[
\sCorr(x + \alpha \mathbf{1}, y) = \sCorr(x,y)
\]
\end{proof}

\subsection{Sample correlation coefficient in simple linear regression}

\begin{theorem}
A linear regression model with one explanatory variable and constant term
\[
y = \beta_1 \mathbf{1} + \beta_2 x + \varepsilon
\]
has the property
\[
\sCorr(y, \hat y) = \sign(\hat \beta_2) \sCorr(y, x)
\]
\end{theorem}

\marginnote{
Assuming the underlying relationship between $x$ and $y$ to be
\[
y_i = \beta_1 + \beta_2 x_i + \varepsilon_i \quad i=1,\ldots,n
\]
where $\varepsilon_i$ is an error term the following holds
\begin{align*}
\sCorr(y, \hat y) &= \frac{\sCov(y, \hat y)}{\sqrt{\sVar(y)\sVar(\hat y)}} \\
&= \frac{\sCov(y, \hat \beta_1 + \hat \beta_2 x)}{\sqrt{\sVar(y)\sVar(\hat \beta_1 + \hat \beta_2 x)}} \\
&= \frac{\sCov(y, \hat \beta_2 x)}{\sqrt{\sVar(y)\sVar(\hat \beta_2 x)}} \\
&= \frac{\hat \beta_2 \sCov(y, x)}{|\hat \beta_2| \sqrt{\sVar(y)\sVar(x)}} \\
&= \sign(\hat \beta_2) \frac{\sCov(y,x)}{\sqrt{\sVar(y)\sVar(x)}} \\
&= \sign(\hat \beta_2) \sCorr(y, x)
\end{align*}
}

\begin{proof}
Firstly, we consider the case when $\hat \beta_2 > 0$.
It has been shown earlier that the correlation coefficient represents the angle
between two random vectors.
So, in order to complete the proof we need to find the appropriate angles and compare them.

However, it seems to be difficult to compare the angles in the three dimensional space.
That is why we start with projecting both $x$ and $y$ onto the plane perpendicular to the vector of all ones (denoted as $\mathbf{1}$) as shown in Figure~\ref{fig:corr_pos_xyc}.
We denote this space as $\Lin^{\perp}(\mathbf{1})$. The resulting vectors are $x - \bar x \cdot \mathbf{1}$  and $y - \bar y \cdot \mathbf{1}$, respectively,
since projection of any vector $\vec{a}$ onto the span of a vector of all ones yields the vector of averages $\vec{\bar a}$.

In order to get the angle between $y$ and $\hat y$ we should start with regressing $y$ on $\Lin(x, \mathbf{1})$.
Then the only thing left is to project $\hat y$ onto $\Lin^{\perp}(\mathbf{1})$ since the $y$ vector has already been projected.
Note that the projected $\hat y$ falls onto tha span of vector $x - \bar x \cdot \mathbf{1}$ as it can be decomposed into a sum $a x + b \mathbf{1}$ where $a, b \in \mathbb{R}$.
The first component $a x$ is projected in the same way as $x$ and $b \mathbf{1}$ yields zero when projected onto the orthogonal space.
The result of this step is shown in Figure~\ref{fig:corr_pos_yhatc}.

\begin{figure*}[h!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/02_simple_regression_coefficient_centred_variables.pdf}
\label{fig:corr_pos_xyc}}
%\hspace{4ex}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/02_simple_regression_coefficient_yhat_projected.pdf}
\label{fig:corr_pos_yhatc}}
\caption{\subref{fig:corr_pos_xyc}: `Centred' $x$ and $y$, i.e., projected onto $\Lin^{\perp}(\mathbf{1})$;
\subref{fig:corr_pos_yhatc}: `Centred' $\hat y$, i.e., projected onto $\Lin^{\perp}(\mathbf{1})$.}
\end{center}
\end{figure*}

Since the projection of $\hat y$ lies exactly on the span of vector $x - \bar x \cdot \mathbf{1}$,
we can conclude that $\cos \varphi = \cos \varphi '$ and to put it another way $\sCorr(x,y) = \sCorr(y, \hat y)$.

Now consider the case when $\hat \beta_2 < 0$.
Note that the sign of $\beta_1$ does not influence the correlation coefficient sign.
The only difference is that now $\hat y$ is projected onto the span of  $x - \bar x \cdot \mathbf{1}$ and not on this vector itself while the projections of $x$ and $y$ remain the same.
Looking at Figure~\ref{fig:corr_negative} we deduce
that the~angle between $y$ and $\hat y$ is compelement to the angle between $x$ and $y$.
Using trigonometric properties, we simplify $\cos(180^{\circ} - \varphi) = -\cos\varphi$
which in turn implies $\sCorr(x,y) = -\sCorr(y,\hat y)$.

\begin{figure}[h!]
\begin{center}
\includegraphics{figures/02_simple_regression_coefficient_negative.pdf}
\label{fig:corr_negative}
\caption{Case of $\beta_2 < 0$.}
\setfloatalignment{b}
\end{center}
\end{figure}
\end{proof}

\subsection{RSS + ESS = TSS}

\marginnote{
Consider a regression model with $n$ observations and $k$ explanatory variables
including a constant unit vector % unit?
\[
y = X \beta + \varepsilon
\]
The OLS estimator for the vector of coefficients $\beta$ is
\[
\hat \beta = (X^T X)^{-1} X^T y
\]
and the residual vector is
\begin{align*}
\hat e &= y - \hat y \\
&= y - X \hat \beta \\
&= y - X (X^T X)^{-1} X^T y
\end{align*}
Then we define residual sum of squares (RSS), explained sum of squares (ESS) and total sum of squares (TSS) as follows:
\begin{align*}
RSS &= \lVert y - \hat y \rVert^2_2 \\
ESS &= \lVert \hat y - \bar y \rVert^2_2 \\
TSS &= \lVert y - \bar y \rVert^2_2 \\
\end{align*}
}

\begin{theorem}
A linear regression model with $n$ observations and $k$ explanatory variables including a constant unit vector
\[
y = X \beta + \varepsilon
\]
has the following property
\[
RSS + ESS = TSS
\]
where $RSS = \lVert y - \hat y \rVert^2_2$, $ESS = \lVert \hat y - \bar y \rVert^2_2$, $TSS = \lVert y - \bar y \rVert^2_2$.
\end{theorem}

\begin{proof}
The proof will be presented for the case of two regressor $x$ and $\mathbf{1}$ in order for the picture to be clear.
However, the same logic applies for the case of $k$ regressors.

We start with depicting the vectors $y \in \mathbb{R}^{n-2}$ and $x, \mathbf{1} \in \mathbb{R}^2$.
Then we project $y$ onto $\Lin(x, \mathbf{1})$ and obtain $\hat y$ which is shown in Figure~\ref{fig:rss}.

From this picture we can immediately derive $\sqrt{RSS}$ as by definition this is the squared difference between $y$ and $\hat y$.

\begin{figure*}[h!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.3\linewidth]{figures/02_rss_ess_tss_yhat.pdf}
\label{fig:rss}}
\hspace{4ex}
\subfigure[]{
\includegraphics[width=0.3\linewidth]{figures/02_rss_ess_tss_sqr_tss_ess.pdf}
\label{fig:tss_ess}}
\subfigure[]{
\includegraphics[width=0.3\linewidth]{figures/02_rss_ess_tss_final.pdf}
\label{fig:final}
}
\caption{\subref{fig:rss}: Residual sum of squares;
\subref{fig:tss_ess}: Total sum of squares and residual sum of squares;
\subref{fig:final}: Illustration of the equality $(\sqrt{RSS})^2 + (\sqrt{ESS})^2 = (\sqrt{TSS})^2$
where $a$ stands for $\sqrt{RSS}$, $b$ — $\sqrt{TSS}$, $c$ — $\sqrt{ESS}$.}
\end{center}
\end{figure*}

So as to visualize $ESS$ and $TSS$ we first need to visualize vector of averages $\bar y$.
Geometrically this means projecting a vector onto a line spanned by vector $\mathbf{1}$.

Now we both project $y$ and $\hat y$ onto $\mathbf{1}$ and following the definition obtain $\sqrt{TSS}$ as the difference vector $y - \bar y$ and $\sqrt{ESS}$ as the vector $\hat y - \bar y$.

The final step is to put everything together.
Note that since $y - \hat y$ is perpendicular to $\Lin(x, \mathbf{1})$ it is also perpendicular to $\hat y - \bar y$ and $\mathbf{1}$ as these vectoros are in $\Lin(x, \mathbf{1})$.
Then, applying the theorem of three perpendiculars we conclude that the foot of vector $y - \bar y$ is the same point as the foot of the vector $\hat y - \bar y$.
Thus, we obtain a right angle triangle and can apply the Pythagorean theorem for the catheti $\sqrt{RSS}$ and $\sqrt{ESS}$ and the
hypotenuse $\sqrt{TSS}$:
\[
\left(\sqrt{RSS}\right)^2 + \left(\sqrt{ESS}\right)^2 = \left(\sqrt{TSS}\right)^2
\]

\marginnote[-5\baselineskip]{
Disclosing parentheses and using the fact that $\hat{y}^T y = \hat{y}^T \hat{y}$
\begin{align*}
\hat{y}^T y &= \beta^T X^T y \\
&=  y^T X (X^T X)^{-1} X^T y \\
\hat{y}^T \hat{y} &= \beta^T X^T X \beta \\
&= y^T X (X^T X)^{-1} X^T X (X^T X)^{-1} X^T y \\
&= y^T X (X^T X)^{-1} X^T y
\end{align*}
we obtain
\begin{align*}
RSS &= y^T y -\hat{y}^T \hat{y} \\
ESS &= \hat{y}^T \hat{y} - \hat{y}^T \bar{y} + \bar{y}^T \bar{y} \\
TSS &= y^T y - 2 y^T \bar y +  \bar{y}^T \bar{y}
\end{align*}
When putting everything together all the terms cancel out which proves
\[
ESS + RSS = TSS
\]
}
\end{proof}

\vspace{3.5cm}
\subsection{Determination coefficient}

\marginnote[-2\baselineskip]{
\begin{align*}
\sCorr^2(y,\hat y) &= \left(\frac{\sCov(y, \hat y)}{\sqrt{\sVar(y)\sVar(\hat y)}}\right)^2 \\
&= \frac{\sCov(y, \hat y) \sCov(y, \hat y)}{\sVar(y)\sVar(\hat y)} \\
&= \frac{\sCov(\hat y + e, \hat y) \sCov(\hat y + e, \hat y)}{\sVar(y)\sVar(\hat y)} \\
&= \frac{\sCov(\hat y, \hat y) + \sCov(e, \hat y)}{\sVar(y)} \\
&\cdot \frac{\sCov(\hat y, \hat y) + \sCov(e, \hat y)}{\sVar(\hat y)} \\
&= \frac{\sVar(\hat y) \sVar(\hat y)}{\sVar(y)\sVar(\hat y)} \\
&= \frac{\sVar(\hat y)}{\sVar(y)} = \frac{ESS}{TSS} = R^2
\end{align*}
}

\begin{theorem}
A linear regression model with $n$ observations and $k$ explanatory variables including a constant unit vector
\[
y = X \beta + \varepsilon
\]
has the following property
\[
R^2 = \sCorr^2(y, \hat y)
\]
\end{theorem}

\begin{proof}
Proving this theorem geometrically means showing that the determination coefficient can be interpreted as some squared angle
which happens to be eqaul to the squared angle between centered $y$ and $\hat y$.

Consider Figure~\ref{fig:final} from the previous proof.
It was shown there that the vectors $y - \bar y$, $y - \hat y$ and $\hat y - \bar y$ form a right triangle.
Having defined the determination coefficient as
\[
R^2 = \frac{ESS}{TSS}
\]
we conclude that its geometric interpretaion is
\[
R^2 = \frac{ESS}{TSS} = \cos^2 \varphi
\]
as shown in Figure~\ref{fig:r_sq_angle}.

\begin{marginfigure}
  \includegraphics{figures/02_determination_coefficient.pdf}
  \caption{Determination coefficient as squared $\cos \varphi$
  where $a$ stands for $\sqrt{RSS}$, $b$ — $\sqrt{TSS}$, $c$ — $\sqrt{ESS}$.}
  \label{fig:r_sq_angle}
\end{marginfigure}

Recall that the sample correlation coefficient two vectors was defined earlier as the angle between these two vectors.
Thus, we conclude that $\sCorr(y, \hat y)$ is the angle between $y$ and $\hat y$ which is also eqaul to $\cos \varphi$.
Finally, squaring both sides, we obtain
\[
R^2 = \sCorr^2(y, \hat y)
\]
\end{proof}

\subsection{Regression line and point of averages}

\marginnote[-1\baselineskip]{
The following decomposition of $\hat y$ holds:
\begin{align*}
\hat y  &= X \hat \beta = X (X^T X)^{-1} X^T y \\
&= X (X^T X)^{-1} X^T X \beta + X (X^T X)^{-1} X^T \varepsilon
\end{align*}
Premultiplying both sides by $X^T$, we obtain:
\begin{align*}
X^T \hat y &=  X^T X (X^T X)^{-1} X^T X \beta \\
&+ X^T X (X^T X)^{-1} X^T \varepsilon \\
&= X^T X \beta + X^T \varepsilon
\end{align*}
This is a system of equations. The first row of $X^T$ is $\mathbf{1}$ vector, so we can write out the first equation:
\[
\sum_{i=1}^n \hat y_i = \sum_{i=1}^{n} \sum_{j=1}^{k} x_{ij} \beta_{j}
\]
From the first equation in the system
\[
X^T \hat y = X^T y
\]
we obtain
\[
\sum_{i=1}^{n} \hat y_i = \sum_{i=1}^{n} y
\]
And this finishes the proof:
\[
\frac{1}{n} \sum_{i=1}^{n} y = \frac{1}{n} \sum_{i=1}^{n} \sum_{j=1}^{k} x_{ij} \beta_{j}
\]
}

\begin{theorem}
In a linear regression model with one explanatory variable and constant term
\[
y = \beta_1 + \beta_2 x + \varepsilon
\]
the point of averages lies on the estimated regression line.
\end{theorem}

\begin{proof}
For the geometrical proof it suffices to show that $\hat y$ is a linear combination of the regressors, which is true by construction,
and that $\frac{1}{n} \sum_{i=1}^{n} \hat y_i = \frac{1}{n} \sum_{i=1}^{n} y$. In order for the pictures to be more clear the proof will be presented for the case of two regressors.

The first step is regressing $y$ on $\Lin(\mathbf{1}, x)$. As shown in Figure~\ref{fig:averages_lin}, we obtain $\hat y$ as a linear combination of $\mathbf{1}$ and $x$.
The next step is to regress both $y$ and $\hat y$ on $\mathbf{1}$ which results in $\bar y$ and $\bar{\hat y}$ correspondingly.
By the theorem of three perpendiculars, $\bar y = \bar{\hat y}$ which is shown in Figure~\ref{fig:averages_bars}.

\begin{figure}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.35\linewidth]{figures/02_averages_yhat_decomposed.pdf}
\label{fig:averages_lin} }
\hspace{4ex}
\subfigure[]{
\includegraphics[width=0.35\linewidth]{figures/02_averages_final.pdf}
\label{fig:averages_bars}}
\caption{\subref{fig:averages_lin}: Regression of $y$ on $\Lin(\mathbf{1},x)$;
\subref{fig:averages_bars}: Regression of $y$ and $\hat y$ on $\mathbf{1}$.}
\setfloatalignment{b}
\end{center}
\end{figure}
\end{proof}

\subsection{Orthogonality of regressors}

\marginnote{
Consider a model
\[
y = \beta_1 x_1 + \beta_2 x_2 + \varepsilon
\]
with centred regressors $x_1$, $x_2$ such that $\bar x_1 = \bar x_2 = 0$.
Suppose we mistakenly believe in a model
\[
y = \beta_1^* x_1 + \varepsilon^*
\]
where $x_2$ is omitted. Then the new estimator
\begin{align*}
\E\left(\hat \beta_1^*\right) &= \E\left(\frac{\sCov(x_1, y)}{\sVar(x_1)}\right) \\
&= \E\left(\frac{\sCov(x_1, \beta_1 x_1 + \beta_2 x_2 + \varepsilon)}{\sVar(x_1)}\right) \\
&= \beta_1 + \beta_2 \frac{\sCov(x_1,x_2)}{\sVar(x_1)} + \E\left(\frac{\sCov(x_1, \varepsilon)}{\sVar(x_1)}\right) \\
&= \beta_1 + \beta_2 \frac{\sCov(x_1,x_2)}{\sVar(x_1)}
\end{align*}
will be unbiased if either $\beta_2 = 0$ or $\sCov(x1,x2) = 0$.
}

\begin{theorem}
Omitting a variable in a regression model does not lead to a bias in coefficient
estimators if either the coefficient of the omitted regressor equals zero
or the regressors are uncorrelated.
\end{theorem}

\begin{proof}
We will not consider the case when the coefficient of the omitted regressor
equals zero because this means that the model is not mis-specified.
The case of orthogonal regressors is less trivial.
However, it can be easily proved once the geometric approach is implemented.

\begin{marginfigure}
\includegraphics[scale=0.7]{figures/02_uncorrelated_regressors.pdf}
\label{fig:uncorrelated_regressors}
\caption{In case of uncorrelated regressors omitting one of them
does not result in bias of the estimator.}
\end{marginfigure}

Condider the following regression model
\[
y = \beta_1 x_1 + \beta_2 x_2 + \varepsilon
\]
where regressors $x_1$ and $x_2$ are uncorrelated, $x_1 \perp x_2$.
When we perform a regression of $y$ onto both $x_1$, $x_2$
we obtain predicted values $\hat y$ which can be broken up into a sum:
\[
\hat y = \hat \beta_1 x_1 + \hat \beta_2 x_2.
\]
Since $x_1$ and $x_2$ are orthogonal it folows that
the difference $\hat y - \hat \beta_1 x_1$ must be orthogonal to $x_1$.
Thus, by the theorem of three perpendiculars the projection of $y$ onto
$x_1$ also results in $\hat \beta_1 x_1$.
\end{proof}

\vspace{4cm}
\subsection{Geometry of formula for OLS estimators}

The following interpretation gives another view on the estimators obtained
by the ordianry least squares method.
Previously, we illustrated the ideas in the observations space, $\mathbb{R}^n$.
Apparantely, the OLS estimators can be depicted in the regressors space, $\mathbb{R}^k$.
This result follows from the Cramer's rule.

Consider a linear model with two regressors:
\[
\hat y = \hat \beta_1 x_1 + \hat \beta_2 x_2.
\]
Let us denote $X = \begin{pmatrix} x_1 \\ x_2 \end{pmatrix}$ and
$\langle X, y \rangle = \begin{pmatrix} \langle x_1,  y \rangle \\
\langle x_2,  y \rangle \end{pmatrix}$.
Then, taking scalar product, we obtain:
\[
\langle X, y \rangle = \hat \beta_1 \langle X, x_1 \rangle + \hat \beta_2 \langle X, x_2 \rangle
\]
All the elements of this equation are illustrated in Figure~\ref{fig:cramers}.

The area of each parallelogram is the determinant of the matrix with columns
which form the corresponding parallelogram.
For instance, the area of parallelogram formed by vectors $\langle X, y \rangle$, $\langle X, x_2 \rangle$
is equal to $\det([\langle X, y \rangle  \langle X,  x_2 \rangle])$.
The parallelogram formed by vectors $\langle X, \hat \beta_1 x_1 \rangle$,
$\langle X, x_2 \rangle$ has the area of $\hat \beta_1 \det([\langle X, x_1 \rangle  \langle X, x_2 \rangle])$.
Since both parallelograms must be of the same area we obtain a formula for $\hat \beta_1$:
\[
\hat \beta_1 = \frac{\det([\langle X, y \rangle  \langle X,  x_2 \rangle])}{ \det([\langle X, x_1 \rangle  \langle X, x_2 \rangle])}.
\]

\begin{marginfigure}
  \includegraphics[scale=0.5]{figures/02_cramers_rule.pdf}
  \label{fig:cramers}
  \caption{Illustration of estimators in $\mathbb{R}^k$ as the ratio of parallelogram areas.}
\end{marginfigure}

So as to generalize this result we introduce a matrix $A = (y | X)$
that has $y$ as its first column and all the regressors as the rest ones.
We will denote this matrix without column $y$ or $x$ as $A_{-y}$, $A_{-x}$, respectively.
Then, we can write out the formula for $\beta_x$:
\[
\hat \beta_x = \frac{\det(\langle A_{-y}, A_{-x} \rangle)}{\det(\langle A_{-y}, A_{-y} \rangle)}.
\]

If there is a regression with unit vector among the regressors and
the Frisch-Waugh-Lovell theorem is applied, the formula above takes the followin form:
\[
\hat \beta_x = \frac{\det(\sCov(A_{-y}, A_{-x}))}{\det(\sVar(A_{-y}))}.
\]

\subsection{Frisch–Waugh–Lovell theorem}

\marginnote{From regression~(\ref{eq:fwl_2}) we get the following estimator:
\begin{align*}
\hat\beta_2 &= ((M_1 X_2)^T M_1 X_2)^{-1}(M_1 X_2)^T M_1 y \\
&= (X_2^T M_1^T M_1 X_2)^{-1}  X_2^T M_1^T M_1 y \\
&= (X_2^T M_1 X_2)^{-1}  X_2^T M_1 y
\end{align*}
As for regression~(\ref{eq:fwl_1}), let us note that due to $y = \hat y + \hat u$ $y$ can be decomposed as follows:
\[
y = Py + My = X_1 \hat \beta_1 + X_2 \hat \beta_2 + My
\]
Premultiplying both sides by $X_2^T M_1$, we obtain:
\begin{align*}
X_2^T M_1 y &= X_2^T M_1 X_1 \hat\beta_1 + X_2^T M_1 X_2 \hat\beta_2 + X_2^T M_1 M y \\
&=  X_2^T M_1 X_2 \hat\beta_2 + X_2^T M_1 M y \\
&= X_2^T M_1 X_2 \hat\beta_2
\end{align*}
On the last step we used the fact that
\begin{align*}
(X_2^T M_1 M y)^T = y^T M^T M_1^T X_2 \\
= y^T M M_1 X_2 = y^T M X_2 = 0^T
\end{align*}
Assuming $X_2^T M_1 X_2$ is invertible, we get the same estimator
\[
\hat\beta_2 = (X_2^T M_1 X_2)^{-1}  X_2^T M_1 y
\]
}

\begin{theorem}
Consider regression
\begin{equation} \label{eq:fwl_1}
y = X_1 \beta_1 + X_2 \beta_2 + u
\end{equation}
where $X_{n \times k} = [X_1 X_2]$, i.e. $X_1$ consists of first $k_1$ columns of $X$ and $X_2$ consists of remaining $k_2$ columns of $X$,
$\beta_1$ and $\beta_2$ are comformable, $k_1 \times 1$ and $k_2 \times 1$ vectors.
Consider another regression
\begin{equation}  \label{eq:fwl_2}
M_1 y = M_1 X_2 \beta_2 + M_1 u
\end{equation}
where $M_1 = I - P_1$ projects onto the~orthogonal complement of the~column space~of
$X_1$ and $P_1 = X_1(X_1^TX_1)^{-1}X_1^T$ is the~projection onto the~column space of~$X_1$.
Then the~estimate of $\beta_2$ from regression~(\ref{eq:fwl_1}) will be the~same
as the~estimate from regression~(\ref{eq:fwl_2}).
\end{theorem}

There are two ways to~visualize the~proof of~the~Frisch-Waugh-Lovell theorem
using geometric concepts. Both are presented below.

\begin{proof}
1.~Consider the following model:
\begin{equation} \label{eq:fwl_proof}
y_i = \beta_1 x_i + \beta_2 z_i + u_i
\end{equation}

We start with a `one-step' regression  and will distinct its coefficients with
an upper index $A$.
The~only step in~obtaining $\beta_1^{A}$ is regressing $y$ on~$\Lin(x,z)$ and
then expanding $\hat y$ as a~linear combination of~basis vectors $x$ and $z$,
which is shown in~Figure~\ref{fig:fwl_1_regression_3d}. Figure~\ref{fig:fwl_1_regression_lin}
depicts $\Lin(x, z)$.

\begin{figure}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.4\linewidth]{figures/02_fwl_v1_yhat_decomposed.pdf}
\label{fig:fwl_1_regression_3d}}
\hspace{4ex}
\subfigure[]{
\includegraphics[width=0.4\linewidth]{figures/02_fwl_v1_yhat_decomposed_lin.pdf}
\label{fig:fwl_1_regression_lin}}
\caption{\subref{fig:fwl_1_regression_3d}: Regression of $y$ on $\Lin(x,z)$;
\subref{fig:fwl_1_regression_lin}: $\Lin(x, z)$.}
\end{center}
\end{figure}

As for the~model~(\ref{eq:fwl_2}) where several regressions are performed consecutively
we start with regressing $y$~on~$z$, resulting in $\tilde{y}$,
which we will refer~to as~`cleansed' $y$.
We will denote its coefficients with an upper index $B$.

\begin{equation}\label{eq:fwl_2_y_clean}
\begin{aligned}
y &= \alpha z + \varepsilon \\
\hat\alpha &= \frac{y^T z}{z^T z} \\
\tilde{y} &= \hat\varepsilon = y - \frac{y^T z}{z^T z}z
\end{aligned}
\end{equation}

Following that, $x$ is regressed on $z$, resulting in $\tilde{x}$ — `cleansed' $x$.

\begin{equation}\label{eq:fwl_2_x_clean}
\begin{aligned}
x &= \gamma z + \nu \\
\hat\gamma &= \frac{x^T z}{z^T z} \\
\tilde{x} &= \hat\nu = x - \frac{x^T z}{z^T z}z
\end{aligned}
\end{equation}

Geometric results of these two steps are presented in~\ref{fig:fwl_2_regression_first}.

Finally, `cleansed' $y$ must be regressed on `cleansed' $x$.
However, it cannot be performed immediately as $\tilde{y}$ and $\tilde{x}$ are skew lines.
So at first, we fix this problem by translation and after that obtain
$\hat\beta_1^{B}\tilde x$ (see Figure~\ref{fig:fwl_2_regression_trans}).

\begin{figure}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.4\linewidth]{figures/02_fwl_v1_cleansed_variables.pdf}
\label{fig:fwl_2_regression_first} }
\hspace{4ex}
\subfigure[]{
\includegraphics[width=0.4\linewidth]{figures/02_fwl_v1_translation.pdf}
\label{fig:fwl_2_regression_trans}}
\caption{\subref{fig:fwl_2_regression_first}: Regression of $y$ on $z$ and of $x$ on $z$;
\subref{fig:fwl_2_regression_trans}: Translation of $\tilde{x}$.}
\end{center}
\end{figure}

Now let us picture all the results in one figure and mark some main points.

\begin{figure}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.4\linewidth]{figures/02_fwl_v1_final.pdf}
\label{fig:fwl_3_3d} }
\hspace{4ex}
\subfigure[]{
\includegraphics[width=0.4\linewidth]{figures/02_fwl_v1_final_lin.pdf}
\label{fig:fwl_3_lin}}
\caption{\subref{fig:fwl_3_3d}: Point A stands for the origin, B — $\hat\gamma z$,
C — $x$, D — $\hat\alpha z$, E — intersection of vector $x$ and line parallel to $\tilde x$,
F — $\hat\beta_1^{A} x$, G — $\hat\beta_1^{B} \tilde{x}$; \subref{fig:fwl_3_lin}: $\Lin(x,z)$.}
\end{center}
\end{figure}

In Figure~\ref{fig:fwl_3_lin} segments $AF$ and $BH = DG$ stand for $\hat\beta_1^{A}x$
and $\hat\beta_1^{B}\tilde x$ respectively, while segments AC and BC represent $x$ and $\tilde{x}$.
Having two congruent angles, triangles ABC and FHC are simillar.
Then, it follows:
\[
\frac{AF}{AC} = \frac{BH}{BC} \Leftrightarrow \frac{\hat\beta_1^{A}x}{x} = \frac{\hat\beta_1^{B}\tilde x}{\tilde x} \Leftrightarrow \hat\beta_1^{A} = \hat\beta_1^{B}
\]

2. Alternatively, we could implement a~concept close to the~partial correlation.
In the~same model~(\ref{eq:fwl_proof}) we wiil treat $z$ vector fixed and again
consecutively cleanse the $x$ and $y$ variables by projecting them onto
the~space orthogonal to $z$, i.e., $\Lin^\perp(z)$ as demonstrated in Figure~\ref{fig:fwl_v2_rcleansed}.
Then we perform a~regression of the~`cleansed' $\tilde y$ on the~`cleansed' $\tilde x$
(see Figure\ref{fig:fwl_v2_regression_cleansed}).

\begin{figure}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/02_fwl_v2_cleansed_variables.pdf}
\label{fig:fwl_v2_rcleansed}}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/02_fwl_v2_cleansed_regression.pdf}
\label{fig:fwl_v2_regression_cleansed}}
\caption{\subref{fig:fwl_2_regression_first}: `Cleansed' variables $\tilde x$ and $\tilde y$;
\subref{fig:fwl_2_regression_trans}: `Cleansed' $\tilde y$ regressed on `cleansed' $\tilde{x}$.}
\end{center}
\end{figure}

Now we show that the~latter regression produces $\hat \beta_1$ coefficient which
is exactly the~coefficient from the~`one-step' regression of~original $y$ onto
original $x$ and $z$. Recall that the~vector $y$ can be split~up into a~sum of
some multiple of $x$ and some multiple of $z$. Since the second term is
the~orthogonal component its projection yields zero. The~multiple of $x$
is equal to $\hat \beta_1$ by construction.

Assume that the~coefficient at~$\tilde x$ is some unknown variable $\hat \delta$.
Then consider the~similar triangles in the $\Lin(x,z)$. From the~proportions
we obtain:
\[
\frac{CE}{CA} = \frac{CD}{CB} \Leftrightarrow \frac{\hat \beta_1 x}{x} = \frac{\hat \delta \tilde x}{\tilde x} \Rightarrow \hat \beta_1 = \hat \delta
\]

\begin{figure}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/02_fwl_v2_similar_triangles.pdf}
\label{fig:fwl_v2_triangles}}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/02_fwl_v2_final.pdf}
\label{fig:fwl_v2_final}}
\caption{\subref{fig:fwl_2_regression_first}: Similar triangles: $\bigtriangleup ABC \sim \bigtriangleup EDC$;
\subref{fig:fwl_2_regression_trans}: Alternative proof for the Frisch-Waugh-Lovell theorem.}
\end{center}
\end{figure}
\end{proof}

\subsection{Duality of regressors and residuals}

The idea of duality is widely used in mathematics.
The concept is to apply some transformation twice and get the~original object.
For example, if $f(a) = 1/a$:
\[
x \stackrel{f}{\to} \frac{1}{x} \stackrel{f}{\to} \frac{1}{1/x} = x
\]
We show that there is duality between regressors and residuals.

\begin{theorem}
Let $x_i$ be a $n \times 1$ regressor,
$u_i$ — a residual in regression of $x_i$ on all the rest regressors,  $i = 1, \ldots, k$.
Consider a transformation of a vector $v$, $f(v) = v/\lVert v \rVert^2$.
Then applying this transformation on the residuals $u_1, \ldots, u_k$ yields
new regressors $v_1, \ldots, v_k$.
Performing $k$ regressions of each $v_i$ on all the rest regressors and
applying the same transformation to the new residuals results in
the original regressors $x_1, \ldots, x_k$.
\end{theorem}

\begin{proof}

\begin{marginfigure}[-2\baselineskip]
\includegraphics[scale=0.7]{figures/02_duality_original_regressors.pdf}
\caption{Two regressors in the unit circle.}
\end{marginfigure}

We start with $2$-dimensional case with two regressors,
and discuss the~case of spaces of higher dimensions later.

As stated in the~theorem we need to keep the~measure of the~lengths of the~regressors.
In order to do this we choose a~basis in $\mathbb{R}^2$ in such a~way that
\begin{align*}
&x_1 = \lambda_1 e_1, \quad \lVert e_1 \rVert = 1 \\
&x_2 = \lambda_2 e_2, \quad \lVert e_2 \rVert = 1
\end{align*}
where $\lambda_1, \lambda_2 \in \mathbb{R}$.

\begin{marginfigure}
\includegraphics[scale=0.7]{figures/02_duality_first_residuals.pdf}
\label{fig:duality_fst_residuals}
\caption{Residuals $\hat{u}_1$ and $\hat{u}_2$.}
\end{marginfigure}

Then we perform two regressions
\begin{align*}
&x_1 = \beta_1 x_2 + u_1 \\
&x_2 = \beta_2 x_1 + u_2
\end{align*}
and get the residuals $\hat{u}_1$, $\hat{u}_2$.
Being orthogonal to $x_2$ and $x_1$, correspondingly, they can be written as follows
\begin{align*}
&\hat{u}_1 = \sin \alpha \cdot \lambda_1 \tilde{e}_1, \quad \lVert \tilde{e}_1 \rVert = 1 \\
&\hat{u}_2 = \sin \alpha \cdot \lambda_2 \tilde{e}_2, \quad \lVert \tilde{e}_2 \rVert = 1
\end{align*}
where $\tilde{e}_1 \perp e_2$ and $\tilde{e}_2 \perp e_1$.

For convenience we translate all the vectors $x_1$, $x_2$, $\hat{u}_1$, $\hat{u}_2$
to the origin of the unit circle as shown in Figure~\ref{fig:duality_fst_residuals_translated}
and after that we invert them.

\begin{marginfigure}
\includegraphics[scale=0.7]{figures/02_duality_inversion.pdf}
\label{fig:duality_inversion}
\caption{Example of inversion for vector $a$.}
\end{marginfigure}

In order to illustrate inversion consider an example with an arbitrary vector $a$.
Knowing its length, the aim is to find such an orthogonal vector $\tilde a$
that the product $\lVert a \rVert^2 \cdot \lVert \tilde a \rVert^2 = 1$.
In other words, we need to find an edge of rectangle with area equal to $1$.
Solving for $\tilde a$, we obtain the length of the inverted vector $a$.
The only thing left is to rotate this inverted vector back
to get a vector $\tilde{\tilde a}$ which satisfies both
\begin{align*}
& \lVert \tilde{\tilde a} \rVert^2 = \frac{1}{\lVert a \rVert^2} \\
& \cos(a, \tilde{\tilde a}) = 1
\end{align*}

\marginnote[-2\baselineskip]{
The transformation stated in the theorem is $f(v) = v / \lVert v \rVert^2$.
Generally speaking, $g(v) = v / (c \cdot \lVert v \rVert^2)$ where $c \in \mathbb{R}$
would also work.
\begin{multline*}
v \stackrel{g}{\to} \frac{v}{c \cdot \lVert v \rVert^2} = w \stackrel{g}{\to} \\
\frac{w}{c \cdot \lVert w \rVert^2} = \frac{\frac{v}{c \cdot \lVert v \rVert^2}}{c \frac{\lVert v \rVert^2}{c^2 \lVert v \rVert^4}} = v
\end{multline*}
}

Having applied the inversion to $\hat{u}_1$, $\hat{u}_2$, we obtained
new vectors $y_1$, $y_2$. Moreover, there is an algebraic expression for them
in terms of rotated basis $\tilde{e}_1$, $\tilde{e}_2$:
\begin{align*}
&\hat{u}_1 = \sin \alpha \cdot \lambda_1 \tilde{e}_1 \Rightarrow y_1 = \frac{1}{\sin \alpha \cdot \lambda_1} \tilde{e}_1 \\
&\hat{u}_2 = \sin \alpha \cdot \lambda_2 \tilde{e}_2 \Rightarrow y_2 = \frac{1}{\sin \alpha \cdot \lambda_2} \tilde{e}_2 \\
\end{align*}

Next, we perform another two regressions:
\begin{align*}
& y_1 = \gamma_1 y_2 + v_1 \\
& y_1 = \gamma_2 y_1 + v_2
\end{align*}

\begin{figure*}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.3\linewidth]{figures/02_duality_first_residuals_translated.pdf}
\label{fig:duality_fst_residuals_translated}}
\subfigure[]{
\includegraphics[width=0.3\linewidth]{figures/02_duality_new_regressors.pdf}
\label{fig:duality_new_regreesors}}
\subfigure[]{
\includegraphics[width=0.3\linewidth]{figures/02_duality_new_residuals.pdf}
\label{fig:duality_new_residuals}}
\caption{\subref{fig:duality_fst_residuals_translated}: Residuals translated to the orgin of the unit circle;
\subref{fig:duality_new_regreesors}: Regressors $v_1$, $v_2$ obtained from inversion of the residuals $\hat{u}_1$, $\hat{u}_2$;
\subref{fig:duality_new_residuals}: Regressions of $v_1$ onto $v_2$ and of $v_2$ onto $v_1$.}
\end{center}
\end{figure*}

There are two things to notice about the new residuals $\hat{v}_1$, $\hat{v}_2$.
First, $\hat{v}_1$ is perpendicular to the line spanned by $\tilde{e}_2$.
Similarly, $\hat{v}_2$ is perpendicular to the line spanned by $\tilde{e}_1$.
This means, that they are parallel to $e_1$, $e_2$, correspondingly,
and once translated, they can be expressed as a multiple of $x_1$, $x_2$.

Second, we can find the lengths of these new residuals from the~right
triangles depicted in Figure~\ref{fig:duality_new_residuals}:
\begin{align*}
& \lVert \hat{v}_1 \rVert = \sin \alpha \cdot \lVert y_2 \rVert = \sin \alpha \cdot \left\lVert \frac{1}{\sin \alpha \cdot \lambda_1} \tilde{e}_1 \right\rVert = \frac{1}{\lambda_1} \\
& \lVert \hat{v}_2 \rVert = \sin \alpha \cdot \lVert y_1 \rVert = \sin \alpha \cdot \left\lVert \frac{1}{\sin \alpha \cdot \lambda_2} \tilde{e}_2 \right\rVert = \frac{1}{\lambda_2}
\end{align*}
Thus, when translated to the origin, the new resiuduals can be rewritten as
\begin{align*}
& \hat{v}_1 = \frac{1}{\lambda_1} e_1 \\
& \hat{v}_2 = \frac{1}{\lambda_2} e_2
\end{align*}

\begin{marginfigure}[4\baselineskip]
\includegraphics[scale=0.6]{figures/02_duality_final.pdf}
\label{fig:duality_final}
\caption{New residuals translated to the origin of the unit circle.}
\end{marginfigure}

The last step is to invert $\hat{v}_1$, $\hat{v}_2$.
Following the same procedure as described above, we finally get the desired result:
\begin{align*}
& \hat{v}_1 = \frac{1}{\lambda_1} e_1 \to \lambda_1 e_1 = x_1 \\
& \hat{v}_2 = \frac{1}{\lambda_2} e_2 \to \lambda_2 e_2 = x_2
\end{align*}
\end{proof}

\vspace{3.5cm}
\subsection{Gauss-Markov theorem}

\begin{theorem}
In the homoskedastic linear regression model the best (minimum-variance) linear
unbiased estimator is given by the ordinary least squares.
\end{theorem}

\begin{proof}

Consider an OLS estimator and an alternative one:
\begin{align*}
\hat{\beta}_{OLS} &= (X^T X)^{-1} X^T y = A^T y \\
\hat{\beta}_{alt} &= A^T_{alt} y
\end{align*}

% Hansen
\marginnote[-4\baselineskip]{
Consider an estimator $\beta$ which is a linear function of $Y$:
\[
\hat \beta = A^T Y
\]
where $A$ is an $n \times k$ function of $X$ such that $A^T X = I_k$.
From
\begin{align*}
\Var(\hat\beta_{OLS}) &= (X^T X)^{-1} \sigma^2 \\
\Var(A^T y) &= A^T A \sigma^2
\end{align*}
it follows that it is sufficient to prove that $A^T A - (X^T X)^{-1}$
is positive semi-definite. Set $C = A - X(X^T X)^{-1}$ and note that $X^T C = 0$, then
\begin{multline*}
A^T A - (X^T X)^{-1} \\
= (C + X(X^T X)^{-1})^T (C + X(X^T X)^{-1}) - (X^T X)^{-1} \\
= C^T C + C^T X(X^T X)^{-1} + (X^T X)^{-1} X^T C + \\
(X^T X)^{-1} X^T X(X^T X)^{-1} - (X^T X)^{-1} \\
= C^T C
\end{multline*}
Matrix $C^T C$ is positive semi-definite since
\[
\forall a \not= 0 \qquad a^T C^T C a = \lVert C a \rVert^2 \geq 0
\]
}

Note that $A^T X = I_{k}$, then the following holds for all $\beta$:
\begin{align*}
A^T X \beta &= \beta \\
A^T_{alt} X \beta &= \beta
\end{align*}
Taking the difference of these equations, we obtain:
\[
\left(A^T_{alt} - A^T\right) X \beta = 0 \Rightarrow \left(A^T_{alt} - A^T\right) \perp X
\]
If we treat the coefficients separately and consider, for instance, $\beta^{(2)}$,
we get the following result
\[
\left(a^T_{alt} - a^T\right) \perp X
\]
where $a_{alt}$ and $a$ are the second columns of matrices
$A_{alt}$ and $A$ correspondingly.
Since $a \in \Lin(\col X)$, it follows that $a_{alt} \not \in \Lin(\col X)$.

\begin{marginfigure}[1\baselineskip]
\includegraphics[scale=0.7]{figures/02_gmt.pdf}
\label{fig:gmt}
\caption{Gauss-Markov theorem for the case of three regressors
where $a^{(1)}$, $a^{(2)}$, $a^{(3)}$ are columns of matrix $A$.}
\end{marginfigure}

Now we can express the variances of both estimators in terms of $a_{alt}$ and $a$:
\begin{align*}
\Var\left(\hat{\beta}^{(2)}_{OLS}\right) &= \Var\left(a^T y\right) = a^T \sigma^2 I_{k} a = \sigma^2 \left\lVert a \right\rVert^2 \\
\Var\left(\hat{\beta}^{(2)}_{alt}\right) &= \Var\left(a^T_{alt} y\right) =  a^T_{alt} \sigma^2 I_{k} a_{alt} = \sigma^2 \left\lVert a_{alt} \right\rVert^2
\end{align*}
Since vectors $a$, $a_{alt}$ and $a - a_{alt}$
form a right triangle and $a_{alt} \not \in \Lin(\col X)$,
the vector $\left\lVert a_{alt} \right\rVert^2$ must be longer than $a$,
and the corresponding estimator must have higher variance.

\end{proof}

\vspace{1cm}

\subsection{Geometry of instrumental variables}

Consifder a model with an endogenity problem, i.e. explanotary variable $x$
is correlated with the error term $u$:
\[
y = \beta x + u
\]
Assume there is an instrument $z$ which is dependent with the problematic regressor $x$
but uncorrelated with the error term $u$.
The 2SLS procedure tells us to perform the following steps.
\begin{enumerate}
  \item Regress $x$ onto $z$ and get the vector of predicted values $\hat x$,
  \item Regress $y$ onto $\hat x$.
\end{enumerate}
These steps result in $\beta_{IV}$ estimator which is illustrated in Figure~\ref{fig:instrumental}.

The same result could be obtained with the oblique projection.
That is projecting $y$ onto $x$ along the vector perpendicular to the span of $z$.

\begin{marginfigure}[10\baselineskip]
\includegraphics[scale=0.85]{figures/02_instr.pdf}
\caption{Geometry of instrumental variables. $A$ stands for $\hat \beta_{IV} \hat x$,
$B$ — $\hat x$, $C$ — $x$, $D$ — $\hat \beta_{IV} x$.}
\label{fig:instrumental}
\end{marginfigure}

The equivalence of these two methods holds due to the similarity of triangles.
Consider a plane $\alpha$ which satisfies the property of being perpendicular
to the span of $z$, $\alpha \perp z$.
Vectors $x$, $z$ and $\hat x$ form a triangle which is denoted as $\bigtriangleup OBC$
where $\overrightarrow{OB} = \hat{x}$, $\overrightarrow{OC} = x$.
In order to get an oblique projection of $y$ onto $x$ we could either
project $y$ directly onto $x$ staying in the plane $\alpha$
or get the same result in two steps. First, project $y$ onto $z$
and then project the result onto $x$ which gives the same outcome
by the theorem of three perpendiculars. Thus, we get another triangle $\bigtriangleup OAD$
where $\overrightarrow{OA} = \hat \beta_{IV} \hat x$, $\overrightarrow{OD} = \hat \gamma x$.
Since triangles $\bigtriangleup OBC$ and $\bigtriangleup OAD$ are similar
it follows that
\[
\frac{OD}{OC} = \frac{OA}{OB}
\]
which means that
\[
\hat \gamma = \hat \beta_{IV}.
\]

\subsection{Geometry of proxy variables}

Consider a model
\[
y = \beta_1 x + \beta_2 w + u
\]
where the error term $u$ is not correlated with the regressors.
Suppose that $w$ is an unobservable variable.
One way to deal with this problem and get a consistent estimator of $\beta_1$ is to use a proxy variable.
In order to clearly state its properties we decompose the unobservable variable
into a sum of a multiple of the proxy ($pr$) and a part that is uncorrelated
with the proxy ($\hat \nu$):
\[
\hat w = \gamma \cdot pr + \hat \nu
\]
Then the proxy variable must satisfy the following properties.
\begin{enumerate}
  \item It must be correlated with the unobservable variable $w$, $pr \not\perp w$.
  \item The error term $\hat u$ must be uncorrelated with the proxy variable, $pr \perp \hat u$.
  \item The error term $\hat \nu$ must be uncorrelated with the regressor $x$, $x \perp \hat \nu$.
\end{enumerate}

\begin{marginfigure}
\includegraphics[scale=0.85]{figures/02_proxy.pdf}
\caption{Geometry of proxy variables.}
\label{fig:proxy}
\end{marginfigure}

To get a consistent estimator of $\beta_1$ we need to regress $y$ onto $x$ and $pr$.
Consistency is illustarted in Figure~\ref{fig:proxy}.
Suppose we could get the $\hat y$ by performing the original regression of
$y$ onto $x$ and $w$. Then, $\hat y$ could be decomposed in a sum of
$\hat \beta_1 x$ and $\hat \beta_2 w$.
Notice, that $\hat y$ is both projection of $y$ onto $w$, $x$ and
onto $w$, $x$, $pr$ due to the property of $\hat \nu$ being orthogonal to $x$.
When projected onto the plane spanned by $x$ and $pr$, the $\hat \beta_1 x$
component stays the same as it is already in this plane
while $\hat \beta_2 w$ projects onto the span of $pr$ by the second property of proxy variable.


\section{Convex Optimisation}
Consider the simple problem of minimizing $f(x)$ subject to $a\le x\le b$ in the case where $f:\mathbb R\to\mathbb R$ is twice differentiable.
\begin{theorem}
    If $x^\star\notin \{a,b\}$ is an optimal solution of the problem, then $f^\prime(x^\star)=0$.
\end{theorem}
\begin{proof}
    For sufficiently small $\epsilon>0$ we can have both $x^\star-\epsilon$ and $x^\star+\epsilon$ are feasible (a fancy way of saying $(x^\star-\epsilon,x^\star+\epsilon)\subset [a,b]$), then by optimality
    $$\frac{f(x^\star)-f(x^\star-\epsilon)}{\epsilon}\le0\le\frac{f(x^\star+\epsilon)-f(x)}{\epsilon}$$
    Sending $\epsilon\to0$ gives the result.
\end{proof}
As well known, there is a partial converse to this theorem.
\begin{theorem}[Sufficient Condition for Optimality]
    Suppose $x^\star$ is feasible and $f^\prime(x^\star)=0$.
    If $f^{\prime\prime}\ge 0$ for all feasible $x$, then $x^\star$ is optimal.
\end{theorem}
\begin{proof}
    By Taylor's Theorem (used correctly!), let $x$ be a feasible solution, then
    $$f(x)=f(x^\star)+f^\prime(x^\star)(x-x^\star)+\frac{1}{2}f^{\prime\prime}(\xi)(x-x^\star)^2$$
    for some $\xi$ in between $x$ and $x^\star$.
    As $f^\prime(x^\star)=0$ and $f^{\prime\prime}(\xi)\ge 0$ by assumption, $f(x)\ge f(x^\star)$.
\end{proof}
The one-dimensional optimisation problems are widely studied and there is not much point to continue talking about it.
Let's move on to higher dimensions.
As it turns out, convexity helps a lot on problems without a functional constraint.
\subsection{General Ideas on Convexity}
Notationally, for a differentiable function $f:\mathbb R^n\to\mathbb R$, we write $Df$ as its gradient and, when $f$ is twice differentiable, $D^2f$ as its Hessian.
Recall that the Hessian is symmetric at $x$ when all the second order partial derivatives are continuous at $x$.
\begin{definition}
    A set $X\subset\mathbb R^n$ is convex if for every pair of points $x,y\in X$ and number $0<p<1$ we have $px+(1-p)y\in X$.\\
    Loosely speaking, the line segment joining $x$ and $y$ is still in $X$.
\end{definition}
One should note that our solution to the previous one dimensional optimisation problem somehow depends on the convexness of the contraint set.
\footnote{Of course, even if it is not, we can sometimes produce similar results as well}
\begin{definition}
    Let $X\subset\mathbb R^n$ be convex.
    A function $f:X\to\mathbb R$ is convex if for every pair of points $x,y\in X$ and $p\in (0,1)$,
    $$f(px+(1-p)y)\le pf(x)+(1-p)f(y)$$
\end{definition}
Respectively, a function is concave if its negative is convex.
This definition is intuitive in one dimension, but might not be that easy to illustrate in higher dimensions on first sight.
We therefore need the following reformulation of the idea.
\begin{theorem}[Supporting Hyperplane]\label{hyperplane}
    Let $X\subset\mathbb R^n$ be convex.
    The function $f:X\to\mathbb R$ is convex iff for every $x\in X$ there exists a vector $\lambda(x)\in\mathbb R^n$ such that
    $$f(y)-f(x)\ge\lambda(x)^\top(y-x)$$
    for all $y\in X$.
\end{theorem}
\begin{remark}
    If $f$ is also differentiable, then we can take $\lambda(x)=Df(x)$.
\end{remark}
One can picture it as if the hyperplane produced by $\lambda(x)$ touches and supports the high-dimensional object that is the graph of the function.
This is the origin of the name.\\
We will prove the case when $f$ is differentiable, but the general case is also true.
\begin{proof}
    Almost immediate but let's write this out.
    First suppose $\lambda(x)$ exists for all $x$.
    Fix $y,z\in X$ and $p\in (0,1)$ and let $x=py+(1-p)z$, then
    $$f(y)-f(x)\ge\lambda(x)^\top(y-x),f(z)-f(x)\ge\lambda(x)^\top(z-x)$$
    So
    $$pf(y)+(1-p)f(z)\ge\lambda(x)^\top(py+(1-p)z-x)+f(x)=f(x)$$
    Conversely, if $f$ is convex, we can just take $\lambda(x)=Df(x)$.
\end{proof}
Hence, for the problem of minimising $f(x)$ subject to $x\in X$ where $X$ is convex and $f:X\to\mathbb R$ is differentiable, we have
\begin{theorem}
    Suppose $x^\star$ is feasible and $Df(x^\star)=0$.
    If $f$ is convex, then $x^\star$ is optimal.
\end{theorem}
\begin{proof}
    Suppose $x$ is feasible, then by convexity of $f$,
    $$f(x)-f(x^\star)\ge(x-x^\star)^\top Df(x^\star)=0$$
    which is what we wanted.
\end{proof}
\begin{definition}
    A symmetric $n\times n$ matrix $A$ is non-negative definite if for every $x\in\mathbb R^n$ we have $x^\top Ax\ge 0$.
\end{definition}
\begin{theorem}
    Let $X\subset\mathbb R^n$ be convex and suppose $f:X\to\mathbb R^n$ is $C^2$.
    If $D^2f$ is nonnegative definite for all $x$, then $f$ is convex.
\end{theorem}
\begin{proof}
    For any $x,y\in X$ we have, by Taylor's Theorem that
    $$f(y)=f(x)+(y-x)^\top Df(x)+\frac{1}{2}(y-x)^\top D^2f(\xi)(y-x)$$
    where $\xi=px+(1-p)y$ for some $p\in(0,1)$.
    Therefore $f(y)-f(x)\ge Df(x)^\top(y-x)$, hence it is convex.
\end{proof}
\begin{definition}
    Given a convex set $X\subset\mathbb R^n$, a function $f:X\to\mathbb R$ is strictly convex if for every $x,y\in X$ where $x\neq y$ and every number $p\in (0,1)$ we have
    $$f(px+(1-p)y)<pf(x)+(1-p)f(y)$$
\end{definition}
\begin{definition}
    An $n\times n$ matrix $A$ is positive definite if $x^\top Ax>0$ for all $x\in\mathbb R^n\setminus\{0\}$.
\end{definition}
\begin{theorem}
    Suppose $f:X\to\mathbb R$ is twice differentiable.
    If $D^2f$ is positive definite over all of $X$, then $f$ is strictly convex.
\end{theorem}
\begin{proof}
    Same idea as in the ordinary convex case.
\end{proof}
\begin{theorem}[Uniqueness of Optimal Solutions]
    Suppose $x^\star$ and $y^\star$ are optimal solutions to the problem of minimising $f(x)$ subject to $x\in X$ with $X$ convex and $f$ strictly convex, then $x^\star=y^\star$.
\end{theorem}
\begin{proof}
    If $x^\star\neq y^\star$, then let $z=(x^\star+y^\star)/2$, then
    $$f(z)<\frac{1}{2}f(x^\star)+\frac{1}{2}f(y^\star)=f(x^\star)$$
    Contradiction.
\end{proof}
\begin{definition}
    A function $f:X\to\mathbb R$ is strongly convex if there exists a constant $m>0$ such that the function $x\mapsto f(x)-m\|x\|^2/2$ is convex.
\end{definition}
This defintion is certainly strange and unintuitive, but as we proceed, we will unveil the reason to introduce this notion.
\begin{theorem}
    Suppose $f$ is twice differentiable.
    Then $f$ is strongly convex if there exists $m>0$ such that for all $x\in X$ the matrix $D^2f(x)-mI$ is non-negative definite.
    Equivalently, $z^\top D^2f(x)z\ge m\|z\|^2$ for all $z\in\mathbb R^n$.
\end{theorem}
\begin{proof}
    Obvious.
\end{proof}
One can check that strongly convex functions are also strictly convex.
\begin{theorem}[Existence of an Optimal Solution]
    Suppose $X\subset\mathbb R^n$ is closed and $f$ is strongly convex, then there exists an optimal solution to the problem of minimising $f(x)$ subject to $x\in X$.
\end{theorem}
\begin{proof}
    Let $g(x)=f(x)-m\|x\|^2/2$ where $m>0$ such that $g$ is convex, then there is a vector $\lambda=\lambda(0)$ such that
    $$g(x)\ge g(0)+\lambda^\top x\ge g(0)-\|\lambda\|\|x\|$$
    by Cauchy-Schwartz.
    So for $\|x\|>R=2\|\lambda\|/m$ we have $f(x)>f(0)$ by rearranging.
    So our problem reduces to minimising $f(x)$ subject to $x\in X\cap B$ where $B$ is the closed ball of radius $R$.
    But $X\cap B$ is closed and bounded, hence is compact, therefore $f$ attains a minimum somewhere in it, which is the optimal sulution we want.
\end{proof}
\begin{theorem}[Gradient Lower Bound]\label{gradient_lower_bound}
    Suppose $f:X\to\mathbb R$ is differentiable and strongly convex with constant $m>0$, then
    $$\|Df(x)\|^2\ge 2m(f(x)-f(y))$$
    for any $x,y\in X$.
\end{theorem}
\begin{proof}
    From the Theorem \ref{hyperplane},
    $$f(y)-f(x)\ge(y-x)^\top Df(x)+\frac{m}{2}\|y-x\|^2$$
    However, by simply conpleting the square, for any $b,z\in\mathbb R^n$,
    $$b^\top z+\frac{m}{2}\|z\|^2\ge-\frac{\|b\|^2}{2m}$$
    Combining the inequalities give the solution.
\end{proof}
\subsection{Computing the Optimal Solution}
Our most familiar example of a convex set is $X=\mathbb R^n$.
We want to find algorithms to compute the optimal solution in this case.
Our first method of interest is called gradient descent.
Suppose $f$ is differentiable.
The rate of change of $f$ at a point $x\in\mathbb R^n$ in direction $u\in\mathbb R^n$ is
$$\lim_{t\to 0}\frac{f(x+tu)-f(x)}{t}=u^\top Df(x)$$
which attains minimum when $u$ is in the direction $-Df(x)$ by Cauchy-Schwartz.\\
So our algorithm will be as follows:
Start with an initial guess $x_0\in\mathbb R^n$ and pick a step suze $t>0$.
For every $k\ge 0$, we use the iteration
$$x_{k+1}=x_k-tDf(x_k)$$
to refine our guess.
Intuitively, this should get closer and closer to an optimal solution.
This might or might not be true, but for a sufficiently nice function $f$ we have the following theorem:
\begin{theorem}
    Suppose $f$ is $C^2$ and there exists positive constants $M>m>0$ such that $mI\preceq D^2f(x)\preceq MI$ for all $x\in\mathbb R^n$.
    Here, two symmetric matrices $A,B$ has $A\preceq B$ iff $x^\top Ax\le x^\top Bx$ for every $x\in\mathbb R^n$.
    The lower bound, in particular, implies the strong convexity, hence existence and uniqueness of $x^\star$.
    Then the gradient algorithm with step size $t=1/M$ has
    $$f(x_k)-f(x^\star)\le\left( 1-\frac{m}{M} \right)^k(f(x_0)-f(x^\star))$$
\end{theorem}
\begin{proof}
    Fix $x,y\in\mathbb R^n$, then by Taylor's Theorem, there is some $p\in (0,1)$ such that for $\xi=px+(1-p)y$ we have
    \begin{align*}
        f(y)&=f(x)+(y-x)^\top Df(x)+\frac{1}{2}(y-x)^\top D^2f(\xi)(y-x)\\
        &\le f(x)+(y-x)^\top Df(x)+\frac{M}{2}\|y-x\|^2
    \end{align*}
    Apply this inequality to $y=x_{k+1},x=x_k$ gives
    \begin{align*}
        f(x_{k+1})-f(x_k)&\le (x_{k+1}-x_k)^\top Df(x_k)+\frac{M}{2}\|x_{k+1}-x_k\|^2\\
        &=\left( -t+\frac{M}{2}t^2 \right)\|Df(x_k)\|^2\\
        &=-\frac{1}{2M}\|Df(x_k)\|^2\\
        &\le-\frac{m}{M}(f(x_k)-f(x^\star))
    \end{align*}
    Rearranging gives
    $$f(x_{k+1})-f(x_k)\le\left( 1-\frac{m}{M} \right)(f(x_k)-f(x^\star))$$
    Using this recursively gives the estimate we need.
\end{proof}
This is certainly a very nice method since the rate of convergence we arrive at looks quite promising.
There is, of course, a better method similar to our familiar formualtion of Newton's method of finding root (since very roughly speaking we are finding a root of the derivative) but in high dimensions.
To confuse people, we shall also call this Newton's method.\\
Suppose $f$ is $C^2$ and let $x_0$ be an initial guess for the optimiser, then by Taylor's Theorem,
$$f(x)\approx f(x_0)+(x-x_0)Df(x_0)+\frac{1}{2}(x-x_0)^\top D^2f(x_0)(x-x_0)$$
as $x\to x_0$ due to the continuity of $D^2f$.
Minimising the quadratic term on the right yields the approximation
$$x^\star\approx x_0-(D^2f(x_0))^{-1}Df(x_0)$$
Formally, we are referring to the recursion
$$x_{k+1}=x_k-(D^2f(x_0))^{-1}Df(x_0)$$
As before, we want to know if it works.
\begin{definition}
    Let $A$ be an $n\times n$ matrix.
    We define the matrix norm $\|A\|$ as
    $$\|A\|=\inf\{a\ge 0:\forall z\in\mathbb R^n,\|Az\|\le a\|z\|\}=\sup_{\|z\|=1}\|Az\|$$
    In particular, if $A$ is nonnegative definite, then $\|A\|$ is the smallest eigenvalue of $A$.
\end{definition}
\begin{theorem}
    Suppose $f$ is twice differentiable and there exists constants $m,L>0$ with $D^2f(x)\succeq mI$ (so $f$ is strongly convex) and
    $$\forall x,y\in\mathbb R^n,\|D^2f(x)-D^2f(y)\|\le L\|x-y\|$$
    (so $D^2f$ is $L$-Lipschitz), then Newton's method guarantees that
    $$f(x_k)-f(x^\star)\le \frac{2m^3}{L^2}\left(\frac{L}{2m^2}\|Df(x_0)\|\right)^{2^{k+1}}$$
\end{theorem}
Note that this convergence rate is way quicker than using gradient descent.
\begin{proof}
    Denote $\Delta x_k=x_{k+1}-x_k=-(D^2f(x_0))^{-1}Df(x_0)$, then we have
    \begin{align*}
        \|Df(x_{k+1})\|&=\|Df(x_{k+1})-Df(x_k)-D^2f(x_k)\Delta x_k\|\\
        &=\left\|\int_0^1(D^2f(x_k+t\Delta x_k)-D^2(x_k))\Delta x_k\,\mathrm dt\right\|\\
        &\le\int_0^1\|(D^2f(x_k+t\Delta x_k)-D^2(x_k))\Delta x_k\|\,\mathrm dt\\
        &\le L\|\Delta x_k\|^2\int_0^1t\,\mathrm dt\\
        &=\frac{1}{2}L\|(D^2f(x_0))^{-1}Df(x_0)\|^2\\
        &\le\frac{L}{2m^2}\|Df(x_k)\|^2
    \end{align*}
    Note here that we have used the fact that $A\succeq mI\implies \|A^{-1}\|\le 1/m$ which is quite obvious.
    Applying this inequality recursively yields
    $$\sqrt{2m(f(x_k)-f(x^\star))}\le\|Df(x_k)\|\le\frac{2m^2}{L}\left( \frac{L}{2m^2}\|Df(x_0)\| \right)^{2^k}$$
    by Theorem \ref{gradient_lower_bound}.
    Rearranging shows what we want.
\end{proof}


\subsection{Martingale property}

For a process with the Martingale property, the expected value of all future variables is the current state.

This only restricts expectations.

\(E(X_{n+1}|X_0,...,X_n)=X_n\)


%---------------------------Skew---------------------------
\section{Skew\label{s:hex-skew}}

To compute the skew, we'll need to compute normalized versions of the principal axes:
\[
\begin{array}{lcl}
\hat{X_1} &=& \frac{\vec X_1}{\normvec{ X_1 }}\\
\hat{X_2} &=& \frac{\vec X_2}{\normvec{ X_2 }}\\
\hat{X_3} &=& \frac{\vec X_3}{\normvec{ X_3 }}
\end{array}
\]
Skew measures the degree to which a pair of vectors are parallel using the dot product.
This means we have three skews to consider for a hexahedron, each of which is the
absolute value of the cosine of the angle between two principal axes:
\[
\begin{array}{lcl}
skew_{12} &=& \left| \hat{X_1} \cdot \hat{X_2} \right|\\
skew_{13} &=& \left| \hat{X_1} \cdot \hat{X_3} \right|\\
skew_{23} &=& \left| \hat{X_2} \cdot \hat{X_3} \right|.
\end{array}
\]
The metric is then the maximum of these skews
\[
  q = \max\left\{ skew_{12}, skew_{13}, skew_{23} \right\}
\]

Note that if $\normvec{X_1}$ or $\normvec{X_2}$ or $\normvec{X_3} \leq DBL\_MIN$, we set $q = DBL\_MAX$.

\hexmetrictable{skew}%
{$1$}%                                        Dimension
{$[0,0.5]$}%                                  Acceptable range
{$[0,1]$}%                                    Normal range
{$[0,DBL\_MAX]$}%                             Full range
{$0$}%                                        Cube
{Adapted from \cite{tf:89}}%                  Citation
{v\_hex\_skew}%                               Verdict function name


\subsection{One-time pad}
\label{preliminaries:sym:otp}

A one-time pad (OTP) is an symmetric encryption scheme where the keys, messages, and ciphertexts are of the same length and the key is never reused or repeated. The scheme is defined as follows~\cite{kiagias:crypto, boneh_crypto}:

\begin{itemize}
  \item Space: $\calm = \calc = \calk = \{0, 1\}^{n}$
  \item Encryption: $Enc_k(m) = k \xor m$
  \item Decryption: $Dec_k(c) = k \xor c$
\end{itemize}

The one-time pad is a \textbf{perfectly secure} scheme~\cite{kiagias:crypto, boneh_crypto}. The main drawback lies in the fact that the key must be at least the length of the original message. If a party wants to encrypt and sent 1GB video to another party, they must have already share a 1GB key. It is proven~\cite{shannon_otp} that perfect security (or perfect secrecy) can only be achieved when the key size is at least as long as the size of the message. As a consequence, perfect security schemes are impossible to use in practice.

\subsection{Stream Ciphers}
\label{preliminaries:sym:stream}

In~\ref{preliminaries:sym:otp} we saw that a symmetric scheme to be perfectly secure, keys and messages must have at least the same length. However, with weaker notion of security we can create symmetric encryption schemes where the size of the key $k$ is much shorter than the original message. Stream ciphers can encrypt or decrypt arbitrary long messages with small fixed length size keys. Instead of using a key of size $n$, a shorter seed $s$ of $l$-bits, where $l$ is much smaller than $n$, is used as the symmetric key. Then from the seed $s$ a $n$-bit key is derived and is used for encryption and decryption. The seed $s$ is expanded to $n$ by a deterministic polynomial algorithm $G$ that maps $l$-bit strings to $n$-bit strings. The algorithm $G$ is called \textbf{pseudorandom generator} (PRG) and the following condition must hold for any PRG~\cite{Katz:2014:IMC:2700550}:

\begin{itemize}
  \item Expansion: On input $s \in \{0, 1\}^{l}$ outputs $k \in \{0, 1\}^{n}$ where $\forall l: n > l$
  \item Pseudorandomness: Lets $\cald$ be a distribution over strings of length $n$. $\cald$ is indistinguishable from the uniform distribution over strings of length $n$. In other words, it is infeasible for any polynomial-time algorithm to tell whether it is given a string sampled according to $\cald$ or uniformly at random.
\end{itemize}

So, with the use of PRGs we can construct encryption schemes with short key sizes and long messages. A stream chipher is defined as follows:

\begin{itemize}
  \item The key generation algorithm $\calg$: Takes as input a security parameter $1^{n}$ and outputs a seed $s \in \{0, 1\}^{l}$ chosen uniformly at random.
  \item The encryption algorithm $\cale$: Takes as input a seed $s \in \{0, 1\}^{l}$ and a message $m \in \{0, 1\}^{n}$ and outputs the ciphertext $c = Enc_s(m) = G(s) \xor m$
  \item The decryption algorithm $\cald$: Takes as input a seed $s \in \{0, 1\}^{l}$ and a cipher $c \in \{0, 1\}^{n}$ and outputs the plaintext $m = Dec_s(c) = G(s) \xor c$
\end{itemize}

An important point, is that $s$ must be chosen uniformly at random and be long enough to withstand brute-force attacks.

There is a number of practical constructed stream ciphers. The most wide used are Salsa20~\cite{Salsa20}, ChaCha~\cite{chacha} and RC4.

\subsection{Diffie–Hellman key exchange}
\label{preliminaries:pub:dh}

The protocol works as follows~\cite{Katz:2014:IMC:2700550, kiagias:crypto}:

\begin{enumerate}
  \item Alice and Bob  with the use of a group generation algorithm $\calg$ agree on the description of a finite group $\G$ with input $1^{n}$. The common input for Alice and Bob is the tuple $(p, m, g)$ where $p$ is a large prime and $g$ is the generator of the finite group $\G$ of order $m$.
  \item Alice choose a random index $x_a \rselect{\Z_m}$ and computes $y_a \leftarrow{g^{x_a}}modp$. Alice sends $y_a$ to Bob.
  \item Bob choose a random index $x_b \rselect{\Z_m}$ and computes $y_b \leftarrow{g^{x_b}}modp$. Alice sends $y_b$ to Bob.
  \item Alice outputs $k = y_b^{x_a}modp = g^{{x_a}{x_b}}modp$
  \item Bob outputs $k = y_a^{x_b}modp = g^{{x_a}{x_b}}modp$
\end{enumerate}

The security of Diffie–Hellman key exchange is based on the difficulty of the discrete log problem (DLOG), which is the problem of computing $x$ given $g^{x}$ in a cyclic group $\G$. A passive adversary cannot compute the private key $k$ because she does not know $x_a$ or $x_b$. To find them she have to compute the discrete log which is assumed to be hard.

\begin{figure}[!hb]
  \centering
  \begin{tikzpicture}
    \matrix (m)[matrix of nodes, column  sep=2cm,row  sep=4mm, nodes={draw=none, anchor=center,text depth=0pt} ]{
    Alice & & Bob\\
    $x_a \rselect{\Z_m}$ & & $x_b \rselect{\Z_m}$ \\
    $y_a \leftarrow{g^{x_a}}modp$ & & $y_b \leftarrow{g^{x_b}}modp$ \\
     & $y_a$ & \\
     & $y_b$ & \\
     $k = y_b^{x_a}modp$ & & $k = y_a^{x_b}modp$ \\
    };

    \draw[shorten <=-1.5cm,shorten >=-1.5cm] (m-1-1.south east)--(m-1-1.south west);
    \draw[shorten <=-1.5cm,shorten >=-1.5cm] (m-1-3.south east)--(m-1-3.south west);
    \draw[shorten <=-1cm,shorten >=-1cm,-latex] (m-4-2.south west)--(m-4-2.south east);
    \draw[shorten <=-1cm,shorten >=-1cm,-latex] (m-5-2.south east)--(m-5-2.south west);

  \end{tikzpicture}
  \caption{Diffie–Hellman key exchange}
  \label{fig:crypto:dh}
\end{figure}

\subsection{The RSA Cryptosystem}
\label{preliminaries:pub:rsa}

The RSA cryptosystem~\cite{rsa} was developed in 1977 at MIT by Ron Rivest, Adi Shamer, and Leonard Adleman and is still the most widely used. It was the first public-key encryption scheme that could both encrypt and sign messages~\cite{kiagias:crypto}.

It works as follows~\cite{Katz:2014:IMC:2700550, kiagias:crypto}:

\begin{itemize}
  \item Key Generation:
    \begin{enumerate}
      \item Select randomly to large primes $p, q$ of length $n$ bits
      \item Compute $N = p*q$
      \item Calculate $\phi(N) = (p - 1)(q - 1)$
      \item Find $e$ such that $gcd(e, \phi(N)) = 1$
      \item Compute $d = e^{-1} mod\phi(N)$
      \item Public key is $(N, e)$ and private key is $(N, d)$
    \end{enumerate}
  \item Encryption: On input a public key $p_k = (N, e)$ and a message $m$ it computes the ciphertext $c$ as $ Enc_{p_k}(m) = m^{e}modN$
  \item Decryption: On input a private key $s_k = (N, d)$ and a ciphertext $c$ it computes the message $m$ as $ Dec_{s_k}(c) = c^{d}modN = m^{ed}modN = m$
\end{itemize}

If $p, q$ are known or obvious any interested party can compute $\phi(N)$ and therefore $d$. The RSA Cryptosystem is secure under the assumption that factorization of $N$ is believed to be hard.

The above mention protocol is not secure as the encryption function is deterministic and for the same input is produces the same output. As mention, a deterministic cryptosystem is not secure. One way to randomize the encryption function, is by appending random padding to message.

\subsection{The El Gamal Cryptosystem}
\label{preliminaries:pub:el_gamal}

The El Gamal Cryptosystem~\cite{el_gamal} is another popular and wide-used encryption scheme. The security of the system is based on the hardness of discrete log problem. It works as follows~\cite{Katz:2014:IMC:2700550, kiagias:crypto}:

\begin{itemize}
  \item Key generation:
    \begin{enumerate}
        \item Run a group generation algorithm $\calg$ to produce the description of a finite group $\G$ with input $1^{n}$. The output is the tuple $(p, m, g)$ where $p$ is a large prime and $g$ is the generator of the finite group $\G$ of order $m$.
        \item Select randomly $x \rselect{\Z_m}$
        \item Calculate $h = g^{x}modp$
        \item The public key is $((p, m, g), h)$
        \item The secret key is $x$
    \end{enumerate}
  \item Encryption: Encrypts a message $m \in \G$
    \begin{enumerate}
      \item Choose randomly $r \rselect{\Z_m}$
      \item Compute $G = g^{r}modp$
      \item Compute $M = mh^{r}modp$
      \item Return $c = (G, M)$
    \end{enumerate}
  \item Decryption: Decrypts a ciphertext $c = (G, M)$
    \begin{enumerate}
      \item Compute $m = M / G^{x} modp$
      \item Return $m$
    \end{enumerate}
\end{itemize}

Another way to express the security of El Gamal scheme is under the decisional Diffie Hellman problem (DDH) which states that the tuples $(g^a, g^b, g^c)$ and $(g^a, g^b, g^{ab})$ are indistinguishable by a probabilistic polynomial-time (PPT) adversary.

\subsection{RSA signatures}
\label{preliminaries:sign:rsa}

As describes in~\ref{preliminaries:pub:rsa} the RSA cryptosystem can be used to sign messages. The RSA digital signature scheme works as follows~\cite{Katz:2014:IMC:2700550, kiagias:crypto}:

\begin{itemize}
  \item Key Generation:
    \begin{enumerate}
      \item Select randomly to large primes $p, q$ of length $n$ bits
      \item Compute $N = p*q$
      \item Calculate $\phi(N) = (p - 1)(q - 1)$
      \item Find $e$ such that $gcd(e, \phi(N)) = 1$
      \item Compute $d = e^{-1} mod\phi(N)$
      \item Verification key is $(N, e)$ and signing key is $(N, d)$
    \end{enumerate}
  \item Sign: On input a sign key $s_k = (N, d)$ and a message $m \in \Z^{*}_{N}$ it computes the signature $\sigma$ as $ Sign_{s_k}(m) = m^{d}modN$
  \item Verify: On input a verification key $p_k = (N, e)$, a message $m \in \Z^{*}_{N}$ and a signature $\sigma \in \Z^{*}_{N}$ it outputs $1$ if and only if $m \stackrel{?}{=} Verify_{p_k}(\sigma, m) = \sigma^{e}modN$
\end{itemize}

The above signature scheme is not secure as an adversary can forge a sign, based on the public key alone, by choosing an arbitrary $\sigma \in \Z^{*}_{N}$ and compute $m = \sigma^{e}modN$. Another attack on the RSA signature scheme allows the adversary to output a forgery on any message of the adversary's choice. One proposal to protect against those attacks, that can be proven secure under certain assumptions, is by applying a cryptographic hash function $H$ to the message before sign it. The minimal requirement for the system to be secure is that $H$ must be collision-resistant.

\subsection{Diffie-Hellman key exchange}
\label{preliminaries:el_curves:dh}

The elliptic curve analogue of Diffie-Hellman key exchange protocol (ECDH) defined in~\ref{preliminaries:pub:dh} can implemented as follows~\cite{elliptic_curves}:

\begin{enumerate}
  \item Alice and Bob agree on elliptic curve $E$ over a finite field $F_p$ for a prime $p$ such that the discrete log problem is hard in $E$ over $F_p$. They also agree on a point $P$ on $E$.
  \item Alice choose a secret integer $a$ and computes $P_a = aP$. Alice sends $P_a$ to Bob
  \item Bob choose a secret integer $b$ and computes $P_b = bP$. Bob sends $P_b$ to Alice
  \item Alice computes $k = aP_b = abP$
  \item Bob computes $k = bP_a = abP$
\end{enumerate}

The security of the protocol is based on ECDLP (§~\ref{preliminaries:el_curves:key_gen}).

\subsection{The El Gamal Cryptosystem}
\label{preliminaries:el_curves:el_gamal}

The elliptic curve analogue of El Gamal cryptosystem defined in~\ref{preliminaries:pub:el_gamal} can implemented as follows~\cite{elliptic_curves_2}:

\begin{itemize}
  \item Key generation: Run elliptic curve key generation algorithm defined in~\ref{preliminaries:el_curves:key_gen} and get a key pair $(Q, d)$.
  \item Encryption: Encrypts a message $m$ represented as a point $M$ in $E$ over $\F_p$
    \begin{enumerate}
      \item Choose randomly $r \rselect{\in \Z_q}$
      \item Compute $G = rP$
      \item Compute $C = M + rQ$
      \item Return $c = (G, C)$
    \end{enumerate}
  \item Decryption: Decrypts a ciphertext $c$
    \begin{enumerate}
      \item Compute $M = C - dG$, and extract $m$ from $M$
      \item return $M$
    \end{enumerate}
\end{itemize}

An eavesdropper adversary who wants to obtain $M$ needs to compute $rQ$; given $Q$, $P$ and $G = rP$ find $r \in Z_q$. This is the elliptic curve analogue of the Diffie-Hellman problem.

\subsection{Enigma}\label{blockchain:impl:enigma}

Enigma is a decentralized computation platform with guaranteed privacy~\cite{DBLP:journals/corr/ZyskindNP15}. It operates through a peer-to-peer network enabling
different parties to store and run computation on data while keeping privacy. Enigma uses a highly optimized version of secure multi-party computation (MPC)
guaranteed by a verifiable secret-sharing scheme~\cite{DBLP:journals/corr/ZyskindNP15}. A blockchain is utilized as the controller of the network, managing access to the data and identities.
The data are stored off-chain, encrypted in a distributed database, and a modified distributed hash table (DTH), accessible through the blockchain, is used for holding only references to the data.

\subsection{zksnarks}\label{zksnarks}

The main idea for constructing zkSNARKs is firstly to transform the generic program as a quadratic equation of polynomials (QAP)~\cite{ggpr}: $p(x)q(x) = s(x)r(x)$, where the equality holds if and only if the program is computed correctly. So the prover wants to convince the verifier that the equality holds.

Succinctness is achieved by random sampling. The verifier chooses a secret evaluation point $x_0$ to reduce the problem from polynomial multiplication and equality to simple number arithmetic checks: $p(x_0)q(x_0) = s(x_0)r(x_0)$. To allow the prover to compute the evaluation of the polynomials at $x_0$, without revealing $x_0$, Homomorphic encryption is used:  $E(p(x_0))E(q(x_0)) = E(s(x_0))E(r(x_0))$.

Finally the prover obfuscates the encrypted values by multiplying with a number so that the verifier can still check the correctness of the structure without knowing the actual encrypted values: $E(k + p(x_0))E(k + q(x_0)) = E(k + s(x_0))E(k + r(x_0))$.

% Introduction

In the big data era, huge amounts of data are constantly being collected and analyzed, evidently, leading innovation and
economic growth~\cite{10.1109/SPW.2015.27}. Data origins are various including wearable and IoT devices.
While data grows exponentially, gathering and storing them imposes high costs and liabilities.

Centralized organizations, amass large quantities of personal and sensitive information on which individuals have little or no control ~\cite{10.1109/SPW.2015.27}. Furthermore, there is a growing public concern about user privacy.

However, data can be private and protected by acts, such as GDPR, making data processing impossible.

Blockchain’s distributed nature eliminates the need for a trusted central authority and enables the connection between trustless entities.
The immutability and auditability of blockchain blocks and transactions, and cryptographical guarantees provided, can enforce liability, transparency and accountability without compromising privacy and security.
In addition, blockchain can significantly reduce data transaction costs between entities and increase transaction efficiency.
Like all technology, blockchain has limitations. It is argued that blockchain is not suited for high performance transactions or as a database replacement. Due to the decentralized nature of blockchain and its necessity for a consensus mechanism the transaction rate remains quite low compared to financial services~\cite{Sompolinsky2015,Zohar:2015:BUH:2817191.2701411}.
Blockchain is not made for big data. The amount of data that blockchain can store and process is very limited so off-chain data frameworks are needed.


\section{Asymptotic Hypothesis tests}

Two hypotheses ($\Theta_0$ disjoint set from $\Theta_1$):
$\begin{cases}
		H_0: \theta \epsilon \Theta_0\\
		H_1: \theta \epsilon \Theta_1\\
\end{cases}$. Goal is to reject $H_0$ using a test statistic.\\

A test $\psi$ has \textbf{level $\alpha$} if $\alpha_{\psi}(\theta) \leq \alpha, \forall \theta \in \Theta_0.$ and \textbf{asymptotic level $\alpha$} if $\lim _{n \to \infty } P_\theta ( \psi = 1) \leq \alpha$.\\

\textbf{A hypothesis-test} has the form

\begin{align*}
\psi = \textbf{1} \lbrace  T_n \geq c \rbrace
\end{align*}
for some test statistic $T_n$ and threshold $c \in \mathbb{R}$. Threshold $c$ is usually $q_{\alpha/2}$\\
\textbf{Rejection region:}
\begin{align*}
R_{\psi} = \lbrace T_n > c \rbrace
\end{align*}
\textbf{Symmetric about zero and acceptance Region interval:}
\begin{align*}
\psi = \mathbf{1}\{  |T_n| - c > 0 \} .
\end{align*}

\textbf{Power of the test}:\\
\begin{align*}
\pi_\psi = \inf_{\theta \in \Theta_1}(1 - \beta_\psi(\theta))
\end{align*}
Where $\beta_\psi$ is the probability of making a Type2 Error and $inf$ is the maximum.\\
\textbf{Two-sided test}:
\begin{align*}
H_1&: \theta \neq \Theta_0\\
\mathbf{1}&(|T_ n| > q_{\alpha /2})
\end{align*}
\textbf{One-sided tests}:
\begin{align*}
H_1&: \theta > \Theta_0\\
\mathbf{1}&(T_ n < -q_{\alpha})
H_1&: \theta < \Theta_0\\
\mathbf{1}&(T_ n > q_{\alpha})
\end{align*}
\textbf{Type1 Error:}\\
Test rejects null hypothesis $\psi = 1$ but it is actually true $H_0 = TRUE$ also known as the level of a test.\\
\textbf{Type2 Error:}\\
Test does not reject null hypothesis $\psi = 0$ but alternative hypothesis is true $H_1 = TRUE$\\
\textbf{Example:} Let $X_1, \ldots , X_ n \stackrel{i.i.d.}{\sim } \text {Ber}(p^*)$. Question: is $p^* = 1/2$.\\
$H_0: p^* = 1/2; H_1:p^* \neq 1/2$\\
If asymptotic level $\alpha$ then we need to standardize the estimated parameter $\hat{p} = \overline{X}_ n$ first.\\
\begin{align*}
T_n &= \sqrt{n}\frac{\left| \overline{X}_ n - 0.5\right|}{\sqrt{0.5(1 - 0.5)}}\\
\displaystyle \psi _{n} &= \displaystyle \mathbf{1}\left(T_n>q_{\alpha /2} \right)
\end{align*}
where $q_{\alpha /2}$ denotes the $q_{\alpha /2}$ quantile of a standard Gaussian, and $\alpha$ is determined by the required level of $\psi$. Note the absolute value in $T_n$ for this two sided test.\\
\textbf{Pivot:}\\
Let $T_n$ be a function of the random samples $X_1,\dots ,X_ n,\theta$. Let $g(T_ n)$ be a random variable whose distribution is the same for all $\theta$ . Then, $g$ is called a pivotal quantity or a pivot.\\
\textbf{Example:} let $X$ be a random variable with mean $\mu$ and variance $\sigma^2$ . Let $X_1,\dots ,X_ n$ be iid samples of $X$. Then,
\[\displaystyle  g_ n \triangleq \frac{\overline{X_ n} - \mu }{\sigma }\]
is a pivot with $\theta = \left[\mu ~ ~  \sigma ^2\right]^ T$ being the parameter vector (not the same set of paramaters that we use to define a statistical model).
\subsection{P-Value}
The (asymptotic) p-value of a test $\psi_{\alpha}$ is the smallest (asymptotic) level $\alpha$ at which $\psi_{\alpha}$ rejects $H_0$. It is random since it depends on the sample. It can also interpreted as the probability that the test-statistic $T_n$ is realized given the null hypothesis.\\

If $pvalue \leq \alpha$ , $H_0$ is rejected by $\psi_{\alpha}$ at the (asymptotic) level $\alpha$\\

The smaller the p-value, the more confidently one can reject $H_0$.\\
\textbf{Left-tailed p-values:}
\begin{align*}
pvalue&=\mathbb{P}(X\leq x|H_0)\\
&=\mathbf{P}( Z < T_{n,\theta _0}(\overline{X}_ n)))\\
&=\Phi (T_{n,\theta _0}(\overline{X}_ n))\\
&Z\sim \mathcal{N}(0,1)
\end{align*}
\textbf{Right-tailed p-values:}
\begin{align*}
	pvalue&=\mathbb{P}(X\geq x|H_0)
\end{align*}
\textbf{Two-sided p-values:}
If asymptotic, create normalized $T_n$ using parameters from $H_0$. Then use $T_n$ to get to probabilities.
\begin{align*}
&pvalue=2 min\{\mathbb{P}(X\leq x|H_0),\mathbb{P}(X\geq x|H_0)\}\\
&\mathbb{P}(\lvert Z\rvert > \lvert T_{n,\theta _0}(\overline{X}_ n)\rvert = 2(1-\Phi(T_n))\\
&Z \sim N(0,1)
\end{align*}
\subsection{Comparisons of two proportions}

Let $X_1,\dots ,X_ n \stackrel{iid}{\sim} Bern(p_x)$ and  $Y_1,\dots ,Y_ n \stackrel{iid}{\sim} Bern(p_y)$ and be $X$ independent of $Y$. $\hat{p}_x= 1/n \sum_{i=1}^{n} X_i$ and $\hat{p}_x= 1/n \sum_{i=1}^{n} Y_i$\\

$H_0: p_x = p_y; H_1: p_x \neq p_y$

To get the asymptotic Variance use multivariate Delta-method. Consider $\hat{p}_x - \hat{p}_y = g(\hat{p}_x,\hat{p}_y); g(x,y)= x -y$, then

$\sqrt(n) (g(\hat{p}_x,\hat{p}_y) - g(p_x-p_y)) \xrightarrow[n \rightarrow \infty]{(d)} N(0,\nabla g(p_x-p_y)^T \Sigma \nabla g(p_x-p_y))$\\

$\Rightarrow N(0,p_x(1-px) + p_y(1-py))$

\section{Non-asymptotic Hypothesis tests}

\subsection{Chi squared}
The $\chi _ d^2$ distribution with $d$ degrees of freedom is given by the distribution of $Z_1^2 + Z_2^2 + \cdots + Z_ d^2,$ where $Z_1, \ldots , Z_ d \stackrel{iid}{\sim } \mathcal{N}(0,1)$

If $V \sim \chi^2_k:$\\

$\mathbb{E}= \mathbb{E}[Z_1^2] + \mathbb{E}[Z_2^2] + \ldots + \mathbb{E}[Z_d^2] = d$\\

$Var(V) = Var(Z_1^2) + Var(Z_2^2) + \ldots + Var(Z_d^2) = 2d$

\textbf{Cochranes Theorem:}\\
If $X_1, ..., X_n \stackrel{iid}{\sim} N(\mu,\sigma^2)$, then sample mean $\bar{X}_n$ and the sample variance $S_n$ are independent. The sum of squares of $n$ variables follows a chi squared distribution with (n-1) degrees of freedom:
\begin{align*}
\frac{n S_ n}{\sigma ^2} \sim \chi _{n -1}^2
\end{align*}
If formula for unbiased sample variance is used:\\
\begin{align*}
\frac{(n-1) S_ n}{\sigma ^2} \sim \chi _{n -1}^2
\end{align*}
\subsection{Student's T Test}
Non-asymptotic hypothesis test for small samples (works on large samples too), data must be gaussian.\\

\textbf{Student's T distribution} with $d$ degrees of freedom:
$t_d := \frac{Z}{\sqrt{V/n}}$ where $Z \sim \mathcal{N}(0,1)$ and $V \sim \chi^2_k$ are independent.\\

\textbf{Student's T test (one sample + two-sided):}\\

Let $X_1, ..., X_n \stackrel{iid}{\sim} N(\mu,\sigma^2)$ and suppose we want to test $H_0: \mu = \mu_0 = 0$ vs. $H_1: \mu \neq 0$.\\

Test statistic follows Student's T distribution:

\begin{align*}
T_n &= \frac{Z}{\tilde{S}} \\
&= \frac{\bar{X}-\mu}{\frac{\hat{\sigma}}{\sqrt{n}}}\\
&=\frac{\displaystyle \sqrt{n}\frac{\bar{X}_n - \mu_0}{\sigma}}{\displaystyle \sqrt{\frac{\tilde{S}_n}{\sigma^2}}}\\
&\sim \frac{N(0,1)}{\sqrt{\frac{\chi^2_{n-1}}{n-1}}}\\
&\sim t_{n-1}
\end{align*}

Works bc. under $H_0$ the numerator $N(0,1)$ and the denominator $\frac{\tilde{S}_n}{\sigma^2} \sim \frac1{n-1}\chi^2_{n-1}$ are independent by Cochran's Theorem.\\

Student's T test at level $\alpha$:
\begin{align*}
\psi_\alpha = \textbf{1}\{|T_n| > q_{\alpha/2}(t_{n-1})\}
\end{align*}

\textbf{Student's T test (one sample, one-sided):}
\begin{align*}
\psi_\alpha = \textbf{1}\{T_n > q_\alpha(t_{n-1})\}
\end{align*}

\textbf{Student's T test (two samples, two-sided):}

Let $X_1, ..., X_n \stackrel{iid}{\sim} N(\mu_X,\sigma^2_X)$ and $Y_1, ..., Y_n \stackrel{iid}{\sim} N(\mu_Y,\sigma^2_Y)$, suppose we want to test $H_0: \mu_X = \mu_Y$ vs $H_1: \mu_X \neq \mu_Y$.

\begin{align*}
T_{n, m} = \frac{\bar{X}_n - \bar{Y}_m}{\displaystyle \sqrt{\frac{\hat{\sigma^2}_X}n + \frac{\hat{\sigma^2}_Y}m}}
\end{align*}

\textbf{Welch-Satterthwaite formula:}\\

When samples are different sizes we need to finde the Student's T distribution of: $T_{n, m} \sim t_N$\\

Calculate the degrees of freedom for $t_N$ with:
\begin{align*}
N = \frac{\displaystyle \left(\frac{\hat{\sigma^2}_X}n + \frac{\hat{\sigma^2}_Y}m\right)^2}{\displaystyle \frac{\hat{\sigma^2}^2_X}{n^2(n-1)} + \frac{\hat{\sigma^2}^2_Y}{m^2(m-1)}} \geq \min(n, m)
\end{align*}

$N$ should be rounded down.

\subsection{Walds Test}

Squared distance of $\widehat{\theta}_ n^{MLE}$ to true $\theta_0$ using the fisher information $I(\widehat{\theta}_ n^{MLE})$ as metric.

Let $\, X_1, \ldots , X_ n \stackrel{iid}{\sim } \mathbf{P}_{\theta ^*}$ for some true parameter $\theta ^* \in \mathbb {R}^ d$ and the  maximum likelihood estimator $\widehat{\theta }_ n^{MLE}$ for $\theta ^*$.\\

Test $H_0: \displaystyle  \theta ^* = \mathbf{0}$ vs $H_1: \displaystyle  \theta ^* \neq \mathbf{0}$\\

Under $H_0$, the asymptotic normality of the MLE $\widehat{\theta }_ n^{MLE}\,$ implies that:\\

$\left\lVert \sqrt{n}\, \mathcal{I}(\mathbf{0})^{1/2}(\widehat{\theta }_ n^{MLE}- \mathbf{0}) \right\rVert ^2 \xrightarrow [n\to \infty ]{(d)} \chi ^2_ d\,$\\

\textbf{Test statistic:}
\begin{align*}
T_n = &
n(\widehat{\theta}_ n^{MLE} - \theta_0)^\top I(\widehat{\theta}_ n^{MLE}) (\widehat{\theta}_ n^{MLE} - \theta_0)\\
&\xrightarrow [n\to \infty ]{(d)} \chi ^2_ d
\end{align*}

\textbf{Wald test} of level $\alpha$:
\begin{align*}
\psi_\alpha = \mathbf{1}\{T_n > q_\alpha(\chi^2_d)\}
\end{align*}

%\textbf{Wald test with one dimensional model}
%\begin{align*}
%W&=\frac{(\widehat{\theta}_ n^{MLE} - \theta_0)^2}{Var(\widehat{\theta})}\\
%&=nI(\widehat{\theta) (\widehat{\theta }^{\text {MLE}} -\theta _0)^2
%\end{align*}

\subsection{Likelihood Ratio Test}
Parameter space $\Theta \subseteq \mathbb{R}^d$ and $H_0$ is that parameters $\theta_{r+1}$ through $\theta_d$ have values $\theta_c^{r+1}$ through $\theta^c_d$ leaving the other $r$ unspecified. That is:\\
$H_0: (\theta_{r+1}, ..., \theta_d)^T = \theta_{r+1...d} = \theta_0$\\

\textbf{Construct two estimators:}
\begin{align*}
\widehat{\theta}_n^{MLE} = argmax_{\theta \in \Theta}(\ell_n(\theta))\\
\widehat{\theta}_n^c = argmax_{\theta \in \Theta_0}(\ell_n(\theta))
\end{align*}

\textbf{Test statistic:}
\begin{align*}
T_n = 2 ( \ell(X_1,..X_n|\widehat{\theta}_n^{MLE}) - \ell(X_1,..X_n|\widehat{\theta}_n^c)))
\end{align*}
\textbf{Wilk's Theorem:} under $H_0$, if the MLE conditions are satisfied:
\begin{align*}
T_n& \xrightarrow[n \rightarrow \infty]{(d)} \chi_{d-r}^2\\
\end{align*}
\textbf{Likelihood ratio test} at level $\alpha$:
\begin{align*}
\psi_\alpha = \textbf{1}\{T_n > q_\alpha(\chi^2_{d-r})\}
\end{align*}
\subsection{Implicit Testing}
Todo
\subsection{Goodness of Fit Discrete Distributions}

Let $X_1,...,X_n$ be iid samples from a categorical distribution. Test $H_0: p = p^0$ against $H_1: p \neq p^0$. Example: against the uniform distribution $p^0 = (1/K, \ldots, 1/K)^\top$.\\

\textbf{Test statistic} under $H_0$:
\begin{align*}
T_n = n\sum_{k=1}^K\frac{(\hat{p}_k - p^0_k)^2}{p^0_k} \xrightarrow[n \rightarrow \infty]{(d)} \chi^2_{K-1}
\end{align*}
\textbf{Test at level alpha:}
\begin{align*}
\psi_\alpha = \mathbb{1}\{T_n > q_\alpha(\chi^2_{K-1})\}
\end{align*}
\subsection{Kolmogorov-Smirnov test}
\subsection{Kolmogorov-Lilliefors test}
\subsection{QQ plots}
\textbf{Heavier tails}: below > above the diagonal.\\
\textbf{Lighter tails}: above > below the diagonal.\\
\textbf{Right-skewed}: above > below > above the diagonal.\\
\textbf{Left-skewed}: below > above > below the diagonal.\\