\section{Data normalization in DNNs and CNNs}
The goal of data normalization is twofold:
\begin{enumerate}
\item Data normalization is the organization of data to appear similar across all records and fields.
\item It increases the cohesion of entry types leading to cleansing, lead generation, segmentation, and higher quality data.
\end{enumerate}
Data normalization includes eliminating unstructured data and redundancy (duplicates) in order to ensure logical data storage. When data normalization is done correctly, you will end up with standardized information entry.

In this chapter, we use superscript $i$, $j$, $l$ and $\nu$ to represent the $i$-th data, the $j$-th channel, the $l$-th depth and the $\nu$-th iteration step, respectively.

\subsection{Data normalization in DNNs}
Consider that we have the all training data as
\begin{equation}\label{eq:trainingdata}
(X,Y) := \{(x^i, y^i)\}_{i=1}^N,
\end{equation}
for $x^i \in \mathbb{R}^d$ and $y^i \in \mathbb{R}^k$. Here superscripte $i$ represents the $i$-th data.

Before we input every data into a DNN model, we will apply the following normalization
for all data $x_i$ for each component. The notation $x \sim X$ means that $x$ is a discrete random variable on $X$
with probability
\begin{equation}\label{key}
\mathbb P( x = x^i ) = \frac{1}{N},
\end{equation}
for any $x^i \in X$. The expectation and the variance are
\begin{equation}\label{key}
\mu_X =\mathbb{E}_{x \sim X}[x] = \frac{1}{N}\sum_{i=1}^N x^i,
\quad  \sigma_X= \mathbb{V}_{x\sim X}[x] = \frac{1}{N} \sum_{i=1}^N ( x^i - \mu_X)^T( x^i - \mu_X).
\end{equation}
Define new data
\begin{equation}\label{eq:normlizationData}
\tilde x^i = \frac{x^i - \mu_X }{\sqrt{\sigma_X}},
\end{equation}
where
$
x^i , \tilde x^i , \mu_X, \sigma_X \in \mathbb{R}^d.
$
Finally, we will have a ``new'' data set
\begin{equation}\label{key}
\tilde X = \{\tilde x^1, \tilde x^2, \cdots, \tilde x^N \},
\end{equation}
with unchanged label set $Y$.
To be specific, we rewrite the equation \eqref{eq:normlizationData} into an elementwise form.
Let denote
\begin{equation}\label{key}
x^{i,j} \longleftrightarrow \text{ the j-th component of data } x^i.
\end{equation}
Then we have following formula of for all $j = 1, 2, \cdots, d$
\begin{equation}\label{key}
\tilde x^{i,j}  = \frac{x^{i,j} - \mu_X^j }{\sqrt{\sigma_X^j}},
\end{equation}
where $\mu_X=(\mu_X^1,\mu_X^2,\cdots, \mu_X^d)\in \mathbb{R}^d$, $\sigma_X=(\sigma_X^1,\sigma_X^2,\cdots, \sigma_X^d)\in \mathbb{R}^d$ and
\begin{equation}\label{key}
\mu_X^j =\mathbb{E}_{x \sim X}[x^j] = \frac{1}{N}\sum_{i=1}^N x^{i,j},
\quad  \sigma_X^j = \mathbb{V}_{x\sim X}[x^j] = \frac{1}{N} \sum_{i=1}^N ( x^{i,j} - \mu_X^j)^2.
\end{equation}
Here we note that, by normalizing the data set, we have the next properties
for new data $\tilde x=(\tilde x^1, \tilde x^2, \cdots, \tilde x^N) \in \tilde X$ with component $j = 1,2,\cdots,d$,
\begin{equation}\label{key}
\mathbb{E}_{\tilde X}[\tilde x] = \frac{1}{N} \sum_{i=1}^N \tilde x^i = 0,
\end{equation}
and
\begin{equation}\label{key}
\mathbb{V}_{\tilde X}[\tilde x] = \frac{1}{N} \sum_{i=1}^N (\tilde x^i - \mathbb{E}_{\tilde X}[\tilde x] )^T (\tilde x^i - \mathbb{E}_{\tilde X}[\tilde x] )= 1.
%\begin{pmatrix}
%1\\1\\\vdots\\1
%\end{pmatrix} \in \mathbb{R}^d.
\end{equation}
%\begin{equation}\label{key}
%\mathbb{E}_{\tilde X}[[\tilde x]_j] = \frac{1}{N} \sum_{i=1}^N [\tilde x_i]_j = 0,
%%\begin{pmatrix}
%%0\\0\\\vdots\\0
%%\end{pmatrix} \in \mathbb{R}^d,
%\end{equation}
%and
%\begin{equation}\label{key}
%\mathbb{V}_{\tilde X}[[\tilde x]_j] = \frac{1}{N} \sum_{i=1}^N ([\tilde x_i]_j - \mathbb{E}_{\tilde X}[[\tilde x]_j] )^2 = 1.
%%\begin{pmatrix}
%%1\\1\\\vdots\\1
%%\end{pmatrix} \in \mathbb{R}^d.
%\end{equation}

For the next sections in this chapter, without special notices, we use $X$ data set
as the normalized one as default.

\subsection{Data normalization for images in CNNs}
For images, we have a color image data set $(X,Y) := \{(x^i, y^i)\}_{i=1}^N$.
%where
%\begin{equation}\label{key}
%x^i \in \mathbb{R}^{3 \times m\times n}.
%\end{equation}
We further denote these the $(s,t)$ pixel value for data $x_i$ at channel $j$ as:
\begin{equation}\label{key}
x^{i,j}_{s,t} \longleftrightarrow (s,t) \text{ pixel value for } x^i \text{ at channel } j,
\end{equation}
where $1\le i \le N, 1\le j\le 3, 1\le s \le m$, and  $1\le t\le n$.

Then, the normalization for $x^i$ is defined by
\begin{equation}\label{key}
\tilde x^{i,j}_{s,t} = \frac{x^{i,j}_{s,t} - \mu_X^j }{\sqrt{\sigma_X^j}},
\end{equation}
where %$\bm 1 \in \mathbb{R}^{m\times n}$ with all elements equal to $1$ and
$
x^{i,j}_{s,t}, \tilde x^{i,j}_{s,t}, \mu_X^j, \sigma_X^j \in \mathbb{R}.
$
This means that we need to normalize the data for each channel. Here
\begin{equation}\label{key}
\mu_X^j = \frac{1}{m\times n\times N} \sum_{ 1 \le i \le N} \sum_{1\le s \le m, 1 \le t \le n} x^{i,j}_{s,t}.
\end{equation}
There are two common choices of $\sigma_X^j$:
\begin{enumerate}
\item Batch normalization also uses the
formula below to compute the variance in CNN for each channel.
\begin{equation}\label{key}
\sigma_X^j = \frac{1}{ N \times m\times n} \sum_{ 1 \le i \le N} \sum_{1\le s \le m, 1 \le t \le n} (x^{i,j}_{s,t} -\mu_X^j )^2.
\end{equation}
% we confirmed with Lian by both numerical test and code checking that BN also use the above
%formula to compute the variance in CNN for each channel.
\item
Another way to compute the variance over each channel is to compute the standard deviation on each channel for every data,
and then average them in the data direction.
\begin{equation}\label{key}
\sqrt{\tilde \sigma_X^j} = \frac{1}{ N} \sum_{ 1 \le i \le N}  \left( \frac{1}{m\times n}\sum_{1\le s \le m, 1 \le t \le n} (x^{i,j}_{s,t} - \mu_i^j )^2 \right)^{\frac{1}{2}},
\end{equation}
where
$\displaystyle
\mu_i^j  = \frac{1}{m\times n} \sum_{1\le s \le m, 1 \le t \le n} x^{i,j}_{s,t}.
$
\end{enumerate}

\begin{example}[Comparison of $\sqrt{[\sigma_X]_j}$ and  $\sqrt{[\tilde \sigma_X]_j}$ on CIFAR10.]

On CIFAR10, both ways share the same $\mu_X$ as
\begin{equation}\label{key}
\mu_X = \begin{pmatrix}
0.49140105 & 0.48215663 & 0.44653168
\end{pmatrix}.
\end{equation}
But they had different standard deviation estimates:
\begin{equation}
\begin{aligned}
\sqrt{\sigma_X^j} &= \begin{pmatrix}
0.24703284 & 0.24348499 & 0.26158834
\end{pmatrix} \\
%\sqrt{[\bar \sigma_X]_j} &= \begin{pmatrix}
%0.12835675 & 0.12578563 & 0.1533168
%\end{pmatrix} \\
\sqrt{\tilde \sigma_X^j} &= \begin{pmatrix}
0.20220193 & 0.19931635 & 0.20086373
\end{pmatrix}
\end{aligned}
\end{equation}
This numerical shows that the two normalization are different.
\end{example}

\endinput

\subsubsection{Data normalization for images}
For images, consider we have a color image data set $(X,Y) := \{(x_i, y_i)\}_{i=1}^N$ where
\begin{equation}\label{key}
x_i \in \mathbb{R}^{3 \times m\times n}.
\end{equation}
We further denote these the $(s,t)$ pixel value for data $x_i$ at channel $j$ as:
\begin{equation}\label{key}
[x_i]_{j;st} \longleftrightarrow (s,t) \text{ pixel value for } x_i \text{ at channel } j,
\end{equation}
where $1\le i \le N, 1\le j\le 3, 1\le s \le m$, and  $1\le j\le n$.

Then, the normalization for $x_i$ is defined by
\begin{equation}\label{key}
[\tilde x_i]_{{j;st}} = \frac{[x_i]_{{j;st}} - [\mu_X]_j }{\sqrt{[\sigma_X]_j}},
\end{equation}
where %$\bm 1 \in \mathbb{R}^{m\times n}$ with all elements equal to $1$ and
\begin{equation}\label{key}
[x_i]_{{j;st}}, [\tilde x_i]_{{j;st}}, [\mu_X]_j, [\sigma_X]_j \in \mathbb{R}.
\end{equation}
Here
\begin{equation}\label{key}
[\mu_X]_j = \frac{1}{m\times n\times N} \sum_{ 1 \le i \le N} \sum_{1\le s \le m, 1 \le t \le n} [x_i]_{j;st}.
\end{equation}
and
\begin{equation}\label{key}
[\sigma_X]_j = \frac{1}{ N \times m\times n} \sum_{ 1 \le i \le N} \sum_{1\le s \le m, 1 \le t \le n} ([x_i]_{j;st} -[\mu_X]_j )^2.
\end{equation}
In batch normalization, they also adopt this form.

\subsection{Batch normalization for CNN}
For simplicity, then have the following BN scheme for CNN
\begin{equation}\label{def:BNeq-traningCNN}
\begin{aligned}
[\mu^\ell_{\mathcal B_t}]_{j} & \leftarrow \frac{1}{m \times m_\ell \times n_\ell }\sum_{i=1}^m \sum_{1\le s\le m_\ell, 1\le t \le n_\ell} [f^\ell(x_i)]_{j;st}  &&\text{ mean on channel }j \\
[\sigma^\ell_{\mathcal B_t}]_{j}^2 & \leftarrow \frac{1}{m \times m_\ell \times n_\ell }\sum_{i=1}^m \sum_{1\le s\le m_\ell, 1\le t \le n_\ell} ([f^\ell(x_i)]_{j;st}-[\mu_{\mathcal B_t}]_j)^2   &&\text{ variance on channel }j\\
[\hat f^\ell (x)]_{j;st} & \leftarrow \frac{[f^\ell(x)]_{j,st}-[\mu^\ell_{\mathcal B_t}]_j}{\sqrt{[\sigma^\ell_{\mathcal B_t}]_j^2+\epsilon}}   &&\text{ normalize }\\
[{\rm BN}_{\mathcal B_t}(\tilde f^{\ell})]_{j;st} &:= [\tilde f^\ell(x)]_{j;st}  \leftarrow [\gamma^\ell]_j [\hat f^\ell (x)]_{j;st} + [\beta^\ell]_{j}
&&\text{ scale and shift on channel}
\end{aligned}
\end{equation}

\subsection{Different methods in computing ``standard deviation'' in one channel}
Now basically, we have the following three different ways to
compute the $\sqrt{[\sigma_X]_j}$, different ways correspond to different
assumption for data distribution in one channel.

\subsubsection{1: Whiting via covariance matrix}
For simplicity, we assume data set $ X = \{ x_i~:~ i = 1:N\} \subset \mathbb{R}^d$
sampled from a random vector $x$, then the whiting processing for these data set is
\begin{equation}\label{key}
\tilde x_i = {\rm Cov}^{-\frac{1}{2}}(X)(x_i - \mu),
\end{equation}
where
\begin{equation}\label{key}
\mu = \frac{1}{N} \sum_{i} x_i \approx \mathbb{E}[x] \in \mathbb{R}^d,
\end{equation}
and
\begin{equation}\label{key}
{\rm Cov}(X) = \frac{1}{N} \sum_{i} (x_i - \mu) (x_i - \mu)^T \approx \mathbb{E}[(x-\mathbb{E}[x])(x-\mathbb{E}[x])^T]\in \mathbb{R}^{d\times d}.
\end{equation}
This processing makes
\begin{equation}\label{key}
\tilde \mu = \frac{1}{N} \sum_{i} \tilde x_i = 0 \in \mathbb{R}^d,
\end{equation}
and
\begin{equation}\label{key}
{\rm Cov}(\tilde X) = \frac{1}{N} \sum_{i} (\tilde x_i - \tilde \mu) (\tilde x_i - \tilde \mu)^T = I \in \mathbb{R}^{d\times d}.
\end{equation}

\subsection{2: Normalizing via diagonal matrix}
%If we make the assumption that all components of the random vector $x$ are independent.
%Then, this means that
%\begin{equation}\label{key}
% \mathbb{E}[(x-\mathbb{E}[x])(x-\mathbb{E}[x])^T] \text{ is a diagonal matrix. }
%\end{equation}\\
Here we just want the result that each marginal distribution for $x$ share the same variance, that
is
\begin{equation}\label{key}
\mathbb{V}[[x]_i] = \mathbb{V}[[x]_j].
\end{equation}

This leads us the following element-wise version of normalization:
\begin{equation}\label{norm-D}
\tilde x_i = D^{-\frac{1}{2}}{(x_i - \mu)},
\end{equation}
where
\begin{equation}\label{key}
\mu = \frac{1}{N} \sum_{i} x_i \approx \mathbb{E}[x] \in \mathbb{R}^d,
\end{equation}
and
\begin{equation}\label{key}
D = \frac{1}{N} {\rm Diag}(\sum_{i} (x_i - \mu) \odot (x_i - \mu) ).
\end{equation}

This leads to
\begin{equation}\label{key}
\tilde \mu = \frac{1}{N} \sum_{i} \tilde x_i = 0 \in \mathbb{R}^d,
\end{equation}
and
\begin{equation}\label{key}
 \frac{1}{N} \sum_{i} (\tilde x_i - \tilde \mu) \odot (\tilde x_i - \tilde \mu) = (1,1,\cdots,1)^T,
\end{equation}
which means that $[\tilde x]_i$ and $[\tilde x]_j$ share the same variance as two random variable.

\blue{This normalization is already OK for the assumption of weight initialization in Xaviar's analysis.}

Thus, we may understand all other method as an approximated or modified version for the \eqref{norm-D}.

\subsection{Version 1:  keep $\mu \in \mathbb{R}^d$ but $D$ as a scaler by scaler variance.}
\begin{equation}\label{norm-D}
\tilde x_i = D^{-\frac{1}{2}}{(x_i - \mu)},
\end{equation}
here
\begin{equation}\label{key}
\mu = \frac{1}{N} \sum_{i} x_i \approx \mathbb{E}[x] \in \mathbb{R}^d,
\end{equation}
and
\begin{equation}\label{key}
D = \frac{1}{N}  \sum_{i} \|x_i - \mu\| = \frac{1}{N} \sum_{i} ( \sum_{j} \frac{1}{d}([x_i]_j - [\mu]_j) )^{\frac{1}{2}}.
\end{equation}

\subsubsection{Version 2: let $\mu$  and $D$ both scaler by averaging over all components of $x$.}
In continuous, for mean
\begin{equation}\label{key}
\mu = \frac{1}{d} \sum_t \mathbb{E}[[x]_t],
\end{equation}
this leads to the statistics
\begin{equation}\label{key}
\frac{1}{N \times d} \sum_{i} \sum_{t} [x_i]_t.
\end{equation}
For ``variance'',
\begin{equation}\label{key}
\sigma =  \frac{1}{d} \sum_t \mathbb{E}[([x]_t - \mathbb{E}[[x]_t])^2],
\end{equation}
then this leads to
\begin{equation}\label{key}
\frac{1}{N \times d} \sum_{i} \sum_{t} ([x_i]_t - \mathbb{E}[[x_i]_t])^2.
\end{equation}
This is different from what we did in BN.

However, if we further take $\mathbb{E}[[x_i]_t] = $ some scaler, then it is natural to take
\begin{equation}\label{key}
\mathbb{E}[[x_i]_t] = \mathbb{E}[[x_i]_s] = \frac{1}{N \times d} \sum_{i} \sum_{t} [x_i]_t.
\end{equation}
Then for multi-channel image data, we have:
\begin{equation}\label{key}
[\sigma_X]_j = \frac{1}{ N \times m\times n} \sum_{ 1 \le i \le N} \sum_{1\le s \le m, 1 \le t \le n} ([x_i]_{j;st} -[\mu_X]_j )^2.
\end{equation}
This scheme is adopted in BN.

\subsubsection{Version 3:   let $\mu$  scale but compute $D^{\frac{1}{2}}$ directly}
In continuous, for mean
\begin{equation}\label{key}
\mu = \frac{1}{d} \sum_t \mathbb{E}[[x]_t],
\end{equation}
this leads to the statistics
\begin{equation}\label{key}
\frac{1}{N \times d} \sum_{i} \sum_{t} [x_i]_t.
\end{equation}
This is the same with before.

For standard deviation,
\begin{equation}\label{key}
\sigma^{\frac{1}{2}} =  \mathbb{E} (\frac{1}{d} \sum_t [([x]_t - \mathbb{E}[[x]_t])^2])^{\frac{1}{2}},
\end{equation}
then this leads to
\begin{equation}\label{key}
\frac{1}{N \times d} \sum_{i} \sum_{t} ([x_i]_t - \mathbb{E}[[x_i]_t])^2.
\end{equation}
Then for CNN, we got
\begin{equation}\label{key}
\sqrt{[\tilde \sigma_X]_j} = \frac{1}{ N} \sum_{ 1 \le i \le N} \sqrt{[\tilde \sigma_{X_i}]_j}  =\frac{1}{ N} \sum_{ 1 \le i \le N}  \left( \frac{1}{m\times n}\sum_{1\le s \le m, 1 \le t \le n} ([x_i]_{j;st} -[\mu_{X_i}]_j )^2 \right)^{\frac{1}{2}}.
\end{equation}
The most commonly used standard deviation for CIFAR10 in Pytorch adopts this way.

\subsection{Let $ [\bar x_i]_j  = \frac{1}{m\times n} \sum_{s,t} [x_i]_{j;st} \sim \bar D$ on $\mathbb{R}$ for all $i$ index.}
Then, this leads to the following scheme for $\sqrt{[\sigma_X]_j}$
\begin{equation}
[\bar \sigma_X]_j = \frac{1}{ N } \sum_{ 1 \le i \le N} ([\bar x_i]_{j} -[\mu_X]_j )^2,
\end{equation}
where $[\bar x_i]_j  = \frac{1}{m\times n} \sum_{s,t} [x_i]_{j;st}$.

This formula seems better in mathematics.

However, how to understand this from the normalization viewpoint?

\subsection{Comparison of $\sqrt{[\sigma_X]_j}, \sqrt{[\bar \sigma_X]_j}$ and  $\sqrt{[\tilde \sigma_X]_j}$ on CIFAR10.}

They share the same $\mu_X$ as
\begin{equation}\label{key}
\mu_X = \begin{pmatrix}
0.49140105 & 0.48215663 & 0.44653168
\end{pmatrix}.
\end{equation}
But they had different standard deviation estimates:
\begin{equation}
\begin{aligned}
\sqrt{[\sigma_X]_j} &= \begin{pmatrix}
0.24703284 & 0.24348499 & 0.26158834
\end{pmatrix} \\
\sqrt{[\bar \sigma_X]_j} &= \begin{pmatrix}
0.12835675 & 0.12578563 & 0.1533168
\end{pmatrix} \\
\sqrt{[\tilde \sigma_X]_j} &= \begin{pmatrix}
0.20220193 & 0.19931635 & 0.20086373
\end{pmatrix}
\end{aligned}
\end{equation}

I am very strange about this, because we could also do like the following:
\begin{equation}\label{key}
\sqrt{[\sigma_X]_j} = \frac{1}{ m\times n} \sum_{1\le s \le m, 1 \le t \le n} \left( \frac{1}{m} \sum_{ 1 \le i \le N}  ([x_i]_{j;st} -[\mu_X]_j )^2 \right)^{\frac{1}{2}}.
\end{equation}


\section{Spectrum of Autoregressive Processes}
\subsection{Shortcomings of unbiased ACF}
Based on the Yule-Walker equations, the autoregressive parameters can be obtained by the inversion of autocorrelation matrix, as expressed in Eq.\ref{proof:ar}.
\begin{equation}
\mathbf{r}_{xx}=\mathbf R_{xx}\mathbf a \quad \Rightarrow \quad \mathbf a=\mathbf R^{-1}_{xx}\mathbf {r}_{xx}
\label{proof:ar}
\end{equation}
Therefore, the ACF matrix $\mathbf {R}_{xx}$ is positive semi-definite for biased estimator which is invertible. As to unbiased ACF shown in previous section, the autocorrelation matrix may be singular which can not be inverted.
\subsection{AR Modelling with few samples}
Given the autoregressive parameters $\mathbf a$ =[2.76 -3.81 2.65 -0.92] and the white noise power $\sigma^2=1$, the estimation of AR modelling process with the order $p$ from 2 to 14 is shown in Fig.\ref{fig:1_4_b}. The AR model has a frustrated performance with low order which only detects one peak. With the order increasing, the estimation is approaching to the actual frequency response. However, the AR(4) model perform poorly although it consists with the true order of filter. This is probably caused by the small number of available samples with $N=500$. Therefore, in order to obtain desirable response, either the order of AR model or the number of sample should increase.
\begin{figure}[htbp]
     \centering
     \begin{subfigure}[b]{0.35\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/14/14b1.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.35\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/14/14b2.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.35\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/14/14b3.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.35\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/14/14b4.eps}
     \end{subfigure}
        \caption{AR Model with varying order}
        \label{fig:1_4_b}
\end{figure}
\subsection{AR modelling with increasing samples}
Repeat the process in previous section except applying $10k$ samples to model. When the AR order $p < p_{actrual =4}$, the autoregressive parameters can not be estimated. As shown in Fig.\ref{fig:1_4_c}, the AR(2) model only captures one peak frequency, which is an under-modelling estimation. However, the AR model matches the actual response with two peaks when the order $p$ is larger than 4. Moreover, increasing order ($p>5$) brought less improvement between actual response and estimation. Therefore, the optimal order for the large number of sample is $4^{th}$.
\begin{figure}[htbp]
     \centering
     \begin{subfigure}[b]{0.35\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/14/14c1.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.35\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/14/14c2.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.35\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/14/14c3.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.35\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/14/14c4.eps}
     \end{subfigure}
        \caption{AR Modelling with $10k$ samples}
        \label{fig:1_4_c}
\end{figure}


\section{Useful Mathematical Information}
Here are some helpful mathematical notes.
I also put more mathematical information throughout this document in various section that is not contained here, so be sure to study that information as well.

\subsection{Numerical Data}
\subsubsection{Mathematical Data}
\(\pi\approx3.1\)\\*
\(e\approx2.7\)\\*
\(\mathrm{ln}2\approx.7\)\\*
\(\sqrt{2}\approx1.4\)\\*
\(\sqrt{3}\approx1.7\)\\*
\(\sqrt{10}\approx\pi\)\\\\*
\(\sin(30^{\circ})=\cos(60^{\circ})=\frac{1}{2}=.5\)\\*
\(\sin(45^{\circ})=\cos(45^{\circ})=\frac{1}{\sqrt{2}}\approx.71\)\\*
\(\sin(60^{\circ})=\cos(30^{\circ})=\frac{\sqrt{3}}{2}\approx.87\)
\subsubsection{Physical Data}
Gravitational Constant: \(G\approx6.67\times10^{-11}(Nm^2/kg^2)\)\\*
Proton Mass:  \(m_p\approx1.7\times10^{-27}(kg)\)\\*
Electron Mass: \(m_e\approx9.1\times10^{-31}(kg)\)\\*
Electron Charge: \(e\approx1.6\times10^{-19}(C)\)\\*
Vacuum Permittivity: \(\epsilon_0\approx9\times10^{-12}(C^2/Nm^2)\)\\*
Coulomb's Constant: \(k_e=\frac{1}{4\pi\epsilon_0} \approx 9\times10^8(Nm^2/C^2)\)\\*
Vacuum Permeability: \(\mu_0=4\pi\times10^{-7}(N/A^2)\)\\*
Plank's Constant: \(h\approx6.6\times10^{-34}(Js)\)\\*
Modified Plank's Constant: \(\hbar =\frac{h}{2\pi}\approx10^{-34}(Js)\)\\*
Stefan-Boltzman constant: \(\sigma=5.7\times10^{-8}(W/m^2K^4)\)\\*
Speed of Light: \(c=\frac{1}{\sqrt{\epsilon_0\mu_0}}\approx3\times10^{8}(m/s)\)\\*
\\*
Earth Data:\\*
 Acceleration Due to Gravity: \(g \approx 10(m/s^2)\)\\*
 Year: \(T_{year}\approx\pi\times10^{7}(s)\)\\*
Average Radius: \(R_E\approx6\times10^6 (m)\)\\*
Mass: \(M_E\approx6\times10^{24}(kg)\)\\*
Average distance from the Sun to Earth: \(1(A.U.)\approx1.5\times10^{11}(m)\)\\*
Average distance from the Moon to Earth: \(\sim4\times10^{8}(m)\)\\*
Intensity at Earth's surface: \(\sim1.3\times10^3(W/m^2)\)\\*
Atmospheric Pressure: \(1(atm)\approx10^5(Pa)\)\\*
Mass of Atmosphere: \(\sim5\times10^{18}(kg)\)\\*
Density of Atmosphere at Sea Level: \(\sim1.2(kg/m^3)\)\\*
Number Density of Atmosphere at Sea Level: \(\sim2.5\times10^{25}(molecules/m^3)\)\\*
\(90\%\) of atmosphere is below \(16(km)\)

\subsection{Areas and Volumes}

\subsubsection{Areas}
Circle: \(\pi r^2\)\\*
Triangle: \(\frac{1}{2}bh\)\\*
Function in \(x\)-\(y\) Plane: \(\int_{x_1}^{x_2}f(x)\,\mathrm{d}x\)\\*
Sphere: \(4\pi r^2\)\\*
Cylinder: \(2\pi r^2+2\pi rl\)

\subsubsection{Volumes}
Sphere: \(\frac{4}{3}\pi r^3\)\\*
Cylinder: \(\pi r^2l\)

\subsection{Trigonometric Identities}

\subsubsection{Pythagorean Identity}
\(\sin^2(\theta)+\cos^2(\theta)=1\)

\subsubsection{Double Angle}
\(\sin(2\theta)=2\sin(\theta)\cos(\theta)\)\\*
\(\cos(2\theta)=\cos^2(\theta)-\sin^2(\theta)\)

\subsubsection{Half Angle}
\(\sin^2\left(\frac{\theta}{2}\right)=\frac{1}{2}\left(1-\cos(\theta)\right)\)\\*
\(\cos^2\left(\frac{\theta}{2}\right)=\frac{1}{2}\left(1+\cos(\theta)\right)\)

\subsubsection{Euler's Identity}
\(e^{i\theta}=\cos{\theta}+i\sin{\theta}\)

\subsection{Vector Identities}

\subsubsection{Triple Scalar Product}
\(\mathbf{A}\cdot(\mathbf{B}\times\mathbf{C})=\mathbf{B}\cdot(\mathbf{C}\times\mathbf{A})=\mathbf{C}\cdot(\mathbf{A}\times\mathbf{B})\)

\subsubsection{Triple Vector Product}
\(\mathbf{A}\times(\mathbf{B}\times\mathbf{C})=\mathbf{B}(\mathbf{A}\cdot\mathbf{C})-\mathbf{C}(\mathbf{A}\cdot\mathbf{B})\)

\subsection{Fundamental Theorem of Caculus}
\(\displaystyle \int_{x_1}^{x_2}{\frac{\mathrm{d}f(x)}{\mathrm{d}x}\mathrm{d}x}=f(x_2)-f(x_1)\)\\\\*
\(\displaystyle \int_{\mathbf{a}}^{\mathbf{b}}{(\nabla f)\cdot\mathrm{d}\mathbf{l}}=f(\mathbf{b})-f(\mathbf{a})\)\\\\*
\(\displaystyle \int_{A}{(\nabla\times\mathbf{F})\cdot\mathrm{d}\mathbf{a}}=\oint_{\delta A}{\mathbf{F}\cdot\mathrm{d}\mathbf{l}}\)\\\\*
\(\displaystyle \int_{V}{(\nabla\cdot\mathbf{F})\mathrm{d}V}=\oint_{\delta V}{\mathbf{F}\cdot\mathrm{d}\mathbf{a}}\)\\\\*
In general: \(\displaystyle\int_{\Omega}{\mathrm{d}\omega}=\oint_{\delta\Omega}{\omega}\)\\*
Here \(\omega\) is a (\(n-1\))-form, \(\Omega\) is a manifold of dimension \(n\), and \(\mathrm{d}\) is the exterior derivative

\subsection{Fourier Series}
For function with \(2L\) periodicity:\\*
\(\displaystyle f(x)=\frac{1}{2}a_0+\sum_{n=1}^\infty a_n\cos(nx)+\sum_{n=1}^\infty b_n\sin(nx)\)\\*
\(\displaystyle a_0=\frac{1}{L}\int_{-L}^{L}f(x)\,\mathrm{d}x\)\\*
\(\displaystyle a_n=\frac{1}{L}\int_{-L}^{L}f(x)\cos(nx)\,\mathrm{d}x\)\\*
\(\displaystyle b_n=\frac{1}{L}\int_{-L}^{L}f(x)\sin(nx)\,\mathrm{d}x\)\\\\*
If the function has half- or quarter-wave symmetry, then \(n\) takes on only odd values.

\subsection{Delta Function}
\[\displaystyle\int_{a}^{b}f(x)\delta(x-x_0)\,\mathrm{d}x = \left\{
\begin{array}{l l}
  f(x_0) & \quad \mbox{\(a\leq x_0\leq b\)}\\
  0 & \quad \mbox{otherwise}\\ \end{array} \right. \]
\(\delta(cx)=\frac{1}{|c|}\delta(x)\)

\subsection{Step Function}
\[\theta(x) = \left\{
\begin{array}{l l}
  1 & \quad \mbox{\(x>0\)}\\
  1/2 & \quad \mbox{x=0}\\
  0 & \quad \mbox{\(x<0\)}\\ \end{array} \right. \]
\(\displaystyle\frac{\mathrm{d}\theta}{\mathrm{d}x}=\delta(x)\)

\subsection{Legendre Polynomials}
\(\displaystyle P_l(x)=\frac{1}{2^ll!}\left(\frac{\mathrm{d}}{\mathrm{d}x}\right)^l\left(x^2-1\right)^l\)\\\\*
\(P_l(1)=1\)\\*
\(P_0(x)=1\)\\*
\(P_1(x)=x\)\\*
\(P_2(x)=(3x^2-1)/2\)\\\\*
\(\displaystyle \int_{-1}^{1}P_l(x)P_{l'}(x)\mathrm{d}x=\frac{2}{2l+1}\delta_{ll'}\)

\subsection{Spherical Harmonics}
\(\displaystyle Y^{m}_{l}\left(\theta,\phi\right)=\sqrt{\frac{(2l+1)(l-m)!}{4\pi(l+m)!}}P_{l}^{m}(\cos{\theta})e^{im\phi}\)\\*
where \(P_{l}^{m}\) is the associated Legendre polynomial\\\\*
\(\displaystyle\int_{0}^{\pi}{\int_{0}^{2\pi}{Y_{l}^{m}Y_{l'}^{*m'}\sin{\theta}\mathrm{d}\phi\mathrm{d}\theta}}=\delta_{ll'}\delta_{mm'}\)\\\\*
The concept of an orthonormal set of basis vectors is very important for this test.

\subsection{Common Approximations}
For physical units:\\*
\(1(mi)\approx1.6(km)\)\\*
\(1(rpm)=\frac{\pi}{30}(rad/s)\approx.1(rad/s)\)\\\\*
For small \(\beta\) (special relativity):\\*
\(\gamma\approx1+\frac{1}{2}\beta^2\)\\*
\(\frac{1}{\gamma}\approx1-\frac{1}{2}\beta^2\)\\\\*
For small \(x\):\\*
\(\cos{x}\approx1\)\\*
\(\tan{x}\approx\sin{x}\approx x\)\\*
\(\sqrt{1+x}\approx1+\frac{1}{2}x\)\\*
\(e^x\approx1+x\)\\*
\((1+x)^n\approx1+nx\)


\chapter{Mathematical background}
\label{chpr:math}
The present study starts with an introduction to the mathematical foundations required for its understanding. The reader that is familiar with abstract algebra and elliptic curve cryptography may skip the first two chapters to deal directly with the digital signature schemes presented in Chapter \ref{chpr:dss}.
\\
We start with a brief presentation of the group and field's structures, deferring the introduction to elliptic curves mathematics to the second section. For details and proofs see \cite{RefWork:20} and \cite{RefWork:1}.

\bigskip

\section{Groups and fields}
\label{groupsfields}

\begin{mydef} {\bf (Group)}: A group is a set $G \neq \emptyset$ together with a binary operation $\bullet: G \times G \to G$ (called the 	group law of $G$) that satisfies:
	\begin{itemize}
		\item Closure: $\forall a, b \in G  \Longrightarrow a \bullet b \in G$;
		\item Associativity: $\forall a, b, c \in G  \Longrightarrow (a \bullet b) \bullet c = a \bullet (b \bullet c)$;
		\item Identity: $\exists e \in G \ | \ \forall a \in G, \ e \bullet a = a \bullet e = a$. Such an element $e$ is unique;
		\item Invertibility: $\forall a \in G, \ \exists b \in G \ | \ a \bullet b = b \bullet a = e$. This element is unique and it is commonly denoted either as $a^{-1}$ or $-a$, depending on the notation (multiplicative or additive).
	\end{itemize}
\end{mydef}
\label{def1}
\noindent
As a careful reader may have noticed, it is not required the property of commutativity. This means that, in general, the result of the operation may depend on the order of the operands. If this is not the case, meaning that the group operation is also commutative, the group is called abelian.
\\
Consider now a group $(G, +)$: hereinafter we will stick with the additive notation, unless otherwise specified.
If $G$ is finite, i.e. it has a finite number of elements, we define the order of $G$ as the number of elements in $G$. We can also define the order of an element $g \in G$ to be the smallest integer $k > 0$ such that $kg = 0$, where with 0 we denote the additive identity\footnote{In an additive group it is natural to define the scalar multiplication: given $g \in G$ and $k \in \mathbb{N}, \ k \neq 0$, we have that $kg = g + g + ... + g$, where the summation is repeated $k - 1$ times. In a similar fashion, it is natural to define exponentiation in the multiplicative case.}. If $k$ is the order of $g$, then:
$$ig = jg \Longleftrightarrow i \equiv j \ (\text{mod} \ k).$$
Indeed:
$$ig = jg \ \Longleftrightarrow \ (i - j)g = 0 \ \Longleftrightarrow \ i - j = nk, \ n \in \mathbb{Z},$$
from which the thesis.
\\
Next we state a basic result from group theory, that will turn out to be very useful in the following:
\begin{thm} {\bf (Lagrange's theorem)}:
	Let $G$ be a finite group.
	\begin{enumerate}
		\item Let $H$ be a subgroup\footnote{Given a group $(G, +)$, a subset $H$ is a subgroup of $G$ if $H$ endowed with the restriction of the group operation to $H \times H$ is a group.} of $G$. Then the order of $H$ divides the order of $G$;
		\item Let $g \in G$. Then the order of $g$ divides the order of $G$.
	\end{enumerate}
\end{thm}

\bigskip

\noindent
For the cryptographic applications that we will study, a particular family of groups turns out to be very important: cyclic groups. A cyclic group is a group that is generated by a single element: this means that it contains (at least) an element $g$ such that every other element of the group can be obtained by repeatedly applying the group operation to $g$, i.e. $\forall h \in G, \ \exists n \in \{1, ..., k\} \ | \ h = ng$, where $k$ as before denotes the order of $g$. All the group's elements satisfying this property are called generator of the group.
\\
Two interesting properties of cyclic groups are the following:
\begin{enumerate}
	\item Every cyclic group is abelian;
	\item Every finite group has at least a cyclic subgroup.
\end{enumerate}

\bigskip
\noindent
Now let's get to the mathematical concept of field. A field is a non empty set on which are defined two binary operations, usually called addition and multiplication. It has to be an abelian group under addition (with 0 denoting the additive identity) and the non-zero elements have to form an abelian group under multiplication. Finally, the multiplication has to be distributive over the addition.
\\
More precisely we can give the following definition.
\begin{mydef} {\bf (Field)}: A field is a set $K \neq \emptyset$ endowed with two binary operations $+$ and $*: K \times K \to K$, such that:
\begin{itemize}
	\item $(K, +)$ is an abelian group, whose identity element is denoted by 0. This group is called the additive group of the field;
	\item $(K \text{\textbackslash}\{0\}, *)$ is an abelian group. It is called the multiplicative group of the field, sometimes denoted by $K^{\times}$ or by $K^*$;
	\item $\forall a, b, c \in K: \ a * (b + c) = a * b + a * c$.
\end{itemize}
\end{mydef}

\bigskip
\noindent
Another key role in the following will be played by finite fields: a finite field is simply a field with a finite number of elements. A finite field of order $q$, denoted as $\mathbb{F}_q$, exists if and only if $q = p^k$, where $p$ is a prime number and $k$ is a positive integer. The field of a given order is unique, up to an isomorphism (this means that there are different representations of a unique mathematical object). In a field of order $p^k$, the characteristic of the field (as defined below) is $p$: thus adding $p$ copies of any element results in the additive identity.
\\
In particular we will deal with prime finite fields, i.e. the finite fields whose order is a prime number. Finite fields of order a prime $p$ may be represented by the set $\mathbb{Z}_p = \{0, 1, ..., p-1\}$ with addition and multiplication defined through modular arithmetic, that is: given $a, b \in \mathbb{F}_p$ then $r = a + b$ and $s = a * b$ are integers in $[0, p - 1]$, defined as the remainder of the divisions $(a + b)/p$ and $a*b/p$, respectively. This is usually written as: $a + b \equiv r \ (\text{mod} \ p)$ and $a * b \equiv s \ (\text{mod} \ p)$ or as $a + b \equiv r \ \text{in} \ \mathbb{F}_p$ and $a * b \equiv s \ \text{in} \ \mathbb{F}_p$ . It is easy to see that in this case the additive identity is the integer 0, while the multiplicative identity is the integer 1.
\\
Subtraction and division are defined via the additive and multiplicative inverses:
\begin{itemize}
	\item Subtraction: given $a \in \mathbb{F}_p$, we know by definition that there is a unique element $-a \in \mathbb{F}_p$ such that $a + (-a) = 0 \ (\text{mod} \ p)$. Then we define subtraction as: $a - b = a + (-b) \ (\text{mod} \ p), \ \forall a, b \in \mathbb{F}_p$;
	\item Division: $\forall a \in \mathbb{F}_p \text{\textbackslash} \{0\}, \ \exists a^{-1} \ | \ a * a^{-1} = 1 \ (\text{mod} \ p)$, again by definition. Thus we can define division as: $a / b = a * (b)^{-1}, \ \forall a, b \in \mathbb{F}_p, \ b \neq 0$.
\end{itemize}
Notice that not all the finite fields $\mathbb{F}_q$ are isomorphic to $\mathbb{Z}_q$: consider for example the case $q = 2^2 = 4$. Here $p = 2$ and $k = 2$. Since two is a prime number, we have that there exists a field with four elements. But since four is not a prime number, we cannot represent this field through the set $\mathbb{Z}_4 = \{0, 1, 2, 3\}$. Consider the element 2: 2 * 0 = 0 (mod 4), 2 * 1 = 2 (mod 4), 2 * 2 = 0 (mod 4), 2 * 3 = 2 (mod 4). As we can see it does not admit a multiplicative inverse in $\mathbb{Z}_4$: for this reason $\mathbb{Z}_4$ is not a field and cannot be used as representation of $\mathbb{F}_4$.

\bigskip
\noindent
We end this section giving the definition of field characteristic, that will be useful when defining the equation of an elliptic curve.
\begin{mydef} {\bf (Field characteristic)}: Given a field $(K, \ +, \ *)$, we denote by 0 and 1 the additive and multiplicative identities, respectively.
\\
The characteristic of $K$, denoted as  $\text{char}(K)$, is defined to be the smallest $n \in \mathbb{Z} \text{\textbackslash} \{0\}$ for which the following relation holds:
$$\underbrace{1 + ... + 1}_\text{$n$ times} = 0$$
In case this relation never holds, the field is said to have characteristic zero.
\end{mydef}

\bigskip

\bigskip

\section{Elliptic curves}
\label{ec}
In the most general case, an elliptic curve over a field $K$ is defined through the equation: $$y^2 + a_1xy + a_2y = x^3 + a_3x^2 + a_4x + a_5,$$ where $a_1, ..., a_5 \in K$. This form is called generalized Weierstrass equation, and can be simplified according to the characteristic of the field. In particular, if it is different from two and three we can write:
$$y^2 = x^3 + ax + b, \ a, b \in K.$$
This latter equation is called Weierstrass equation, and it is the most widely used.
\\
We will denote an elliptic curve over a field $K$ through the notation $E(K)$, that represents the set of points satisfying the defining equation. By construction, this set also contains a special point, called the point at infinity (its role will be clarified in the next section):
$$E(K) = \{\infty\}\cup\{(x, y) \in K \times K \ | \ y^2 = x^3 + ax + b\}.$$
\\
To start familiarizing with elliptic curves, it could be useful to consider the case in which $K = \mathbb{R}$, the set of real numbers. In Figure \ref{fig:figure1} we can look at a couple of elliptic curves' real graph.
\begin{figure}
	\noindent
	\makebox[\textwidth]{\includegraphics[width=15cm, height=7cm]{Images/EC.eps}}
	\captionof{figure}{Two examples of elliptic curves over the real numbers.}
	\label{fig:figure1}
	\source{\url{https://en.wikipedia.org/wiki/Elliptic_curve\#Elliptic_curves_over_the_real_numbers}.}
\end{figure}
\noindent
As we can see, the elliptic curves are symmetric with respect to the $x$-axis, so that for every feasible $x$ there are two $y$ values satisfying the Weierstrass equation.
\\
Typically the Weierstrass equation is coupled with the constraint that the discriminant of the cubic $x^3 + ax + b$ is different from zero. This is equivalent to ask that:
$$4a^3 + 27b^2 \neq 0,$$
in order to avoid multiple roots. Geometrically, this requirement means that there are no cusps, self intersections or isolated points\footnote{This requirement has also other implications: it can be shown that, without it, the elliptic curve $E_{ns}(K)$ defined over the non-singular points has an isomorphism with simpler algebraic structures, such as $K$ itself or $K^{\times}$. This problem typically arises when dealing with elliptic curves modulo composites.}.
We can also link the sign of the discriminant to the number of components of the curve: the real graph of a curve has two components if its discriminant is positive, and one component if it is negative, as exemplified in Figure \ref{fig:figure1} (the discriminant is $\Delta = -16(4a^3 + 27b^2)$).

\bigskip
\noindent
We get back for a moment to the generalized Weierstrass equation, and show how to go from there to the more appealing Weierstrass form:
$$y^2 + a_1xy + a_2y = x^3 + a_3x^2 + a_4x + a_5,$$
with $a_1,...,a_5$ constants in $K$. We have said that this is the most generic form for an elliptic curve, but what does it exactly mean? It means that it is always possible to reduce to the Weierstrass equation, unless the characteristic of the field over which the curve is defined is two or three. In these two cases, we need to resort to the general formula.
\\
However, if the characteristic of the field is not two we can divide by two\footnote{Remember that in a field of characteristic two the number two acts as zero, the additive element that has not a multiplicative inverse.} and complete the square: $$\left(y + \frac{a_1x}{2} + \frac{a_2}{2}\right)^2 = x^3 + \left(a_3 + \frac{a_1^2}{4}\right)x^2 + \left(a_4 + \frac{a_1a_2}{2}\right)x + \left(\frac{a_2^2}{4} + a_5\right),$$ which can be written as $$y_1^2 = x^3 + a_1{'}x^2 + a_2{'}x + a_3{'},$$ with $y_1 = y + \frac{a_1x}{2} + \frac{a_2}{2}$ and new constants $a_1{'}, a_2{'}, a_3{'} \in K$. Moreover, if the characteristic is neither three, then we can consider $x_1 = x + \frac{a_1{'}}{3}$:
$$y_1^2 = x^3 + a_1{'}x^2 + a_2{'}x + a_3{'} = \left(x_1 - \frac{a_1{'}}{3}\right)^3 + a_1{'}\left(x_1 - \frac{a_1{'}}{3}\right)^2 + a_2{'}\left(x_1 - \frac{a_1{'}}{3}\right) + a_3{'} = $$ $$= \left(x_1 - \frac{a_1{'}}{3}\right)\left(x_1^2 + \frac{a_1{'}^2}{9} -\frac{2}{3}a_1{'}x_1 + a_1{'}x_1 - \frac{a_1{'}^2}{3} + a_2{'}\right) + a_3{'} = $$ $$=\left(x_1 - \frac{a_1{'}}{3}\right) \left(x_1^2 + \frac{1}{3}a_1{'}x_1 - 	\frac{2}{9}a_1{'}^2 + a_2{'}\right) + a_3{'} = $$ $$= x_1^3 + \cancel{\frac{1}{3}a_1{'}x_1^2} - \frac{2}{9}a_1{'}^2x_1 + a_2{'}x_1 - \cancel{\frac{1}{3}a_1{'}x_1^2} - \frac{1}{9}a_1{'}^2x_1 + \frac{2}{27}a_1{'}^3 - \frac{1}{3}a_1{'}a_2{'} + a_3{'} = $$ $$=x_1^3 + \left(a_2{'} - \frac{1}{3}a_1{'}^2\right)x_1 + \left(\frac{2}{27}a_1{'}^3 - \frac{1}{3}a_1{'}a_2{'} + a_3{'}\right).$$
Through a proper renaming of the constants we get finally to the well known Weierstrass form: $y_1^2 = x_1^3 + ax_1 + b$ for some constants $a$ and $b$ in $K$.
\\
Finally, for the sake of completion, we present the case of the equation $cy^2 = dx^3 + ax + b$ and how to transform it in Weierstrass form. We can multiply both sides by $c^3d^2$ and write $(c^2dy)^2 = (cdx)^3 + (ac^2d)(cdx) + bc^3d^2$. The change of variables $y_1 = c^2dy$ and $x_1 = cdx$ yields to the desired formulation of the equation.

\bigskip

\bigskip

\subsection{The group law}
\label{grouplaw}
This section is devoted to the definition of the addition operation between elements in $E(K)$: this operation turns out to be fundamental since it induces the structure of abelian group on the elliptic curve's point. Moreover, we will see why, in the definition of the curve, we had to consider the point at infinity.
\\
In particular we will consider $K$ to be the set of real numbers, in order to give an easy graphical representation of the operation. This can be done without loss of generality, since the formulas that we will derive can be shown to be valid for every $K$ with $\text{char}(K) \neq 2, 3$.

\bigskip
\noindent
Loosely speaking we can say that:
\begin{itemize}
	\item The points of the elliptic curve are the group's elements;
	\item The identity element is the point at infinity, denoted by $\infty$;
	\item The inverse of a point $Q$ is the one symmetric about the $x$-axis: $Q = (x, y) \Longrightarrow -Q = (x, -y)$;
	\item Addition is given by the following rule: given three aligned points on the curve $Q_1$, $Q_2$ and $Q_3$ their sum is $Q_1 + Q_2 + Q_3 = \infty$.
\end{itemize}

\begin{center}
	\noindent
	\makebox[\textwidth]{\includegraphics[width=20cm, height=5cm]{Images/chord.eps}}
	\captionof{figure}{Geometric interpretation of the addition of EC points.}
	\label{fig:figure2}
	\source{\url{https://en.wikipedia.org/wiki/Elliptic_curve\#The_group_law}.}
\end{center}
\noindent
In order to define properly the addition of EC points we need to consider five different cases. We start taking an elliptic curve $E$ defined by the usual equation in Weierstrass form $y^2 = x^3 + ax + b, \ a,b \in \mathbb{R}$ and two points $Q_1 = (x_1, y_1), \ Q_2 = (x_2, y_2) \in E(\mathbb{R})$. Unless otherwise specified, we assume also that $Q_1, Q_2 \neq \infty$.
\begin{enumerate}
	\item $x_1 \neq x_2$: consider the line passing through $Q_1$ and $Q_2$. In most of the cases (i.e. when neither the points are tangent points) this line will intersect the curve in a third point, that we denote as $-Q_3$. Then take the reflection with respect to the $x$-axis (i.e. change the sign of the $y$ coordinate): denote this final point as $Q_3$.
	\\
	The algorithm specified allows us to give meaning to the formula:
	$$Q_1 + Q_2 = Q_3.$$
	The one outlined above is the geometrical intuition, but we can obtain algebraic formulas for the operation. The line passing through $Q_1$ and $Q_2$ has equation $y = m(x - x_1) + y_1$, where $m = \frac{y_2 - y_1}{x_2 - x_1}$ is its slope. Notice that if $x_1 = x_2$ the line is vertical, but we will treat this case later on. We can find the intersections between the line and the curve through a substitution: $(m(x - x_1) + y_1)^2 = x^3 + ax + b$. We have $m^2(x^2 - 2x_1x + x_1^2) + 2my_1(x - x_1) + y_1^2 = x^3 + ax + b$; this formula can be rearranged in the form: $$x^3 - m^2x^2 + (a + 2m^2x_1 - 2my_1)x + (b + 2my_1x_1 - m^2x_1^2 -y_1^2) = 0.$$
	\\
	In general solving a cubic equation is non trivial, but we have already two roots, namely $x_1$ and $x_2$ since $Q_1$ and $Q_2$ are points on both the line and the curve. To find the third root we notice that, given a cubic polynomial $x^3 + ax^2 + bx + c$ with roots $r, s$ and $t$, we can write: $$x^3 + ax^2 + bx + c = (x - r)(x - s)(x - t) =$$
	$$= x^3 - (r + s + t)x^2 + (rs + rt + st)x - rst.$$
	Therefore $r + s + t = - a$, so that, if we know two roots $r$ and $s$, we can find the third as $t = - a - r - s$.
	\\
	In our case we have $x_1 + x_2 + x_3 = m^2 \ \Longrightarrow \ x_3 = m^2 - x_1 - x_2$. Remembering to change the $y$ coordinate of the intersection point due to the reflection, we get $Q_3$ as: $x_3 = m^2 - x_1 - x_2$ and $y_3 = m(x_1 - x_3) - y_1$.
	\begin{figure}
		\noindent
		\makebox[\textwidth]{\includegraphics[width=15cm, height=7cm]{Images/sum.eps}}
		\captionof{figure}{Graphical representation of the point addition.}
		\label{fig:figure3}
	\end{figure}

	\item $x_1 = x_2 \wedge y_1 \neq y_2$: the line through $Q_1$ and $Q_2$ is vertical, therefore it intersects $E$ at $\infty$. Reflecting the point at infinity across the $x$-axis yields to the same point. Therefore, in this case $Q_1 + Q_2 = \infty$.
	\\
	This is in agreement with what we stated before: if  $x_1 = x_2 \wedge y_1 \neq y_2$ it means that $Q_2 = -Q_1$, and since the point at infinity acts as the identity element we have exactly that $Q_1 + Q_2 = \infty$.

	\item $Q_1 = Q_2 = (x_1, y_1) \wedge y_1 \neq 0$: when two points on a curve are very close one another, the line through them approximates a tangent line. Therefore, when the two points coincide, we take the line through them to be the tangent line. Implicit differentiation allows us to find its slope $m$:
	$$y^2 = x^3 + ax + b \ \Longrightarrow \ \frac{d(y(x)^2)}{dx} = \frac{d}{dx}(x^3 + ax + b) \ \Longrightarrow$$
	$$ \Longrightarrow \ \frac{d(y^2)}{dy}\frac{dy}{dx}=2y\frac{dy}{dx}= 3x^2 + a \ \Longrightarrow $$
	$$\Longrightarrow \ m = \left(\frac{dy}{dx}\right)_{x_1} = \frac{3x_1^2 + a}{2y_1}.$$
	Again, the equation of the line is $y = m(x - x_1) + y_1$. At this point we can proceed as before: the line intersects the curve in a second point. Substituting the equation of the straight line would result in a cubic equation. But we already know two roots, so that we get: $x_3 = m^2 - 2x_1$ and $y_3 = m(x_1 - x_3) - y_1$.
	\\
	\item $Q_1 = Q_2 = (x_1, y_1) \wedge y_1 = 0$: as we can see from the previous point in this case the tangent line is vertical and we set $Q_1 + Q_2 = \infty$.
	\item The last case we have to deal with is when one of the two points is $\infty$. Suppose, without loss of generality, that $Q_2 = \infty$; the line through $Q_1$ and $\infty$ is a vertical line that intersects $E$ in the point $-Q_1$, which is the reflection of $Q_1$ across the $x$-axis. When we reflect $-Q_1$ to get $Q_3 = Q_1 + Q_2$, we get back to $Q_1$. Therefore:
	 $$Q_1 + \infty = Q_1$$
	for all points $Q_1$ on $E$. Of course, we extend this to include $\infty + \infty = \infty$.
\end{enumerate}
Thanks to these algebraic formulas we can notice that when $Q_1$ and $Q_2$ have coordinates in a field $K$ that contains $a$ and $b$, then $Q_1 + Q_2$ also has coordinates in $K$. This means that $E(K)$ is closed under the defined addition operation.

\bigskip
\noindent
Now we state a theorem showing formally that the couple $(E, +)$ is indeed an abelian group. The theorem with the proof can be found in \cite{RefWork:1}.
\begin{thm} The operation $+$ as defined above on an elliptic curve $E$ satisfies the following properties:
	\begin{enumerate}
		\item Commutativity: $Q_1 + Q_2 = Q_2 + Q_1, \ \forall Q_1, Q_2 \in E$;
		\item Identity: $Q + \infty = Q, \ \forall Q \in E$;
		\item Invertibility: $\forall Q \in E, \ \exists Q' \in E \ | \ Q + Q' = \infty$. This point will be denoted by $-Q$;
		\item Associativity: $(Q_1 + Q_2) + Q_3 = Q_1 + (Q_2 + Q_3), \ \forall Q_1, \ Q_2, Q_3 \in E$.
	\end{enumerate}
\end{thm}

\bigskip
\noindent
Although we won't prove the theorem, it can be useful to give a sketch of its proof. The commutativity is obvious, either from the formulas or from the fact that the line passing through $Q_1$ and $Q_2$ is the same as the line passing through $Q_2$ and $Q_1$. The identity property of the point at infinity holds by definition, as the existence of the inverse element.
\\
The tricky part of the proof is related with associativity: either it can be proved working out the annoying computations or through a much more elegant and complex method based on projective coordinates, for which we again refer to the bibliography.

\bigskip
\noindent
Now that we have clear in mind what does addition between EC points mean, we can define subtraction and scalar multiplication:
\begin{itemize}
	\item Subtraction: given $Q, R \in E$, we set $Q - R = Q + (-R)$. We can do this since everything is perfectly defined in the right hand side of the equation;
	\item Scalar multiplication: $\forall k \in \mathbb{N}$\textbackslash$\{0\}, \ kQ = Q + Q + ... + Q$, where the addition is repeated $k - 1$ times.
\end{itemize}
Notice that to compute the scalar multiplication for a large $k$ it is inefficient to add $Q$ to itself repeatedly. There are several faster approaches, the simplest called the {\bf Double and Add Algorithm}.
We stress this fact since, as we will see later on, scalar multiplication and its computational asymmetry are the key ingredients for elliptic curve cryptography: the fact is that it is easily computable\footnote{The double and add algorithm requires polynomial time in the number of bits $n$ representing the numbers involved, compared to the exponential time required by the naive approach; that is, assuming that the point doubling and the point addition require a constant time, we need $O(n^{\alpha}), \ \alpha > 1$ elementary operations, compared to $O(2^n)$. In particular on average it requires $n$ doublings and $0.5 * n$  additions, since the binary expansion of a random integer has an equal number of 1's and 0's.}
but its inversion, the so called discrete logarithm (DL), although not provable to be hard, is believed to be so: this means that in general we have no efficient algorithm to compute it.
\\
\\
{\bf Double and Add Algorithm}: To compute $kQ$, start with the binary representation of $k$: $k = k_0 + 2k_1 + 2^2k_2 + ... + 2^nk_n$, where $k_0,...,k_n \in \{0, 1\}$.

\begin{algorithm}
	\caption{Double and Add algorithm}
	\label{alg:double_add}
	\begin{algorithmic}[1]
		\Procedure{double\_and\_add}{$k, \ Q$}
		\State $R \gets \infty$
		\For {$i\gets 1,n$}
		\If{$k_i = 1$}
		\State $R \gets Q + R$
		\EndIf
		\State $Q \gets Q + Q$
		\EndFor
		\State \textbf{return} $R$
		\EndProcedure
	\end{algorithmic}
\end{algorithm}
\noindent
The algorithm can be further sped up through the pre-computation of a vector of multiples of $Q$: this approach is not always feasible, since the point $Q$ is not always fixed\footnote{The point $Q$ is fixed, for example, when computing the public key from a private key. For a thorough exposition of these concepts we refer to Chapter \ref{chpr:ecc}.}.


\section{2-Dimensional Diffusion}

The two dimensional diffusion equation is given as:
\begin{equation}
  \frac{\partial u}{\partial t} = v\frac{\partial^2 u}{\partial x^2} + v\frac{\partial^2 u}{\partial y^2}
\end{equation}

We approximate this using a forward difference in time and a central difference in space:
\begin{equation}
  \frac{u^{n+1}_{i, j} - u^{n}_{i, j}}{\Delta t} = \nu\frac{ u^{n}_{i, j-1} - 2  u^{n}_{i, j} +  u^{n}_{i, j+1}}{\Delta x^2} + \nu\frac{ u^{n}_{i-1, j} - 2  u^{n}_{i, j} +  u^{n}_{i+1, j}}{\Delta y^2}
\end{equation}

Rearranging to solve for $u^{n+1}_{i, j}$ yields:
\begin{equation}
u^{n+1}_{i, j} = u^{n}_{i, j} + \frac{\nu\Delta t}{\Delta x^2}(u^{n}_{i, j-1} - 2  u^{n}_{i, j} +  u^{n}_{i, j+1}) + \frac{\nu\Delta t}{\Delta y^2}(u^{n}_{i-1, j} - 2  u^{n}_{i, j} +  u^{n}_{i+1, j})
\end{equation}


\section{$LU$ factorization}

An $LU$ factorization of a matrix involves writing the given matrix as the
product of a lower triangular matrix $L$ which has the main diagonal consisting
entirely of ones, and an upper triangular matrix $U$ in the indicated
order. This is the version discussed here but it is sometimes the case that
the $L$ has numbers other than 1 down the main diagonal. It is still a
useful concept. The $L$ goes with ``lower'' and the $U$ with ``upper''.

It turns out many matrices can be written in this way and when this is
possible, people get excited about slick ways of solving the system of
equations, $AX=B$. It is for this reason that you want to study the
$LU$ factorization. It allows you to work only with triangular
matrices. It turns out that it takes about half as many operations to
obtain an $LU$ factorization as it does to find the {\rref}.

First it should be noted not all matrices have an $LU$ factorization and so
we will emphasize the techniques for achieving it rather than formal proofs.
\index{LU decomposition!non-existence}
\index{LU factorization}

\begin{example}{A matrix with NO $LU$ factorization}{}
Can you write $\begin{mymatrix}{rr}
0 & 1 \\
1 & 0
\end{mymatrix} $ in the form $LU$ as just described?
\end{example}

\begin{solution}
To do so you would need
\begin{equation*}
\begin{mymatrix}{rr}
1 & 0 \\
x & 1
\end{mymatrix} \begin{mymatrix}{rr}
a & b \\
0 & c
\end{mymatrix} =\allowbreak \begin{mymatrix}{cc}
a & b \\
xa & xb+c
\end{mymatrix} =\begin{mymatrix}{rr}
0 & 1 \\
1 & 0
\end{mymatrix} .
\end{equation*}

Therefore, $b=1$ and $a=0$. Also, from the bottom rows, $xa=1$ which can't
happen and have $a=0$. Therefore, you can't write this matrix in the form $%
LU$. It has no $LU$ factorization. This is what we mean above by saying the
method lacks generality.

Nevertheless the method is often extremely useful, and we will describe
below one the many methods used to produce an $LU$ factorization when
possible.
\end{solution}


\section{Task C}
\label{sec:task-c}

The equivalent internal forces are given as
\begin{equation} \tag{9.15b}
  \label{eq:int-force}
  \ubar{\bm{T}}_{a}^{(e)} = \int_{v^{(e)}} \utilde{\sigma} \cdot \ubar{\nabla} N_{a}\, dv
\end{equation}
For a CST element of thickness \(t\) the same expression in local coordinates reads
\begin{equation}
  \label{eq:int-force-local}
  \ubar{\bm{T}}_{a}^{(e)} = \int \int \utilde{\sigma} \cdot \ubar{\nabla} N_{a} \, t
  \, dx \, dy =
  \int_{-1}^{1} \int_{-1}^{1} \utilde{\sigma} \cdot \ubar{\nabla} N_{a} \, t
  \det \left( \frac{\partial \ubar{\bm{x}}}{\partial \ubar{\bm{\xi}}} \right) \,
  d\xi \, d\eta
\end{equation}
Apply Gaussian quadrature rule to compute the integral numerically:
\begin{equation}
  \label{eq:int-force-numer}
  \ubar{\bm{T}}_{a}^{(e)} \approx
  \sum_{i=1}^{\text{nip}} W_{i} \utilde{\sigma} \cdot \ubar{\nabla} N_{a} \, t
  \det \left( \frac{\partial \ubar{\bm{x}}}{\partial \ubar{\bm{\xi}}} \right),
\end{equation}
where \(W_{i}\) is a weight for each of the integration points.
For a CST element, the number of integration points is 1 with coordinates
\(\xi = 1/3\), \(\eta = 1/3\) and the weight \(W = 0.5\).

The Matlab implementation of this task can be found in
\texttt{get\_intern\_eq\_forces.m} (see section \ref{app:matlab-code}).

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../main"
%%% End:


\documentclass{article}
\usepackage{amsmath, amssymb, bm}
\title{Sweet \texttt{Cython} code for planet search finding}
\author{@\texttt{dfm} \& @\texttt{mirca}}

\begin{document}
\maketitle
\section{Why period search algorithms are important?}
\subsection{Statistical assumptions}
We assume that $y_i \sim \mathcal{N}(m(t_i), \sigma_i)$, $i=1, 2, ..., n$, where
\begin{equation}
m(t_i) =
\left\{
    \begin{array}{ll}
        l, & \text{if}~t_0 \leq t_i \leq t_0 + w ,\\
        h, & \text{otherwise}.
    \end{array}
\right.
\end{equation}
such that $h>l$, in which $t_0$ is the transit time, $w$ is the transit duration,
$h$ is the mean (not quite) flux level out-of-transit, $l$ is the mean (not quite)
flux level in-transit.

Then the log-likelihood function of $\bm{y} \triangleq (y_1, y_2, ..., y_n)$,
assuming $y_i$'s are pair-wise idependent, can be written as
\begin{align}
    \log p(\bm{y}| h, l) = -\dfrac{1}{2}\sum_{i=1}^{n}\left(\dfrac{y_i - m(t_i)}{\sigma_i}\right)^2,
\end{align}
up to an additive constant.

Note further that, the log-likelihood function can be expressed as
\begin{align}
    \log p(\bm{y}| h, l) = -\dfrac{1}{2}\left[\sum_{i \in I}\left(\dfrac{y_i - l}{\sigma_i}\right)^2
                              + \sum_{i \in I^{c}}\left(\dfrac{y_i - h}{\sigma_i}\right)^2\right],
\end{align}
in which $I\triangleq\left\{i | t_0 \leq t_i \leq t_0 + w \right\}$ and $I^{c}$ denotes the complement of $I$.

And the maximum likelihood estimator for $h$ and $l$, denoted as $h^{\star}$ and $l^{\star}$,
respectively, can be written as
\begin{equation}
    l^{\star}(\bm{y}) = \dfrac{\displaystyle\sum_{i \in I}\dfrac{y_i}{\sigma^2_i}}{\displaystyle\sum_{i\in I}\dfrac{1}{\sigma^2_i}},
    ~~h^{\star}(\bm{y}) = \dfrac{\displaystyle\sum_{i \in I^{c}}\dfrac{y_i}{\sigma^2_i}}{\displaystyle\sum_{i\in I^{c}}\dfrac{1}{\sigma^2_i}}.
\end{equation}

Note that
\begin{align}
    \mathrm{var}\left(l^{\star}(\bm{y})\right) = \dfrac{\displaystyle\sum_{i \in I}\dfrac{\mathrm{var}(y_i)}{\sigma^4_i}}
    {\displaystyle\left(\sum_{i\in I}\dfrac{1}{\sigma^2_i}\right)^{2}}
    = \displaystyle\left(\sum_{i\in I}\dfrac{1}{\sigma^2_i}\right)^{-1}\\
    \mathrm{var}\left(h^{\star}(\bm{y})\right) = \dfrac{\displaystyle\sum_{i \in I^c}\dfrac{\mathrm{var}(y_i)}{\sigma^4_i}}
    {\displaystyle\left(\sum_{i\in I^c}\dfrac{1}{\sigma^2_i}\right)^{2}}
    = \displaystyle\left(\sum_{i\in I^c}\dfrac{1}{\sigma^2_i}\right)^{-1}
\end{align}

The maximum likelihood for the transit depth, denoted as $d^{\star}$, can then be written as
\begin{equation}
    d^{\star}(\bm{y}) = h^{\star}(\bm{y}) - l^{\star}(\bm{y}).
\end{equation}

Then the variance on $d^{\star}$ can be expressed as
\begin{align}
    \mathrm{var}\left(d^{\star}\right) &= \mathrm{var}\left(h^{\star}\right) + \mathrm{var}\left(l^{\star}\right)\\
    &= \displaystyle\left(\sum_{i\in I}\dfrac{1}{\sigma^2_i}\right)^{-1}
    + \displaystyle\left(\sum_{i\in I^c}\dfrac{1}{\sigma^2_i}\right)^{-1}.
\end{align}

Then the signal to noise ratio of the transit depth can be computed as
\begin{equation}
    \mathrm{SNR}_{d^{\star}}= \dfrac{d^{\star}}{\sqrt{\displaystyle\left(\sum_{i\in I}\dfrac{1}{\sigma^2_i}\right)^{-1}
                                            + \displaystyle\left(\sum_{i\in I^c}\dfrac{1}{\sigma^2_i}\right)^{-1}}}.
\end{equation}

\end{document}


% s,mask

\subsection{Masking}

One subtle point is that we usually display images
as a rectangular grid of $\ty{nx} \times \ty{ny}$ pixels,
but usually the iterative algorithms
work on column vectors.
Often only a subset of the pixels are updated,
as illustrated in
\fref{fig,reg,mask}.
A logical array called the \ty{mask}
specifies which pixels are to be updated.
The function call
\ty{x_col = x_array(mask(:))}
extracts the relevant pixel elements
``within the mask''
into a column vector.
Conversely, the call
\ty{x_array = embed(x_col, mask)}
puts the elements of the column vector
back into the appropriate places
in the array.

\newcommand{\fdir}{book/c-reg/fig}
%\infig{fig,reg,mask}{0.4}

\begin{figure}
\cent{
\input{\fdir/fig,reg,mask,pic,6,5}
}
\caption{
A $6 \times 5$
image lattice with approximately circular FOV.
Only the $\np = 8$ pixels with indices shown are estimated.
}
\label{fig,reg,mask}
\end{figure}

The following code illustrates these ideas.

\begin{verbatim}
mask = false(6,5); mask([9 10 14:17 21 22]) = true;
xc = [10:5:45]';
xa = embed(xc, mask)
\end{verbatim}

\noindent
Here is the resulting output.

\begin{verbatim}
0     0     0     0     0
0     0    20     0     0
0    10    25    40     0
0    15    30    45     0
0     0    35     0     0
0     0     0     0     0
\end{verbatim}

\noindent
Conversely,
typing the following
produces a vector that is the same as the original \ty{xc} vector.
\begin{verbatim}
xa(mask)
\end{verbatim}

When first working on image reconstruction problems,
it is quite tempting to disregard
using any such mask
and try to reconstruct
the ``full'' image
because it seems simpler.
However,
estimating more parameters than are needed
to describe the object
usually requires more computation time
and often degrades the conditioning
of the problem
leading to slower convergence.


The grammar used in this section is the following:

\begin{align*}
    &S \to AB\\
    &A \to a\\
    &B \to BB|a|b\\
\end{align*}

This grammar matches every string that begins with an `a', no matter what comes next.

\subsubsection{Number of iterations/recusive calls for each parser}

It is very interesting because with this grammar the top-down algorithm never uses the stored values in its table, no matter the pattern used.
At first one could think that this is a very bad thing, indeed since the top-down without the table is exactly the same as the naive parser it means that using this grammar they will both have the same behaviour.
But what is interesting is that the naive parser with this grammar and a string of size $n$ reacts like so:
$$
recursive \text{ } calls =
\begin{cases}
    2n - 1 &\text{if string matches the grammar}\\
    n &\text{otherwise}
\end{cases}
$$

It is very easy to demonstrate those two results:
\begin{enumerate}
    \item In every pattern that does start with an `a' the number of recursive calls is $2n - 1$ because:
        \begin{itemize}
            \item[$-$] The initial call is $parse(S, 0, n)$.
            \item[$-$] The function will the recursively call $parse(A, 0, 1)$, which will return true since the \textit{terminal} `a' is equal to the character 0 of the string which is also `a'.
            \item[$-$] Then the function will call a second time $parse(B, 1, n)$.
            \item[$-$] That function will call $parse(B, 1, 2)$. That will return true no matter the character since `B' possesses both \textit{terminals} `a' and `b'.
            \item[$-$] Then it will call $parse(B, 2, n)$. This will repeat itself until $parse(B, n - 1, n)$ is called. At that moment it will return true.
            \item[$-$] At that moment the number of recursive calls is $(n - 1) * 2$ plus the initial call $1$, for a total of $2n - 1$ recursive calls. Note that every 2 steps this algorithm goes 1 step deeper into the recursive stack, this is an important fact for the experimentations.
        \end{itemize}
    \item In every pattern that does not start with an `a' the number of recursive calls is the size of the string $n$ because:
        \begin{itemize}
            \item[$-$] The initial call is $parse(S, 0, n)$. That function will enter in a loop for k going from $1$ to $n - 1$.
            \item[$-$] The function will then recursively call $parse(A, 0, 1)$, which returns false since the \textit{terminal} `a' is not equal to the character 0 of the string, which is `b'.
            \item[$-$] Then the function will call $parse(A, 0, k)$ until the end of the for loop. The calls will return false each time since `A' does not possess any \textit{non-terminals} rules.
            \item[$-$] So there is $1$ call plus $n - 1$ calls made by the first called function, for a total of $n$ recursive calls. Note that all the sub calls are made by the first called function, which limits the maximum reached recursive depth to only 2, it will be interesting to see in the experimentations how that has a very important impact.
        \end{itemize}
\end{enumerate}

With a behaviour like this one the naive parser is very efficient with that grammar.
Also in this section only the naive parser and both bottom-up parsers will be considered, since the naive and top-down one are the same.

With this grammar the best and worst case scenarios for the `string' bottom-up parser are:
\begin{itemize}
    \item[$-$] The worst case is a string following this pattern: `$b\string^ n$'.
    \item[$-$] The best case is a string following this pattern: `$a\string^ n$'.
    \item[$-$] Every case that is a mix between `a's and `b's will have a running time situated between the best and worst case.
        The more `b's there is in the string the closer the running time will be from the worst case and conversely with `a's.
\end{itemize}

It is now poosible to compare the anticipated behaviours of the `boolean' bottom-up and the naive parser, in both cases, using the function \ref{eq:bottom-up_iterations} and the two expressions from above.

\FloatBarrier
\begin{figure}[h]
\begin{tikzpicture}
\begin{groupplot}[group style={group size=2 by 1},height=0.5\textwidth,width=0.5\textwidth, domain=0:1000]
\nextgroupplot[title=`Boolean' bottom-up, xlabel=string size, ylabel=iterations]
\addplot[red]{3 * x + ((2 / 6) * ((3 * (x^2) * (x + 1)) - (x * (x + 1) * ((2 * x) + 1))))};
\nextgroupplot[title=Naive, legend pos=north west,xlabel=string size, ylabel=recusive calls]
\addplot[green]{2 * x - 1};
\addlegendentry{Case 1}
\addplot[blue]{x};
\addlegendentry{Case 2}
\end{groupplot}
\end{tikzpicture}
\caption{Anticipation of the behaviours of the parsers, grammar 2}
\end{figure}
\FloatBarrier

Since the number of recursive calls of the naive parser with that grammar is linear it should be very effective compared to the `boolean' bottom-up one.
The naive parser should also be more effective than the `string' bottom-up parser since this one also follows a power function.

\subsubsection{Comparing the efficiency}

With that grammar there is only two cases to compare, the case in which the string matches the grammar and the case when it doesn't, except for the `string' bottom-up which has a behaviour depending on the pattern but for which those two patterns are the extrem cases.

The first used pattern is a case in which the strings match the grammar: `$a\string^ n$'.

\FloatBarrier
\begin{figure}[h]
\begin{tikzpicture}
    \begin{groupplot}[group style={group size=2 by 2},height=0.5\textwidth,width=0.5\textwidth]
    \nextgroupplot[title=Both bottom-up counters, ylabel=iterations, legend pos = north west]
    \addplot coordinates {
        (50, 187575)
        (100, 1500150)
        (200, 12000300)
        (400, 96000600)
        (600, 324000900)
        (800, 768001200)
        (1000, 1500001500)};
    \addlegendentry{String}
    \addplot coordinates {
        (50, 41800)
        (100, 333600)
        (200, 2667200)
        (400, 21334400)
        (600, 72001600)
        (800, 170668800)
        (1000, 333336000)};
    \addlegendentry{Boolean}
    \nextgroupplot[title=Both bottom-up running time, legend pos = north west, ylabel=seconds]
    \addplot coordinates {
        (50, 0.011391)
        (100, 0.08719)
        (200, 0.668587)
        (400, 5.41536)
        (600, 17.6169)
        (800, 41.9493)
        (1000, 82.579)};
    \addlegendentry{String}
    \addplot coordinates {
        (50, 0.001673)
        (100, 0.011914)
        (200, 0.101755)
        (400, 0.816538)
        (600, 3.0517)
        (800, 8.58051)
        (1000, 18.6903)};
    \addlegendentry{Boolean}
    \nextgroupplot[title=Naive counter, xlabel=string size, ylabel=recursive calls]
    \addplot coordinates {
        (50000, 99999)
        (100000, 199999)
        (200000, 399999)
        (300000, 599999)
        (400000, 799999)};
    \nextgroupplot[title=Naive running time, xlabel=string size, ylabel=seconds]
    \addplot coordinates {
        (50000, 0.001854)
        (100000, 0.002961)
        (200000, 0.005776)
        (300000, 0.007833)
        (400000, 0.010237)};
    \end{groupplot}
\end{tikzpicture}
\caption{Both bottom-up and naive parsers behaviours, grammar 2, case 1}
\end{figure}
\FloatBarrier

The running time of the naive parser compared to the two bottom-up ones with this grammar is so small that it is needed to use different string sizes in order to get relevant results.

With this grammar and that case the `boolean' bottom-up parser is faster than the `string' version.

A limitation of the top-down parser is that it needs to allocate memory for its table, but with strings of the size that are used in this experimentation it is not possible to allocate the memory.
So even if the behaviour is theoricaly exactly the same as with the naive parser in practice with a classic computer it is not possible to parse strings that long with the top-down parser.

The running time of the naive parser is linear, which makes sense since its number of recursive calls is linear too.

An interesting thing is that even if the running time of the naive parser remains very small with big string sizes the maximum size of the recursive stack was reached for strings bigger than 410000 characters.
This can be explained by the fact that when the string size $n$ increases the algorithm goes deeper and deeper in the stack: 1 more level of depth every 2 recursive calls.
This is not an issue that we will be able to see with a pattern that does not starts with a `b', indeed in that case as explained in section 3.2.2.1, case 2, the maximum recursive depth accessed is 2, which will allow the parser to parse longer strings.

The second used pattern is a case in which the strings do not match the grammar: `$b\string^ n$'.

\FloatBarrier
\begin{figure}[h]
\begin{tikzpicture}
    \begin{groupplot}[group style={group size=2 by 2},height=0.5\textwidth,width=0.5\textwidth]
    \nextgroupplot[title=Both bottom-up counters, ylabel=iterations, legend pos = north west]
    \addplot coordinates {
        (50, 62625)
        (100, 500250)
        (200, 4000500)
        (400, 32001000)
        (600, 108001500)
        (800, 256002000)
        (1000, 500002500)};
    \addlegendentry{String}
    \addplot coordinates {
        (50, 41800)
        (100, 333600)
        (200, 2667200)
        (400, 21334400)
        (600, 72001600)
        (800, 170668800)
        (1000, 333336000)};
    \addlegendentry{Boolean}
    \nextgroupplot[title=Both bottom-up running times, legend pos = north west, ylabel=seconds]
    \addplot coordinates {
        (50, 0.005147)
        (100, 0.043097)
        (200, 0.329631)
        (400, 2.56894)
        (600, 8.39397)
        (800, 20.0372)
        (1000, 39.8311)};
    \addlegendentry{String}
    \addplot coordinates {
        (50, 0.001673)
        (100, 0.011914)
        (200, 0.101755)
        (400, 0.816538)
        (600, 3.0517)
        (800, 8.58051)
        (1000, 18.6903)};
    \addlegendentry{Boolean}
    \nextgroupplot[title=Naive counter, xlabel=string size, ylabel=recursive calls]
    \addplot coordinates {
        (50000000, 50000000)
        (100000000, 100000000)
        (200000000, 200000000)
        (300000000, 300000000)
        (400000000, 400000000)
        (500000000, 500000000)
        (600000000, 600000000)};
    \nextgroupplot[title=Naive running time, xlabel=string size, ylabel=seconds]
    \addplot coordinates {
        (50000000, 0.262654)
        (100000000, 0.524298)
        (200000000, 1.05584)
        (300000000, 1.56371)
        (400000000, 2.08152)
        (500000000, 2.61476)
        (600000000, 3.13239)};
    \end{groupplot}
\end{tikzpicture}
\caption{Both bottom-up and naive parsers behaviour, grammar 2, case 2}
\end{figure}
\FloatBarrier

The curves of the `boolean' are the same as with the last case since no matter the pattern used that parser will always have the same behaviour with that grammar.
This case is the best case for the `string' bottom-up parser and the running time is indeed lower than with the first case, however it still takes more time than the `boolean' version.

With that case it was possible to parse a word of 600 millions character in about 3 seconds with the naive parser, which is very impressive.
The parser was maybe able to parse the word in some seconds but the longest part was in fact the time to generate the string and also the time to pass such string in parameter to some functions.
The running time is once again linear as expected.
\\
\\
With both the best and the worst case scenarios for the `string' bottom-up parser it is possible to display the area between the two curves.
The running time of that parser will always be contained in the blue area and the more `$b$'s there is in the given string the more the running time will follow a close curve from the worst case and conversely.

\FloatBarrier
\begin{figure}[h]
    \centering
    \begin{tikzpicture}
        \begin{axis}[title=`String' bottom-up running times, ylabel=seconds, xlabel=string size, legend pos = north west]
        \addplot[mark=*, green, name path=plot1] coordinates {
        (50, 0.005147)
        (100, 0.043097)
        (200, 0.329631)
        (400, 2.56894)
        (600, 8.39397)
        (800, 20.0372)
        (1000, 39.8311)};
        \addlegendentry{Best case}
        \addplot[mark=*, red, name path=plot2] coordinates {
        (50, 0.011391)
        (100, 0.08719)
        (200, 0.668587)
        (400, 5.41536)
        (600, 17.6169)
        (800, 41.9493)
        (1000, 82.579)};
        \addlegendentry{Worst case}
        \addplot[blue!30] fill between[of=plot1 and plot2];
        \addlegendentry{All cases}
        \end{axis}
    \end{tikzpicture}
    \caption{Running time of the `string' bottom-up parser, grammar 3}
\end{figure}
\FloatBarrier


\paragraph{Range} is very prone to outliers.
$ R = \max(X) - \min(X) $

\paragraph{Interquantile range}
$ IQR = Q_3 - Q_1 $

\paragraph{Variance} \hspace{0pt}

\vspace{-30pt}
\begin{gather*}
\var X = s^2 = \frac{1}{n-1}\sum_{i=1}^n \left( X_i - \mean \right)^2
\end{gather*}

\paragraph{Standard deviation}
$s = \sqrt{s^2} = \sqrt{Var}$

\paragraph{Coefficient of variation} \hspace{0pt}

\vspace{-20pt}
\begin{gather*}
cv = \frac{s}{\mean}
\end{gather*}

\paragraph{Median absolute deviation} \hspace{0pt}

\vspace{-20pt}
\begin{gather*}
\mad = \med \left( |X_1 - \med_X|, \ldots, |X_n - \med_X| \right)
\end{gather*}

\vspace{-10pt} %\noindent
where $\med_X$ is the sample median.


\section{Exercises in Discrete Random Variables}\label{S:xsDiscreteRVs}
\begin{ExerciseList}
\Exercise[label={xRV1}]
One number in the following table for the probability function of a random variable $X$ is incorrect.
Which is it, and what should the correct value be?
$$
\begin{array}{c|ccccc}
x&1&2&3&4&5\\\hline
\P(X=x)&0.07&0.10&1.10&0.32&0.40
\end{array}
$$
\Answer
$\P(X=3)$ does not satisfy the condition that $0\leq \P(A)\leq1$ for any event $A$.
If $\Omega$ is the sample space, then $\P(\Omega)=1$ and so  the correct probability is
\[
\P(X=3)\;=\;1-0.07-0.10-0.32-0.40\;=\;0.11 \enspace .
\]

\Exercise
Let $X$ be the number of years before a particular type of machine will need replacement.
Assume that $X$ has the probability function $f(1)=0.1$, $f(2)=0.2$, $f(3)=0.2$, $f(4)=0.2$, $f(5)=0.3$.
\be
\item Find the distribution
  function, $F$,  for $X$, and graph both $f$ and $F$.

\item  Find the probability that the machine needs to be
  replaced during the first 3 years.

\item  Find the probability that the machine needs no
  replacement during the first 3 years.
\ee
\Answer
\be
\item  Tabulate the values for the probability mass function  as follows: $$
\begin{array}{c|ccccc}
x&1&2&3&4&5\\\hline
\P(X=x)&0.1&0.2&0.2&0.2&0.3
\end{array}
$$so the  distribution function is:
\[F(x)\; = \;\P(X \leq x) \;=\;
\begin{cases}
 0 & \text{ if }  0 \leq   x < 1\\
 0.1 & \text{ if } 1 \leq  x < 2\\
0.3 & \text{ if } 2 \leq x < 3\\
 0.5 & \text{ if } 3 \leq x < 4\\
0.7 & \text{ if }  4 \leq x < 5\\
 1& \text{ if }    x \geq 5
\end{cases}
\]

%The graphs of $f(x)$ and $F(x)$ for random variable $X$ are shown below:

%TODO\centering   \makebox{\includegraphics[width=6.5in]{figures/fandFfor5YearMachine.png}}

\item The probability that the machine needs to be replaced during the
  first 3 years is:
$$\P(X \leq  3)\;=\;\P(X=1)+\P(X=2)+\P(X=3)\;=\;0.1+0.2+0.2\;=\;0.5\,.$$
(This answer is easily seen  from the distribution function of $X$.)
\item The probability that the machine needs no replacement during the
first three years is
\[ \P(X >  3)\;=\;\, 1-\P(X \leq  3)\,=\, 0.5 \,.\]
\ee

\Exercise
Of 200 adults, 176 own one TV set, 22 own two TV sets, and 2 own three TV sets.
A person is chosen at random. What is the probability mass function of $X$,  the number of TV sets owned by that person?
\Answer
Assuming that the probability model is being built from the observed relative frequencies, the probability mass function is:
$$
f(x)\;=\;\begin{cases}\frac{176}{200}&x=1\\\frac{22}{200}&x=2\\\frac{2}{200}&x=3\end{cases}$$

\Exercise
Suppose a discrete random variable $X$ has probability function give by
$$\begin{array}{c|cccccccccccc}
x&3&4&5&6&7&8&9&10&11&12&13\\\hline
 \P(X=x)&0.07&0.01&0.09&0.01&0.16&0.25&0.20&0.03&0.02&0.11&0.05
\end{array}
$$
\be
\item[(a)] Construct a row of cumulative probabilities for this table, that
  is, find the distribution function of $X$.
\item[(b)] Find  the following  probabilities.
\bcols{3}
\be
\item[(i)]$\P(X\leq 5)$
\item[(ii)] $\P(X<12)$
\item[(iii)] $\P(X>9)$
\item[(iv)] $\P(X\geq 9)$
\item[(v)] $\P(4 <  X\leq 9)$
\item[(vi)] $\P(4<X<11)$
\ee
\ecols
\ee

\Answer
\begin{itemize}
\item[(a)]  $$\begin{array}{c|cccccccccccc}
x&3&4&5&6&7&8&9&10&11&12&13\\\hline
F(x) =\P(X\leq x)&0.07&0.08&0.17&0.18&0.34&0.59&0.79&0.82&0.84&0.95&1.00
\end{array}
$$

\medskip
\item[(b)]
\be
\item[(i)]$\P(X\leq 5)= F(5) = 0.17$ \\[3pt]
\item[(ii)]$\P(X<12)=\P(X\leq11) = F(11) =0.84$\\[3pt]
\item[(iii)] $\P(X>9)= 1 - \P(X\leq 9) = 1- F(9) =1-0.79=0.21$ \\[3pt]
\item[(iv)] $\P(X\geq 9)=1-\P(X <  9)=1-\P(X\leq 8)=1-0.59=0.41$ \\[3pt]
\item[(v)] $\P(4 <  X\leq 9)= F(9) - F(4) =0.79-0.08=0.71$ \\[3pt]
\item[(vi)] $\P(4<X<11)=\P(4 <  X\leq 10)= F(10) -F(4) =0.82- 0.08=0.74$ \\[3pt]
\ee
\end{itemize}

\Exercise
A box contains 4 right-handed and 6 left-handed screws.
Two screws are drawn at random without replacement. Let $X$ be the number of left-handed screws drawn.
Find the probability mass function for $X$, and then calculate  the following probabilities:
\be
\item $\P(X\leq 1)$
\item $\P(X\geq 1)$
\item $\P(X>1)$
\ee
\Answer
Since  we are sampling without replacement,
\begin{eqnarray*}
\P(X=0) &=& \frac{4}{10}\cdot\frac{3}{9} = \frac{2}{15}\quad (\text{one way of drawing two right screws}), \\
\P(X=1) &=& \frac{6}{10}\cdot\frac{4}{9}+\frac{4}{10}\cdot\frac{6}{9} = \frac{8}{15}\quad(\text{two ways of drawing one left and one  right screw}),\\
\P(X=2) &=& \frac{6}{10}\cdot\frac{5}{9} = \frac{1}{3}\quad(\text{one way of drawing two left screws}).
\end{eqnarray*}
So the probability mass function of $X$ is:
\[f(x)\;=\;\P(X=x)\;=\;
\begin{cases}
\frac{2}{15}  & \text{if } x=0\\[3pt]
\frac{8}{15} & \text{if } x=1\\[3pt]
\frac{1}{3}  & \text{if } x=2
\end{cases}\]

The required probabilities are:
\be
\item
$$\P(X\leq1)\;=\;\P(X=0)+\P(X=1)\;=\;\frac{2}{15}+\frac{8}{15}\;=\;\frac{2}{3}$$
\item
$$\P(X\geq 1)\;=\;\P(X=1)+\P(X=2)\;=\;\frac{8}{15}+\frac{1}{3}\;=\;\frac{13}{15}$$
\item
$$\P(X>1)\;=\;\P(X=2)\;=\;\frac{1}{3}$$
\ee

\Exercise
Suppose that  a random variable $X$ has geometric  probability mass function,
  \[f(x)\;=\;\frac{k}{2^x}\quad (x=0,1,2,\dots)\,.\]
\be
\item Find the value of  $k$.
\item  What is  $\P(X\geq 4)$?
\ee
\Answer
\be
\item
Since $f$ is a probability mass function,  $$\sum_{x=0}^\infty\frac{k}{2^x}\;=\;1\,,
\qquad \text{that is,}\qquad k\,\sum_{x=0}^\infty\frac{1}{2^x}\;=\;1\ \enspace.$$
Now $\displaystyle \sum_{x=0}^\infty\frac{1}{2^x}$ is a geometric series with common ratio $r= \frac{1}{2}$ and first term $a=1$,
and so has  sum
\[ S\;=\; \frac{a}{1-r} \;=\;\frac{1}{1-\frac{1}{2} } \;=\; 2\]
Therefore, \[  2 k\;=\; 1 \,, \quad \text{that
  is,}\quad k\;=\; \frac{1}{2}\,.\]

\item From (a), the probability mass function of $f$ is
  $$f(x)\;=\;\frac{\frac{1}{2}}{2^{x}}\;=\;\frac{1}{2^{x+1}}\,.\quad
  (x=0,1,2,\dots)$$ Now
\[\P(X\geq 4)\;=\;1-\P(X<4)\;=\;1-\P(X\leq3)\] where
\ba{\P(X\leq3)&=\;\sum^3_{x=0}\frac{1}{2^{x+1}}\\[3pt]
&=\;\frac{1}{2}+\frac{1}{4}+\frac{1}{8}+\frac{1}{16}\\[3pt]
&=\;\frac{8}{16}+\frac{4}{16}+\frac{2}{16}+\frac{1}{16}\\[3pt]
&=\;\frac{15}{16}\enspace.
}
That is,  $\P(X\geq4)\,=\,\displaystyle \frac{1}{16}$.
\ee

\Exercise
Four fair coins are tossed simultaneously.
If we count the number of heads that appear then we have a  binomial random variable,  $X=$ {\it the number of heads}.
\be
\item Find the probability mass
  function of  $X$.
\item  Compute the probabilities of obtaining no heads, precisely 1
  head, at least 1 head, not more than 3 heads.
\ee
\Answer
Note that  $\theta=\frac{1}{2}$ here.
\be
\item $X$ has probability mass function
\[f(x)\;=\;\begin{cases}
\displaystyle\displaystyle \binom{4}{0}\frac{1}{2}^0\frac{1}{2}^4=\frac{1}{16}&x=0\\[16pt]
\displaystyle\displaystyle\binom{4}{1}\frac{1}{2}^1\frac{1}{2}^3=\frac{4}{16}&x=1\\[16pt]
\displaystyle\displaystyle\binom{4}{2}\frac{1}{2}^2\frac{1}{2}^2=\frac{6}{16}&x=2\\[16pt]
\displaystyle\binom{4}{3}\frac{1}{2}^3\frac{1}{2}^1=\frac{4}{16}&x=3\\[16pt]
\displaystyle\binom{4}{4}\frac{1}{2}^4\frac{1}{2}^0=\frac{1}{16}&x=4
\displaystyle\end{cases}
\]

\item The required probabilities are:

$\displaystyle \P(X=0)=f(0)=\frac{1}{16}$\\[3pt]
$\displaystyle \P(X=1)=f(1)=\frac{4}{16}$\\[3pt]
$\displaystyle \P(X\geq1)=1-\P(X=0)=1-f(0)=\frac{15}{16}$\\[3pt]
$\displaystyle \P(X\leq3)=f(0)+f(1)+f(2)+f(3)=\frac{15}{16}$
\ee

\Exercise
The distribution of blood types in a certain  population is as follows:
$$
\begin{array}{c|cccc}
\text{Blood type}&\text{Type } O&\text{Type } A&\text{Type } B& \text{Type }AB\\\hline
\text{Proportion}&0.45&0.40&0.10&0.05
\end{array}
$$
A random sample of 15 blood donors is observed from this
population. Find the probabilities of the following events.

\be
\item Only one type $AB$ donor is included.
\item At least three of the donors are type $B$.
\item More than ten of the donors are \emph{either} type $O$ \emph{or} type $A$.
\item Fewer that five of the donors are \emph{not} type $A$.
\ee
\Answer
\be
\item  If the random variable $X$ denotes the number of type $AB$ blood donors
  in the sample of 15, then $X$ has a binomial distribution with $n=15$
  and $\theta=0.05$.  Therefore
\[\P(X=1) \;=\;\binom{15}{1} (0.05)^1  (0.95)^{14}\;=\;0.366 \quad (\text{3 sig. fig.}) \,.\]

\medskip
\item  If the random variable $X$ denotes the number of type $B$  blood donors
  in the sample of 15, then $X$ has a binomial distribution with $n=15$
  and $\theta=0.10$.  Therefore
\ba{\P(X\geq 3) &\;=\;  1\, - \,\P(X=0)\, -\, \P(X=1)\, -\, \P(X=2) \\[3pt]
 &\;=\;  1\; - \; \binom{15}{0} (0.1)^0  (0.9)^{15}\;-\; \binom{15}{1}
 (0.1)^1  (0.9)^{14}\;-\; \binom{15}{2} (0.1)^2  (0.9)^{13}\\[3pt]
&\;=\;  1\,-\, 0.2059 \,-\, 0.3432 \,-\, 0.2669\\[3pt]
&\;=  \;0.184 \quad (\text{to 3 sig. fig.})
}

\medskip
\item If the random variable $X$ denotes the number of type  $O$ or type $A$ blood donors
  in the sample of 15, then $X$ has a binomial distribution with $n=15$
  and $\theta=0.85$.  Therefore
\ba{\P(X > 10 ) &\;=\;  \P(X=11)\, + \, \P(X=12)\, +\,  \P(X=13) \, + \, \P(X=14)\, + \, \P(X=15)  \\[3pt]
 &\;=\;   \binom{15}{11} (0.85)^{11}  (0.15)^{4}\;+\; \binom{15}{12}
 (0.85)^{12}  (0.15)^{3}\\[3pt]
&\;+\; \binom{15}{13} (0.85)^{13}  (0.15)^{2}\;+\; \binom{15}{14} (0.85)^{14}  (0.15)^{1}\;+\; \binom{15}{15} (0.85)^{15}  (0.15)^{0}\\[3pt]
&\;=\;  0.1156 \,+\,0.2184 \,+\,0.2856 \,+\, 0.2312\,+\,0.0874\\[3pt]
&\;=\;0.938 \quad (\text{to 3 sig. fig.})
}

\medskip
\item If the random variable $X$ denotes the number of blood donors that
  are \emph{not} of type $A$ blood donors
  in the sample of 15, then $X$ has a binomial distribution with $n=15$
  and $\theta=0.6$.  Therefore
\ba{\P(X <  5) &\;=\;  \P(X=0)\, +\, \P(X=1)\, + \, \P(X=2)\, + \, \P(X=3)\, + \, \P(X=4) \\[3pt]
 &\;=\;   \binom{15}{0} (0.6)^0  (0.4)^{15}\;+\; \binom{15}{1}
 (0.6)^1  (0.4)^{14}\;+\; \binom{15}{2} (0.6)^2  (0.4)^{13}\\[3pt]
&\;+\; \binom{15}{3} (0.6)^3  (0.4)^{12}\;+\; \binom{15}{4} (0.6)^4  (0.4)^{11}\\[3pt]
&\;=\;  0.0000 \,+\,  0.0000\,+\, 0.0003\,+\, 0.0016\,+\,0.0074\\[3pt]
&\;=\;0.009 \quad (\text{to 3 DP.})
}
\ee

\Exercise
If the probability of hitting a target in a single shot is $10\%$ and 10 shots are fired independently, what is the probability that the target will be hit at least once?
\Answer
This is a Binomial experiment with parameters $\theta=0.1$ and $n=10$, and so
\[\P(X\geq 1) = 1-\P(X<1) = 1-\P(X=0) \enspace ,\]
where
\[\P(X=0) = \binom{10}{0}0.1^0 0.9^{10} \approxeq 0.3487 \enspace .\]

 Therefore, the probability that the target will be hit at least once is
 \[1- 0.3487 \approxeq 0.6513 \enspace .\]

%question  (poisson)
\Exercise
Suppose that a certain type of magnetic tape contains, on the average, 2 defects per 100 meters.
What is the probability that a roll of tape 300 meters long will contain  no defects?
\Answer
Since 2 defects exist on every 100 meters, we would expect 6 defects on a 300 meter tape.
If $X$ is the number of defects on a 300 meter tape, then $X$ is Poisson with $\lambda = 6$ and so the probability of zero defects is
$$\P(X=0;6)\;=\;\frac{6^0}{0!}e^{-6}\;=\;0.0025\enspace.$$

%question  (poisson)
\Exercise
In 1910, E.~Rutherford and H.~Geiger showed experimentally that the number of alpha particles emitted per second in a radioactive process is a random variable $X$ having a Poisson distribution. If the average number of particles emitted per second is  0.5, what is the probability of observing two or more particles during any given second?
\Answer
Since $X$ is $\poisson(\lambda)$ random variable with $\lambda=0.5$, $\P(X\geq2)$
is the probability of observing two or more particles during any
given second. $$\P(X\geq 2)\;=\;1-\P(X<2)\;=\;1-\P(X=1)-\P(X=0)\enspace,$$
where $\P(X=1)$ and $\P(X=0)$ can be carried out by the Poisson
probability mass function $$\P(X=x)\;=\; f(x)\;=\;\frac{\lambda^x}{x!}e^{-\lambda}\enspace.$$
Now \[\P(X=0)\;=\;\frac{0.5^0}{0!}\,\times \,e^{-0.5} \;=\; 0.6065\] and
\[ \P(X=1)\;=\;\frac{0.5^1}{1!}\,\times \, e^{-0.5}\;=\; 0.3033\] and so
$$\P(X\geq 2)\;=\; 1- 0.9098\;= \;0.0902\enspace.$$

%question  (poisson)
\Exercise
The number of lacunae (surface pits) on specimens of steel, polished and examined in a metallurgical laboratory, is thought to have a Poisson distribution.
\be
\item Write down the formula for the probability that a specimen has $x$
  defects, explaining the meanings of the symbols you use.
\item Simplify the formula in the case $x=0$.
\item In a large homogeneous collection of specimens, 10\% have one or more lacunae. Find (approximately) the percentage having exactly two.
\item Why might the Poisson distribution not apply in this situation?\\[4pt]
[{\scriptsize HINT: Recall the {\em emphasised sentence} in THINKING POISSON and what the continuum on which the number of events occur is for the problem, and what could possibly go wrong in your imagination of the manufacturing process of the steel specimens (normally you need to melt and manipulate iron with other elements and cast them in moulds and this needs energy and raw materials of possibly varying quality and the machines used in the process could break down, etc.) to violate the Poisson assumption about the occurrence of pits on the surface of the specimens.}]
\ee
\Answer
\be
\item The Probability mass function for $\poisson(\lambda)$ random variable $X$ is
\[\P(X=x)\;=\;f(x;\lambda)\;=\; \frac{e^{-\lambda}\lambda^x}{x!}\]
where $\lambda$ is the mean number of lacunae per specimen and $X$ is
the random variable ``number of lacunae on a specimen''.

\item If $x=0$ then $x!=0!=1$ and $\lambda^x=\lambda^0=1$, and the formula becomes
$\displaystyle \P(X=0) \,=\, e^{-\lambda}$.

\item Since $\P(X \geq 1) = 0.1$, \[\P( X=0)\;=\; 1 -  \P(X \geq 1)\;=\; 0.9\,.\]

Using (b) and solving for $\lambda$ gives:
\[ e^{-\lambda}\;=\; 0.9 \quad \text{that is,}\quad \lambda \;=\; -\ln(0.9) \;=\; 0.1\;
(\text{approximately}\,.)\]

Hence \[\P( X=2 )\;=\;   \frac{e^{-0.1} (0.1)^2}{2!}  \;=\;0.45\% \;
(\text{approximately}\,.) \]

\item Occurrence of lacunae may not always be independent. For example, a machine malfunction may cause them to be clumped.
\ee

\end{ExerciseList}


\section{Standing wave in 3D}

You are singing in a 1\,m $\times$ 2\,m $\times$ 3\,m shower room.\\
\begin{enumerate}[label=\alph*)] % a), b), c), ...

\item (3 pts) The sound waves generated are pressure waves propagating in air with some speed $c_s$. What is the wave equation for the displacement $\xi$ from equilibrium position in 3D?

\item (2 pts) Assuming the walls are immovable, what are the boundary conditions of the wave equation? Justify your answer.
\\

Functions of the form $\xi(x,y,z,t)=A\sin(k_x x)\sin(k_y y)\sin(k_z z)\cos(\omega t)$ solve the wave equation.

\item (3 pts) Determine $\omega$ as a function of $k_x, k_y, k_z$.

\item (2 pts) What are the 3 lowest normal mode frequencies?

\end{enumerate}

% \newpage
% \section{Solutions}
%
% \begin{enumerate}[a)]
% 	\item The wave equation is
% 	\begin{equation}
% 	c_s^2 \nabla^2 \xi =\partial_t^2 \xi
% 	\end{equation}
%
%
% 	\item Dirichlet BC, $\xi|_{\partial}=0$. No displacement at the boundary.
%
%
% 	\item The dispersion relation can be written as
% 	\begin{equation}
% 	\omega^2 = c_s^2 (k_x^2+k_y^2+k_z^2)
% 	\end{equation}
%
%
% 	\item The normal mode frequencies are determined by
%
% 	\begin{equation}
% 	\omega^2 = c_s^2 ((\frac{n_x\pi}{1})^2+(\frac{n_y\pi}{2})^2+(\frac{n_z\pi}{3})^2)
% 	\end{equation}
%
% 	Hence by a simple counting, we have the 3 lowest 3-tuple: $(1,1,1), (1,1,2), (1,2,1)$, ordered as $(n_x,n_y,n_z)$.
%
%
% \end{enumerate}


\subsection{Trigonometric Identities}
There are numerous trigonometric identities, including those relating to shift/periodicity, Pythagoras type identities, double-angle formulas, half-angle formulas and addition formulas.
We list these below.

\begin{enumerate}
\item[1.] \dfont{Shifts and periodicity}
$$\begin{array}{|rcl|rcl|rcl|}
\hline
~&~&~&~&~&~&~&~&~\\
\ds{\sin (\theta + 2\pi)} & = & \ds{\sin \theta} &
\ds{\cos (\theta + 2\pi)} &= & \ds{\cos \theta} &
\ds{\tan (\theta + 2\pi)} &= & \ds{\tan \theta} \\
~&~&~&~&~&~&~&~&~\\
\hline
~&~&~&~&~&~&~&~&~\\
\ds{\sin (\theta + \pi)} &= & \ds{-\sin \theta} &
\ds{\cos (\theta + \pi)} &= & \ds{-\cos \theta} &
\ds{\tan (\theta + \pi)} &= & \ds{\tan \theta} \\
~&~&~&~&~&~&~&~&~\\
\hline
~&~&~&~&~&~&~&~&~\\
\ds{\sin (-\theta)} &= & \ds{-\sin \theta} &
\ds{\cos (-\theta)} &= & \ds{\cos \theta} &
\ds{\tan (-\theta)} &= & \ds{-\tan \theta}  \\
~&~&~&~&~&~&~&~&~\\
\hline
~&~&~&~&~&~&~&~&~\\
\ds{\sin \left(\theta +\frac{\pi}{2}\right) } &= &  \ds{\cos \theta} &
\ds{\cos \left(\theta -\frac{\pi}{2}\right)} &= &  \ds{\sin \theta} &
\ds{\tan \left(\frac{\pi}{2}-\theta\right)} &= & \ds{\cot \theta}  \\
~&~&~&~&~&~&~&~&~\\
\hline
\end{array}$$
\end{enumerate}

\begin{multicols}{2}
\begin{enumerate}
\item[2.] \dfont{Pythagoras type formulas}

$ \begin{array}{|rcl|} \hline
&& \\
\ds{\sin ^2 \theta + \cos ^2 \theta} & = & \ds{1}\\
&& \\
\ds{\tan^2 \theta + 1} & = & \ds{\sec^2 \theta} \\
&& \\
\ds{1 + \cot ^2 \theta}  & = & \ds{\csc^2 \theta} \\
&& \\ \hline
\end{array} $

\vspace{5mm}
\item[3.] \dfont{Addition and subtraction formulas}

$\begin{array}{|rcl|} \hline
&& \\
\ds{\sin(\theta+\phi)} & = & \ds{\sin\theta\cos\phi + \cos\theta\sin\phi} \\
&& \\
\ds{\cos(\theta+\phi)} & = & \ds{\cos\theta\cos\phi - \sin\theta\sin\phi} \\
&& \\
\ds{\tan(\theta+\phi)} & = & \ds{\frac{\tan\theta + \tan\phi}{1 - \tan\theta\tan\phi}}\\
&& \\
\ds{\sin(\theta-\phi)} & = & \ds{\sin\theta\cos\phi - \cos\theta\sin\phi} \\
&& \\
\ds{\cos(\theta-\phi)} & = & \ds{\cos\theta\cos\phi + \sin\theta\sin\phi} \\
&& \\ \hline
\end{array} $

\item[4.] \dfont{Double-angle formulas}

$\begin{array}{|rcl|} \hline
&& \\
\ds{\sin(2\theta)} & =  & \ds{2\sin\theta\cos\theta} \\
&& \\
\ds{\cos(2\theta)} & = & \ds{\cos^2\theta - \sin^2\theta}\\
& =  & \ds{2\cos^2\theta - 1} \\
& = & \ds{1-2\sin^2\theta}\\
&& \\ \hline
\end{array}$

\vspace{4mm}
\item[5.] \dfont{Half-angle formulas}

$ \begin{array}{|rcl|} \hline
&& \\
\ds{\cos^2\theta} & = & \ds{\frac{1+\cos(2\theta)}{2}} \\
&& \\
\ds{\sin^2\theta} & = & \ds{\frac{1-\cos(2\theta)}{2}} \\
&& \\ \hline
\end{array} $

\end{enumerate}
\end{multicols}

\begin{example}{Double Angle}{DoubleAngle}
Find all values of $x$ with $0\leq x\leq \pi$ such that $\sin 2x=\sin x$.
\end{example}

\begin{solution}
Using the double-angle formula $\sin 2x = 2\sin x \cos x$ we have:

\hspace{3cm} $$\begin{array}{rcl}
\ds{2\sin x\cos x} & = & \ds{\sin x} \\
\ds{2\sin x\cos x- \sin x} & = & \ds{0} \\
\ds{\sin x \, (2\cos x - 1)} & = & \ds{0} \\
\end{array}$$

Thus, either \hspace{1cm} $\,\, \sin x=0 \,\, $ or $\,\, \cos x = 1/2\,\, $. \\

For the first case when $\,\, \sin x = 0 \,\, $, we get $\, x=0 \, $ or $\, x=\pi \,$.
For the second case when $\,\,\cos x = 1/2 \,\,$, we get $\, x=\pi/3 \,$ (use the special triangles and CAST rule to get this).
Thus, we have three solutions: $x=0, \, \pi/3, \, \pi$.
\end{solution}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS630: Database Management Systems
% Copyright 2014 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 1}

Write \textbf{relational algebra} expressions for the following queries:

\begin{enumerate}[label=(\alph*)]

\item Find the ages of employees who work in a department named 'Catering'.

\textbf{Solution:}
$$ \pi_{E.age}(\pi_{W.eid}(\pi_{D.did}(\sigma_{D.dname = 'Catering'}D)\Join W)\Join E) $$
\item Find salaries of employees that work at least 30\% of their time in departments that have budget at least \$50000.

\textbf{Solution:}
$$ \pi_{E.salary}(\pi_{W.eid}(\sigma_{W.pct\_time \geq 30}(\pi_{D.did}(\sigma_{D.budget \geq 50000}D)\Join W))\Join E) $$

\item Find the salaries of department managers.

\textbf{Solution:}
$$ \pi_{E.salary}(\sigma_{E.eid = D.managerid}((D \Join_{W.did = D.did} W)\Join_{W.eid = E.eid} E)) $$

\item Find the names of employees who work in the \textit{Marketing} department or who spend more than half of their time in a single department.

\textbf{Solution:}
$$\rho(E1,\pi_{W.eid}(\pi_{D.did}(\sigma_{D.dname ='Marketing'}D)\Join W))$$
$$\rho(E2,\pi_{E.eid}(\sigma_{W.pct\_time \geq 50}(E \Join W)))$$
$$\pi_{E.ename}((E1 \cup E2)\Join E) $$

\item Find the \texttt{eid} of managers who manage exactly one department.

\textbf{Solution:}
\begin{enumerate}
\item M is defined as \texttt{eid} of managers who manage more than two department.
$$\rho(D1(1\rightarrow did2, 2\rightarrow dname2, 3\rightarrow budget2, 4\rightarrow managerid2),Department) $$
$$\rho(M,\pi_{managerid}(Department \Join_{(managerid = managerid2 \wedge did \neq did2)} D1 )) $$
\item desired list $M^\prime$ is obtained by subtracting M from the list of all managers.
$$\rho(M^\prime,\pi_{E.eid}(\pi_{W.eid}((\pi_{managerid}Department - M)\Join_{managerid = eid} Works)\Join Employee))$$
\end{enumerate}
\end{enumerate}