\section{08/26}
\subsection{Birthday Paradox}
The probability that no two people share a birthday is
\begin{align*}\prob{E_1 \cap E_2 \cap \dots \cap E_n}
    &= \prob{E_1}\prob{E_2 \given E_1}\cdots \prob{E_n \given E_1 \cap E_2 \cap \dots \cap E_{n - 1}}\\
    &= 1 \times \left(1 - \frac{1}{365}\right) \times \left(1 - \frac{2}{365}\right) \times \cdots \times \left(1 - \frac{365 - n + 1}{365}\right)
    \shortintertext{Applying the inequality $1 - x \leq e^{-x}$ when $\abs{x} < 1$}\\
    &\leq 1 \cdot e^{-\frac{1}{365}} \cdot e^{-\frac{2}{365}} \cdots e^{-\frac{n - 1}{365}}\\
    &= e^{-\frac{n(n-1)}{2\cdot365}}
\end{align*}
Setting
\[e^{-\frac{n(n-1)}{2\cdot365}} \leq \frac{1}{2}\]
yields $n \geq 23$.

\subsection{Randomized Algorithms}
\begin{problem}{Toy Problem}{}
Let $A$ be an array of $n$ numbers with the property that at least one number
occurs at least $\sfrac{n}{2}$ times and the rest are distinct. Our goal is to
find the element with duplicate entries.
\end{problem}

Any deterministic algorithm will require at least $\sfrac{n}{2} + 1$ operations.
However, we can construct a randomized algorithm that only requires
$\bigO{\log{n}}$ operations.

We do this by \emph{random sampling}. Assume that choosing a single element in
$\set{a_1, a_2, \dots, a_n}$ requires $\bigO{1}$ time and that we sample
\emph{with replacement}. We will frequently use sampling with replacement
because it allows the samples to be independent, and therefore simplifies the
analysis. Now, consider the following algorithm:

\begin{algorithm}
    \caption{Randomized Algorithm for Toy Problem}
    \begin{algorithmic}[1]
        \Function{Rand-Duplicate-Toy}{\texttt{arr}}
            \Loop\label{line:toyloop}
                \State $\texttt{sample}_1 \gets \Call{Sample}{\texttt{arr}}$
                \State $\texttt{sample}_2 \gets \Call{Sample}{\texttt{arr}}$
                \If{$\texttt{sample}_1 \neq \texttt{sample}_2$}
                    \State \Return $\texttt{sample}_1$
                \EndIf
            \EndLoop
        \EndFunction
    \end{algorithmic}
\end{algorithm}

This algorithm simply chooses two elements (not necessarily distinct) at random
from the array and repeats until the two elements are identical, after which it
outputs the sampled element as the duplicate.

Let us analyze the probability that this algorithm outputs the duplicate value
after a single iteration. Suppose the two values chosen are $a_i$ and $a_j$.
Then
\begin{align*}\prob{a_i = a_j \text{ and } i \neq j}
    &= \frac{\sfrac{n}{2}}{n} \cdot \frac{\sfrac{n}{2} - 1}{n}\\
    &= \frac{1}{2} \left(\frac{1}{2} - \frac{1}{n}\right)\\
    &= \frac{1}{4} - \frac{1}{2n}\\
    &\geq \frac{3}{2} \hbox{ for $n \geq 5$}
\end{align*}
What is the probability, then, that our algorithm fails after $k$ iterations of
\cref{line:toyloop}? These events are independent, hence the probability is
bounded above by
\begin{align*}\prob{\text{failure}}
    &\leq \left(1 - \frac{3}{20}\right)^k\\
    &\leq e^{-\frac{3}{20}k}
    \shortintertext{Setting $k = \frac{20}{3}\ln{n}$,}\\
    &= e^{-\ln{n}}\\
    &=\frac{1}{n}
\end{align*}

\subsection{Bayes' Law}
\begin{theorem}{Bayes' Law}{}
    For any events $A$ and $B$,
    \[\prob{A \given B} = \frac{\prob{B \given A} \prob{A}}{\prob{B}}\]
\end{theorem}

\subsection{Concepts in Randomized Algorithms}
\begin{definition}{}{}
    An algorithm is \emph{Monte Carlo} if it has a non-zero probability of
    outputting an incorrect solution. If an algorithm will output the correct
    solution with probability 1, it is \emph{Las Vegas}.
\end{definition}

\begin{definition}{High Probability}{}
    Given an input of size $n$, we say an event occurs \emph{with high
    probability} (whp) if it occurs with probability $1 - \sfrac{1}{n^c}$ for
    some $c > 0$.
\end{definition}

\begin{problem}{Modified Toy Problem}{}
    Let $A$ be an array of $n$ numbers with the property that \emph{either} all
    elements are distinct \emph{or} $\sfrac{n}{2}$ elements are repeated.
\end{problem}

Let us define
\begin{align*}
    E_1 &= \hbox{event that exactly $\sfrac{n}{2}$ elements are distinct}\\
    E_2 &= \hbox{event that all $n$ elements are distinct}
\end{align*}

At first, our ``belief'' in $E_1$ and $E_2$ are approximately the same, i.e., we
have no information to determine which event is more likely. However, let us
assume we have sampled two elements with replacement and determined that they
are not equal (call this event $F$). We can update our ``belief'' via Bayes'
Law.
\begin{align*}\prob{E_1 \given F}
    &= \frac{\prob{F \given E_1} \prob{E_1}}{\prob{F}}\\
    &\leq \frac{\left(1 - \frac{3}{20}\right) \cdot \frac{1}{2}}{\left(1 - \frac{3}{20}\right) \cdot \frac{1}{2} + \frac{1}{2}}\\
    &= \frac{17}{37}
\end{align*}
Thus, we see that the probability has shifted from ``50-50'' to skewing \emph{in
favor} of the belief that $E_2$ has occurred.

\subsection{Law of Total Probability}
\begin{theorem}{Law of Total Probability}{}
    For any event $A$ and any countable partition of the sample space $B_1$,
    $B_2$, \dots, that is, the events $B_i$ are pairwise disjoint and their
    union forms the entire sample space, we have
    \[\prob{A} = \sum_i \prob{A \cap B_i} = \sum_i \prob{A \given B_i} \prob{B_i}\]
\end{theorem}

\subsection{Naive Bayesian Classifier}
A Naive Bayesian Classifier is a classic example of how we can apply
probabilistic techniques and Bayes' Law in practice. Suppose we have a
\emph{feature set} $e$, a vector of boolean $n$ boolean variable. These could
be, for example, whether a certain keyword appears in an email in an attempt to
detect spam. Then, assuming the independence of the boolean variables (this is
why it is referred to as \emph{naive}), we have
\begin{align*}\prob{E \given e = (b_1, b_2, \dots, b_n)}
    &= \frac{\prob{e = (b_1, b_2, \dots, b_n) \given E} \prob{E}}{\prob{e = (b_1, b_2, \dots, b_n)}}\\
    &= \frac{\prob{b_1 \given E} \cdot \prob{b_2 \given E} \cdots \prob{b_n \given E} \cdot \prob{event}}{\prob{e = (b_1, b_2, \dots, b_n)}}
\end{align*}


\chapter{Equations}
\section{Quadratic equations}
\begin{dfn}
A quadratic equation is an equation of the form
\begin{equation}
  \label{quad}
  ax^2 + bx + c = 0
\end{equation}
where \( a, b \) and \( c \) are constants and \( a \neq 0 \).
\end{dfn}
\begin{thm}
A quadratic equation (\ref{quad}) has two solutions for the variable \( x \):
\begin{equation}
  \label{root}
  x_{1,2} = \frac{-b \pm \sqrt{b^2-4ac}}{2a}
\end{equation}
\end{thm}
\begin{dfn}
\label{disc}
The \emph{discrimimant} of a quadratic equation (\ref{quad})
is called \( \Delta \) and it is defined as:
\[
  \Delta = b^2 - 4ac
\]
\end{dfn}
\begin{lem}
If the discrimimant \( \Delta \) is zero, then the equation (\ref{quad}) has a double solution: (\ref{root}) becomes:
\[
  x = - \frac{b}{2a}
\]
\end{lem}
\begin{proof}
The equation \eqref{quad} together with the definition \ref{disc} becomes:
\begin{align*}
  x_{1,2} &= \frac{-b \pm \sqrt{b^2-4ac}}{2a} \\
          &= \frac{-b \pm \sqrt{\Delta}}{2a} \\
          &= \frac{-b \pm \sqrt{0}}{2a} \\
          &= -\frac{b}{2a} \\
\end{align*}
and there exists only one solution.
\end{proof}


\subsection{Variation of Parameters for Systems}
\noindent
Let $X$ be a fundamental matrix for the homogeneous system.
\begin{equation*}
	\vec{x_h}' = A\vec{x_h}.
\end{equation*}
That is,
\begin{equation*}
	\vec{x_h} = X\vec{c}
\end{equation*}
where
\begin{equation*}
	\vec{c} = \begin{bmatrix}
	C_1 \\
	\vdots \\
	C_n
	\end{bmatrix}
\end{equation*}
and the entries of matrix $A$ can be any continuous functions of $t$.

\noindent
We are looking for the particular solution $\vec{x_p}$, to the system
\begin{equation*}
	\vec{x} = A\vec{x} + \vec{f}
\end{equation*}
where $\vec{x_p}$ is of the form
\begin{equation*}
	\vec{x_p} = X\vec{v}
\end{equation*}
where $\vec{v}$ is a vector of functions of $t$ that we'll have to find.

\noindent
Differentiating $\vec{x_p}$,
\begin{equation*}
	\vec{x_p}' = X\vec{v}' + X'\vec{v}.
\end{equation*}
From the system we're trying to solve we know that
\begin{equation*}
	X\vec{v}' + X'\vec{v} = A(X\vec{v}) + \vec{f}.
\end{equation*}
Since $X' = AX$,
\begin{equation*}
	X\vec{v}' = \vec{f}.
\end{equation*}

\noindent
Since the columns of $X$ are always linearly independent, we know that $X^{-1}$ always exists.
Multiplying by $X^{-1}$,
\begin{equation*}
	\vec{v}' = X^{-1}\vec{f}.
\end{equation*}
Integrating with respect to $t$,
\begin{equation*}
	\vec{v} = \int{X^{-1}\vec{f} \mathrm{d}t}.
\end{equation*}
So,
\begin{equation*}
	\vec{x_p} = X\int{X^{-1}\vec{f} \mathrm{d}t},
\end{equation*}
and
\begin{equation*}
	\vec{x} = X\vec{c} + X\int{X^{-1}\vec{f} \mathrm{d}t}.
\end{equation*}

\begin{example}
	Find the general solution to the system by variation of parameters
	\begin{equation*}
		\vec{x}' = \begin{bmatrix}
			2 & -3 \\
			1 & -2
		\end{bmatrix}\vec{x} + \begin{bmatrix}
			e^{2t} \\
			1
		\end{bmatrix}
	\end{equation*}
	given the fundamental matrix
	\begin{equation*}
		X = \begin{bmatrix}
			3e^t & e^{-t} \\
			e^t & e^{-t}
		\end{bmatrix}.
	\end{equation*}
\end{example}
\noindent
Finding $\vec{x_h}$ is simply multiplying $X$ by a vector, so we'll save that for the end and focus on $\vec{x_p}$. First we need to find $X^{-1}$. The details are left out, but the process is identical to that for matrices of numbers.
\begin{equation*}
	X^{-1} = \begin{bmatrix}
		\frac{1}{2}e^{-t} & -\frac{1}{2}e^{-t} \\
		-\frac{1}{2}e^t & \frac{3}{2}e^t
	\end{bmatrix}.
\end{equation*}
Multiplying by $\vec{f}$.
\begin{equation*}
	X\vec{f} = \begin{bmatrix}
		\frac{1}{2}e^t - \frac{1}{2}e^{-t} \\
		-\frac{1}{2}e^{3t} + \frac{3}{2}e^t
	\end{bmatrix}.
\end{equation*}
Integrating with respect to $t$,
\begin{equation*}
	\int{X^{-1}\vec{f} \mathrm{d}t} = \begin{bmatrix}
		\frac{1}{2}e^t + \frac{1}{2}e^{-t} \\
		-\frac{1}{6}e^{3t} + \frac{3}{2}e^t
	\end{bmatrix}.
\end{equation*}
Note that although we are doing an indefinite integral, we don't have a constant (or in this case a vector) of integration.
Multiplying by $X$ to obtain $\vec{x_p}$,
\begin{equation*}
	\vec{x_p} = X\int{X^{-1}\vec{f} \mathrm{d}t} = \begin{bmatrix}
		\frac{4}{3}e^{2t} + 3 \\
		\frac{1}{3}e^{2t} + 2
	\end{bmatrix}.
\end{equation*}
Finding $\vec{x_h}$,
\begin{equation*}
	\vec{x_h} = X\vec{c} = \begin{bmatrix}
		3C_1e^t + C_2e^{-t} \\
		C_1e^t + C_2e^{-t}
	\end{bmatrix}.
\end{equation*}
Putting $\vec{x_h}$ and $\vec{x_p}$ together to get the general solution,
\begin{equation*}
	\vec{x} = \begin{bmatrix}
		3C_1e^{3t} + C_2e^{-t} + \frac{4}{3}e^{2t} + 3 \\
		C_1e^t + C_2e^{-t} + \frac{1}{3}e^{2t} + 2
	\end{bmatrix}.
\end{equation*}


\subsection{Heteroskedasticity-Consistent (HC) standard errors}

\subsubsection{Variance of OLS estimators}

\(Var [\hat \theta ]=(X^TX)^{-1}X^T\Omega X(X^TX)^{-1}\)

\subsubsection{Robust standard errors for heteroskedasticity}

\(\Omega_{ij}=\delta_{ij}\epsilon_i\epsilon_j\)

These are also known as the Eicker-Huber-White standard errors, or the White correction.

These are also refered to as robust standard errors.


\subsection{Definition and properties}

When we take the dot product of two vectors, the result is a
scalar. For this reason, the dot product is also called the
\textbf{scalar product}. Sometimes it is also called the \textbf{inner
  product}. The definition is as follows%
\index{dot product}%
\index{vector!dot product}%
\index{scalar product|see{dot product}}%
\index{inner product|seealso{dot product}}.

\begin{definition}{Dot product}{dot-product}
  Let $\vect{u}=\begin{mymatrix}{c}
    u_1 \\
    u_2 \\
    \vdots \\
    u_n
  \end{mymatrix}$, $\vect{v}= \begin{mymatrix}{c}
    v_1 \\
    v_2 \\
    \vdots \\
    v_n
  \end{mymatrix}$ be two vectors in $\R^n$. We
  define their \textbf{dot product} as
  \begin{equation*}
    \vect{u}\dotprod \vect{v} = u_1v_1+u_2v_2+\ldots+u_nv_n.
  \end{equation*}
\end{definition}

\begin{example}{Compute a dot product}{dot-product}
  Find $\vect{u} \dotprod \vect{v}$ for $\vect{u}=\mat{1,2,0,-1}^T$
  and $\vect{v}=\mat{0,1,2,3}^T$.
\end{example}

\begin{solution}
  We have
  \begin{eqnarray*}
    \vect{u} \dotprod \vect{v}
    &=&
        (1)(0) + (2)(1) + (0)(2) + (-1)(3) \\
    &=&
        0 + 2 + 0 + -3 \\
    &=&
        -1.
  \end{eqnarray*}
\end{solution}

The dot product satisfies a number of important properties.

\begin{proposition}{Properties of the dot product}{properties-dot-product}
  \index{dot product!properties}%
  \index{properties of dot product}%
  \index{vector!dot product!properties}%
  \index{vector!properties of dot product}%
  The dot product satisfies the following properties, where
  $\vect{u},\vect{v},\vect{w}$ are vectors and $k,\ell$ are
  scalars.
  \begin{itemize}
  \item $\vect{u}\dotprod\vect{v}=\vect{v}\dotprod\vect{u}$.
  \item $\vect{u}\dotprod\vect{u}\geq 0$, and $\vect{u}\dotprod\vect{u}=0$ if and only if $\vect{u}=\vect{0}$.
  \item $(k\vect{u}+\ell\vect{v})\dotprod\vect{w}=k(\vect{u}\dotprod\vect{w})+\ell(\vect{v}\dotprod\vect{w})$.
  \item $\vect{u}\dotprod(k\vect{v}+\ell\vect{w})
    =k(\vect{u}\dotprod \vect{v})+\ell(\vect{u}\dotprod\vect{w})$.
  \item $\vect{u}\dotprod\vect{u}=\norm{\vect{u}}^2$.
  \end{itemize}
\end{proposition}

The proof is left as an exercise. Note that, by the last part of the
proposition, we can also use the dot product to find the length of a
vector.

\begin{example}{Length of a vector}{dot-product-length}
  Use a dot product to find $\norm{\vect{u}}$, where
  \begin{equation*}
    \vect{u}
    =
    \begin{mymatrix}{r}
      2 \\
      1 \\
      4 \\
      2
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  By the last part of Proposition~\ref{prop:properties-dot-product}, we have
  $\norm{\vect{u}} = \sqrt {\vect{u} \dotprod \vect{u}}$. We have
  $\vect{u} \dotprod \vect{u} = 2^2+1^2+4^2+2^2 = 25$, and therefore
  $\norm{\vect{u}} = \sqrt{\vect{u} \dotprod \vect{u}} = \sqrt{25} = 5$.
\end{solution}


\chapter{Basic Numerical Methods}

\section{Roundoff and Truncation Errors}

\newthought{Floating point numbers} can be represented using either single- (32 bits) or double-precision (64 bits). Single-precision usually stores out to six or seven decimal places while double-precision can usually store up to 16 decimal places. The main pitfall of roundoff error is when two numbers are treated as being equal when the user has not intended for this to be the case. For example, say $\epsilon$ is the smallest number in single-precision that we can store and $\xi<\epsilon$. If for two single-precision floating point numbers $a,b>\epsilon$ we attempt to perform the operation
\begin{equation}
	\frac{b}{(a+\xi)-a},
\end{equation}
we will have some serious problems as our computer will treat $a+\xi$ as $a$, resulting in $0$ in the denominator. Using double-precision often helps to alleviate these problems. Roundoff error depends on the hardware (and probably software) we are using. Thus, it can be persistent across many different parts of the program we are writing.

Truncation error, unlike roundoff error, is generally \textit{not} dependent on our choice of hardware, but rather on our choice of method (i.e. Euler, Runge-Kutta, etc.). Consider the Taylor series
\begin{equation}
	f(x+h) = f(x) + hf^{\prime}(x) + \frac{1}{2}h^2f^{\prime\prime}(x) + \ldots
\end{equation}
This can be rearranged to solve for the derivative of $f$,
\begin{equation}
	f^{\prime}(x)=\frac{f(x+h)-f(x)}{h} + \mathcal{O}(h),
	\label{eq:right_deriv}
\end{equation}
where $\mathcal{O}(h)$ represents the truncation error of order $h$. How is this applied in numerical integration? Consider the example of projectile motion. Our equations of motion can be written as
\begin{align}
	\frac{\mathrm{d}\mathbf{v}}{\mathrm{d}t} &= \mathbf{a}(\mathbf{r},\mathbf{v}), \\[0.5em]
	\frac{\mathrm{d}\mathbf{r}}{\mathrm{d}t} &= \mathbf{v},
\end{align}
where $\mathbf{a}$ is the acceleration. Let's rewrite these equations of motion in terms of Eq. \ref{eq:right_deriv},
\begin{align}
	\frac{\mathbf{v}(t+\tau) - \mathbf{v}(t)}{\tau} + \mathcal{O}(\tau) &= \mathbf{a}(\mathbf{r}(t),\mathbf{v}(t)) \\[0.5em]
	\frac{\mathbf{r}(t+\tau) - \mathbf{r}(t)}{\tau} + \mathcal{O}(\tau) &= \mathbf{v}(t)
\end{align}
and then solving for the updated values of $\mathbf{v}$ and $\mathbf{r}$,
\begin{align}
	\mathbf{v}(t+\tau) &= \mathbf{v}(t) + \tau\mathbf{a}(\mathbf{r}(t),\mathbf{v}(t)) + \mathcal{O}(\tau^2), \\[0.5em]
	\mathbf{r}(t+\tau) &= \mathbf{r}(t) + \tau\mathbf{v}(t) + \mathcal{O}(\tau^2).
\end{align}
Thus, the truncation error for an Euler scheme is quadratic in $\tau$, the timestep. However, we should be careful to draw the distinction between \textit{local} and \textit{global} truncation error. The global error is the sum of all the local truncation errors made at each iteration and is highly dependent on our choice of scheme. Consider a projectile motion problem on $t\in(0,T)$ solved using the Euler method such that the number of steps is $N_{\tau}=T/\tau$. The global error is thus expressed as
\begin{equation}
	\varepsilon_{G}\propto N_{\tau}\times \varepsilon_L.
\end{equation}
In the Euler method case, $\varepsilon\propto T/\tau\times\mathcal{O}(\tau^2)=\mathcal{O}(\tau)$. Thus, the global truncation error of the Euler scheme is $\mathcal{O}(\tau)$.

\section{Curve Fitting and Interpolation}

\newthought{Suppose we have $N$ data points} $(x_i,y_i)$ and we want to fit them to a curve $Y(x;\{a_j\})$ where $\{a_j\}$ is a set of $M$ adjustable parameters. If each point $y_i$ has an associated error $\sigma_i$, the fit parameters can be determined by minimizing the $\chi^2$-function
\begin{equation}
	\label{eq:chi_squared}
	\chi^2(\{a_j\})=\sum^{N}_{i=1}\left(\frac{\Delta_i}{\sigma_i}\right)= \sum^{N}_{i=1}\frac{[Y(x_i;\{a_j\}) - y_i]^2}{\sigma_i^2}.
\end{equation}
The set of parameters $\{a_j\}$ are determined by finding all $a_j$ which satisfy $\partial\chi^2/\partial a_j=0$. Now, say we want to fit $N$ data points to a function of the form
\begin{equation}
	\label{eq:gen_func_lsq}
	Y(x;\{a_j\}) = \sum_{j=1}^Ma_jY_j(x).
\end{equation}
To find $\{a_j\}$, we compute $\partial \chi^2/\partial a_j=0$ which for Eq. \ref{eq:gen_func_lsq} can be expressed as
\begin{align}
	\sum^N_{i=1}\frac{1}{\sigma_i^2}Y_j(x_i)\left(\sum_{k=1}^Ma_kY_k(x_i) - y_i\right) = 0 \\[0.5em]
	\sum^N_{i=1}\sum^M_{k=1}\frac{Y_jYk}{\sigma_i^2} = \sum^N_{i=1}\frac{Y_jy_i}{\sigma_i^2},
\end{align}
for $j=1,\ldots,M$. We can rewrite this equation as
\begin{equation}
	\sum^N_{i=1}\sum^M_{k=1}A_{ij}A_{ik}a_k = \sum^N_{i=1}A_{ij}\frac{y_i}{\sigma_i},
\end{equation}
where $A_{ij}=Y_j(x_i)/\sigma_i$ are the matrix elements of the so-called design matrix. In matrix form, this can be expressed as
\begin{equation}
	(\mathbf{A}^T\mathbf{A})\mathbf{a} = \mathbf{A}^T\mathbf{b},
\end{equation}
where $b_i=y_i/\sigma_i$. This can be easily inverted to solve for $\mathbf{a}$. The estimated errors for $a_j$ can be calculated using the equation
\begin{equation}
	\sigma_{a_j} = \sqrt{C_{jj}},
\end{equation}
where $\mathbf{C} = (\mathbf{A}^T\mathbf{A})^{-1}$ \cite{garcia_numerical_2000}. How does one evaluate whether the curve $Y$ is a good fit to the data $y$? On average, we can say that the difference between the data and the fit should be roughly equal to the error bar such that $\sigma_i\approx|y_i - Y(x_i)|$. Plugging this into Eq. \ref{eq:chi_squared} then gives that $\chi^2\approx N$. However, recalling that we have $M$ degrees of freedom, our condition should be revised such that $\chi^2\approx N-M$ implies an adequate fit. $\chi^2\gg N-M$ suggests that a poor choice of $Y$ or very small $\sigma_i$ and $\chi^2\ll N-M$ implies either overfitting or that the error bars may be too large \cite{garcia_numerical_2000}.
%
\par \hl{Still need something about interpolation}

\section{Numerical Integration}
\section{Basic Linear Algebra: Analytical and Numerical}
\section{Ordinary Differential Equations: Analytical and Numerical}
\section{Partial Differential Equations: Analytical and Numerical}


% !TEX root = lectures.tex
\section{Math Basics}
\bigskip

\subsection{Scalars, Vectors and Matrices}

A scalar is something with a value that is independent of coordinate system. Examples are mass, or the relative time between events. A vector has magnitude and direction. Under rotation, the magnitude stays the same but the direction changes. Scalars have no spatial index, whereas a three-dimensional vector has 3 indices, e.g. the position $\vec{r}$ has components $r_1,r_2,r_3$, which are often referred to as $x,y,z$.

There are several categories of changes of coordinate system. The observer can translate the origin, might move with a different velocity, or might rotate his/her coordinate axes. For instance, a particle's position vector changes when the origin is translated, but its velocity does not. When you study relativity you will find that quantities you thought of as scalars, such as time or electric potential, are actually parts of four-dimensional vectors and that changes of the velocity of the reference frame act in a similar way to rotations.

In addition to vectors and scalars, there are matrices, which have two indices. One also has objects with 3 or four indices. These are called tensors of rank $n$, where $n$ is the number of indices. A matrix is a rank-two tensor. The Levi-Civita symbol, $\epsilon_{ijk}$ used for cross products, is a third-rank tensor.

\subsubsection*{Unit Vectors}

Also known as basis vectors, unit vectors point in the direction of the coordinate axes, have unit norm, and are orthogonal to one another. Sometimes this is referred to as an orthonormal basis,
\begin{equation}
\hat{e}_i\cdot\hat{e}_j=\delta_{ij}=\left(\begin{array}{ccc}
1 & 0 & 0\\
0& 1 & 0\\
0 & 0 & 1
\end{array}\right).
\end{equation}
Here, $\delta_{ij}$ is unity when $i=j$ and is zero otherwise. This is called the unit matrix, because you can multiply it with any other matrix and not change the matrix. The ``dot'' denotes the dot product, $\vec{A}\cdot\vec{B}=A_1B_1+A_2B_2+A_3B_3=|A||B|\cos\theta_{AB}$. Sometimes the unit vectors are called $\hat{x}$, $\hat{y}$ and $\hat{z}$. Vectors can be decomposed in terms of unit vectors,
\begin{equation}
\vec{r}=r_1\hat{e}_1+r_2\hat{e}_2+r_3\hat{e}_3.
\end{equation}
The vector components $r_1$, $r_2$ and $r_3$ might be called $x$, $y$ and $z$ for a displacement of $v_x$, $v_y$ and $v_z$ for a velocity.

\subsubsection*{Rotations}

Here, we use rotations as an example of matrices and their operations. One can consider a different orthonormal basis $\hat{e}'_1$, $\hat{e}'_2$ and $\hat{e}'_3$. The same vector $\vec{r}$ mentioned above can also be expressed in the new basis,
\begin{equation}
\vec{r}=r'_1\hat{e}'_1+r'_2\hat{e}'_2+r'_3\hat{e}'_3.
\end{equation}
Even though it is the same vector, the components have changed. Each new unit vector $\hat{e}'_i$ can be expressed as a linear sum of the previous vectors,
\begin{equation}
\hat{e}'_i=\sum_j U_{ij}\hat{e}_j,
\end{equation}
and the matrix $U$ can be found by taking the dot product of both sides with $\hat{e}_k$,
\begin{eqnarray}
\nonumber
\hat{e}_k\cdot\hat{e}'_i&=&\sum_jU_{ij}\hat{e}_k\cdot\hat{e}_j\\
\label{eq:lambda_angles}
\hat{e}_k\cdot\hat{e}'_i&=&\sum_jU_{ij}\delta_{jk}=U_{ik}.
\end{eqnarray}
Thus, the matrix lambda has components $U_{ij}$ that are equal to the cosine of the angle between new unit vector $\hat{e}'_i$ and the old unit vector $\hat{e}_j$.
\begin{equation}
U = \left(\begin{array}{ccc}
\hat{e}'_1\cdot\hat{e}_1& \hat{e}'_1\cdot\hat{e}_2& \hat{e}'_1\cdot\hat{e}_3\\
\hat{e}'_2\cdot\hat{e}_1& \hat{e}'_2\cdot\hat{e}_2& \hat{e}'_2\cdot\hat{e}_3\\
\hat{e}'_3\cdot\hat{e}_1& \hat{e}'_3\cdot\hat{e}_2& \hat{e}'_3\cdot\hat{e}_3
\end{array}\right),~~~~~U_{ij}=\hat{e}'_i\cdot\hat{e}_j=\cos\theta_{ij}.
\end{equation}
Note that the matrix is not symmetric, $U_{ij}\ne U_{ji}$. One can also look at the inverse transformation, by switching the primed and unprimed coordinates,
\begin{eqnarray}
\label{eq:inverseU}
\hat{e}_i&=&\sum_jU^{-1}_{ij}\hat{e}'_j,\\
\nonumber
U^{-1}_{ij}&=&\hat{e}_i\cdot\hat{e}'_j=U_{ji}.
\end{eqnarray}
The definition of transpose of a matrix, $M^{t}_{ij}=M_{ji}$, allows one to state this as
\begin{eqnarray}
\label{eq:transposedef}
U^{-1}&=&U^{t}.
\end{eqnarray}
A tensor obeying Eq. (\ref{eq:transposedef}) defines what is known as a unitary, or orthogonal, transformation.

The matrix $U$ can be used to transform any vector to the new basis. Consider a vector
\begin{eqnarray}
\vec{r}&=&r_1\hat{e}_1+r_2\hat{e}_2+r_3\hat{e}_3\\
\nonumber
&=&r'_1\hat{e}'_1+r'_2\hat{e}'_2+r'_3\hat{e}'_3.
\end{eqnarray}
This is the same vector expressed as a sum over two different sets of basis vectors. The coefficients $r_i$ and $r'_i$ represent components of the same vector. The relation between them can be found by taking the dot product of each side with one of the unit vectors, $\hat{e}_i$, which gives
\begin{eqnarray}
r_i&=&\sum_j \hat{e}_i\cdot\hat{e}'_j~r'_j.
\end{eqnarray}
Using Eq. (\ref{eq:inverseU}) one can see that the transformation of $r$ can be also written in terms of $U$,
\begin{eqnarray}
\label{eq:rotateR}
r_i&=&\sum_jU^{-1}_{ij}~r'_j.
\end{eqnarray}
Thus, the matrix that transforms the coordinates of the unit vectors, Eq. (\ref{eq:inverseU}) is the same one that transforms the coordinates of a vector, Eq. (\ref{eq:rotateR}).

\example\label{ex:rotmat}
Find the rotation matrix $U$ for finding the components in the primed coordinate system given from those in the unprimed system, given that the unit vectors in the new system are found by rotating the coordinate system by and angle $\phi$ about the $z$ axis.\\
Solution:\\
In this case
\begin{eqnarray*}
\hat{e}'_1&=&\cos\phi \hat{e}_1-\sin\phi\hat{e}_2,\\
\hat{e}'_2&=&\sin\phi\hat{e}_1+\cos\phi\hat{e}_2,\\
\hat{e}'_3&=&\hat{e}_3.
\end{eqnarray*}

By inspecting Eq. (\ref{eq:lambda_angles}),
\[
U=\left(\begin{array}{ccc}
\cos\phi&-\sin\phi&0\\
\sin\phi&\cos\phi&0\\
0&0&1\end{array}\right).
\]
\exampleend

Under a unitary transformation $U$ (or basis transformation) scalars are unchanged, whereas vectors $\vec{r}$ and matrices $M$ change as
\begin{eqnarray}
r'_i&=&U_{ij}~ r_j, ~~({\rm sum~inferred})\\
\nonumber
M'_{ij}&=&U_{ik}M_{km}U^{-1}_{mj}.
\end{eqnarray}
Physical quantities with no spatial indices are scalars (or pseudoscalars if they depend on right-handed vs. left-handed coordinate systems), and are unchanged by unitary transformations. This includes quantities like the trace of a matrix -- the matrix itself had indices but none remain after performing the trace.
\begin{eqnarray}
{\rm Tr} M&\equiv& M_{ii}.
\end{eqnarray}
Because there are no remaining indices, one expects it to be a scalar. Indeed one can see this,
\begin{eqnarray}
{\rm Tr} M'&=&U_{ij}M_{jm}U^{-1}_{mi}\\
\nonumber
&=&M_{jm}U^{-1}_{mi}U_{ij}\\
\nonumber
&=&M_{jm}\delta_{mj}\\
\nonumber
&=&M_{jj}={\rm Tr} M.
\end{eqnarray}
A similar example is the determinant of a matrix, which is also a scalar.

\subsubsection*{Vector and Matrix Operations}
\begin{itemize}\itemsep=0pt
\item Scalar Product (or dot product): For vectors $\vec{A}$ and $\vec{B}$,
\begin{eqnarray*}
\vec{A}\cdot\vec{B}&=&A_iB_i=|A||B|\cos\theta_{AB},\\
|A|&\equiv& \sqrt{\vec{A}\cdot\vec{A}}.
\end{eqnarray*}
Note that the summation sign is inferred any time there are repeated indices. For example,
\begin{eqnarray}
A_iB_i&=&\sum A_iB_i.
\end{eqnarray}
\item Multiplying a matrix $C$ times a vector $\vec{A}$:
\[
(CA)_i=C_{ij}A_j.
\]
For the $i^{\rm th}$ element one takes the scalar product of the $i^{\rm th}$ row of $C$ with the vector $\vec{A}$.
\item Mutiplying matrices $C$ and $D$:
\[
(CD)_{ij}=C_{ik}D_{kj},
\]
This means the one obtains the $ij$ element by taking the scalar product of the $i^{\rm th}$ of $C$ with the $j^{\rm th}$ column of $D$.
\item Vector Product (or cross product) of vectors $\vec{A}$ and $\vec{B}$:
\begin{eqnarray*}
\vec{C}&=&\vec{A}\times\vec{B},\\
C_i&=&\epsilon_{ijk}A_jB_k.
\end{eqnarray*}
Here $\epsilon$ is the third-rank anti-symmetric tensor, also known as the Levi-Civita symbol. It is $\pm 1$ only if all three indices are different, and is zero otherwise. The choice of $\pm 1$ depends on whether the indices are an even or odd permutation of the original symbols. The permutation $xyz$ or $123$ is considered to be $+1$. For the 27 elements,
\begin{eqnarray}
\epsilon_{ijk}&=&-\epsilon_{ikj}=-\epsilon_{jik}=-\epsilon_{kji}\\
\nonumber
\epsilon_{123}&=&\epsilon_{231}=\epsilon_{312}=1,\\
\nonumber
\epsilon_{213}&=&\epsilon_{132}=\epsilon_{321}=-1,\\
\nonumber
\epsilon_{iij}&=&\epsilon_{iji}=\epsilon_{jii}=0.
\end{eqnarray}
You used cross products extensively when studying magnetic fields. Because the matrix is anti-symmetric, switching the $x$ and $y$ axes (or any two axes) flips the sign. If the coordinate system is right-handed, meaning the $xyz$ axes satisfy $\hat{x}\times\hat{y}=\hat{z}$, where you can point along the $x$ axis with your extended right index finger, the $y$ axis with your contracted middle finger and the $z$ axis with your extended thumb. Switching to a left-handed system flips the sign of the vector $\vec{C}=\vec{A}\times\vec{B}$. Note that $\vec{A}\times\vec{B}=-\vec{B}\times\vec{A}$. The vector $\vec{C}$ is perpendicular to both $\vec{A}$ and $\vec{B}$ and the magnitude of $\vec{C}$ is given by
\[
|C|=|A||B|\sin\theta_{AB}.
\]
Vectors obtained by the cross product of two real vectors are called pseudo-vectors because the assignment of their direction can be arbitrarily flipped by defining the Levi-Civita symbol to be based on left-handed rules. Examples are the magnetic field and angular momentum. If the direction of a real vector prefers the right-handed over the left-handed direction, that constitutes a violation of parity. For instance, one can polarize the spins (angular momentum) of nuclei with a magnetic field so that the spins preferentially point along the direction of the magnetic field. This does not violate parity because both are pseudo-vectors. Now assume these polarized nuclei decay and that electrons are one of the products. If these electrons prefer to exit the decay parallel vs. antiparallel to the polarizing magnetic field, this constitutes parity violation because the direction of the outgoing electron momenta are a real vector. This is precisely what is observed in weak decays.

\item Differentiation of a vector with respect to a scalar: For example, the acceleration is $d\vec{v}/dt$:
\[
(d\vec{v}/dt)_i=\frac{dv_i}{dt}.
\]

\item Angular velocity: Choose the vector $\vec{\omega}$ so that the motion is like your fingers wrapping around your thumb on your right hand with $\vec{\omega}$ pointing along your thumb. The magnitude is $d|\phi|/dt$.

\item Gradient operator $\nabla$: This is the derivative $\partial/\partial x$, $\partial/\partial y$ and $\partial/\partial z$, where $\partial _x$ means $\partial/\partial_x$. For taking the gradient of a scalar $\Phi$,
\[
{\rm\bf grad}~\Phi, (\nabla\Phi(x,y,z,t))_i=\partial/\partial r_i\Phi(\vec{r},t)=\partial_i\Phi(\vec{r},t).
\]
For taking the dot product of the gradient with a vector, sometimes called a divergence,
\[
div \vec{A}, \nabla\cdot\vec{A}=\partial_i A_i.
\]
For taking the vector product with another vector, sometimes called curl, $\nabla\times\vec{A}$,
\[
{\rm\bf curl}~\vec{A}, (\nabla\times\vec{A})_i=\epsilon_{ijk}\partial_j A_k(\vec{r},t).
\]
\item The Laplacian is referred to as $\nabla^2$ and is defined as
\[
\nabla^2=\nabla\cdot\nabla=\frac{\partial^2}{\partial x^2}+\frac{\partial^2}{\partial y^2}+\frac{\partial^2}{\partial z^2}.
\]
\end{itemize}

\subsubsection*{Some identities}

Here we simply state these, but you may wish to prove a few. They are useful for this class and will be essential when you study E\&M.
\begin{eqnarray}
\vec{A}\cdot(\vec{B}\times\vec{C})&=&\vec{B}\cdot(\vec{C}\times\vec{A})=\vec{C}\cdot(\vec{A}\times\vec{B})\\
\nonumber
\vec{A}\times(\vec{B}\times\vec{C})&=&(\vec{A}\cdot\vec{C})\vec{B}-(\vec{A}\cdot\vec{B})\vec{C}\\
\nonumber
(\vec{A}\times\vec{B})\cdot(\vec{C}\times\vec{D})&=&(\vec{A}\cdot\vec{C})(\vec{B}\cdot\vec{D})
-(\vec{A}\cdot\vec{D})(\vec{B}\cdot\vec{C})
\end{eqnarray}

\example
The height of a hill is given by the formula
\[
z(x,y)=2xy-3x^2-4y^2-18x+28y+12.
\]
Here $z$ is the height in meters and $x$ and $y$ are the east-west and north-south coordinates. Find the position $x,y$ where the hill is the highest, and give its height.

{\bf Solution:} The maxima or minima, or inflection points, are given when $\partial_x z=0$ and $\partial_yz=0$.
\begin{eqnarray*}
\partial_xz(x,y)&=&2y-6x-18=0,\\
\partial_yz(x,y)&=&2x-8y+28=0.
\end{eqnarray*}
Solving for two equations and two unknowns gives one solution $x=-2,y=3$. This then gives $z=72$. Although this procedure could have given a minimum or an inflection point you can look at the form for $z$ and see that for $x=y=0$ the height is lower, therefore it is not a minimum. You can also look and see that the quadratic contributions are negative so the height falls off to $-\infty$ far away in any direction. Thus, there must be a maximum, and this must be it. Deciding between maximum or minimum or inflection point for a general problem would involve looking at the matrix $\partial_i\partial_j z$ at the specific point, then finding the eigenvalues of the matrix and seeing whether they were both positive (minimum) both negative (maximum) or one positive and one negative (saddle point).

\exampleend

\subsubsection*{Gauss's Theorem and Stokes's Theorem}

For an integral over a volume $V$ confined by a surface $S$, Gauss's theorem gives
\[
\int_V dv~\nabla\cdot\vec{A}=\int_Sd\vec{S}\cdot\vec{A}.
\]
For a closed path $C$ which carves out some area $S$,
\[
\int_C d\vec{\ell}\cdot\vec{A}=\int_Sd\vec{s} \cdot(\nabla\times\vec{A})
\]
Stoke's law can be understood by considering a small rectangle, $-\Delta x<x<\Delta x$, $-\Delta y<y<\Delta y$. The path integral around the edges is
\begin{eqnarray}
\int_C d\vec{\ell}\cdot\vec{A}&=&2\Delta y[A_y(\Delta x,0)-A_y(-\Delta x,0)]-2\Delta x[A_x(0,\Delta y)-A_x(0,-\Delta y)]\\
\nonumber
&=&4\Delta x\Delta y\left\{
\frac{A_y(\Delta x,0)-A_y(-\Delta x,0)}{2\Delta x}-\frac{A_x(0,\Delta y)-A_x(0,-\Delta y)}{2\Delta y}\right\}\\
\nonumber
&=&4\Delta x\Delta y\left\{\frac{\partial A_y}{\partial x}-\frac{\partial A_x}{\partial y}\right\}\\
&=&\Delta S \cdot \nabla\times\vec{A}.
\end{eqnarray}
Here $\Delta S$ is the area of the surface element.

\subsubsection*{Some Notation}
From here on in, we may use bold face to denote vectors, e.g. ${\bf v}$ instead of $\vec{v}$. We also might use dots over quantities to represent time derivatives, e.g. $\dot{\bf v}=d{\bf v}/dt$. As mentioned above, repeated indices infer sums, e.g.,
\begin{eqnarray}
x_iy_i&=&\sum_i x_iy_i=\vec{x}\cdot\vec{y},\\
\nonumber
x_iA_{ij}y_j&=&\sum_{ij}x_iA_{ij}y_j,
\end{eqnarray}

\subsection{Exercises}

\begin{enumerate}

\item All physicists must become comfortable with thinking of oscillatory and wave mechanics in terms of expressions that include the form $e^{i\omega t}$.
\begin{enumerate}
\item Perform Taylor expansions in powers of $\omega t$ of the functions $\cos\omega t$ and $\sin\omega t$.
\item Perform a Taylor expansion of $e^{i\omega t}$.
\item Using parts (a) and (b) show that $e^{i\omega t}=\cos\omega t+i\sin\omega t$.
\item Show that $\ln(-1)=i\pi$.
\end{enumerate}

\item One of the many uses of the scalar product is to find the angle between two given vectors. Find the angle between the vectors $\vec{b}=(1,2,4)$ and $\vec{c}=(4,2,1)$ by evaluating their scalar product.

\item Use the product rule to show that
\[
\frac{d}{dt}(\vec{r}\cdot\vec{s})=\frac{d\vec{r}}{dt}\cdot\vec{s}+\vec{r}\cdot\frac{d\vec{s}}{dt}.
\]

\item Multiply the rotation matrix in Example \ref{ex:rotmat} by its transpose to show that the matrix is unitary or orthogonal, i.e. you get the unit matrix.

\item Find the matrix for rotating a coordinate system by 90 degrees about the $x$ axis.

\begin{comment}

\item Consider a rotation where in the new coordinate system the three axes are at angles $\alpha$, $\beta$ and $\gamma$ relative to the original $x$ axis. Show that
\[
\cos^2\alpha+\cos^2\beta+\cos^2\gamma=1.
\]

\item Find the rotation matrix for each of the three rotations, then find the matrix for the combined rotations.
First rotate by 90$^\circ$ around the $y$ axis to get to the primed system. Then rotate the primed system 90$^\circ$ about the new $z$ axis to get to the doubly primed system. Then find the matrix that rotates the doubly primed coordinate system 90$^\circ$ about the new $x$ axis to get to the triply primed system. Finally, find the matrix that performs all three operations sequentially.

\end{comment}

\item Consider a parity transformation which reflects about the $x=0$ plane. Find the matrix that performs the transformation. Find the matrix that performs the inverse transformation.

\item Show that the scalar product of two vectors is unchanged if both undergo the same rotation. Use the fact that the rotation matrix is unitary, $U_{ab}=U^{-1}_{ba}$.

\item Show that the product of two unitary matrices is a unitary matrix.

\item Show that
\[
\sum_k\epsilon_{ijk}\epsilon_{klm}=\delta_{il}\delta_{jm}-\delta_{im}\delta_{jl}.
\]

\item Consider a cubic volume $V=L^3$ defined by $0<x<L$, $0<y<L$ and $0<z<L$. Consider a vector $\vec{A}$ that depends arbitrarily on $x,y,z$. Show how Gauss's law,
\[
\int_V dv\nabla\cdot\vec{A}=\int_Sd\vec{S}\cdot\vec{A},
\]
is satisfied by direct integration. I.e., you should use the fact that $\int_a^b dx~(d/dx)f(x)=f(b)-f(a)$.

\item Consider the function $z=3x^2-4y^2+12xy-6x+24$. Find any maxima or minima and determine whether it is a maximum or a minimum or an inflection point.

\item A real $n-$dimensional symmetric matrix $\lambda$ can always be diagonalized by a unitary transformation, i.e. there exists some unitary matrix $U$ such that,
\begin{eqnarray}
U_{ij}\lambda_{jk}U^{-1}_{km}&=&\tilde{\lambda}_{im}=\left(\begin{array}{cccc}
\tilde{\lambda}_{11}&0&\cdots&0\\
0&\tilde{\lambda}_{22}&\cdots&0\\
\vdots& & \ddots & \vdots\\
0& \cdots &\cdots &\tilde{\lambda}_{nn}
\end{array}\right).
\end{eqnarray}
The values $\tilde{\lambda}_{ii}$ are referred to as eigenvalues. The set of $n$ eigenvalues are unique, but their ordering is not -- there exists a unitary transformation that permutes the indices.

Consider a function $f(x_1,\cdots,x_n)$ that has the property,
\begin{eqnarray}
\left.\partial_i f(\vec{x})\right|_{\vec{x}=0}=0,
\end{eqnarray}
for all $i$. Show that if this function is a minimum, and not a maximum or an inflection point, that the $n$ eigenvalues of the matrix
\begin{eqnarray}
\lambda_{ij}&\equiv&\left.\partial_i\partial_j f(\vec{x})\right|_{\vec{x}=0},
\end{eqnarray}
must be positive.

\item
\begin{enumerate}
\item For the unitary matrix $U$ that diagonalizes $\lambda$ as shown in the previous problem. Show that each row of the unitary matrix represents an orthogonal unit vector by using the definition of a unitary matrix.
\item Show that the vector
\begin{eqnarray}
x_i^{(k)}&\equiv& U_{ki}=(U_{k1},U_{k2},\cdots,U_{kn}),
\end{eqnarray}
has the property that
\begin{eqnarray}
\lambda_{ij}x^{(k)}_j&=&\tilde{\lambda}_{kk}x^{(k)}_i.
\end{eqnarray}
These vectors are known as eigenvectors, as they have the property that when multiplied by $\lambda$ the resulting vector is proportional ( same direction) as the original vector. Because one can transform to a basis, using $U$, where $\lambda$ is diagonalized, in the new basis the eigenvectors are simply the unit vectors.
\end{enumerate}
\end{enumerate}


\subsection{Classic regression trees}

In a classical regression tree, we follow a decison process as before, but the outcome is real number.

Within each leaf, all inputs are assigned that same number.

\subsubsection{Training}

With a regression problem we cannot split nodes the same way as we did for classification.

Instead by split by the residual sum of squares.


\section{Results}
All the implemented transforms where tested for correctness by comparing the output
of my implementation to other reference-implementations, like matlab's or scipy's.
The implementation was also tested for speed depending on the length of the input.
For a given input length the FFT algorithm was performed multiple times with random numbers
and the
runtime was measured and averaged. The number of times the algorithm was evaluated
was chosen, such that the uncertainty of the average was below \SI{10}{\nano\second}.
While the actual numbers will be more or less uninteresting, it is possible to see
the $\symcal{O}(N\log N)$ scaling for powers of 2, as shown in \autoref{fig:timesp2}.
Especially for higher-length input the scaling is fulfilled, but for short length
input there is too much overhead, and thus the numbers are not following $N\log N$ perfectly there.

\begin{figure}[t]
    \centering
    \includegraphics[width=\linewidth]{build/plots/times_p2.pdf}
    \caption{Runtime of my FFT implementation in double-logarithmic-scaling depending on the length of the input sequence.
        Each random-number input is transformed multiple time and the results were averaged.
        The function $t(N)=c\times N\log N$ is fitted to the measurements, as we expect this scaling for
        input lengths in the form of $2^p$. }
    \label{fig:timesp2}
\end{figure}

For arbitrary sized input, the actual runtime of this algorithm depends heavily on
the number of two's in the prime-factor decomposition of the input-length.
In easier words: the FFT of 48 numbers should be roughly twice as fast as 56, even though
56 is the higher number.
The reason is 48 is divisible by 2 4 times ($48=2^4\cdot 3$), where 56 is only divisble by
2 3 times ($56=2^3\cdot 7$).
This is also clearly seen in the data, as \autoref{fig:times} shows.
\begin{figure}[t]
    \centering
    \includegraphics[width=.5\textwidth]{build/plots/times_lin_c.pdf}
    \caption{Averaged runtime of the implemented FFT algorithm on random numbers depending on the input length.
        A clear separation of the runtime scaling is seen. The data-points are colored
        depending on the number of times the input length is divisible by 2.}
    \label{fig:times}
\end{figure}

\subsection{Depicting the Fourier Space}
A digital image is nothing more than $N=h\cdot w\cdot C$ numbers $x_{k,l,c}$
\footnote{$k=0,\dots, h-1\quad l=0,\dots,w-1\quad c=0,\dots,C-1$}
between $0$ and $1$.
Where $w$ is the width, $h$ is the height and $C$ is the number of channels of the image.
There are a lot of different types of images out there, with different channel numbers and interpretation
of them (different color encodings, transparent images, etc.), but we will focus on the
case of grayscale $C=1$ or colored RGB $C=3$ pictures.
The interpretation of the number in grayscale images is the illumination of the pixel ranging from $0=$~off~$=$~black
to $1=$~on~$=$~white.
In colored RGB images, we have three channels: Red (R), Green (G), Blue (B). Where we can light up each color separately.
Because of how the eye works, we are able to trick the brain into perceiving most of the colors by just
mixing these three.
The numbers are usually not stored as floating point numbers, but as unsigned integers, most commonly 8-bit unsigned integers.
This means that 1 is stored as 255 and 0 as 0 and all the numbers are mapped in an equally spaced fashion.
If we have multiple channels, we will Fourier transform each channel separately and
equation \eqref{eqn:dft2d} gets the form
\begin{equation*}
    X_{k,l,c} = \frac{1}{\sqrt{hw}}\sum_{n=0}^{h-1}\sum_{m=0}^{w-1}x_{n,m,c} \ e^{-i{2\pi}\ \left(\!\frac{kn}{h}+\frac{lm}{w}\right)}.
\end{equation*}
In the Fourier space each pixel is a complex number.
We usually visualize the magnitude and the phase (the argument) of each number.
Because the magnitude of the Fourier transformed numbers is not guaranteed
to be between 0 and 255, we scale the Fourier transformed numbers in such a way,
that the highest is 255 and the lowest is 0
\begin{equation*}
    \abs {X'_{k,l,c}} = 255\frac{\abs {X_{k,l,c}}-\min_{n,m} \abs {X_{n,m,c}}}{\max_{n,m} \abs {X_w{n,m,c}}-\min_{n,m} \abs {X_{n,m,c}}}.
\end{equation*}
We also scale the phases to the range of $(0, 255)$
\begin{equation*}
    \arg {X'_{k,l,c}} = 255\frac{\arg X_{k,l,c}}{2\pi}.
\end{equation*}
This mode of visualization can be seen in \autoref{subfig:ftm_n} and \autoref{subfig:ftp_n}.
\begin{figure}[htbp]
    \centering
    \begin{subfigure}[h]{.9\linewidth}
        \centering
        \includegraphics[width=\linewidth]{images/webb.png}
        \caption{Original}
    \end{subfigure}
    \begin{subfigure}[h]{.9\linewidth}
        \centering
        \includegraphics[width=\linewidth]{build/output/webb_mag_n.png}
        \caption{Fourier Transform Magnitude}
        \label{subfig:ftm_n}
    \end{subfigure}
    \begin{subfigure}[h]{.9\linewidth}
        \centering
        \includegraphics[width=\linewidth]{build/output/webb_phase_n.png}
        \caption{Fourier Transform Phase}
        \label{subfig:ftp_n}
    \end{subfigure}
    \begin{subfigure}[h]{.9\linewidth}
        \centering
        \includegraphics[width=\linewidth]{build/output/webb_mag_log.png}
        \caption{Fourier Transform magnitude in logarithmic space}
        \label{subfig:ftm_log}
    \end{subfigure}
    \begin{subfigure}[h]{.9\linewidth}
        \centering
        \includegraphics[width=\linewidth]{build/output/webb_mag.png}
        \caption{FT Magnitude in logarithmic space and rearranged}
        \label{subfig:ftm}
    \end{subfigure}
    \caption{An example of the Fourier transform. The image is the first calibration picture of the new James-Webb space telescope\cite{webbimg}. }
    \label{fig:fourier_example_n}
\end{figure}

As you can see in the example image, the Fourier magnitude is mostly black, and the phases appear to be noise to the eye.
To better distinguish small features, we will now switch to logarithmic color-space for the magnitude
\begin{equation*}
    \abs {X''_{k,l,c}} = 255\frac{\log \abs {X_{k,l,c}}-\min_{n,m} \log\abs {X_{n,m,c}} } {\max_{n,m} \log\abs {X_{n,m,c}}-\min_{n,m} \log\abs {X_{n,m,c}}}.
\end{equation*}
This is depicted in \autoref{subfig:ftm_log}.
Finally, to better resemble the symmetries in the Fourier space and
to have low frequencies in the middle of the picture, and high frequencies outside,
the indices are shifted
\begin{equation}
    k' = \lfloor k+h/2\rfloor \mod h \qquad l'=\lfloor l+w/2\rfloor\mod w.
    \label{eqn:shift}
\end{equation}
This will be our method of visualization from now on and can be seen in \autoref{subfig:ftm}.

\subsection{Editing in Fourier Space}
If we now modify the numbers in the Fourier space and transform the modified numbers back,
we can achieve complex-looking outcome.
For example, if we take the high frequencies in the image away, we are only left with the
"long-range" information, but the "short-range" information is lost.
This means the image will appear to be blurred.
Taking away the high frequencies can be done in multiple ways.
We can just remove the highest frequencies combining the axes by calculating
$(k'-h/2)^2+(l'-w/2)^2 < r^2$ for some $r$ and setting the pixel in Fourier space to 0, if
this evaluates to false. We can also do this for each axis independently
($(k'-h/2)^2<r_y^2$, $(l'-w/2)^2<r_x^2$).
With these hard cut-offs it is easy to see periodic artifacts in the image.
If we do not want that, we can apply a smooth filter.
A selection of different blurring techniques can be seen
in \autoref{fig:blur_2} to \autoref{fig:blur_smooth}.

\begin{figure}[htbp]
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/dune_mag.png}
        \caption{Fourier space}
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{images/dune.png}
        \caption{Original}
    \end{subfigure}\
    \caption{A movie poster of Dune (2021) which serves as an example image being blurred in this section \cite{dune}.}
    \label{fig:dune_orig}
\end{figure}

\begin{figure}[htbp]
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/dune_blur2_mask.png}
        \caption{Edit in Fourier space }
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/dune_blur2.png}
        \caption{Transformed back to real space}
    \end{subfigure}\
    \caption{The poster above blurred by only keeping the innermost frequencies with a radius of $20\%$ of the width.}
    \label{fig:blur_2}
\end{figure}
\begin{figure}[htbp]
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/dune_blur1_mask.png}
        \caption{Edit in Fourier space }
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/dune_blur1.png}
        \caption{Transformed back to real space}
    \end{subfigure}\
    \caption{The poster blurred by only keeping the innermost frequencies with a radius of $10\%$ of the width. This is a circular filter.}
    \label{fig:blur_1}
\end{figure}
\begin{figure}[htbp]
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/dune_rect_mask.png}
        \caption{Edit in Fourier space }
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/dune_blur_rect.png}
        \caption{Transformed back to real space}
    \end{subfigure}\
    \caption{The poster blurred by only keeping the $10\%$ innermost frequencies along each axis independently. This is a rectangular filter.}
    \label{fig:blur_rect}
\end{figure}
\begin{figure}[htbp]
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/dune_blur_smooth_mask.png}
        \caption{Edit in Fourier space }
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/dune_blur_smooth.png}
        \caption{Transformed back to real space}
    \end{subfigure}\
    \caption{The poster blurred by only multipliying each pixel in the Fourier space with $\exp(-0.001 ((k'-h/2)^2+(l'-w/2)^2))$, where $k', l'$ are the shifted indices from above. This is a Gaussian filter.}
    \label{fig:blur_smooth}
\end{figure}

\FloatBarrier

If we do the opposite and take away some low frequencies, we are "sharpening" the image.
We must not understand "sharpening" in the literal meaning, like focusing the lens of a camera,
since we can not generate new information in the image. We can only enhance the short range information.
There are a couple of examples below in \autoref{fig:blackhole} to \autoref{fig:blackhole_sharp_g_g}.
You will notice the colors in the images are distorted, since we completely treat the
color channels separately. In \autoref{fig:blackhole_sharp_g_g} you can see a grayscale version
of the edit, where the colors might be less distracting.

If we take it to the extreme and remove most low frequencies, we can seethe edges in the image, this
is demonstrated in \autoref{fig:edges}.
\begin{figure}
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/blackhole_sharp_smooth_mask.png}
        \caption{Fourier space}
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{images/blackhole.png}
        \caption{Original}
    \end{subfigure}\
    \caption{The first image of a black hole, taken in 2021.
        This appears not to be sharp and is taken as an example to experiment with sharpening in Fourier space in this section. }
    \label{fig:blackhole}
\end{figure}
\begin{figure}
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/blackhole_sharp1_mask.png}
        \caption{Edit in Fourier space}
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/blackhole_sharp1.png}
        \caption{Transformed back to real space}
    \end{subfigure}\
    \caption{The black hole image sharpened by applying a circular filter with a radius of 1\% of the height.}
    \label{fig:blackhole_sharp}
\end{figure}
\begin{figure}
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/blackhole_sharp3_mask.png}
        \caption{Edit in Fourier space}
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/blackhole_sharp3.png}
        \caption{Transformed back to real space}
    \end{subfigure}\
    \caption{The black hole image sharpened by applying a circular filter with a radius of 3\% of the height.}
    \label{fig:blackhole_sharp3}
\end{figure}
\begin{figure}
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/blackhole_sharp_smooth_mask.png}
        \caption{Edit in Fourier space}
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/blackhole_sharp_smooth.png}
        \caption{Transformed back to real space}
    \end{subfigure}\
    \caption{The black hole image sharpened by applying a Gaussian filter of $1-\exp(-0.01 ((k'-h/2)^2+(l'-w/2)^2))$.}
    \label{fig:blackhole_sharp_g}
\end{figure}
\begin{figure}
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/blackhole_sharp_smooth_mask_g.png}
        \caption{Edit in Fourier space}
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/blackhole_sharp_smooth_g.png}
        \caption{Original}
    \end{subfigure}\
    \caption{The black hole image sharpened by applying a Gaussian filter of $1-\exp(-0.01 ((k'-h/2)^2+(l'-w/2)^2))$ in black and white.}
    \label{fig:blackhole_sharp_g_g}
\end{figure}
\begin{figure}
    \centering
    \begin{subfigure}[h]{.32\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{images/A.png}
        \caption{Original }
    \end{subfigure}
    \begin{subfigure}[h]{.32\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/A_sharp_smooth_mask.png}
        \caption{Edit}
    \end{subfigure}
    \begin{subfigure}[h]{.32\linewidth}
        \centering
        \includegraphics[width=.9\linewidth]{build/output/A_sharp_smooth.png}
        \caption{Back to real space}
    \end{subfigure}
    \caption{An image sharpened with a Gaussian filter of $1-\exp(-0.001 ((k'-h/2)^2+(l'-w/2)^2))$ to demonstrate edge-detecting capabilities of Fourier edits.}
    \label{fig:edges}
\end{figure}
\FloatBarrier

\subsection{Diffraction}
As already mentioned, we can
"simulate" Fraunhofer diffraction by displaying the magnitude of the Fourier transformed image.
In the images in this section, you will be able to realize a couple of known features
of diffraction images.
Lines in the diffraction images will be perpendicular to lines in the aperture (the original image).
For two double slits of width $b$ and distance $a$ the analytical expression for Fraunhofer diffraction
is
\begin{equation*}
    I(\theta) = I_0 \left(\frac{\sin(c b \sin\theta)}{c b \sin\theta} \cos(c a \sin\theta)\right)^2.
\end{equation*}
Which is calculated by solving equation \eqref{eqn:fraunhofer}.
With some constant $c$. From this we can read that the smaller the slits are, the wider the
encapsulating intensity is (since $b < a$) and the closer the slits are, the smaller the distance between
the peaks is. This can be seen in \autoref{fig:double_slit}.

This calculation is rather simple, but also more complex apertures can be investigated, like a circular
aperture, seen in \autoref{fig:circular}. The analytical solution to this is a Bessel function of the radius.

\begin{figure}[htbp]
    \centering
    \begin{subfigure}[h]{\linewidth}
        \centering
        \includegraphics[width=.49\linewidth]{images/hole.png}
        \includegraphics[width=.49\linewidth]{build/output/hole_mag.png}
        \caption{Circular aperture.}
    \end{subfigure}
    \begin{subfigure}[h]{\linewidth}
        \centering
        \includegraphics[width=.49\linewidth]{images/hole_tiny.png}
        \includegraphics[width=.49\linewidth]{build/output/hole_tiny_mag.png}
        \caption{Circular aperture with smaller radius.}
    \end{subfigure}
    \caption{Fraunhofer diffraction image (right) for two different circular apertures (left).}
    \label{fig:circular}
\end{figure}

\begin{figure}[htbp]
    \centering
    \begin{subfigure}[h]{\linewidth}
        \centering
        \includegraphics[width=.49\linewidth]{images/double_slit_small_norm.png}
        \includegraphics[width=.49\linewidth]{build/output/double_slit_small_norm_mag.png}
        \caption{Double slit.}
    \end{subfigure}
    \begin{subfigure}[h]{\linewidth}
        \centering
        \includegraphics[width=.49\linewidth]{images/double_slit_small_close.png}
        \includegraphics[width=.49\linewidth]{build/output/double_slit_small_close_mag.png}
        \caption{Double slit where the slits are closer.}
    \end{subfigure}
    % \begin{subfigure}[h]{\linewidth}
    %     \centering
    %     \includegraphics[width=.49\linewidth]{images/double_slit_small_wide.png}
    %     \includegraphics[width=.49\linewidth]{build/output/double_slit_small_wide_mag.png}
    %     \caption{Simulated diffraction image (right) of aperture image (left).}
    % \end{subfigure}
    \begin{subfigure}[h]{\linewidth}
        \centering
        \includegraphics[width=.49\linewidth]{images/double_slit_small_widest.png}
        \includegraphics[width=.49\linewidth]{build/output/double_slit_small_widest_mag.png}
        \caption{Double slit where the slits are further apart.}
    \end{subfigure}
    \begin{subfigure}[h]{\linewidth}
        \centering
        \includegraphics[width=.49\linewidth]{images/double_slit_norm_widest.png}
        \includegraphics[width=.49\linewidth]{build/output/double_slit_norm_widest_mag.png}
        \caption{Double slit with wider slits, but same distance as last image.}
    \end{subfigure}
    \caption{Different images of double slits with different features. The aperture image is on the left and the diffraction image is on the right.}
    \label{fig:double_slit}
\end{figure}

\subsection{Compression}
In this section we will very quickly review a naive compression algorithm, based on the discrete cosine transform.
Here the images will not be displayed in the transformed space, since they were very uninformative in my testing.
They appeared to look like mostly a single color with very subtle variation.
The idea of our naive compression is to sort the cosine-space coefficient by magnitude and just
set the lowest of them to zero. Then we only need to save the nonzero coefficients.
The results of removing the lowest coefficients in this way can be seen in \autoref{fig:compr}.

It is fairly obvious, that this compression is not looking very good, except at low compression rates.
But would it save space on disk? To answer this question I implemented a very simple binary format,
saving the cosine coefficients and their indices in a binary, compressed format. Sadly, except
at absurdly high compressions, this did not save disk space.
On the contrary: The file size actually increased, since now that we are not saving every
pixel, we also have to save the indices, which takes additional space.
The actual \texttt{.jpg} format is a little bit more complicated.
It performs the DCT on each $8\times 8$ tile in the image and instead of
setting small amplitudes to 0, it tries to find a smaller, finite set of amplitudes,
so that setting the real amplitudes to the one closest to it in the smaller set of amplitudes, does
not change too much.
\begin{figure}[htbp]
    \centering
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=\linewidth]{build/plots/A9.png}
        \caption{10\% compression}
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=\linewidth]{build/plots/A5.png}
        \caption{50\% compression}
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=\linewidth]{build/plots/A2.png}
        \caption{80\% compression}
    \end{subfigure}
    \begin{subfigure}[h]{.49\linewidth}
        \centering
        \includegraphics[width=\linewidth]{build/plots/A1.png}
        \caption{90\% compression}
    \end{subfigure}
    \caption{The same image with different "compression levels". 10\% compression means that the lowest 10\% of the
        cosine coefficients have been set to 0.}
    \label{fig:compr}
\end{figure}


Galois theory involves studying polynomials over a field, which are ubiquitous in algebra and number theory. So we
start with the basics of fields.
\begin{defn}
Recall that a \term{field} $k$ is a commutative ring with $1$ such that $1\ne 0$ and every nonzero element is
invertible, and $1\ne 0$.
\end{defn}
That is: in a field we can add, subtract, multiply, and divide, though as usual we can't divide by zero. A ``field
homomorphism'' (meaning a structure-preserving map) is just a ring homomorphism; we ask to preserve addition and
multiplication, and subtraction and division come for free.
\begin{comp}{exm}{enumerate}
	\item The rational numbers $\Q$, the real numbers $\R$, and the complex numbers $\C$ are all fields.
	\item Let $p$ be prime. Then, $\Z/p$ is a field, called the \term{finite field of order $p$} and denoted
	$\F_p$.
	\begin{proof}
	First, we show $\Z/p$ is an integral domain: if $m\cdot n\equiv 0\bmod p$, then $m\cdot n = xp$ for some $x$,
	so $p$ divides either $m$ or $n$. Thus, either $m$ or $n$ is $0$ in $\Z/p$.

	Now, for any nonzero $m\in\Z/p$, consider the multiplication map $\vp_m\cdot:\Z/p\to\Z/p$ sending $n\mapsto
	mn$. If $a,b\in\Z/p$ are such that $\vp_m(a) = \vp_m(b)$, then $\vp_m(a-b) = m\cdot(a-b) = 0$. Since $m$ is
	nonzero and $\Z/p$ is an integral domain, then $a = b$, and so $\vp_m$ is injective. Since $\Z/p$ is finite, an
	injective map from $\Z/p$ to itself is a bijection. Thus, there's a unique $n\in\Z/p$ such that $\vp_m(n) = 1$,
	or $1 = mn$. That is, every nonzero element is invertible.
	\end{proof}
	\item If $k$ is any field, we can form the \term{field of rational functions} in $k$, denoted $k(x)$, to be
	ratios $p/q$ for polynomials $p,q\in k[x]$ with $q\ne 0$. We'd like them to be ``in lowest terms,'' but this is
	a clunky definition and it's simpler to just say that two rational functions $p/q$ and $p'/q'$ are the same if
	we can cross-multiply: $pq' = p'q$.
\end{comp}
There's not a whole lot we can say about a field $k$ in total generality, without knowing more about it, but we
know $1\in k$, and therefore $1+1\in k$, and $1+1+1\in k$, and so on. These numbers might all be distinct, like for
$\Q$, or might not be, like for $\F_p$.
\begin{defn}
The \term{characteristic} $\chr(k)$ of a field $k$ is the smallest multiple of $1$ that is equal to $0$ in $k$, or
is $0$ if no such multiple exists.
\end{defn}
For example, $\chr(\F_p) = p$, and $\chr(\Q) = \chr(\R) = 0$.
\begin{ex}
Show that for any field $k$, $\chr(k)$ is either $0$ or a prime number.
\end{ex}
The characteristic is an important property of a field: many important things in Galois theory are different in the
characteristic $0$ case and the characteristic $p$ case.

One of the consistent lessons of algebra is to study objects by looking at their homomorphisms. In the case of
fields, morphisms are the setting of Galois theory.
\begin{lem}
Let $\vp:k\to K$ be a homomorphism of fields. Then, $\vp$ is injective.
\end{lem}
\begin{proof}
The kernel $\ker(\vp)\subset k$ is an ideal of $k$. Since $k$ is a field, its only ideals are $0$ and $k$ itself.
If $\ker(\vp) = 0$, then $\vp$ is injective, as desired; if $\ker(\vp) = k$, then $\vp(1) = 0$, which is
impossible, because ring morphisms must send $1$ to $1$.
\end{proof}
This is very different than for other kinds of algebraic objects: no interesting kernels and no interesting
quotients.
\begin{defn}
If $k$ is a field, a \term{field extension} of $k$ is a homomorphism $i:k\inj L$, often written
$L/k$.\footnote{This notation looks like a quotient, but since we will never take the quotient by a field
extension, this is not ambiguous. We will take quotients of rings, however.}
\end{defn}
Galois theory is the study of field extensions and relations between them.
\begin{defn}
Let $k$ be a field and $i_K:k\inj K$ and $i_L:k\inj L$ be field extensions. An \term{embedding}
(sometimes said to be an \term{embedding over $k$}) is a field homomorphism $j:K\inj L$ such that the following
diagram commutes:
\[\xymatrix{
	K\ar[rr]^j && L\\
	& k.\ar[ul]_{i_K}\ar[ur]^{i_L}
}\]
That is, $i_L = j\circ i_K$.
\end{defn}
Embeddings keep track of how one field lies as a subfield of another.
\begin{defn}
If $k$ is a field, its \term{prime subfield} is the subfield of $k$ generated by $1$.
\end{defn}
That is, the prime subfield is the smallest field containing $1$ inside $k$, meaning it must contain $1+1$,
$1+1+1$, and so forth. If $\chr(k) = 0$, this generates a copy of $\Z$ inside $k$, and therefore $\Q$ also, since
we can invert all nonzero elements of $\Z$. That is, if $\chr(k) = 0$, then the prime subfield of $k$ is $\Q$.
In the same way, if $\chr(k) = p$, then its prime subfield is $\F_p$.
\begin{cor}
If $K/k$ is a field extension, then $\chr(k) = \chr(K)$.
\end{cor}
This is because the prime subfield of $K$ contains the prime subfield of $k$.
\begin{comp}{exm}{enumerate}
\label{fieldextexm}
	\item Since $\Q$ is a subfield of $\R$, the inclusion $\Q\inj\R$ is a field extension. In the same way,
	$\R\inj\C$ is a field extension; this fixes $\Q$, so it's an embedding over $\Q$.
	\item\label{Gaussrat} The \term{field of Gaussian rationals} is $\Q(i) = \set{a+bi\mid a,b\in\Q}$.
	\begin{ex}
	Show that $\Q(i)$ is a field.
	\end{ex}
	If $a\in\Q$, $a = a+0i\in\Q(i)$, so $\Q\inj\Q(i)$ is another example of a field extension. One can form a
	similar definition for, e.g.\ $\Q(\sqrt 3)$ or $\Q(\sqrt{-2})$, but we'll soon define something much more
	general.
\end{comp}
\begin{aside}[Categorical language in field theory]
The modern formulation of abstract algebra tends to use categorical language, defining categories of algebraic
objects such that useful constructions satisfy universal properties. However, this is uncommon for field
theory, as the category of fields is poorly behaved: few products exist (e.g.\ the ring $\Q\times\Q$ is not a
field), there are no initial or final objects, and the category is disconnected, since a map of fields must
preserve the characteristic. Specializing to the category of fields of a given characteristic fixes some, but not
all, of these problems.

Nonetheless, there are a few places where words from category theory will simplify things, and I'll try to mention
them as they happen. Since they may require knowledge beyond what I assume for these notes, they will also be
marked as asides.

Fixing a base field $k$, we can define the \term{category of field extensions} $\FExt_k$ to be the category whose
objects are field extensions $K/k$ and whose morphisms are embeddings $K\inj L$ over $k$.

{\color{red}TODO}: talk about the poset? Eventually will be a lattice.
\end{aside}
\begin{lem}
If $i:k\inj K$ is a field extension, then $K$ is a $k$-vector space.
\end{lem}
\begin{proof}
We need to define an action of $k$ on $K$, which will just be multiplication: if $\lambda\in k$ and $x\in K$, let
$\lambda\cdot x = i(\lambda)x$. The field axioms of $K$ imply (since $k$ is realized as a subfield of $K$) that
multiplication satiafies the axioms for a vector space.
\end{proof}
\begin{defn}
\label{finite_ext}
If $K/k$ is a field extension, its \term{degree}, written $[K:k]$, is the dimension of $K$ as a $k$-vector space.
If this is a finite number, $K/k$ is said to be a \term{finite extension}; otherwise, it's an \term{infinite
extension}.
\end{defn}
This is analogous to the index of a subgroup $[G:H]$. For example, $[\C:\R] = 2$, so $\C/\R$ is a finite extension,
but $\R/\Q$ is an infinite extension.

%The first important reason we care about field extensions is to discuss roots of polynomials. For example, $x^2-2$
%doesn't have a root in $\Q$, but it has $2$ roots in $\Q(\sqrt 2)$. This enables questions about polynomials to be
%turned into questions about field extensions, which can be solved with algebraic methods.
% \begin{thm}
% \label{rootinext}
% Let $f\in k[x]$ be a nonconstant polynomial. Then, there's an extension $K/k$ such that $f$ has a root in $K$.
% \end{thm}
% Algebraically, what does it actually mean that $f$ has a root in $K$, given that $f$ is only defined over $K$? The
% extension $i:k\inj K$ induces a map $i_*:k[x]\inj K[x]$ by applying $i$ to each coefficient of a polynomial.
% Technically, one says that $i_*f$ has a root in $K$, but there's rarely if ever a need to keep the two separate, so
% we identify a polynomial with its image over an extension field.
%
% Another way of thinking of this is that a polynomial defines a function: $x^2+1$, for example, is a function
% $\Q\to\Q$ that doesn't vanish. However, we can define the same function with the same coefficients on $\Q(i)$, and
% there it vanishes for $x = \pm i$.
% \begin{proof}
% Without loss of generality, assume $f$ is irreducible: if not, replace $f$ with one of its irreducible factors. A
% field containing a root of a factor of $f$ must then contain a root of $f$.
%
% Since $f$ is nonconstant and irreducible, the ideal $(f)\subset k[x]$ is maximal, and therefore $K = k[x]/(f)$ is a
% field. Let $j:k\inj k[x]$ send $a\in k$ to the constant polynomial $a$, and $\pi:k[x]\surj K$ be the canonical
% projection, taking everything mod $(f)$. Then, $\pi\circ j: k\to K$ is a field homomorphism, so it's an extension.
%
% Finally, we need to produce a root. Consider $\alpha = \pi(x)\in K$. Since $\pi$ is a ring homomorphism, it
% commutes with polynomials, and therefore $p(\alpha) = \pi(p(x))$, and this is $p(x)\bmod p(x) = 0$. Hence, $\alpha$
% is a root of $f$.
% \end{proof}
% This is a very useful way of producing field extensions, as we'll see later. In particular, it generalizes
% Example~\ref{fieldextexm}\eqref{Gaussrat}: $\Q(i)$ is isomorphic to $\Q[x]/(x^2+1)$, and similarly $\Q(\sqrt 3)
% \cong \Q[x]/(x^2-3)$. Similarly, $\C \cong \R[x]/(x^2+1)$ under the isomorphism $a+bi\mapsto a+bx$. We could use
% this to \emph{define} the complex numbers concretely: rather than supposing that we had a square root of $-1$,
% we've constructed the field $\R[x]/(x^2+1)$, where $x^2 = -1$. In the same way, any polynomial in $\Q[x]$ has a
% root in an extension of $\Q$, which motivates the existence of algebraic numbers.

{\color{red}TODO}: treat this in a unified way with adjoining an element; explain what it means to ``adjoin a
square root of $2$,'' and that this is algebraically indistinguishable (maybe this should be another section, and
then irreducibility criteria is a third section).

\subsection*{Irreducibility criteria}
The key of Theorem~\ref{rootinext} is that $f$ is irreducible; if we take $k[x]/(f)$ for a reducible $f$,
$(f)$ isn't even prime, so the resulting quotient isn't a field, or even an integral domain! So in practice, we
need to know when a polynomial is irreducible. Here are a few criteria.
\begin{lem}
\label{cubirred}
If $f\in k[x]$ has degree $2$ or $3$, then $f$ is irreducible iff it has no roots.
\end{lem}
Of course, the hypothesis is necessary: $(x+2)^2\in\Q[x]$ is reducible, but has no roots.
\begin{thm}[Rational root theorem]
{\color{red}TODO}
\end{thm}
\begin{lem}[Gauss' lemma]
\label{gausslem}
Let $R$ be a UFD and $k$ be its field of fractions. If $f\in R[x]$, then if $f$ is reducible in $k[x]$, then it's
reducible in $R[x]$.
\end{lem}
This is most often used when $R = \Z$, so $k = \Q$: then, it says that \emph{an $f\in\Z[x]$ is irreducible iff it's
irreducible in $\Q[x]$}.
\begin{prop}
Let $R$ be an integral domain and $I\subset R$ be a proper ideal. If $f\in R[x]$ and $f\bmod I$ is irreducible in
$(R/I)[x]$, then $f$ is irreducible over $R$.\footnote{The converse is untrue: $x^4+1$ is irreducible over $\Z$,
but reducible modulo every prime.}
\end{prop}
This is most often used when $R = \Z$ and $I = (p)$ for some prime $p$.
\begin{prop}[Eisenstein's criterion]
Let $R$ be an integral domain and $\p\subset R$ be a prime ideal. If $f(x) = x^n + a_{n-1}x^{n-1} + \dotsb + a_1x +
a_0$ is such that $a_{n-1},\dotsc,a_0\in\p$ and $a_0\not\in\p^2$, then $f$ is irreducible in $R[x]$.
\end{prop}
This is generally used for $R = \Z$ and $\p = (p)$ for a prime number $p$. In this case, it says the following.
\begin{cor}
Let $f\in\Z[x]$ be given by $f(x) = x^2 + a_{n-1}x^{n-1} + \dotsc + a_1x + a_0$, and suppose there's a prime number
$p\in\Z$ such that $p\mid a_i$ for $i = 0,\dotsc,n-1$, and $p^2\nmid a_0$. Then, $f$ is irreducible in $\Z[x]$.
\end{cor}
By Lemma~\ref{gausslem}, this also implies $f$ is irreducible in $\Q[x]$. The most common application of
Eisenstein's criterion is to show that a given polynomial in $\Z[x]$ is irreducible over $\Q$.

{\color{red}TODO} fill in these proofs.

Doing a few exercises using these results will probably be more helpful than reading their proofs.
\begin{proof}[Proof of Lemma~\ref{cubirred}]
If $f$ is reducible, then $f = gh$, where $g$ and $h$ are polynomials of degree at least $1$. Since $\deg(g) +
\deg(h)\le 3$, this means at least one of $g$ or $h$ has degree exactly $1$. Without loss of generality, assume
it's $g$, so $g(x) = ax + b$ for $a,b\in k$ and $a\ne 0$; then, $-b/a$ is a root of $g$, and therefore of $f$.

Conversely, if $f$ has a root, then it's reducible.
\end{proof}

In the next few sections, we'll develop the theory of a few nice kinds of field extensions.


\subsection{Normal form}

This is where a formula is shown using only:

\begin{itemize}
\item And / Conjunction- \(\land \)
\item Or / Disjunction   - \(\lor \)
\item Negation - \(\neg \)
\end{itemize}

The conjunctive normal form (CNF) is where a formula is converted to a normal form with the following layout:

$a \land b \land c \land d$

These letters can represent complex sub-formulae, in normal form.

Statements in this form are easier to evaluate, as each subformula can be evaluated separately. The statement is true only if all formulaes within are also true.

The disjunctive normal form (DNF) is similar for \(\lor \).

$a \lor b \lor c \lor d$

\subsection{Properties of the normal form}

The normal binary operators are commutitive - \(A\land B\Leftrightarrow B\land A\) and \(A\lor B\Leftrightarrow B\lor A\)

Both binary operators are associative - \((A\land B)\land C\Leftrightarrow A\land (B\land C)\) and \((A\lor B)\lor C\Leftrightarrow A\lor (B\lor C)\)

Negation is complementary.

\(A\land \neg A\Leftrightarrow F \)

\(A\lor \neg A\Leftrightarrow T\)

Normal binary operators are absorbative.

\(A\land (A\lor B)\Leftrightarrow A\)
\(A\lor (A\land B)\Leftrightarrow A\)

Identity.

\(A\land T\Leftrightarrow A\)

\(A\lor F \Leftrightarrow A\)

Distributivity.

\(A\land (B\lor C)\Leftrightarrow (A\land B)\lor (A\land C)\)

\(A\lor (B\land C)\Leftrightarrow (A\lor B)\land(A\lor C)\)


\subsection{Likelihood function}

We want to estimate parameters. One way of looking into this is to look at the likelihood function:

\(L(\theta ; X)=P(X|\theta )\)

The likelihood function shows the chance of the observed data being generated, given specific parameters.

If this has high peaks then it provides information that \(\theta \) is located in this region.

\subsection{IID}

For multiple events, the likelihood function is:

\(L(\theta ; X)=P(X|\theta )\)

\(L(\theta ; X)=P(A_1 \land B_2 \land C_3 \land D_4…|\theta )\)

If the events are independent, that is the chance of a flip doesn’t depend on any other outcomes, then:

\(L(\theta ; X)=P(A_1|\theta ).P(B_2|\theta ).P(C_3|\theta ).P(D_4|\theta )...\)

If the events are identically distributed, the chance of flipping a head doesn’t change across flips (for example the heads side doesn’t get heavier over time) then:

\(L(\theta ; X)=P(A|\theta ).P(B|\theta ).P(C|\theta ).P(D|\theta )...\)

\(L(\theta ; X)=\prod_{i=1}^n P(X_i|\theta )\)


\section{Methods}
\subsection{Using Conditional Mutual Information}
\begin{itemize}
\item Following \cite{Liu} in order to determine whether methylation $X$ and expression $Y$ of a gene exhibit an L--shape, the conditional Mutual Information $cMI(t)$ for different choices of threshold $t$ is computed.
\[
\mathit{cMI}(t)=I(X,Y|X>t)P(X>t) + I(X,Y|X\le t)P(X\le t)
\]
\item If the relation between methylation and expression shows an L-shape  as $t$ moves from 0 to 1, $\mathit{cMI}(t)$ first decreases and then increases, its value approaching zero when $t$ coincides with the reflection point.
\end{itemize}


\subsection{Normal matrices}

\(M^*M=MM^*\)

All symmetrix matrices are normal

All hermetitian matrices (inc subset symmetric) are normal

Normal matrix never defective


\lab{Algorithm}{Pseudorandom Number Generators}{Pseudorandom Number Generators}
\label{Ch:PRNG}

\objective{This section teaches about how to build pseudorandom number generators and the differences between the different kinds of PRNGs}

\section*{Linear congruential genterators}

%Lab \ref{PRNG}

Lotteries, most board games, and statistics need random numbers. In real life, we roll dice, take balls out of a bag, or spin a wheel. in order to rpresent random number on computer we use  psuedorandom number generators (PRNGs). The numbers are not truly random, but they are based on a complex formula that makes them look ``random." Another thing is that these generators have to be fast for their aplication. So you have to have something that is fast and looks random.

One of the oldest of theses is the Linear Congruiental Generator (LCG). The Generator is defined by the recurrence relation:
$X_{n+1}=(a*X_n + c)$ mod $m$ where

\begin{itemize}
\item[] $X$ is a sequence of pseudorandom values, and
\item[] $m$, $0<m$ is the modulus
\item[] $a$, $0<a<m$ is the multiplier
\item[] $c$, $0\leq c<m$ is the increment
\item[] $X_0$, $0\leq X_0 <m$ is the seed
\end{itemize}

each of these are interger constansts.

\begin{problem}
Write a LCG that produces a number an array of psuedorandom numbers between $0.0$ and $1.0$. Let the arguments be size of array and let a,c,mod,and seed be optional arguments. I recommend $a=1103515245$, $c=12345$, $m=2^{31}-1$, and $seed=4329$ as the defualt values.
\end{problem}

\begin{problem}
Write a LCG that produces a number an array of psuedorandom numbers of integers between two input arguments. Do it by calling your agorithm from problem one and mutiplying it by the values and casting the array as as int using the .astype() function. Let the arguments be size of array and the two intergers. Let a,c,mod,and seed continue to be optional arguments.
\end{problem}

One easy way to ``see" if your generator is random is to look at a bitmap of the output. In python, use the plt.imshow() function to see a bitmap of the array produced by your LCG. Resize your output to be $512 \times 512$.

\begin{problem}
For what values of $a$, $c$, and $m$ does your LCG have a definite pattern.
\end{problem}

This method is not rigorist. There are several ways to test the randomness of your output.

The length over with your random number generator repeats is called the period. The period is at most m, but it may be shorter based on the values of a and c.

According to the Hull-Dobell Theorem (find a source, used wikipedia), The LCG will only have a full period if and only if,
1. $c$ and $m$ are relatively prime,
2. $a-1$ is divisible by all prime factors of $m$,
3. $a-1$ is a multiple of 4 if $m$ is a multiple of 4

\begin{problem}
Test values of $a$,$c$, and $m$ that fit these requirements.
\end{problem}

This algorithm is used to generate random numbers in Java, C++ and other places.

\section*{Mersenne Twister and Bitwise operations}

All numbers can be represented in bits as a base two number. Computers are optimized to work with numbers in that manner. There are operations that work on the the bit representation of the two number. These are XOR, OR, and AND.

AND - if both numbers have a 1 in the ith place then the ith place is 1. Otherwise the ith place is 0.

OR - if one or both numbers have a 1 in the ith place then the ith place is 1. Otherwise the ith place is 0.

XOR - if only one of the two numbers has a 1 in the ith place then the ith place is 1. If both or neither of the numbers has a 1 in the ith place, the ith place is 0.

In addition you can shift the bitwise number over anumber of values. For example, shifting 10100 to the right by one yields 1010 and shifting it to the left by one yields 101000. This can be done by $\ll$ and $\gg$ in python.

The mersenne twister PRNG does a series of these operations to generate random numbers. the Random class in python uses the mersenne twister algorithm.

\begin{problem}
Look at the bitmap of output of sp.rand(512,512). Can you see any patterns?
\end{problem}

\begin{problem}
Analyse the first of 10,000 outputs of the random class. Can you see a period?
\end{problem}


\subsection{Multiplying a matrix and a vector}

One of the most important uses of a matrix is to multiply a matrix by
a vector. In fact, this is one of the reasons matrices were invented.
Let us start by considering an alternative way of writing a system of
linear equations.

\begin{definition}{The vector form of a system of linear equations}{vector-form}
  Suppose we have a system of equations given by
  \begin{equation*}
    \begin{array}{ccc}
      a_{11}x_1 + \ldots + a_{1n}x_n &=& b_1, \\
      a_{21}x_1 + \ldots + a_{2n}x_n &=& b_2, \\
      \vdots \\
      a_{m1}x_1 + \ldots + a_{mn}x_n &=& b_m.
    \end{array}
  \end{equation*}
  We can express this system in \textbf{vector form}%
  \index{system of linear equations!vector form}%
  \index{vector form!of a system of equations}, which is as follows:
  \begin{equation*}
    x_1 \begin{mymatrix}{c} a_{11} \\ a_{21} \\ \vdots \\ a_{m1} \end{mymatrix}
    + x_2 \begin{mymatrix}{c} a_{12} \\ a_{22} \\ \vdots \\ a_{m2} \end{mymatrix}
    + \ldots
    + x_n \begin{mymatrix}{c} a_{1n} \\ a_{2n} \\ \vdots \\ a_{mn} \end{mymatrix}
    ~=~
    \begin{mymatrix}{c} b_1 \\ b_2 \\ \vdots \\ b_m \end{mymatrix}.
  \end{equation*}
\end{definition}

Notice that each vector used here is one column from the corresponding
augmented matrix. There is one vector for each variable in the system,
along with the constant vector. The left-hand side is a linear
combination of column vectors. Linear combinations of column vectors
are so important that we introduce a special notation for them.

\begin{definition}{The product of a matrix and a vector, by columns}{matrix-times-vector-columns}
  \index{multiplication!matrix times vector}%
  \index{matrix!multiplication!matrix times vector}%
  The product of an $m\times n$-matrix $A$ and an $n$-dimensional
  column vector $\vect{x}$ is an $m$-dimensional column vector,
  defined as a linear combination of the columns of $A$ as follows:
  \begin{equation*}
    A\vect{x} \,~=~
    \begin{mymatrix}{cccc}
      a_{11} & a_{12} & \cdots & a_{1n} \\
      a_{21} & a_{22} & \cdots & a_{2n} \\
      \vdots & \vdots & \ddots & \vdots \\
      a_{m1} & a_{m2} & \cdots & a_{mn} \\
    \end{mymatrix}
    \begin{mymatrix}{c}
      x_1 \\
      x_2 \\
      \vdots \\
      x_n
    \end{mymatrix}
    ~=~
    x_1 \begin{mymatrix}{c} a_{11} \\ a_{21} \\ \vdots \\ a_{m1} \end{mymatrix}
    + x_2 \begin{mymatrix}{c} a_{12} \\ a_{22} \\ \vdots \\ a_{m2} \end{mymatrix}
    + \ldots
    + x_n \begin{mymatrix}{c} a_{1n} \\ a_{2n} \\ \vdots \\ a_{mn} \end{mymatrix}.
  \end{equation*}
\end{definition}

In other words, we can think of the vector $\vect{x}$ as encoding
instructions for how to take a linear combination of the columns of
$A$. The product $A\vect{x}$ is computed by taking $x_1$ times the
first column of $A$, plus $x_2$ times the second column of $A$, and so
on. For this to work, $A$ must have the same number of columns as
$\vect{x}$ has components.

\begin{example}{Multiplying a matrix and a vector, by columns}{matrix-times-vector-columns}
  Compute the product
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 2 & 3 \\
      4 & 5 & 6
    \end{mymatrix} \begin{mymatrix}{r}
      7 \\
      8 \\
      9
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We have
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 2 & 3 \\
      4 & 5 & 6
    \end{mymatrix} \begin{mymatrix}{r}
      7 \\
      8 \\
      9
    \end{mymatrix}
    ~=~
  7\begin{mymatrix}{c}
    1 \\
    4
  \end{mymatrix}
  + 8 \begin{mymatrix}{c}
    2 \\
    5
  \end{mymatrix}
  + 9 \begin{mymatrix}{c}
    3 \\
    6
  \end{mymatrix}
  = \begin{mymatrix}{c}
    50 \\
    122
  \end{mymatrix}.
\end{equation*}
\end{solution}

There is another way of looking at the product of a matrix and a
vector. Instead of looking at the columns of $A$, we can look at the
rows.

\begin{proposition}{The product of a matrix and a vector, by rows}{matrix-times-vector-rows}
  Let $A$ be an $m\times n$-matrix and let $\vect{x}$ be an $n$-dimensional
  column vector. The product $A\vect{x}$ can also be written like this:
  \begin{equation*}
    A\vect{x} \,~=~
    \begin{mymatrix}{cccc}
      a_{11} & a_{12} & \cdots & a_{1n} \\
      a_{21} & a_{22} & \cdots & a_{2n} \\
      \vdots & \vdots & \ddots & \vdots \\
      a_{m1} & a_{m2} & \cdots & a_{mn} \\
    \end{mymatrix}
    \begin{mymatrix}{c}
      x_1 \\
      x_2 \\
      \vdots \\
      x_n
    \end{mymatrix}
    ~=~
    \begin{mymatrix}{c}
      a_{11}x_1+a_{12}x_2+\ldots+a_{1n}x_n \\
      a_{21}x_1+a_{22}x_2+\ldots+a_{2n}x_n \\
      \vdots \\
      a_{m1}x_1+a_{m2}x_2+\ldots+a_{mn}x_n \\
    \end{mymatrix}.
  \end{equation*}
\end{proposition}

\begin{example}{Multiplying a matrix and a vector, by rows}{matrix-times-vector-rows}
  Compute the product
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 2 & 3 \\
      4 & 5 & 6
    \end{mymatrix} \begin{mymatrix}{r}
      7 \\
      8 \\
      9
    \end{mymatrix}
  \end{equation*}
  by rows.
\end{example}

\begin{solution}
  We have
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 2 & 3 \\
      4 & 5 & 6
    \end{mymatrix} \begin{mymatrix}{r}
      7 \\
      8 \\
      9
    \end{mymatrix}
    ~=~
    \begin{mymatrix}{c}
      1\cdot 7 + 2\cdot 8 + 3\cdot 9 \\
      4\cdot 7 + 5\cdot 8 + 6\cdot 9
    \end{mymatrix}
    ~=~
    \begin{mymatrix}{c}
      50 \\
      122
    \end{mymatrix}.
  \end{equation*}
  Note that the is exactly the same answer as before.
\end{solution}

When we use Definition~\ref{def:matrix-times-vector-columns}, we
calculate the product by looking at one column of $A$ at a time. When
we use Proposition~\ref{prop:matrix-times-vector-rows}, we calculate the
product by looking at one row of $A$ at a time.  As the above examples
show, both methods give exactly the same answer. Please convince
yourself that this is true in general. This ability to switch back and
forth between a column-based viewpoint and a row-based viewpoint is
one of the central tools of linear algebra.

Using the above operation, we can also write a system of linear
equations in \textbf{matrix form}. In this form, we express the system
as a matrix multiplied by a vector.

\begin{definition}{The matrix form of a system of linear equations}{matrix-form}
  Suppose we have a system of equations given by
  \begin{equation*}
    \begin{array}{c}
      a_{11}x_1 + \ldots + a_{1n}x_n = b_1, \\
      a_{21}x_1 + \ldots + a_{2n}x_n = b_2, \\
      \vdots \\
      a_{m1}x_1 + \ldots + a_{mn}x_n = b_m.
    \end{array}
  \end{equation*}
  Then we can express this system in \textbf{matrix form}%
  \index{system of linear equations!matrix form}%
  \index{matrix form!of a system of equations}, which is as follows:
  \begin{equation*}
    \begin{mymatrix}{cccc}
      a_{11} & a_{12} & \cdots & a_{1n} \\
      a_{21} & a_{22} & \cdots & a_{2n} \\
      \vdots & \vdots & \ddots & \vdots \\
      a_{m1} & a_{m2} & \cdots & a_{mn}
    \end{mymatrix}
    \begin{mymatrix}{c}
      x_1 \\
      x_2 \\
      \vdots \\
      x_n
    \end{mymatrix}
    =
    \begin{mymatrix}{c}
      b_1 \\
      b_2 \\
      \vdots \\
      b_m
    \end{mymatrix}.
  \end{equation*}
\end{definition}

The matrix form of a system of equations is therefore written as
$A\vect{x}=\vect{b}$, where $A$ is the coefficient matrix%
\index{coefficient matrix}%
\index{matrix!coefficient matrix} of the system, $\vect{x}$ is an
$n$-dimensional column vector constructed from the variables of the
system, and $\vect{b}$ is an $m$-dimensional column vector constructed
from the constant terms of the system. Any system of linear equations
can be written in this form.


\subsection{Span}

\subsubsection{Span function}

We can take a subset \(S\) of \(V\). We can then make linear combinations of these elements.

This is called the linear span - \(span (S)\).


%!TEX program = xelatex

\documentclass[a4paper,12pt]{article}
\usepackage{fontspec}
\usepackage{polyglossia}
\usepackage{csquotes}
\setdefaultlanguage{english}
\usepackage[titletoc,toc,title]{appendix}
\usepackage{mathpazo}
\setmainfont
     [ BoldFont       = texgyrepagella-bold.otf ,
       ItalicFont     = texgyrepagella-italic.otf ,
       BoldItalicFont = texgyrepagella-bolditalic.otf ]
     {texgyrepagella-regular.otf}
\usepackage{setspace}

\setstretch{1.10}
\usepackage{amsmath, amsfonts, amssymb, amsthm, dsfont,cancel}
\usepackage{graphicx, float}
\usepackage{tikz, color, xcolor} % For artists
\usepackage[hmargin=3.5cm,vmargin=3cm]{geometry}
\usepackage{setspace}
\usepackage{enumerate} % For (i) (ii) items in enumerate

\newcommand\bhat[1]{\hat{b}_#1}
\newcommand\sumin{\sum_{i=1}^N}

\title{Quick proof on univariate OLS estimator}
\author{Antoine Mayerowitz}

\begin{document}

\maketitle

We suppose that the data consist of $N$ observations $\{y_i, x_i\}_{i=1}^N$ where each observation $i$ consist of a dependent variable $y_i$ and an explanatory variable $x_i$. We suppose a linear relationship in our variables such that
\begin{equation}
	y_i = b_0 + b_1 x_i + \varepsilon_i
\end{equation}
Where $b_0$ and $b_1$ are scalars, shared by all observations and $\varepsilon_i$ is the error term.

Our goal is to find $b_0$ and $b_1$ such that the \emph{sum of squared of residuals}(SSR) is minimized. Where the SSR is defined by
\begin{align*}
	SRR &= \sumin e^2 \\
			&= \sumin (y_i-\bhat{0}-\bhat{1}x_i)^2
\end{align*}

Mathematicaly we seek

\begin{equation}
	\min_{\bhat{0},\bhat{1}} \sumin (y_i-\bhat{0}-\bhat{1}x_i)^2
\end{equation}

As we need to minimize a quadratic function, we take the derivative with respect to both argument and equalize to 0.

\[
\begin{cases}
	\dfrac{\partial \; SSR}{\partial \bhat{0}} = -2 \sumin (y_i-\bhat{0}-\bhat{1}x_i) \\
	~\\
	\dfrac{\partial \; SSR}{\partial \bhat{1}} = -2 \sumin x_i (y_i-\bhat{0}-\bhat{1}x_i) \\
\end{cases}
\]

Then we equalize both equations to 0
\[
\begin{cases}
	-2 \sumin (y_i-\bhat{0}-\bhat{1}x_i) &= 0 \\
	-2 \sumin x_i (y_i-\bhat{0}-\bhat{1}x_i) &= 0 \\
\end{cases}
\]

We now have our system of 2 equations with 2 unknowns that we need to solve for $\bhat{0}$ and $\bhat{1}$

We simplify
\[
\begin{cases}
	\cancel{-2} \sumin (y_i-\bhat{0}-\bhat{1}x_i) &= 0 \\
	\cancel{-2} \sumin x_i (y_i-\bhat{0}-\bhat{1}x_i) &= 0 \\
\end{cases}
\]

Rewriting

\[
\begin{cases}
	\sumin y_i &= \sumin \bhat{0} + \bhat{1} \sumin x_i\\
	\sumin x_i y_i &= \bhat{0} \sumin x_i  + \bhat{1} \sumin x_i^2 \\
\end{cases}
\]

Using the fact that $\sum_{k=1}^K a = K\times a$ and that $\bhat{0}$ do not depends on $i$ we get

\[
\begin{cases}
	\sumin y_i &= N \bhat{0} + \bhat{1} \sumin x_i\\
	\sumin x_i y_i &= \bhat{0} \sumin x_i  + \bhat{1} \sumin x_i^2 \\
\end{cases}
\]

We multiply the first equation by $\frac{1}{N}$

\[
\begin{cases}
	\frac{1}{N}\sumin y_i &= \frac{1}{N} N \bhat{0} + \bhat{1} \frac{1}{N} \sumin x_i\\
	\sumin x_i y_i &= \bhat{0} \sumin x_i  + \bhat{1} \sumin x_i^2 \\
\end{cases}
\]

We define the average $z$ by $\bar{z} := \frac{1}{N}\sumin z_i$.

\[
\begin{cases}
	\bar{y} &= \bhat{0} + \bhat{1} \bar{x}\\
	\sumin x_i y_i &= \bhat{0} \sumin x_i  + \bhat{1} \sumin x_i^2 \\
\end{cases}
\]

We rewrite to get an expression of $\bhat{0}$
\[
\begin{cases}
	\bhat{0} &= \bar{y} - \bhat{1} \bar{x}\\
	\sumin x_i y_i &= \bhat{0} \sumin x_i  + \bhat{1} \sumin x_i^2 \\
\end{cases}
\]

We substitute the first equation in the second
\[
\begin{cases}
	\bhat{0} &= \bar{y} - \bhat{1} \bar{x}\\
	\sumin x_i y_i &= (\bar{y} - \bhat{1} \bar{x}) \sumin x_i  + \bhat{1} \sumin x_i^2 \\
\end{cases}
\]

Developing

\[
\begin{cases}
	\bhat{0} &= \bar{y} - \bhat{1} \bar{x}\\
	\sumin x_i y_i &= \bar{y} \sumin x_i - \bhat{1} \bar{x} \sumin x_i  + \bhat{1} \sumin x_i^2 \\
\end{cases}
\]

We factor by $\bhat{1}$
\[
\begin{cases}
	\bhat{0} &= \bar{y} - \bhat{1} \bar{x}\\
	\sumin x_i y_i &= \bar{y} \sumin x_i + \bhat{1} \left(\sumin x_i^2 - \bar{x} \sumin x_i \right)\
\end{cases}
\]

Isolating $\bhat{1}$

\[
\begin{cases}
	\bhat{0} &= \bar{y} - \bhat{1} \bar{x}\\
	~ \\
	\bhat{1} &= \dfrac{\sumin x_i y_i - \bar{y} \sumin x_i}{\sumin x_i^2 - \bar{x} \sumin x_i}
\end{cases}
\]

After a bit of algebra (see appendix), one can simplify to

\[
\begin{cases}
	\bhat{0} &= \bar{y} - \bhat{1} \bar{x}\\
	~ \\
	\bhat{1} &= \dfrac{\sumin (y_i - \bar{y}) (x_i - \bar{x})}{\sumin (x_i - \bar{x})^2}
\end{cases}
\]

Which is strictly equivalent to

\[
\begin{cases}
	\bhat{0} &= \bar{y} - \bhat{1} \bar{x}\\
	~ \\
	\bhat{1} &= \dfrac{Cov(x,y)}{Var(x)}
\end{cases}
\]

\clearpage
\begin{appendices}
\section{Simplifying $\bhat{1}$}
\subsection{Finding the covariance}
\[
\sumin x_i y_i - \bar{y} \sumin x_i =  N \times Cov(x,y)
\]
This proof need some tricks and heavily play with the fact that $\sumin x_i = \bar{x}$ and that $\sumin \bar{y} = N \bar{y}$. For example, one could write
\[
	\bar{y} \sumin x_i = N\bar{y}\bar{x} = \sumin \bar{x}\bar{y} =\sumin y_i \bar{x}
\]

\begin{proof}
	\begin{align*}
		\sumin x_i y_i - \bar{y} \sumin x_i &= \sumin x_i y_i - N\bar{y}\bar{x}\\
		&= \sumin x_i y_i - N\bar{y}\bar{x} - N\bar{y}\bar{x} +  N\bar{y}\bar{x}\\
		&= \sumin x_i y_i - \sumin x_i \bar{y} - \sumin \bar{x} y_i +  \sumin \bar{y}\bar{x} \\
		& = \sumin (x_i y_i -  x_i \bar{y} - \bar{x} y_i + \bar{y}\bar{x}) \\
		&= \sumin (y_i - \bar{y}) (x_i - \bar{x}) \\
		&= N \times Cov(x,y)
	\end{align*}
\end{proof}

\subsection{Finding the variance}
\[
\sumin x_i^2 - \bar{x} \sumin x_i = N \times Var(x)
\]
This proof is very similar to the one for covariance.

\begin{proof}
	\begin{align*}
		\sumin x_i^2 - \bar{x} \sumin x_i &= \sumin x_i^2 - N\bar{x}\bar{x}\\
		&= \sumin x_i^2 - 2N\bar{x}\bar{x} + N\bar{x}\bar{x} \\
		&= \sumin x_i^2 - 2 \bar{x} \sumin x_i + N\bar{x}^2 \\
		&= \sumin (x_i^2 - 2\bar{x}x_i + \bar{x}^2) \\
		&= \sumin (x - \bar{x})^2 \\
		&= N \times Var(x)
	\end{align*}
\end{proof}

\end{appendices}

\end{document}


\subsection{Substitution}

If we have a tautology, then we can substitute the formula of any propositional variable with any formula to arrive at any other tautology.

For example, we know that \(\theta \lor \neg \theta \) is a tautology. This means that an arbitrary formula for \(\theta \) is also a tautology.

An example is \((\gamma \land \alpha )\lor \neg (\gamma \land \alpha )\), which we know is a tautology, without having to examine each variable.

\subsection{Syntactic consequence}

Let us call the first formula \(A\) and the second \(B\). We can then say:

\(A\vdash B\)

This says that: if \(A\) is true, then we can deduce that \(B\) is true using steps such as substitution.

\subsection{Modus Ponens}

Modus Ponens is a deduction rule. This allows us to use stpes other than substitution to derive new tautologies.

If \(A\) implies \(B\), and \(A\) is true, then \(B\) is also true.

\((\theta \rightarrow \gamma )\land \theta \Rightarrow \gamma \)

That is, if we can show that the following are true:

\(\theta \rightarrow \gamma \)

\(\theta \)

We can infer that the following is also true:

\(\gamma \)

\subsection{Theory}

Results derived from substitution or induction are called theorems. Theorems often divided into:

\begin{itemize}
\item Theorems - important results
\item Lemmas - results used for later theorems
\item Corollaries - readily deduced from a theorem
\end{itemize}

We take a set of axioms, as true, and a deduction rule which enables us to derive additional formulae, or theorems. The collection of axioms and theorems is known as the theory.