\chapter{Applications of the Fast-Fourier-Transform}
\label{AppendixFFT}

The Fast-Fourier-Transform~\cite{BrighamFFTBook} is a versatile numerical
\index{Fourier transform!fast}
tool. It is an efficient implementation of the discrete
Fourier transform that maps between a function in a canonical variable and
a function in the conjugate variable, e.g.\
space ($x$) and wavenumber ($k$, equal to momentum with $\hbar=1$),
or time ($t$) and angular frequency ($\omega$). For these two examples, the
discrete Fourier transform $F$ of a function $f$ reads as
\begin{align}
  F(\omega_j)
  &= \sum_{n=0}^{N-1} f(t_n) \ee^{-\ii \frac{2\pi j n}{N}}
  \approx \sum_{n=0}^{N-1} f(t_n) \ee^{-\ii \omega_j t_n}\,,
  \\
  F(k_j)
  &= \sum_{n=0}^{N-1} f(x_n) \ee^{-\ii \frac{2\pi j n}{N}}
  \approx \sum_{n=0}^{N-1} f(x_n) \ee^{-\ii k_j x_n}\,,
\end{align}
where $N$ is the sampling size, and using the correspondence
\begin{equation}
  w_j t_n \approx \frac{2 \pi j n}{N}\,,
  \qquad
  k_j x_n \approx \frac{2 \pi j n}{N}\,.
  \label{eq:discrete_to_cont_FTT}
\end{equation}

This transformation is widely implemented in numerical libraries as
$F(\omega_j) \equiv \FFT f(t_n)$ with the inverse transform defined by
\begin{equation}
  f(t_n) = \frac{1}{N} \FFT^{-1} F(\omega_j)
         = \frac{1}{N} \FFT^{-1} \FFT f(t_n)\,.
\end{equation}
The FFT scales as $N \log N$ with the sampling size~\cite{DuhamelSP1990}.

\section{The Frequency Grid}

When using the output of the FFT routine, e.g.\ as the spectrum of
a time-dependent signal $f(t_n)$, it is important to understand exactly which
angular frequency values $\omega_j$ the resulting amplitudes $F(\omega_j)$
correspond to. This differs between odd and even $N$, a detail that is often
neglected and can lead to subtle errors in numerical calculations.

If the original signal is of duration $T = t_{N-1} - t_0$, and has a sampling
rate $dt = \frac{T}{N-1}$, the result of a call to the FFT routine is
an array of $N$ complex numbers, containing the amplitudes for angular
frequencies between $-\omega_{\max}$ and $+\omega_{\max}$, with
\begin{equation}
  \omega_{\max} = \begin{cases}
     \frac{\pi}{dt} = \frac{(N-1)\pi}{T}
                      & \text{if $N$ even} \\
     \frac{N-1}{N} \frac{\pi}{dt} = \frac{N-1}{N} \frac{(N-1)\pi}{T}
                      & \text{if $N$ odd}
                  \end{cases}
\end{equation}

The layout of the frequency array also depends on whether $N$ is odd or even. In
any case, the frequency array consists of two parts: the first sub-array of
length $l$ contains the amplitudes of the positive frequencies, the remaining sub-array
of length $N-l$ (running from $l+1$ to $N$) contains the amplitudes for the
negative frequencies.
\begin{itemize}[noitemsep,nolistsep]

  \item $N$ even.

  For even $N$, there are $l=N/2$ positive frequencies, and the values
  correspond to $$0, d\omega, \dots, \omega_{\max}-d\omega,$$
  followed by $$-\omega_{\max}, -\omega_{\max}+d\omega, \dots, -d\omega.$$

  \item $N$ odd.

  For odd $N$, there are $l=N/2 + 1$ positive frequencies, and the values
  correspond to $$0, d\omega, \dots, \omega_{\max},$$ followed by
  $$-\omega_{\max}, -\omega_{\max}+d\omega, \dots, -d\omega.$$
\end{itemize}

The spectral resolution is
\begin{equation}
  d\omega = \begin{cases}
      \frac{2 \omega_{\max}}{N} = \frac{2\pi}{T} - \frac{2\pi}{N T}
            & \text{if $N$ even} \\
      \frac{2 \omega_{\max}}{N-1} = \frac{2\pi}{T}
            & \text{if $N$ odd}
        \end{cases}\,.
\end{equation}
With $t_n  = n \frac{T}{N}$ and $\omega_j = j \, d\omega$,
Eq.~\eqref{eq:discrete_to_cont_FTT} is recovered.  If $f(t) \in \Real$, then $F(\omega) = F(-\omega)^{*}$. For complex signals, on the other hand, the positive and the negative part of the spectrum are \emph{not} equivalent.
\section{Derivatives and the Kinetic Operator}

The FFT can be used to calculate the derivative of a signal
$f(x)$ evenly sampled at $N$ points.
\begin{equation}
  \partdifquo{x} \equiv \frac{1}{N} \FFT^{-1} \left( \ii k \right) \FFT
\end{equation}
The kinetic operator in one Cartesian dimension is
\begin{equation}
  \Op{T} = \frac{\Op{p}^2}{2m}
         = -\frac{\hbar}{2m} \frac{\partial^2}{\partial x^2}\,.
\end{equation}
Using the FFT, this becomes
\begin{equation}
\begin{split}
  \Op{T}&= -\frac{\hbar^2}{2m} \frac{1}{N} \FFT^{-1} \ii k
             \FFT \frac{1}{N} \FFT^{-1} \ii k \FFT \\
        &= \frac{\hbar^2}{2m} \frac{1}{N}\FFT^{-1} k^2 \FFT\,.
\end{split}
\end{equation}

\section{Cosine-Transform and Chebychev Coefficients}

For $f(x) = e^{x}$, the Chebychev coefficients can be derived analytically
to be proportional to the Bessel functions, see
section~\ref{subsec:chebychev} in chapter~\ref{chap:numerics}. For a general
function, however, the coefficients are calculated via a cosine
transform~\cite{NdongJCP09}. Since the Chebychev polynomials $P_n$ are defined
only in the interval $[-1, 1]$, the function $f(x)$ must be rescaled as $f(x)
= \tilde{f}(\xi)$ with $\xi \in [-1,1]$. The coefficients are then calculated as
\begin{equation}
  a_n = \frac{2 - \delta_{n,0}}{\pi}
        \int_{-1}^{+1} \frac{\tilde{f}(\xi) P_n(\xi)}{\sqrt{1-\xi^2}} \dd \xi
      \approx \frac{2 - \delta_{n,0}}{\pi}
        \sum_{k=0}^{N-1} w_k f_k \cos\left( n \theta_k \right)\,,
\end{equation}
with $f_k \equiv f(x_k) = \tilde{f}(\xi_k)$, $\theta_k \equiv \arccos(\xi_k)$,
and weights $w_k$ that depend on the choice of the sampling points $x_k$.
The approximation becomes exact for $N \rightarrow \infty$.
There are two possible choices of sampling points:
\begin{itemize}

  \item Gauss-Lobatto-Chebychev grid
        (``closed interpolation'': $\xi_k \in [1, -1]$)
  \begin{equation}
    \xi_k = \cos\left( \frac{k \pi}{N-1}
                \right), \qquad
    w_k = \left(\frac{1}{2}\right)^{\delta_{n,0}}
          \left(\frac{1}{2}\right)^{\delta_{n,N-1}}
          \frac{\pi}{N-1}\,.
    \label{eq:gauss_lobatto}
  \end{equation}
  The $\xi_k$ are the extrema of the Chebychev polynomials, plus
  endpoints. The resulting formula for the coefficients $a_n$ is defined as
  a discrete cosine transform of type I (DCT I) and may be implemented via the
  FFT~\cite{RaoDCTBook1990}. The complete calculation of the coefficients is
  shown in Algorithm~\ref{al:ChebyCoeffs}.

  \item Gauss-Chebychev (``open interpolation'': $\xi_k \in (1, -1)$)
  \begin{equation}
    \xi_k = \cos\left( \frac{\left(k + \frac{1}{2}\right) \pi}{N}
                \right), \qquad
    w_k = \frac{\pi}{N}\,.
    \label{eq:gauss_cheby}
  \end{equation}
  The $\xi_k$ are the roots of the Chebychev polynomials. The resulting
  formula for the $a_n$ is defined as a discrete cosine transform of type II
  (DCT II). There is no direct mapping to the FFT in this case.
\end{itemize}

\begin{algorithm}
  \caption{{\sc ChebychevCoefficients}
  for expansion of $f(x)$.
  \label{al:ChebyCoeffs}
  }
  \begin{algorithmic}[1]
    \Statex
    \Require{$f(x)$ with $x \in [x_{\min}, x_{\max}]$;
             maximum number $n_{\max}$  of coefficients
    }
    \Ensure{Array of Chebychev coefficients $[a_0\dots a_n]$, $n < n_{\max}$
    allowing to approximate $f(x)$ to predefined precision.}
    \Statex
    \Procedure{ChebyCoeffs}{$f(x)$}
     \State $\Delta = x_{\max} - x_{\min}$
     \State $\alpha = \frac{1}{2} \Delta$;
            $\beta = \alpha + x_{\min}$
     \State $F_{0:2 n_{\max}-3} = 0$
            \Comment{allocation to size $2(n_{\max}-1)$}
     \For{$k = 0:n_{\max}-1$}
        \State $\xi_{k} = \cos\left( \frac{k \pi}{n_{\max} - 1} \right)$
        \Comment{$\xi_k = +1, \dots, -1$}
        \State $F_k = f(\alpha \xi_{k} + \beta)$
     \EndFor
     \For{$k = 1:n_{\max}-2$} \Comment{mirror, without endpoints}
        \State $F_{n_{\max}-1+k} = F_{n_{\max}-1-k}$
     \EndFor
     \State $F$ = \Call{FFT}{$F$}
     \State $F = F / (n_{max}-1)$
     \State $F_0 = \frac{1}{2} F_0$
     \State $F_{n_{\max}-1} = \frac{1}{2} F_{n_{\max}-1}$
     \For{$i=0: n_{\max}-1$}
       \State $a_i = F_i$
       \IIf{$\Abs{a_i} < $~limit} exit loop with $n=i$
     \EndFor
     \State \Return $[a_0, \dots a_n]$
   \EndProcedure
  \end{algorithmic}
\end{algorithm}


\lab{Diffie-Hellman Key Exchange}{Diffie-Hellman Key Exchange}
\label{lab:DiffieHellman}
\objective{Understand a method of creating a shared secure key with insecure communications.}

In cryptography, a \emph{key} is a piece of secret information that is required to either encrypt or decrypt a message.
A cipher (or code) is \emph{symmetric} if it uses the same key for both encryption and decryption.
People have used symmetric ciphers since ancient history.
For example, the ancient Greeks used substitution ciphers, the Nazis used Enigma, and people today use AES and Twofish.

For Alice and Bob to use a symmetric cipher, they must first agree on a key.
A \emph{key exchange algorithm} is a way to do this so that even if an enemy Eve intercepts all the information passed between Alice and Bob, she will not be able to deduce the key.
The Diffie-Hellman key exchange was the first such algorithm, and it is frequently used today.

\section*{One-Time Pad (Optional)}
In this section we introduce a basic symmetric cipher with modern uses: the one-time pad.
This cipher is outlined below.
\begin{itemize}
\item The key is a number.
\item Encrypt a message by
\begin{enumerate}
\item converting it to a number
\item then adding the key \emph{bitwise}. This means that binary digits are added with no carrying; for example, $1010+11=1001$.
\end{enumerate}
\item Decrypt the ciphertext by
\begin{enumerate}
\item adding the key to it bitwise
\item then converting the resulting number back into text.
\end{enumerate}
\end{itemize}
The security of a one-time pad is compromised when the key is reused to encrypt another message.

For example, suppose we wish to encrypt the message SECRET.
One way to convert this message to a number is to use the substitution cipher $A=10, B=11, \ldots, Z=35$.
With this rule, SECRET=281412271429.
To encrypt this message, we must choose a key that is larger than 281412271429.
We choose 987654321000.
We perform the bitwise addition in Python with the caret operator \li{^}.
\begin{lstlisting}
>>> message = 281412271429
>>> key = 987654321000
>>>
>>> ciphertext = message^key    # Encode the message
>>> ciphertext
706282163757
>>>
>>> message == ciphertext^key    # Decode the message
True
\end{lstlisting}

\begin{problem}
For this problem, use the substitution cipher $A=11, \ldots$ described above, with the additional rule \texttt{' '}=36.
\begin{enumerate}
\item Encrypt the message `PRIVATE INFO' with the key 987654321098765432109876.
\item Decrypt the message 153931672663401143 with the key 12345678901234567890.
\end{enumerate}
\end{problem}

\section*{Diffie-Hellman key exchange algorithm}
The Diffie-Hellman key exchange uses \emph{primitive roots} modulo a prime.

\subsection*{Primitive roots}
Let $p$ be a prime number.
A positive number $a$ is a \emph{primitive root modulo p} if the positive powers $a^1, a^2, \ldots, a^{p-1}$ generate all nonzero congruence classes modulo $p$.
For example, let $p=7$.
Then $a=2$ is not a primitive root, since
\[
2^1=2, \quad 2^2=4, \quad 2^3 \equiv 1, \quad 2^4 \equiv 2, \quad 2^5 \equiv 4, \quad 2^6 \equiv 1 \pmod{7}.
\]
The powers of $2$ never hit the classes $3, 5,$ or $6$ modulo 7.
However, $3$ is a primitive root since
\[
3^1=3, \quad 3^2\equiv2, \quad 3^3 \equiv 6, \quad 3^4 \equiv 4, \quad 3^5 \equiv 5, \quad 3^6 \equiv 1 \pmod{7}.
\]
That is, each congruence class $\pmod{7}$ appears as a power of 3.

\subsection*{Diffie-Hellman algorithm}

Here is an outline of how Alice and Bob can perform a Diffie-Hellman key exchange:

\begin{itemize}[$\bullet$]

\item Alice and Bob (publicly) agree on a prime $p$ and a primitive root $g$.

\item Alice and Bob each secretly choose an integer, $x$ and $y$ respectively.

\item Alice computes $A = g^x \pmod{p}$ while Bob computes $B = g^y \pmod{p}$.

\item Alice and Bob exchange $A$ and $B$ (still keeping $x$ and $y$ secret).

\item Alice computes $B^x \pmod{p}$ and Bob computes $A^y \pmod{p}$. They get the same answer,
\[k \equiv B^x \equiv (g^y)^x \equiv (g^x)^y \equiv A^y \pmod{p}.\]

\end{itemize}
Now Alice and Bob can use $k$ as a secure key for a symmetric cipher.
One advantage of this algorithm is that Alice and Bob can use the same prime $p$ and root $g$ to generate a new secure key, simply by picking new integers $x$ and $y$.

The key idea of the Diffie-Hellman algorithm is that $(g^x)^y\equiv (g^y)^x \pmod{p}$.
We do an example of this in Python with $p=41$, $g=6$, $x=10$, and $y=13$.
The function \li{pow(a, b, n)} computes $a^b \pmod{n}$.
\begin{lstlisting}
>>> A = pow(6, 10, 41)
>>> B = pow(6, 13, 41)
>>> pow(A, 13, 41)
32
>>> pow(B, 10, 41)
32
\end{lstlisting}

\begin{problem}
With a partner, perform a Diffie-Hellman key exchange with $p = 21929$ and $g=3$.
\end{problem}

The Diffie-Hellman can also be used to create a secure key shared by $n>2$ people.
If $n$ people choose private exponents $x_1, \ldots, x_n$, then the secure key will be $g^{x_1 \ldots x_n}$.

\begin{problem}
Work out a method by which $n$ people can use Diffie-Hellman to create a secure key.
Remember that only the $i^{th}$ person knows the $i^{th}$ exponent $x_i$.
With a group of at least 3 people, perform such an exchange with $p = 21929$ and $g=3$.
\end{problem}

\subsection*{Security of Diffie-Hellman}
Why is the Diffie-Hellman algorithm secure?
Suppose a third party, Eve, intercepts $p$, $g$, $A$, and $B$
If she can compute $x$ and $y$, she easily compute $k$ just as Alice and Bob did.
Then Eve must compute $x$ satisfying $A\equiv g^x \pmod{p}$.
This computation is known as taking a \emph{discrete logarithm} of $A$ and there are no fast algorithms for doing it.
When $p$ is large, say 100s of digits, calculating a discrete logarithm is effectively impossible.

\section*{Practical Considerations}
For Diffie-Hellman to be a practical key exchange algorithm, we need an efficient way to find primitive roots and perform modular exponentiation.

\subsection*{Finding primitive roots}
The only way to find the primitive roots of $p$ is to test numbers $a<p$ until a primitive root is found.
One test to see if $a$ is a primitive root is as follows.

First, factor $p-1$ as a product of primes, say $p=p_1^{k_1}\ldots p_n^{k_n}$.
Then, for each prime $p_i$ in the factorization, compute $a^{(p-1)/p_i} \pmod{p}$.
If for any $i$ we find $a^{(p-1)/p_i}\equiv 1 \pmod{p}$, then $a$ is NOT a primitive root.
Otherwise, $a$ is a primitive root.

As an example we look for a primitive root of $p=41$.
We factor $p-1=40$ using the function \li{factorint()} from SymPy.
\begin{lstlisting}
>>> from sympy import factorint
>>> factorint(40)
{2: 3, 5: 1}
\end{lstlisting}
This output means that $40=2^3\cdot5^1$.
Then the powers to check are $40/2=20$ and $40/5=8$.
The following code shows that $2$ is not a primitive root modulo 41, since $2^{20}\equiv 1 \pmod{41}$.
\begin{lstlisting}
>>> pow(2, 20, 41)
1
\end{lstlisting}
However, 6 is a primitive root modulo 41, as the following code shows.
\begin{lstlisting}
>>> pow(6, 8, 41)
10
>>> pow(6, 20, 41)
40
\end{lstlisting}

The idea behind this algorithm is that if $a^n \equiv 1 \pmod{p}$ for some $n<p-1$, then $a$ will not be a primitive root.
Instead, powers of $a$ will repeat and miss some congruence classes, as they did in the example of $2 \pmod{7}$.
For ``group-theoretic'' reasons, if $a^n \equiv 1 \pmod{p}$ for some $n<p-1$ (i.e., $a$ is not a primitive root), then there must be some $p_i$ such that $a^{(p-1)/p_i} \equiv 1 \pmod{p}$.
So, instead of checking all powers of $a$ modulo $p$, we only need to check the powers $(p-1)/p_i$ for all $i$.

\begin{problem}
Write the following function to test if $a$ is a primitive root modulo $p$.
\begin{lstlisting}
def is_primitive(root, mod):
    '''Determine whether `root' is a primitive root modulo `mod'.

    INPUTS:
    root - A positive integer.
    mod  - A prime integer.

    Return a Boolean value.
    '''
\end{lstlisting}
\end{problem}

\subsection*{Fast modular exponentiation}
The na\"ive way to compute $123^{361} \pmod{19673}$ is to multilply 123 by itself 361 times, producing an enormous number, and then mod this number by 19673.
A slightly better way is to take the modulus after each multiplication by 123, since this way we do not store such a large number.

\begin{problem}\label{prob:power1}
Write the following function to implement the algorithm for modular exponentiation outlined above.
\begin{lstlisting}
def power1(base, exp, mod):
    ''' Return base^exp modulo `mod'.

    Multiply `base' by itself `exp' times, taking the modulus after each
    multiplication.

    Example:
    >>> power1(5064, 361, 19673)
    994
    '''
\end{lstlisting}

Time your function against Python's built in \li{pow()} function.
What is the difference in speed?
\end{problem}

If you did Problem \ref{prob:power1}, you know that Python's exponentiation method is orders of magnitude faster than the na\"ive method outlined above.
This increase in performance is acheived with \emph{binary exponentiation} (or exponentiation by squaring).

This idea of this method is as follows.
To compute $a^{361} \pmod{19673}$, we first write 361 in binary as 101101001. Thus,
\begin{equation}\label{equ:pow}
a^{361} = a^{2^0+2^3+2^5+2^6+2^8} = a^{2^0}a^{2^3}a^{2^5}a^{2^6}a^{2^8} \pmod{19673}.
\end{equation}
We can quickly compute the integers
\[
a=a^{2^0},\; a^2=a^{2^1}, \;a^{2^2}, \;\ldots, \;a^{2^8} \pmod{19673}
\]
by squaring each term to get the next.
Then, to compute \eqref{equ:pow}, we simply multiply the appropriate terms from this series.

\begin{problem}
Implement binary exponentiation with the following function.
\begin{lstlisting}
def power2(base, exp, mod):
    ''' Return base^exp modulo `mod'.

    Compute the result using the right-to-left binary method.

    Example:
    >>> power2(5064, 361, 19673)
    994
    '''
\end{lstlisting}
Do not at any point use Python's built-in exponentiation methods.
Compare the speed of this function to that of the function in Problem \ref{prob:power1} and Python's \li{pow()} function.
Even with a na\"ive implementation of binary exponentiation, you should see enormous improvement over the \li{power1()} function.
\end{problem}


\section{Linearization \& Newton's Method}

\subsection{Linearization}
As we've seen, tangent lines intersect their function at most once: at the point of tangency.
However, we know that differentiable functions are locally linear, so we'd expect the tangent line to be a decent approximation of the function near the point of tangency.

\begin{definition}
	If $f$ is differentiable at $a$, then the approximating function
	\begin{equation*}
		L(x) = f^\prime(a)(x-a) + f(a)
	\end{equation*}
	is the linearization of $f$ at $a$.
\end{definition}

\begin{example}
	Find the linearization of $f(x) = \ln{(x+1)}$ at $x=0$.
	How accurate is this approximation at $x=0.1$?
\end{example}
\begin{answer}
	Following the definition,
	\begin{align*}
		f(0) = \ln{(0+1)} = 0 \\
		f^\prime(x) &= \frac{1}{x+1} \\
		f^\prime(0) &= \frac{1}{0+1} = 1 \\
		L(x) &= 1(x-0) + 0 = x.
	\end{align*}

	Calculating the error at $x=0.1$,
	\begin{align*}
		L(0.1) &= 0.1 \\
		f(0.1) &\approx .0953 \\
		\text{\% error} &= \frac{\abs{L(0.1)-f(0.1)}}{L(0.1)}100\text{\%} \approx 4.7\text{\%}.
	\end{align*}

	So, we can see the linear approximation is pretty good.
\end{answer}

We call the difference in $x$ between the point of tangency and the point we're trying to approximate the differential.
\begin{definition}
	Let $y=f(x)$ be a differentiable function.
	The differential $\d{x}$ is an independent variable.
	The differential $\d{y}$ is $\d{y} = f^\prime(x)\d{x}$.
\end{definition}

$\mathrm{d}y$ is the approximated change in $y$ expected by the linearization for some given change in $x$, $\d{x}$.

\begin{example}
	Find $\d{y}$ for $y=\frac{2x}{1+x^2}$, $x=-2$, and $\d{x} = 0.1$.
\end{example}
\begin{answer}
	\begin{align*}
		y^\prime &= \frac{-4x^2}{\left(1+x^2\right)^2} + \frac{2}{1+x^2} \\
		y^\prime(-2) &= -6/25 \\
		\d{y} &= (-6/25)(0.1) = -0.024.
	\end{align*}
\end{answer}

\subsection{Newton's Method}
We can use the fact that the tangent line approximates the function to find the zeroes of functions.
Starting with an initial guess $x_0$ for the $x$ value of the zero, we look at the the tangent line at $x_0$ and find where it intersects the $x-axis$.
\begin{align*}
	L_0(x) &= f^\prime(x_0)(x - x_0) + f(x_0) \\
	0 &= f^\prime(x_0)(x - x_0) + f(x_0) \\
	-f^\prime(x_0)(x - x_0) &= f(x_0) \\
	x - x_0 &= -\frac{f(x_0)}{f^\prime(x_0)} \\
	x &= x_0 - \frac{f(x_0)}{f^\prime(x_0)}.
\end{align*}
This $x$ value serves as our next guess for the zero.
We repeat this process, until we find the zero or are satisfied with our error\footnote{For most well-behaved functions, Newton's Method can get within a small margin of error or a zero relatively quickly. There is also a generalized, sometimes faster version of Newton's Method that approximates the function with higher-order polynomials than just lines.}.
This yields a recursive formula
\begin{equation*}
	x_{n+1} = x_n - \frac{f(x_n)}{f^\prime(x_n)}.
\end{equation*}


\chapter{The Basics of Regression \label{chapter:regression}}

Classification is a form of supervised learning in which the outcome is a category. \textbf{Regression}\index{regression} is another form of supervised learning in which the outcome is a numeric value. For example, it may be a lab value, physical characteristic (height, weight, etc.), or numeric measurement (e.g. oxygen saturation).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Visualizing the Regression Problem \label{section:visualizingreg}}

Let's consider the same setup from Section~\ref{section:visualizingclass} but this time with a quantitative outcome: a ``recurrence biomarker'' that indicates the likelihood of recurrence of disease.

Again, we have data on two predictors: a disease severity score ($x_1$), which characterizes the severity of the illness for which the patient was originally treated, and a social determinants score ($x_2$), which characterizes a patient's socioeconomic status. We have measurements of $x_1$ and $x_2$ on the same $200$ patients as in Section~\ref{section:visualizingclass}.
\begin{center}
\includegraphics[width=0.65\textwidth]{img/esl-reg-just-data.png}
\end{center}

This is a plot of the data in a single plane. The color represents the value of the recurrence biomarker -- the height of the point above the plane. We want to design a model that will predict the value of the biomarker ($y$) based on the values of the two predictors, $x_1$ and $x_2$. These plots show the \textbf{univariate} relationship of each predictor with the outcome.
\begin{center}
\includegraphics[width=0.45\textwidth]{img/esl-reg-x1.png}
\includegraphics[width=0.45\textwidth]{img/esl-reg-x2.png}
\end{center}

\begin{question}{question:influence}
Which of the two predictors, $x_1$ or $x_2$, appears to more strongly influence the value of the recurrence biomarker? Explain your reasoning using evidence from the preceding three plots.
\end{question}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Three Regression Algorithms}

\subsection{Linear Regression \label{ssect:linreg}}

The regression analogue of logistic regression is \textbf{linear regression}\footnote{The terminology here is confusing. When we learn about generalized linear models in Chapter~\ref{chapter:glms}, you'll see why logistic regression has the word ``regression'' in its name even though it's a classification algorithm.}. Linear regression creates a hyperplane that slices through the cloud of training datapoints such that it passes as close as possible, on average, to the data. This is, of course, easiest to see when the feature space is two-dimensional, as it is here:

\begin{center}
\includegraphics[width=0.65\textwidth]{img/esl-reg-linear.png}
\end{center}
The three lines shown here sit on the hyperplane learned by the linear regression model. They are located at heights corresponding to the 25th, 50th, and 75th percentiles of the outcome, $y$ (the biomarker value). The plane tilts downward toward the upper left corner of the $x_1 \times x_2$ grid and upward toward the bottom right corner. It may be helpful to visualize grabbing the $x_1 \times x_2$ plane and rotating/translating it so that it passes through the middle of the training data. Here is a summary of the trained linear regression model:

\begin{center}
\includegraphics[width=0.7\textwidth]{img/linear-regression-model-output.png}
\end{center}

At each point $(x_1, x_2)$ in the feature space, the model's predicted value of the recurrence biomarker, $\hat{y}$, is
$$ \hat{y} = 49.8600 + 10.4372 x_1 - 1.8824 x_2 $$

\begin{question}{}
Compare and contrast the output from the linear regression model with the output from the logistic regression model in Chapter~\ref{chapter:classification}. What looks the same? What looks different? What is being predicted in each case?
\end{question}

\subsection{K Nearest Neighbors (KNN)}

Regression using KNN works very similarly to KNN for classification. In classification, we allow the nearest $K$ points to vote on the label of a new test point. In regression, we \textbf{interpolate} between the values of the surrounding points to come up with the value of $y$ for a test point. Typically this is done just by averaging the $y$ values of the nearest $K$ points, but you can also do something more sophisticated, like weight their contributions by distance to the test point. Here is a contour plot of the regression surface produced by KNN ($K=15$) for our example:

\begin{center}
\includegraphics[width=0.65\textwidth]{img/esl-reg-knn-15.png}
\end{center}

\noindent The contours are again drawn at the 25th, 50th, and 75th percentiles of the outcome, $y$. This looks like a bit of a mess compared to the linear regression plot, but at the same time, the KNN algorithm is able to capture arbitrarily complex relationships between $x_1$, $x_2$, and $y$ that can be missed by other regression algorithms.

\subsection{Decision Tree}

Decision tree regression is similar to decision tree classification except that the output at each leaf is not a class label or the probability of membership in the positive training class (both of which are shown on the tree in Section~\ref{ssect:class_decision_tree}), but a numeric value. That value corresponds to the mean outcome value for the points in that leaf.
\vspace{-7mm}
\begin{center}
\includegraphics[width=0.6\textwidth]{img/esl-decision-tree-just-tree-reg.png}
\end{center}
\vspace{-7mm}
The predicted biomarker values for a decision tree trained on this dataset (created using the \texttt{rpart} package in R with default parameters) are shown here:
\begin{center}
\includegraphics[width=0.65\textwidth]{img/esl-reg-decision-tree.png}
\end{center}
You can see that the decision tree always chooses to split on $x_1$, the disease severity score, rather than $x_2$. Revisit Question~\ref{question:influence} to remind yourself of why this is. The regression surface produced by the decision tree looks like a set of stairs climbing higher and higher as one moves from left to right across the $x_1 \times x_2$ plane. The predicted value of $y$, the recurrence biomarker, is constant within each stair.
\vspace{5mm}

\begin{question}{}
Compare this decision tree with the decision tree for the classification problem in Chapter~\ref{chapter:classification}. What is the same? What is different?
\end{question}

\begin{question}{}
This \textbf{regression tree} has eight leaves. What region of the feature space does each leaf correspond to?
\end{question}

\begin{question}{}
What are the advantages and disadvantages of each of these three regression algorithms (linear regression, KNN, regression tree)?
\end{question}


\subsection{Z-test for variable significance}

\subsubsection{The standard score}

We may want to see how different a mean statistic is from a specific value.

The standard score allows us to measure this, by taking this distance and standardising by the standard deviation.

\(z=\dfrac{\bar x-x_0}{\sigma }\)

This requires us to know the standard deviation, which is in general not known.

If the sample size is large, we know this converges to the normal distribution through the central limit theorem.

\subsubsection{The Z-test}

We can see how likely our statistic was to be produced if it was drawn from a normal distribution with mean \(x_0\) and standard deviation \(s_0\).

\subsubsection{P-values}

This is the chance of the statistic being produced by chance.


% !TeX root = ../main.tex

\chapter{Theory}
\label{ch:theory}

\section{SVM}
A \acrfull{svm} is a machine learning algorithm which tries to separate binary-class training data. The \gls{svm} achieves this by searching for a hyperplane or a set of hyperplanes to separate the points of training data in a multidimensional space. There might be several possible hyperplanes that sufficiently separate the data but a \gls{svm} searches for the hyperplane with the biggest perpendicular distance between the hyperplane and the borders of both classes that should be separated. The space between the hyperplane and the closest of the training data points is defined as the margin. If the algorithm finds the maximal margin it has also found the optimal hyperplane. See figure \ref{fig:svmHyperplaneSeparation} for an example of possible hyperplane separations.

\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.5]{figures/theorySVM_hyperplanes}
	\caption{Separation of two classes by hyperplanes in 2 dimensional space. $H_1$ does not separate, $H_2$ and $H_4$ separate but the margin is very slim, $H_3$ separates with a much better margin.}
	\label{fig:svmHyperplaneSeparation}
\end{figure}

\subsection*{Formal Definition}
Since a \Gls{svm} is a binary classifier a data point $x$ has to either belong into class A or class B. Let the training dataset of $n$ points be represented as $x_0,\dots,x_n$ and the target values be $y_0,\dots,y_n \in {-1,+1}$. The values of $y_n$ should be
\begin{equation}
	y_i=
		\begin{cases}
		+1 & \text{if } x_i \in \text{class } {\color{red}A} \\
		-1 & \text{if } x_i \in \text{class } {\color{blue}B}
		\end{cases}.
\end{equation}
The optimal hyperplane should separate all vectors $x_i$ with a value of $y_i=1$ {(class ${\color{red}A}$)} and those with a value of $y_i=-1$ {(class ${\color{blue}B}$)} so that the distance between those two groups is maximal \cite{Thome2012}. Let the hyperplane be
\begin{equation}
	D(x)=W \bullet x+b
\end{equation}

where $W$ is a vector normal to the hyperplane (see figure \ref{fig:svmHyperplaneDistance}) and $b$ is a bias term \cite{Boser1992}. $D(x)$ has to be calculated so that
\begin{equation}
	x \in
	\begin{cases}
	\text{class } {\color{red}A} & \text{if } D(x) > 0 \\
	\text{class } {\color{blue}B} & \text{if } D(x) < 0
	\end{cases}
\end{equation}
is true. The distance between a point $X$ and the hyperplane $D(x)$ is given as
\begin{equation}
\frac{D(X)}{\|W\|}
\end{equation}
which is illustrated in figure \ref{fig:svmHyperplaneDistance} \cite{Thome2012}. Assuming that the data is linear separable {(as in figure \ref{fig:svmHyperplaneSeparation})} we can then select two hyperplanes ${\color{red}D(x)>0}$ and ${\color{blue}D(x)<0}$ so that there are no points in between them and try to maximize the distance between those two hyperplanes {(in figure \ref{fig:svmHyperplaneSeparation} $H_2$ and $H_4$)} which is then given as
\begin{equation}
\frac{2}{\|W\|}.
\end{equation}
To maximize the distance we must therefore minimize $\|W\|$ but with the constraint {(\ref{eq:svmHyperplaneCondition})} that no data gets between the two hyperplanes {(hard-margin classification)}. Both constraints
\begin{equation}
	\begin{split}
	x_i \bullet w+b \geq +1 \quad \text{if } {\color{red}y_i=+1}\\
	x_i \bullet w+b \leq -1 \quad \text{ if } {\color{blue}y_i=-1}
	\end{split}
\label{eq:svmHyperplaneCondition}
\end{equation}
can be combined to $y_i(x_i\bullet w + b)\geq 1\quad \forall i, i=1\dots n$. The solution for a maximum margin can therefore be found by solving
\begin{equation}
\begin{split}
\operatorname*{arg\,min}_{w, b}	\left\{\frac{1}{2}\|W\|^2\right\} \text{subject to} \\
y_i(x_i\bullet w + b)-1\geq 0 \quad \forall i, i=1\dots n
\end{split}
\end{equation}
with the Lagrangian multiplier method \cite{Thome2012}.

If the data is not linear separable we have to relax the constraints from \ref{eq:svmHyperplaneCondition} and allow some points in the margin {(soft-margin-classification)}.

Classification of unknown data is now simply done through a calculation of the decision function $D(x)$
\begin{equation}
\text{prediction}
	\begin{cases}
	x \in \text{class } {\color{red}A} & \text{if } D(x) > 0 \\
	x \in \text{class } {\color{blue}B} & \text{if } \text{otherwise}
	\end{cases}
\end{equation}

\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.4]{figures/theorySVM_distanceToW}
	\caption{Visualization of optimal margin and hyperplane in 2 dimensional space. The distance of Point $X$ to $D(x)$ is $\frac{D(X)}{\|W\|}$. Source Bosner et al. \cite{Boser1992}.}
	\label{fig:svmHyperplaneDistance}
\end{figure}
\subsection*{Kernel Trick}
However, there are still cases where the data can not be separated by a linear function as shown in figure \ref{fig:svmKernelTrick}. In this case the data is mapped into another higher dimensional feature space {(Kernel trick)} so that it becomes linear separable again. The function that maps the data is called kernel. For evaluation in chapter \ref{ch:discussion} the following four kernels from OpenCV \cite{bradski2000opencv} and sklearn \cite{Pedregosa2011} were used:
\begin{itemize}
	\item Linear kernel: $k(x,y)=x*y$
	\item Polynomial kernel: $k(x,y)=(\gamma \left\langle x, y \right\rangle + coef_0)^\text{degree}$
	\item Radial basis function kernel: $k(x,y)=\exp(-\gamma ||x-y||^2)$
	\item Additive $\chi^2$ kernel: $ k(x,y)=-\sum \left[ (x-y)^2 / (x+y)\right] $
\end{itemize}
\begin{figure}
	\centering
	\subfloat[Data not separable]{\includegraphics[width=55mm]{figures/theorySVM_kernelTrick1}}
	\subfloat[Data separable]{\includegraphics[width=55mm]{figures/theorySVM_kernelTrick2}}
	\subfloat[Data separable with hyperplane]{\includegraphics[width=55mm]{figures/theorySVM_kernelTrick3}}
	\caption{Illustration of Kernel trick. Data in {(a)} is not separable but if the data gets mapped into a higher dimensional space {(b)} a linear hyperplane can be fitted {(c)}. Figures plotted with code adapted from \texttt{http://www.eric-kim.net/eric-kim-net/posts/1/kernel\_trick.html}}
	\label{fig:svmKernelTrick}
\end{figure}

\subsection*{Multiclass SVMs}
\label{subsec:svmMulticlass}
In case of a multiclass \gls{svm} problem the most popular approach is to use several different binary class \glspl{svm} instead of only one multiclass \gls{svm} \cite{Hsu2002, Duan2005, Thome2012}. There are two common solutions for the multiclass problem: One-VS-Rest and One-VS-One.

An One-VS-Rest multiclassifier with $n$ classes consists of $n$ distinct \glspl{svm}. Each \gls{svm} is trained on one of the classes as class A {(positive features)} and all other samples as class B {(negative features)}. Classification is done in a "winner-takes-it-all" fashion meaning that the \gls{svm} with the highest output function assigns the class label.

In the One-VS-One approach $n\frac{n-1}{2}$ \glspl{svm} are constructed. Each class is paired in a \gls{svm} with each other class. Classification is done in a "majority-vote" fashion so each \gls{svm} votes for its class and the class with the most votes is assigned as the class label.
\newpage
\section{K-Nearest Neighbors}
The \gls{knn} algorithm is a lazy classification algorithm which stores the complete training data in the learning phase \cite{Keller1985}. In the classification phase it searches for the $k$ points in the training data that are nearest to the point that should be classified. The label that gets the majority of votes from the $k$ closest points {(neighbors)} is the result of the classification. See figure \ref{fig:knn3} for a three class example and $k=3$.

The most popular metric to get the distance between the neighbors is the Euclidean distance, however, the accuracy can greatly be improved by using a learned distance metric such as the Mahanalobis distance which tries to maximize the margin between classes \cite{Weinberger2005}.

\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.45]{figures/theoryKNN_3}
	\caption{Example for a knn classification with $k=3$. The three nearest elements to the purple pentagon {(point that should be classified)} are marked with a square. Since two of the three neighbors are triangles the new point should also be a triangle.}
	\label{fig:knn3}
\end{figure}

\section{Convolutional Neural Networks}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.35]{figures/theoryCNN_model}
	\caption{Sample architecture of a \gls{cnn}. Source: Christodoulidis and Anthimopoulos \cite{Christodoulidis2015}}
	\label{fig:cnnModel}
\end{figure}
A \acrfull{cnn} is a multi-layer-feed-forward neural network that emulates the processes in the visual cortex by employing feature detection with small convolutional filters. \glspl{cnn} generally outperform other gradient based learning techniques \cite{LeCun1998} and are currently state-of-the-art \cite{Russakovsky2015} for image recognition.

\glspl{cnn} usually consist of a succession of trainable convolutional, dense and pooling layers. Normally, the first layers comprise of several convolutional and pooling layers for feature extraction. For the classifications of features extracted in the previous layers, the network has at least one dense layer of neurons before the last layer. The last layer is usually the output layer and for the standard classification task the number of neurons is equal to the number of classes to classify so that the output of a neuron is the probability of a specific class at the same time. Figure \ref{fig:cnnModel} shows an example of a \gls{cnn} architecture with two convolutional and pooling layers and a dense, fully connected layer before the output layer.

\subsection*{Convolutional Layers}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.45]{figures/theoryCNN_filter}
	\caption{96 convolutional filters of size $3\times 11\times 11$ in the first layer. Filters trained on \gls{ilsvrc}-2010. Source: Krizhevsky et al. \cite{Krizhevsky2012}}
	\label{fig:cnnFilter}
\end{figure}
Convolutional layers contain multiple convolutional filters or kernels. They convolve the image to extract image features. Each filter convolves the output of the previous layer. In case of the first layer the filters operate directly on the input image. In later layers they collect more complex features from previous layers. Convolution happens by calculating the dot-product of the input matrix with the convolutional filter. Filters in \glspl{cnn} usually overlap which helps to get a better translation invariance since each pixel is convolved multiple times by the same filter. The main goal of convolution in \glspl{cnn} is to get distinctive image features. These convolutional filters are not handcrafted like filters in typical feature detectors. They learn the best filters by learning large quantities of data and change over the time and are ultimately able to extract features that are characteristic for the problem. Often, these kernels extract simple features like horizontal, vertical or diagonal edges {(figure \ref{fig:cnnFilter})}. For food, prominent first layer kernels typically include many color kernels that specialize in extracting a range of similar colors \cite{Christodoulidis2015}. The basic idea behind convolutional filters is that many features like edges or corners appear in multiple locations in images. By using the filters as a sliding window, one filter is able to detect the feature in any position.

Therefore, the main befit of using convolutional filters is the reduction of learnable weights. Filters are very small {(7x7 as the maximum size for a current \gls{cnn} \cite{Szegedy2014})} but are applied on the whole image in a sliding window approach. This means that a single convolutional filter shares weights for the image which significantly improves learning because it reduces the number of parameters to learn \cite{LeCun1998}. GoogLeNet, a 22-layer \gls{cnn}, uses \gls{rgb} input images with a size of 224x224 \cite{Szegedy2014}. A normal Neural Net without convolutions would not be able to handle this input size.

\subsection*{Pooling Layers}
Once a feature is detected the exact position becomes less important. Only the spatial relation to other features remains valuable. Knowing the exact position of a feature may even be harmful for generalization because the model becomes less invariant to position \cite{LeCun1998}. One way to solve this problem and reduce the number of weights to learn is subsampling. In \glspl{cnn} this normally occurs in pooling layers. Currently, the most common pooling layer type is max-pooling. This filter basically divides the input into $z\times z$ blocks and extracts the maximum value of each block. A popular max-pooling layer instance is a $z=2$, stride $s=2$ layer. This layer downsamples the input by two. Recently, overlapping max-pooling layers have become popular \cite{Szegedy2014}. They do not downsample as much as non-overlapping pooling layers {(in fact $z=3$ and $s=2$ outputs the same dimensions as the input)} but they seem to slightly decrease the error \cite{Krizhevsky2012}.

\subsection*{Overfitting and Dropout}
\label{subsec:overfittingDropout}
Although \glspl{cnn} need fewer parameters to train high dimensional inputs than conventional neural nets, they also have the tendency to overfitt like conventional nets do. Overfitting is a common problem in machine learning that occurs if models are very complex. An overfitted model is optimized across the training data and describes or "fits" this data so well that it can not explain unseen data \cite{Falkenauer1998}. Figure \ref{fig:overfittingNet}a shows an overifitting neural network. After epoch 80 the error on the training set continues to decrease but the error on the validation set stops to decrease which means that although the net gets better at predicting the training data it gets worse for predicting unseen data.

Dropout is a novel way for neural nets to reduce the problem of overfitting by randomly dropping out nodes from the model. This forces the net to generalize better \cite{Srivastava2014}.

\subsection*{Learning Rate and Momentum}
\label{subsec:learningMomentum}
Neural networks try to find the global minimum of their cost function by adjusting the network weights. If there is only one minimum this is fairly easy as the network only has to adjust the weights towards the downward gradient. Real world scenarios, however, are much more complex and include many local minima as well.

Learning rate and momentum are measurements of how weights are adjusted in backpropagation in search of the global minima. The learning rate denotes the magnitude of the weight change. That means that networks with a high learning rate advance faster along the gradient. Momentum is a term that multiplies a fraction of the previous weight update to the weight adjustment. By applying momentum, the steps the network takes on the gradient towards the minimum get bigger with each iteration. This helps to increase the speed of learning and may also prevent the network from getting stuck on local minima and saddle points because of the "momentum" the network simply "steps over" these points.

\section{Feature detectors and descriptors}
A common approach in image classification is the use of feature detectors and feature descriptors. A feature detector detects interest points or keypoints in images. Ideally those keypoints are invariant to image transformations like rotation, scale or illumination changes so that the points can be found even if the image is rotated or taken from another perspective. In most cases, keypoints are corners, as corners can be localized quite easily. Knowing that a corner exists, however, does not help much with recognition. An algorithm also needs to describe the characteristics of an interest point so that it is possible to compare different interest points. The part of describing a keypoint is done by keypoint descriptors.

\subsection[HOG]{Histograms of Oriented Gradients (HOG)}
\begin{figure}
	\centering
	\includegraphics[scale=0.45]{data/images/theory/theoryHOG_VisualizationGradients}
	\caption{Visualization of HOG descriptor.}
	\label{fig:hogVisualization}
\end{figure}

\acrfull{hog} is a popular image descriptor for human detection, developed by Dalal and Triggs in 2005 \cite{Dalal2005}. The algorithm makes use of small contrast changes by describing the distribution of local gradients. This approach is believed to be based on biological processes of neurons in the primary visual cortex \cite{Lowe2004}.

The first step is to divide the image into small blocks with a size of 16x16 pixels and a 50\% overlap for better results. Each block is then subdivided by four 8x8 pixel cells. For each of these cells, gradient intensities are computed by applying the 1-D centered point derivative convolution mask horizontally and vertically:
\begin{equation}
	D_x=
	\begin{bmatrix}
	-1 & 0 & 1
	\end{bmatrix}
	\quad
	D_y=
	\begin{bmatrix}
	-1 \\ 0 \\ 1
	\end{bmatrix}
\end{equation}
The magnitude of the gradients is
\begin{equation}
	m=\sqrt{(I\ast D_x)^2+(I\ast D_y)^2}
\end{equation}
where $I$ is the image cell and the gradient orientation is given as
\begin{equation}
	\theta = \arctan \left[\ \frac{(I\ast D_y)}{(I\ast D_x)}\right]\text{.}
\end{equation}
To achieve a better illumination invariance against shadowing, it is useful to contrast-normalize each cell.

To get a feature vector for each cell, a 9-bin histogram of the gradient orientations {(0\degree - 180\degree)} is calculated. The gradient orientations $\theta$ are then scaled by the corresponding magnitude $m$. Figure \ref{fig:hogVisualization} shows a visualization of gradient histograms. The gradient histogram in the center row on the right, is a single horizontal line meaning that the gradients in this cell are all horizontal which is logical because the original image in this cell is a horizontal edge.

To get the feature vector for the whole image all histograms are concatenated to form a large gradient histogram.

\gls{hog} is not rotation invariant which does not matter for human detection as humans tend to always have the same upright orientation. However, for food, rotation invariance is quite important.

\subsection[LBP]{Local Binary Pattern (LBP)}
\begin{figure}[ht]
	\centering
	\includegraphics[width=\linewidth]{figures/theoryLBP_coding}
	\caption{Example for a LBP coding of the center pixel with value 6 (a). The neighboring pixels are thresholded against the center value (b) and multiplied by values in (c) with the result in (d). Source: Ojala et al. \cite{Ojala1999}}
	\label{fig:lbpCoding}
\end{figure}
\acrfull{lbp} is a simple texture descriptor which was originally proposed by Ojala et al. in 1994 \cite{Ojala1994}. For the calculation of the \gls{lbp} descriptor the 8 neighbors in a 3x3 block for each pixel are taken into account {(fig \ref{fig:lbpCoding}a)}. The neighbors of the center pixel are then thresholded by comparing the value {(grayscale intensity)} of the pixel in the center to the intensities of the neighboring pixels. Pixels with a higher or equal value are getting labeled as "1" and "0" otherwise {(fig \ref{fig:lbpCoding}b)}. Each position in the 3x3 block is then assigned a weight of $2^n$ {(fig \ref{fig:lbpCoding}c)} where $n$ is ascending from left-to-right, top-to-bottom. In recent publications these weights are labeled clockwise ascending. The binary values are than multiplied by the corresponding weights {(fig \ref{fig:lbpCoding}d)}. The values are then summed up and assigned as the new value for this "texture unit". The values can then be inserted into a histogram to form a 256-dim feature vector since there are 256 possible LBP values {($2^8=256$)}. Local binary patterns are invariant to grayscale changes because \gls{lbp} describes spatial texture structure but not contrast intensities \cite{Ojala1999}.

In the following years \gls{lbp} has been adapted and extended with additional contrast information \cite{Ojala1999} or different neighborhood sizes to achieve rotation invariance with uniform \glspl{lbp} \cite{Ojala2002}.

\subsection[SIFT]{Scale invariant feature transform (SIFT)}
\label{subsec:sift}
\acrfull{sift} is a very popular local image feature detector, descriptor and matcher by Lowe from 1999 \cite{Lowe1999}. It is invariant to uniform image scaling, translation and rotation and partially invariant to affine distortions, 3D viewpoint and illumination changes. It is also invariant to noise, partial occlusion and clutter \cite{Lowe1999}.

\subsubsection*{Keypoint detection}
%\begin{figure}
%	\centering
%	\begin{minipage}{.5\textwidth}
%		\centering
%		\includegraphics[width=0.45\linewidth]{data/images/theory/theorySIFT_gaussianScales}
%		\captionof{figure}{Image with different Gaussian scales.}
%		\label{fig:gaussianScales}
%	\end{minipage}%
%	\begin{minipage}{.5\textwidth}
%		\centering
%		\includegraphics[width=0.45\linewidth]{data/images/theory/theorySift_dog}
%		\captionof{figure}{Example of a Difference of Gaussians with $\sigma=1$ and $\sigma=3$.}
%		\label{fig:dog}
%	\end{minipage}
%\end{figure}

\begin{figure}
	\centering
	\begin{minipage}{.5\textwidth}
		\centering
		\includegraphics[width=0.45\linewidth]{figures/theorySIFT_scaleSpaceFiltering}
		\captionof{figure}{Sequence of Gaussian smoothings of a 1D graph with $\sigma$ increasing from bottom to top. Source Witkins \cite{Witkin1983}}
		\label{fig:scaleSpaceFiltering}
	\end{minipage}%
	\begin{minipage}{.5\textwidth}
			\centering
			\includegraphics[width=0.45\linewidth]{figures/theorySIFT_interestPointDetection}
			\captionof{figure}{Interest points are detected by searching for minima and maxima. The value of the center pixel {(marked with X)} is compared to its 26 neighbors {(marked with circles)} on the current and adjacent scales. Source: Lowe \cite{Lowe2004}}
			\label{fig:interestPointDetection}
	\end{minipage}
\end{figure}

The basic idea of \gls{sift}'s keypoint extraction follows Witkins scale-space filtering \cite{Witkin1983}. Witkins found out that by applying Gaussian smoothings at different scales of $\sigma$ on a 1D-graph he could find robust edges depending on the scale of $\sigma$ {(figure \ref{fig:interestPointDetection}a)}. Since this process can also be applied to 2D images scale-space filtering is a good way to find scale invariant interest points {(also called \gls{log})}. \gls{sift}, however, uses a much more computation efficient approximation of \gls{log} called \acrfull{dog} \cite{Lowe2004}. \gls{dog} can be computed by subtracting two adjacent scales from each other. Figure \ref{fig:dogScaleFiltering} shows the process of creating \glspl{dog} at different scales. \gls{sift} samples 3 scales per octave. After each octave the image size is halved and \gls{dog} is applied again on the smaller image {(figure \ref{fig:dogScaleFiltering})}.

\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.3]{figures/theorySIFT_dogScaleFiltering}
	\caption{For each octaves the image is blurred with Gaussian smoothings at rising scales. A \gls{dog} is formed by subtracting two adjacent scales. After each octave the image size is halved and the process repeated. Source: Lowe \cite{Lowe2004}}
	\label{fig:dogScaleFiltering}
\end{figure}

To check if a pixel qualifies as an interest point at a certain scale, \gls{sift} compares the values of the 3x3 neighboring pixels. If the pixel has the maximum or minimum value \gls{sift} then compares the neighbors on the scales above and below. If the pixel has indeed the lowest or highest value across all 27 points it is taken as a potential interest point {(figure \ref{fig:interestPointDetection})}. The cost of this operation is reasonably low because most of the pixels will be eliminated after the first few comparisons \cite{Lowe2004}. To filter weak interest points, \gls{sift} tries to eliminate low contrast points and points on edges since edges are not very robust features. After the elimination of weak points, \gls{sift} computes a 32-bin gradient orientation histogram in the interest point neighborhood. Orientations are weighted by the magnitude of the gradient. Each bin represents 10\degree of the 360 possible orientations. The bin with the highest peak is then selected as the dominant orientation for this keypoint. Bins with at least 80\% of the highest bin value are selected to form an additional new keypoint.

\subsubsection*{Keypoint Description}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.2]{figures/theorySIFT_descriptor}
	\caption{Creation of the \gls{sift} descriptor. Around the keypoint a 16x16 region {(this image is only half the size)} is taken and gradient orientations are calculated. The region forms 16 4x4 blocks with 8-bin gradient histograms. Source Lowe \cite{Lowe2004}}
	\label{fig:siftDescriptor}
\end{figure}
The \gls{sift} keypoint descriptor relies on gradient orientation histograms. They are more stable features than just raw intensity values and less sensitive to 3D rotation. To calculate a descriptor for a keypoint, a neighborhood region of 16x16 pixels is selected oriented along the dominant orientation of the keypoint. The 16x16 region is then divided into 4x4 blocks and for each of these blocks a 8-bin gradient orientation histogram is calculated {(similar to the keypoint extraction)}. This results in a 128 dimensional feature vector for each keypoint {($16\text{ blocks}*8\text{ bins per histogram}=128$)} \cite{Lowe2004}. Figure \ref{fig:siftDescriptor} shows the process of the descriptor creation.

\subsection[SURF]{Speeded up Robust Features (SURF)}
\label{subsec:surf}
\gls{surf} is a feature detector and descriptor which was proposed in 2006 \cite{Bay2006} and later revised in 2008 by Bay et al. \cite{Bay2008}. It was developed with the goal to provide a faster alternative for the popular \gls{sift} algorithm without trade offs in recognition performance.

\subsubsection*{Keypoint detection}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.2]{figures/theorySURF_boxFilters}
	\caption{Left half: discretized and cropped Gaussian second order partial derivates in $y$-direction and $xy$-direction. Right half: corresponding box filter approximations. Gray ares are equal to zero. Source Bay et al. \cite{Bay2008}}
	\label{fig:surfBoxFilters}
\end{figure}
Like \gls{sift}, \gls{surf} also uses scale spaces to detect interest points. \gls{surf}, however, relies on an approximation of the Hessian matrix

The first step is to compute the integral image. In integral images, pixel values are the sum of all pixels within a square between the pixel and the origin. These representations are efficient to compute and allow to find the sum of pixel values for any sized box within the image with only four arithmetic operations. Throughout the algorithm this benefit is leveraged to speed up detection and description.

Bay et al. approximate the Hessian matrix using box filters {(in the right of figure \ref{fig:surfBoxFilters})}. By using integral images in combination with box filters the process of scale space creation can be sped up significantly. The benefit of box filters contrary to Gaussians which have to be applied iteratively is, that box filters can be applied directly on the original image which makes parallelization possible. In addition, \gls{surf} does not downscale after each octave. It scales the image up which has the advantage of not causing aliasing. To localize interest points across scales, \gls{surf} applies non-maximal suppression on a 3x3x3 neighborhood.

\subsubsection*{Keypoint Description}
To achieve rotation invariance, \gls{surf} computes a dominant orientation for each interest point from a circular 6$s$ neighborhood by calculating Haar-wavelet responses in $x$ {(horizontal wavelet response)} and $y$ {(vertical wavelet response)} directions with $s$ being the scale of the interest point. Again, the combination of integral images and wavelets helps to calculate Haar-wavelets efficiently. The dominant orientation is calculated by the use of a sliding rotating window with an angle of $\frac{\pi}{3}$. The wavelet responses are then represented as vectors weighted with a Gaussian of $\sigma=2.5s$. Horizontal and vertical vectors are added and the highest sum is the dominant interest point orientation.

To describe a keypoint, \gls{surf} forms a square neighborhood region oriented along the dominant orientation with a size of 20$s$ {(so keypoints at higher scales include a bigger neighborhood)}. The region is then divided into 4x4 smaller sub-regions. In each of these sub-regions \gls{surf} computes features at 5x5 evenly spaced sample points. Contrary to \gls{sift}, \gls{surf} does not utilize orientation histograms but again Haar-wavelet responses in horizontal $d_x$ and vertical $d_y$ direction relative to the dominant keypoint orientation. $d_x$ and $d_y$ are then summed up over each subregion. Those two and the sum of absolute response values $|d_x|$ and $|d_y|$ form the four dimensional feature vector for each subregion. Therefore, each keypoint has a 64 dimensional feature vector taking into account that there are 16 subregions each with a four dimensional vector. To achieve additional invariance to contrast and illumination changes the feature vector is turned into a unit vector.

\subsection[ORB]{Oriented FAST and Rotated BRIEF (ORB)}
\gls{orb} is a combination of the corner detection algorithm \gls{fast} \cite{Rosten} and the binary descriptor \gls{brief} \cite{Calonder2010}. \gls{orb} was introduced by Rublee and Bradski in 2011 \cite{Rublee2011} as a replacement for \gls{sift} for low-power devices and real time performance.

\subsubsection*{FAST Keypoint Detection}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.3]{figures/theoryFAST_corners}
	\caption{FAST corner detection. Interest points are selected if at least 12 adjoining pixel values that are either above or below the center pixels intensity {(dashed line)}. Source: Rosten and Drummond \cite{Rosten}.}
	\label{fig:fastCorners}
\end{figure}
\gls{fast} is a corner detection algorithm by Rosten and Drummond from 2006 that does not rely on Gaussians or scale spaces. \gls{fast} compares 16 pixel intensities arranged on a circle with radius 9 around a possible corner point in the center. It is an interest point if it has at least 12 adjoining pixel values that are either above or below the center pixels intensity {(figure \ref{fig:fastCorners})}. \gls{fast}, however, is prone to have many responses along edges which are weaker features so \gls{brief} tries to filter these edge responses by using a Harris corner measure. To achieve scale invariance \gls{brief} applies \gls{fast} on different image pyramid scale spaces. To also make the keypoints rotation invariant, \gls{brief} calculates the intensity centroid which assumes that a corner's maximum intesity is offset from the actual center of the corner. This makes it possible to calculate the orientation by creating the vector from the intensity centroid to the actual corner point.

\subsubsection*{BRIEF Keypoint Description}
\gls{brief} is a binary descriptor from 2010 by Calonder et al. \cite{Calonder2010}. Unlike \gls{sift} or \gls{surf}, a binary descriptor creates the descriptor values by comparing intensity values of pixels in the neighborhood of the keypoint. If the intensity of pixel $a$ is lower than the intensity of pixel $b$ than the test yields "1" and "0" otherwise. To create the feature vector \gls{brief}  performs 256 binary tests and concatenates the results. The pixels that are compared are chosen by a Gaussian distribution around the center of the keypoint. Prior to these tests the image is smoothed. Additional in-plane rotation invariance is achieved by the proposed "steered" \gls{brief}. Steered \gls{brief} rotates the matrix of binary tests by a rotation matrix along the keypoint orientation. A lookup table with precomputed \gls{orb} features is computed with a 12\degree  interval. However, due to the very rotation by steered \gls{brief}, the binary tests loose most of their variance so the expressiveness of the descriptor suffers. To prevent this, Rublee and Bradski proposed an algorithm to "learn" the optimal location of the pixel-pairs that are compared for the binary tests.

\subsection[CenSurE]{Center Surround Extremas (CenSurE)}
\begin{figure}
	\centering
	\includegraphics[scale=0.2]{figures/theoryCenSurE_filters}
	\caption{Center surround filters. From left to right: decreasing accuracy and computational cost.  Source Agrawal et al. \cite{Agrawal2008}}
	\label{fig:censureFilters}
\end{figure}
The \gls{censure} feature detector was proposed by Agrawal et al. in 2008 as a framework of several bi-level filters for interest point detection \cite{Agrawal2008}. \gls{censure} is scale and rotation invariant and similar to the way \gls{sift} and \gls{surf} perform interest point detection. Contrary to the aforementioned keypoint detectors, \gls{censure} does not up- or downscale the image in scale space and therefore does not inherit the disadvantages of up- and downscaling like inaccurate keypoint locations.

To detect corners \gls{censure} employs approximations of \gls{log} with center-surround discrete bi-level filters. The closest approximation of \gls{log} is a circular bi-level filter as shown in figure \ref{fig:censureFilters}a. Bi-level means that the filter only contains two values namely $1$ or $-1$. The circular filter is rotation invariant due to its symmetry but it is also expensive to compute. Therefore, \gls{censure} uses octagons and boxes as approximations of the circular filter. Like \gls{surf}, \gls{censure} also leverages integral images for box filters and applies them on five scales. Since a major shortcoming of box filters is the lack of rotation invariance Agrawal et al. also proposed an octagon shaped filter which resembles the circle filter much more closely. Integral images, however, do not work with diagonal edges and thus do not work with octagons. To make computation still possible Agrawal et al. introduced "slanted" integral images which are able to construct any trapezoidal area in the same amount of time as rectangular integral images.

To detect interest points, \gls{censure} performs non-maximal suppression over scale space across a 3x3x3 neighborhood. Weak interest points are filtered by threshold and edges are filtered the Harris corner measure.

\section{Bag of Words}
\acrfull{bow}, bag of features, bag of visual words or bag of keypoints is a simple, yet powerful image feature quantization method. The original idea of a \gls{bow} approach for images came from texture classification \cite{Leung2001}. Leung and Malik found out that by quantification of small texture patches {(textons)} in histograms they were able to accurately classify different textures. In 2004 Csurka et al. refined this approach and applied it to general image classification \cite{Csurka2004}.

The idea behind a \gls{bow} is very simple: During learning the \gls{bow} clusters image descriptors using $k$-means clustering. The clustered representation of the image descriptors is called vocabulary. The vocabulary size is $k$, so using $k=1000$ for the k-means clustering results in a vocabulary of 1000 image features. Choosing $k$ is a trade-off between speed and accuracy and has to be determined during experiments. In an ideal world, a \gls{bow} clusters image features together that are unique and expressive so that we would have a vocabulary of bread, salad, meat and noodle features.

During classification the \gls{bow} gets a set of image descriptors and matches these features with the previously learned vocabulary. The output of this process is a histogram of vocabulary occurrences. Referring to the ideal world example, the \gls{bow} would get a set of features including many that look like bread, some that look like meat and a few that are similar to the salad texton. The output would be a histogram with a large value for the bread bin and some smaller values for the salad and meat bins. Giving this occurrence vector to a trained SVM would reveal that the image might be a Hamburger.

\section{Methodology}
\subsection{Performance Measurements}

\subsubsection*{Precession - Recall}
The most used measure for the precision of food classifiers is the average accuracy which is calculated by dividing the number of correct matches and the total number of samples. Accuracy, however, gives no information about the underlying conditions. It is a measure of overall performance. To have a higher chance of suggesting the correct items, future systems may present a list of options that the user can chose from. Intuitively, the accuracy is much higher if a classifier can present a list of items with high confidences instead of only one item because the problem is much easier. Accuracy, however, does not measures how easy a problem is. If a classifier were able to suggest all classes as options the accuracy would always be 100\% although the results are not useful at all.

The combination of precision and recall objectively measures the actual relevance and performance of a classifier for a class of images because it includes the amount of considered items and the correct predictions. In this case the amount of considered items changes based on how many items the classifier can suggest. Precision and recall is defined as:

\begin{equation}
Precision = \frac{T_p}{T_p+F_p} \quad Recall = \frac{T_p}{T_p+F_n}.
\end{equation}

\begin{itemize}
	\item True positives $T_P$ is the number of correctly classified images of a class.
	\item False positives $F_P$ are all images that the classifier predicted to be positive but are in reality negative. {(Type I Error)}
	\item False negatives $F_N$ are all images that are positive {(belong to the class)} but are labeled as negative {(do not belong to class)} {(Type II Error)}
\end{itemize}

A high recall means that many images were matched correctly and a high precision denotes a low number of incorrectly classified images. The bigger the area under the Precision-Recall curve the better the classifier.

\subsubsection*{Null Error Rate}
The null error rate is a baseline for any classification task that calculates the accuracy if a classifier would just predict the class with the most images.

\subsubsection*{Confusion Matrix}
Confusion matrices are one of the most important metrics to understand why a classifier struggles with certain classes while getting a high precision with others. As the name suggests, a confusion matrix tells if the classifier "confuses" two classes.

A confusion matrix for $n$ classes is always a $n \times n$ matrix where columns represent the actual images classes and rows represent the predicted image classes so if the diagonal of the matrix has high values this means that the classifier makes correct predictions.

\subsubsection*{Categorical Cross-Entropy}
The categorical cross-entropy $L_i$ is an error function that is used for the training of neural networks in classification tasks as the objective function. It is more versatile than the accuracy or the \gls{mse} because it takes the deviations of the predicted label $p_{i,j}$ and the actual label $t_{i,j}$ into account and weights the "closeness" of the prediction with the logarithm. For classification, cross entropy is more useful than \gls{mse} because \gls{mse} gives too much emphasis on incorrect predictions. The categorical cross entropy function is defined as:

\begin{equation}
L_i = - \sum_{j} t_{i,j}\log(p_{i,j})
\end{equation}

The loss values that are used for the discussion of results for neural networks are the average values of the categorical cross-entropy {(\gls{ace})}.

\subsection{Cross Validation}
Cross validation is one of the most essential techniques to evaluate real-world classification performance. Classifiers like \glspl{svm} or neural networks are always better on data they have already seen. This is called overfitting {(see section \ref{subsec:overfittingDropout})}. By training and testing on the same data the classification performance would be much better than the actual real world performance. To test if a classifier can actually work with samples it has not seen cross validation divides the dataset into different partitions.

For most tasks it is sufficient to divide the dataset into a training and a test set. The data in the training set is used to train the classifier and the test data is used to evaluate it with data is has not seen before.

\subsubsection*{k-fold Cross Validation}
To make the classification evaluation even more robust, $k$-fold cross validation is used. By applying $k$-fold cross validation the dataset is randomly partitioned into $k$ different parts. $k-2$ parts are used for training and two parts are used for the evaluation. This process is repeated $k$-times and after each iteration the parts are exchanged so that at the end, each sample was used for training and for validation. Calculating the mean of the $k$ evaluations gives a much more robust measurement because the evaluation does not depend on the difficulty of the test partitions.


\documentclass{article}
\usepackage[utf8]{inputenc}

\title{Messy Integral (@Vibingmath)}
\author{Shreenabh Agrawal}
\date{\today}
\usepackage{amsmath}
\usepackage{geometry}
\geometry{a4paper, portrait, margin=1in}
\usepackage{natbib}
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{relsize}
\usepackage[makeroom]{cancel}
\begin{document}

\maketitle

\section{Question}
$$\LARGE{{\int\limits_{1}^{e}}\frac{x-x\ln x+1}{x\left(x+1\right)^{2}+x\ln^{2}x}\: dx}$$
\section{Solution}

Let us take substitution $\ln x = t$ so that $dx = e^{t}dt$, thus our integral becomes

$$\LARGE{\int\limits_{0}^{1}}\ \frac{\left(e^{t}\ -\ te^{t}\ +\ 1\right)}{\cancel{e^{t}}\left[\left(e^{t}+1\right)^{2}\ +\ t^{2}\right]}\: {\cancel{e^{t}}}\:dt$$
Now dividing both numerator and denominator by $t^2$ we get,
$$\mathlarger{\LARGE{\int\limits_{0}^{1}}\ \frac{\frac{e^{t}}{t^{2}}+\ \frac{1}{t^{2}}-\frac{e^{t}}{t}}{\left(\frac{e^{t}}{t}+\frac{1}{t}\right)^{2}+\ 1}\ dt}$$
Now taking a final substitution $$\frac{e^{t}}{t}+\frac{1}{t} = u$$ so that $$\left(\frac{e^{t}}{t}-\frac{e^{t}}{t^{2}}-\frac{1}{t^{2}}\right)dt\ =\ du$$
The integral simplifies as:
$$\LARGE{\int\limits_{e+1}^{\infty}\frac{1}{1+u^{2}}\ du}$$
Plugging in the limits, the answer is $$\boxed{=\tan^{-1}\left(\frac{1}{1+e}\right)}$$
\end{document}


\chapter{Sets}
\label{chapter:sets}
\marginurl{%
  Sets:\\\noindent
  Introduction to Mathematical Reasoning \#6
}{youtu.be/bshBV2H4Sqo}
\section{The Intuitive Definition of a Set}
A set is one of the two most important concepts in mathematics. Many mathematical
statements involve ``an integer $n$'' or ``a real number $a$''. Set theory
notation provides a simple way to express that $a$ is a real number or that $a$
is a natural number. However, this language is much more expressible and it is
impossible to imagine modern mathematics without this notation.

As in the previous chapter it is difficult to define a set formally so we give
a less formal definition which should be enough to use the notation.
A \emph{set} is a well-defined collection of objects. Important examples of
sets are:
\begin{enumerate}
  \item $\R$ a set of reals,
    \nomenclature[S]{$\R$}{denotes the set of all real numbers}
  \item $\Z$ the set of integers\footnote{``Z'' stands for the German word
    Zahlen (``numbers'').},
    \nomenclature[S]{$\Z$}{denotes the set of all integers}
  \item $\N$ the set of natural numbers\footnote{%
      Note that in the literature
      there are two different traditions: in one $0$ is a natural number, in
      another it is not; in this book we are going to assume that $0$ is not a
      natural number.
    },
    \nomenclature[S]{$\N$}{denotes the set of all integers greater than $0$}
  \item $\Q$ a set of rational numbers,
    \nomenclature[S]{$\Q$}{denotes the set of all rational numbers}
  \item $\mathbb{C}$ a set of complex numbers.
    \nomenclature[S]{$\C$}{denotes the set of all complex numbers}
\end{enumerate}
Usually, sets are denoted by a single letter.

Objects in a set are called \emph{elements} of the set and we denote the
statement ``x is in the set $E$''\footnote{%
  The symbol $\in$ was first used by Giuseppe Peano 1889 in his work
  ``Arithmetices principia, nova methodo exposita''. Here he wrote on page X:
  ``The symbol $\in$ means is. So $a \in b$ is read as a \emph{is} a b;
  \dots''
  The symbol itself is a stylized lowercase Greek letter epsilon
  (``$\epsilon$''), the first letter of the word  \textgreek{esti}, which means
  ``is''.
}
by the formula $x \in E$ and the negation of this statement by $x \not\in E$.
For example, we proved that $\sqrt{2} \not\in \Q$.
\nomenclature[S]{$x \in X$}{says that $x$ is an element of $X$}
\nomenclature[S]{$x \not\in Y$}{says that $x$ is not an element of $Y$}

\section{Basic Relations Between Sets}
Many problems in mathematics are problems of determining whether two descriptions
of sets are describing the same set or not. For example, when we learn how to
solve quadratic equations of the form $ax^2 + bx + c = 0$ ($a, b, c \in \R$) we
learn how to list the elements of the set $\set[ax^2 + bx + c = 0]{x \in \R}$.

We say that two sets $A$ and $B$ are equal if they contain the
same elements (we denote it by $A = B$). If all the elements of $A$ belong to
$B$ we say that $A$ is a subset of $B$ and denote it by
$A \subseteq B$\footnote{%
  In the literature there are three symbols for ``subset'': $\subseteq$,
  $\subsetneq$, and $\subset$. $A \subseteq B$ means that $A$ is a subset of
  $B$ and we allow $A = B$ and $A \subsetneq B$ means that $A$ is a subset of
  $B$ and we forbid $A = B$. However, there is a problem with the third symbol,
  some people use it as a synonym of $\subseteq$ and some use it as a synonym of
  $\subsetneq$. Due to this ambiguity we are going to avoid using it in this
  book.
}.

For example, $\Q \subseteq \R$ since any rational number is
also a real number. A special set is an empty set i.e. the set that does not
have elements, we denote it $\emptyset$.

\nomenclature[S]{$A \subseteq B$}{says that $A$ is a subset of $B$}

\nomenclature[S]{$\emptyset$}{denotes the set that does not have elements}

\begin{exercise}
  \label{exercise:inclusion}

  Which of the following sets are included in which? Recall that a number is
  prime iff it is an integer greater than $1$ and divisible only by $1$ and
  itself.
  \begin{enumerate}
    \item The set of all positive integers less than $10$.
    \item The set of all prime numbers less than $11$.
    \item The set of all odd numbers greater than $1$ and less than $6$.
    \item The set of all positive integers less than $10$.
    \item The set whose only elements are $1$ and $2$.
    \item The set whose only element is $1$.
    \item The set of all prime numbers less than $11$.
  \end{enumerate}
\end{exercise}

\subsection{Diagrams}
If we think of a set $A$ as represented by all the points within a circle or
any other closed figure, then it is easy to represent the notion of $A$ being a
subset of another set $B$ also represented by all the points within a circle.
We just put a circle labeled by $A$ inside of the circle labeled by $B$. We can
also diagram an equality by drawing a circle labeled by both $A$ and $B$. (see
fig.~\ref{figure:euler-diagram}). Such diagrams are called Euler diagrams and
it is clear that one may draw Euler diagrams for more than two sets.

\begin{figure}
    \centering
    \subfloat[$B \subseteq A$]{
      \begin{tikzpicture}[scale=0.6]
        \draw (20:2cm) circle (3cm) node [yshift=8ex] {$A$};
        \draw (0:2cm) circle (1.5cm) node {$B$};
      \end{tikzpicture}
    }
    \qquad\qquad
    \subfloat[$A = B$] {
      \begin{tikzpicture}[scale=0.6]
        \draw (20:2cm) circle (3cm) node [yshift=8ex] {$A, B$};
      \end{tikzpicture}
    }
    \caption{Euler diagrams for subset and equality relations}
    \label{figure:euler-diagram}
\end{figure}

\subsection{Descriptions of Sets}
In this section we describe how to define new sets, this notation is also
known as \emph{set-builder notation}.

\paragraph{Listing elements.} The simplest way to define a set is just to list
the elements. For example
\begin{enumerate}
  \item $\set{1, 2, \pi}$ is the set consisting of three elements 1, 2, and
    $\pi$, and
  \item $\set{1, 2, 3, \dots}$ is the set of all positive integers i.e. it is
    the set $\N$.
\end{enumerate}

\paragraph{Conditional definitions.}
We may also describe a set using some constraint e.g we may list all the even
numbers using the following formula $\set[n \text{ is even}]{n \in \Z}$
(we read it as ``the set of all integers $n$ such that $n$ is even'').

Using this we may also define the set of all integers from $1$ to $m$, we
denote it $[m]$; i.e. $[m] = \set[0 < n \le m]{n \in \N}$.
\nomenclature[S]{$[n]$}{denotes the set of all the integers from $1$ to n}

\paragraph{Constructive definitions.} Another way to construct a set of all
even numbers is to use the constructive definition of a set:
$\set[k \in \Z]{2k}$.

We may also describe a set of rational numbers using this description:
$\Q = \set[a \in \Z, b \in \N]{a / b}$ (note that we may also use a mix of
a conditional and constructive definitions,
$\Q = \set[a, b \in \Z, b \neq 0]{a / b}$).

\begin{exercise}
  Describe a set of perfect squares using constructive type of definition.
\end{exercise}

\subsection{Disjoint Sets}
Two sets are \emph{disjoint} iff they do not have common elements. We also
say that two sets are \emph{overlapping} iff they are not disjoint i.e. they
share at least one element.

More generally, $A_1$, \dots, $A_\ell$ are pairwise disjoint iff $A_i$ is
disjoint with $A_j$ for all $i \neq j \in [\ell]$

\begin{exercise}
  Of the sets in Exercise~\ref{exercise:inclusion}, which are disjoint from
  which?
\end{exercise}

\section{Operations over Sets.}
Another way to describe a set is to apply operation to other sets. Let $A$ and
$B$ be sets.
\begin{figure}
  \centering
  \subfloat[$A \cup B$]{
    \begin{tikzpicture}[scale=0.8]
        \fill[yellow] (45:2cm) circle (1.5cm);
        \fill[yellow] (0:2cm) circle (1.5cm);

        \draw (45:2cm) circle (1.5cm) node [above] {$A$};
        \draw (0:2cm) circle (1.5cm) node [below] {$B$};
    \end{tikzpicture}
  }
  \qquad\qquad
  \subfloat[$A \cap B$]{
    \begin{tikzpicture}[scale=0.8]
      \begin{scope}
          \clip (45:2cm) circle (1.5cm);
          \fill[yellow] (0:2cm) circle (1.5cm);
      \end{scope}
      \draw (45:2cm) circle (1.5cm) node [above] {$A$};
      \draw (0:2cm) circle (1.5cm) node [below] {$B$};
    \end{tikzpicture}
  }
  \vskip 0.25cm
  \subfloat[$A \setminus B$]{
    \begin{tikzpicture}[scale=0.8]
      \begin{scope}[even odd rule]
        \clip (0:2cm) circle (1.5cm) (-0.25,-0.25) rectangle (3,3);
        \fill[yellow] (45:2cm) circle (1.5cm);
      \end{scope}
      \draw (45:2cm) circle (1.5cm) node [above] {$A$};
      \draw (0:2cm) circle (1.5cm) node [below] {$B$};
    \end{tikzpicture}
  }
  \qquad\qquad
  \subfloat[$A \Delta B$]{
    \begin{tikzpicture}[scale=0.8]
      \begin{scope}[even odd rule]
        \clip (0:2cm) circle (1.5cm) (45:2cm) circle (1.5cm);
        \fill[yellow] (45:2cm) circle (1.5cm);
        \fill[yellow] (0:2cm) circle (1.5cm);
      \end{scope}

      \draw (45:2cm) circle (1.5cm) node [above] {$A$};
      \draw (0:2cm) circle (1.5cm) node [below] {$B$};
    \end{tikzpicture}
  }
  \caption{Euler diagrams for set operations}
\end{figure}

The first example of the operations on sets is the \emph{union} operation.
The union of $A$ and $B$ is the set containing all the elements of $A$ and all
the elements of $B$ i.e.
$A \cup B = \set[x \in A \text{ or } x \in B]{x}$\footnote{%
  Note that this definition is not correct since in the conditional definitions
  we have to specify the set $x$ belongs to and we cannot do this here.
}.
\nomenclature[S]{$A \cup B$}{denotes the union of two sets $A$ and $B$}

Another example of such an operation is \emph{intersection}. The
intersection of $A$ and $B$ is the set of all the elements belonging to both
$A$ and $B$ i.e $A \cap B = \set[x \in A \text{ and } x \in B]{x}$\footnote{%
  You may notice that in the definition of the union we use disjunction and
  in the definition of intersection we use conjunction. Actually this is the
  reason the symbol of the conjunction is similar to the symbol of intersection
  and the symbol of the disjunction is similar to the symbol of union.
}.
\nomenclature[S]{$A \cap B$}{denotes the intersection of two sets $A$ and $B$}

The third operation we are going to discuss this lecture is
\emph{set difference}. If $A$ and $B$ are some sets, then
$A \setminus B = \set[x \in A \text{ and } x \not\in B]{x}$.
\nomenclature[S]{$A \setminus B$}{denotes the difference of two sets $A$ and $B$}

The last operation is \emph{symmetric difference}. If $A$ and $B$ are some
sets, then $A \Delta B = (A \setminus B) \cup (B \setminus A)$. Note that
alternatively $A \Delta B = (A \cup B) \setminus (A \cap B)$

\begin{exercise}
  Describe the set
  $\set[n \text{ is even}]{n \in \N} \cap \set[n \in \N]{3n}$.
\end{exercise}

\begin{theorem}
\label{theorem:set-equalities}
  Let $A$, $B$, and $C$ be some sets. Then we have the following identities.
  \begin{description}
    \item[(associativity)] $A \cup (B \cup C) = (A \cup B) \cup C$ and
      $A \cap (B \cap C) = (A \cap B) \cap C$.
    \item[(commutativity)] $A \cup B = B \cup A$ and $A \cap B = B \cap A$.
    \item[(distributivity)] $A \cup (B \cap C) = (A \cup B) \cap (A \cup C)$
      and $A \cap (B \cup C) = (A \cap B) \cup (A \cap C)$.
  \end{description}
\end{theorem}
\begin{proof}
  One may prove these properties using the Euler diagrams. Alternatively they
  can be proven by definitions. Let us prove only the first part of the
  distributivity, the rest is Exercise~\ref{exercise:set-equalities}.

  Our proof consists of two parts in the first part we prove that
  $A \cup (B \cap C) \subseteq (A \cup B) \cap (A \cup C)$.
  Suppose that $x \in A \cup (B \cap C)$. Then $x \in A$ or $x \in (B \cap C)$.
  \begin{itemize}
    \item If $x \in A$, then $x \in (A \cup B)$ and $x \in (A \cup C)$ i.e.
      $x \in ((A \cup B) \cap (A \cup C))$.
    \item If $x \in (B \cap C)$, then $x \in B$ and $x \in C$. Which implies
      that $x \in (A \cup B)$ and $x \in (A \cup C)$. As a result,
      $x \in ((A \cup B) \cap (A \cup C))$.
  \end{itemize}
\end{proof}

\begin{exercise}
\label{exercise:set-equalities}
  Prove the rest of the equalities in Theorem~\ref{theorem:set-equalities}.
\end{exercise}

Probably the most difficult concept connected to sets is
the concept of a power set. Let $A$ be some set, then the set of
all possible subsets of $A$ is denoted by $2^A$ (sometimes this set is denoted
by $\mathcal{P}(A)$) and called the power set of $A$. In other words $2^A =
\set[B \subseteq A]{B}$.

\nomenclature[S]{$2^A$}{denotes the set of all the subsets of the set $A$}

\begin{warning}
  Please do not forget about two extremal elements of the power set $2^A$: the
  empty set and $A$ itself.
\end{warning}

\noindent For example if $A = \set{1, 2, 3}$, then
\[
  2^A = \set{\emptyset, \set{1}, \set{2}, \set{3}, \set{1, 2},
  \set{1, 3}, \set{2, 3}, \set{1, 2, 3}}.
\]

\section{The Well-ordering Principle}
Using the set notation we may finally justify the proof of the statement
that $2^n > n$ for all positive integers $n$ from the video about mathematical
induction. In order to do this let us first formulate the following theorem.
\begin{theorem}
\label{theorem:well-ordering}
  Let $A \subseteq \Z$ be a non-empty set. We say that $b \in \Z$ is a lower
  bound for the set $A$ iff $b \le a$ for all $a \in A$. Additionally, we say
  that the set $A$ is bounded if there is a lower bound for $A$.

  Given this, if $A$ is bounded, then there is a lower bound $a \in A$ for the set $A$
  (we say that $a$ is the minimum of the set $A$).
\end{theorem}
Note that this theorem also states that any subset of natural numbers have a
minimum.

Recall that we wish to prove that $2^n > n$ for all positive $n$.
Assume that it is not true, in this case the set
$A = \set[2^n < n]{n \in \N}$ is non-empty. Denote by $n_0$ the minimum of the
set $A$, $n_0$ exists by Theorem~\ref{theorem:well-ordering}. We may consider
the following two cases.
\begin{itemize}
  \item If $n_0 = 1$, then it leads to a contradiction since $2 = 2^1 > 1$.
  \item Otherwise, note that $1 \le n_0 - 1 < n_0$, hence,
    $2^{n_0 - 1} > n_0 - 1$. So $2^{n_0} > 2n_0 - 2 \ge n_0$. Which is a
    contradiction with the definition of $n_0$.
\end{itemize}

Finally, we prove Theorem~\ref{theorem:well-ordering}.
\begin{proof}[Proof of Theorem~\ref{theorem:well-ordering}]
  Let $b$ be a lower bound for the set $A$. Assume that there is no minimum of
  the set $A$. Let $P(n)$ be the statement that
  $n \notin A$.

  First, we are going to prove that $P(n)$ is true for all
  $n \ge b$. The base case is true since if $b \in A$, then $b$ is the minimum
  of $A$ which contradicts to the assumption that there is no minimum of $A$.
  The induction step is also clear, by the induction hypothesis we know that
  $P(b)$, \dots, $P(k)$ are true, hence, $(k + 1) \in A$ implies that $k + 1$
  is the minimum of $A$.

  Now we prove that $A$ is empty. Assume the opposite i.e. assume that there is
  $x \in A$. Note that $x \ge b$ since $b$ is a lower bound of $A$. However,
  $P(x)$ is true which implies that $x \notin A$. Therefore the assumption was
  false and $A$ is empty, but this contradicts to the fact that $A$ is non-
  empty.
\end{proof}

\begin{chapterendexercises}
  \exercise Find the power sets of $\emptyset$, $\set{1}$, $\set{1, 2}$,
    $\set{1, 2, 3, 4}$. How many elements in each of this sets?
  \exercise[recommended] Prove that
    \begin{itemize}
      \item $A \subseteq B \iff A \cup B = B$,
      \item $A \subseteq B \iff A \cap B = A$.
    \end{itemize}
  \exercise Let $A$ be a subset of a set $U$ we call this set a universe.
    We say that the set $\overline{A} = U \setminus A$ is a complement of $A$
    in $U$. Show the following equalities
    \begin{itemize}
      \item $\overline{\overline{A}} = A$.
      \item $\overline{A \cup B} = \overline{A} \cap \overline{B}$.
      \item $\overline{A \cap B} = \overline{A} \cup \overline{B}$.
    \end{itemize}
  \exercise[recommended] Let us define an intersection of more than two sets as follows.
    Let $A_1$, \dots, $A_n$ be some sets. Then
    \begin{itemize}
      \item $\bigcap_{i = 1}^1 A_i = A_1$ and
      \item $\bigcap_{i = 1}^{k + 1} A_i = \left( \bigcap_{i = 1}^k A_i \right)
        \cap A_{k + 1}$.
    \end{itemize}
    \nomenclature[S]{$\bigcap_{i = 1}^k A_i$}{denotes $A_1 \cap \dots \cap
    A_k$}

    Show that $\bigcap_{i = 1}^n \set[i \le x \le n]{x \in \N} =
    \set{n}$ for all integers $n > 0$.
    \begin{solution}
      There are two solutions for this problem which provide two important
      points of view.

      \begin{enumerate}
        \item We may prove the statement using induction by $k$. Note that the
          statement does not involve $k$ so we need to prove a bit different
          statement, we prove
          $\bigcap_{i = 1}^k \set[i \le x \le n]{x \in \N} =
            \set{k, k + 1, \dots, n}$.
          The base case for $k = 1$ is clear since
          $\bigcap_{i = 1}^1 \set[i \le x \le n]{x \in \N} =
            \set[1 \le x \le n]{x \in \N} = \set{1, 2, \dots, n}$.

          The induction step is also easy to see. By the induction hypothesis
          $\bigcap_{i = 1}^k \set[i \le x \le n]{x \in \N} =
            \set{k, k + 1, \dots, n}$.
          Note that
          \begin{multline*}
            \bigcap_{i = 1}^{k + 1} \set[i \le x \le n]{x \in \N} = \\
            \left(
              \bigcap_{i = 1}^k \set[i \le x \le n]{x \in \N}
            \right) \cap
            \set[k + 1 \le x \le n]{x \in \N} =  \\
            \set{k, k + 1, \dots, n} \cap \set{k + 1, \dots, n} =
            \set{k + 1, \dots, n}.
          \end{multline*}
          Thus for $k = n$,
          $\bigcap_{i = 1}^n \set[i \le x \le n]{x \in \N} = \set{n}$.
        \item Another approach is to prove the statement using induction by $n$.
          The base case is also clear,
          $\bigcap_{i = 1}^1 \set[i \le x \le n]{x \in \N} = \set{1}$.

          Let us now prove the induction step. By the induction hypothesis
          $\bigcap_{i = 1}^k \set[i \le x \le k]{x \in \N} = \set{k}$.
          Hence,
          \begin{multline*}
            \bigcap_{i = 1}^k \set[i \le x \le k + 1]{x \in \N} = \\
            \bigcap_{i = 1}^k
              \left(
                \set[i \le x \le k]{x \in \N} \cup \set{k + 1}
              \right)
            \stackrel{*}{=} \\
            \left(\bigcap_{i = 1}^k \set[i \le x \le k]{x \in \N} \right) \cup
            \set{k + 1} =
            \set{k, k + 1};
          \end{multline*}
          note that all these equalities are clear except the equiality marked
          by $*$, we prove it later.

          Thus by the definition of the union
          \begin{multline*}
            \bigcap_{i = 1}^{k + 1} \set[i \le x \le k + 1]{x \in \N} = \\
            \left(
              \bigcap_{i = 1}^k \set[i \le x \le k + 1]{x \in \N}
            \right) \cap \\
            \set[k + 1 \le x \le k + 1]{x \in \N}= \\
            \left(
              \bigcap_{i = 1}^k \set[i \le x \le k + 1]{x \in \N}
            \right) \cap
            \set{k + 1} = \\
            \set{k, k + 1} \cap \set{k + 1} =
            \set{k + 1}.
          \end{multline*}

          Now we need to prove that if $A_1, \dots, A_\ell$, and $B$ are some sets,
          then $\bigcap_{i = 1}^\ell (A_i \cup B) =
          \left(\bigcap_{i = 1}^\ell A_i\right) \cup B$.
          We prove it also using induction by $\ell$. For $\ell = 1$, the
          statement is clear since $\bigcap_{i = 1}^1 (A_i \cup B) = A_1 \cup B
          = \left(\bigcap_{i = 1}^\ell A_i\right) \cup B$.
          Now we need to check the induction step. By the induction hypothesis
          $\bigcap_{i = 1}^k (A_i \cup B) =
            \left(\bigcap_{i = 1}^k A_i\right) \cup B$. Note that
          \begin{multline*}
            \bigcap_{i = 1}^{k + 1} (A_i \cup B) =
            \bigcap_{i = 1}^k (A_i \cup B) \cap (A_{k + 1} \cup B) = \\
            \left(\left(\bigcap_{i = 1}^k A_i\right) \cup B\right) \cap
              (A_{k + 1} \cup B) =  \\
            \left(
              \left(
                \bigcap_{i = 1}^k A_i
              \right) \cap A_{k + 1}
            \right) \cup B =
            \left(\bigcap_{i = 1}^{k + 1} A_i\right) \cup B.
          \end{multline*}
      \end{enumerate}
    \end{solution}
  \exercise Let us define a union of more than two sets as follows.
      Let $A_1$, \dots, $A_n$ be some sets. Then
      \begin{itemize}
        \item $\bigcup_{i = 1}^1 A_i = A_1$ and
        \item $\bigcup_{i = 1}^{k + 1} A_i =
          \left( \bigcup_{i = 1}^k A_i \right) \cup A_{k + 1}$.
      \end{itemize}
      \nomenclature[S]{$\bigcup_{i = 1}^k A_i$}{denotes $A_1 \cup \dots \cup
      A_k$}

      Show that $\bigcup_{i = 1}^n \range{i} = \range{n}$ for all integers
      $n > 0$.
      \begin{solution}
        First, note that $\range{k - 1} \cup \range{k} = \range{k}$ for any
        $k > 1$.

        Now we are ready to prove the statement, we prove it using induction by
        $n$. The base case is true since $\bigcup_{i = 1}^1 \range{i} =
        \range{1}$ by the definition. The induction step is also true since by
        the induction hypothesis $\bigcup_{i = 1}^{k - 1} \range{i} =
        \range{k - 1}$ and by the definition of the union
        $\bigcup_{i = 1}^k \range{i} =
        \left(\bigcup_{i = 1}^{k - 1} \range{i}\right) \cup \range{k}$.
        Hence, $\bigcup_{i = 1}^k \range{i} =
          \left( \bigcup_{i = 1}^{k - 1} \range{i} \right) \cup \range{k} =
          \range{k - 1} \cup \range{k} = \range{k}$.
      \end{solution}
  \exercise[recommended] Let $\Omega$ be some set and $A_1, \dots, A_n \subseteq \Omega$.
    Show that $\bigcup_{i = 1}^n A_i =
      \set[{\exists i \in \range{n}\ x \in A_i}]{x \in \Omega}$.
  \exercise Let $A_1$, \dots, $A_n$ be some sets.
    Show that $\bigcup_{i = 1}^n (A_i \cap B) =
    \left(\bigcup_{i = 1}^n A_i\right) \cap B$.
  \exercise Show that $A \Delta (B \Delta C) = (A \Delta B) \Delta C$.
  \exercise[recommended]
    Let $\mathbb{R}^{m \times n}$ be the set of all matrices $m \times n$ and
    $\mathbb{R}^n$ be the set of $n$ dimensional vectors.
    Show that for any matrix $A \in \mathbb{R}^{m \times n}$
    ($n > m$) there is a nonzero vector $x \in \mathbb{R}^n$ such that $Ax = 0$.
\end{chapterendexercises}