\chapter{Bayesian theory}
\section{Probability theory and statistics}
\subsection[Sigma-algebra]{$\boldsymbol\sigma$-algebra}
%
The following mathematical object is necessary in order to define probability properly:

\begin{definition}
  Let $X$ be a set. A $\sigma$-algebra is a set $Y$ of subsets of $X$ satisfying:
  \begin{enumerate}
    \item $\emptyset \in Y$
    \item $Z \in Y \implies Z^C \in Y$ where $Z^C$ denotes the complement of $Z$, $X \setminus Z$.
    \item $\left(\bigcup_{i=1}^\infty Z_i\right) \in Y$ where $n \in \Nat$ and $Z_i \in Y$ where $i = 1, 2, \ldots$.
  \end{enumerate}
\end{definition}
When $X$ is a finite set, we may limit the third condition to a finite union.

\begin{example}
  Let $X \coloneqq \set{a, b, c, d}$ and $Z \coloneqq \set{\set{a}}$. We extend $Z$ to a $\sigma$-algebra $Y$:
  \[ Y = \set{\set{}, \set{a,b,c,d}, \set{a}, \set{b,c,d}} \]
\end{example}
The notion of the $\sigma$-algebra is essential to define the notion of the probability space and the random variable rigorously.
In this thesis, our discussion is rigorous when $X$ is a finite set.
When $X$ is an infinite set, we need a full general discussion of measure theory.
Then our discussion will sometimes be intuitive or informal.

\subsection{Basic definitions}
\label{sec:bp-def}
%
Probability theory is concerned with random experiments and random phenomena. Probability in its basic form is the fraction of events with a certain outcome to the total number of events.
%
\begin{definition}
  \label{def:prob}
  \index{Probability space}
  \index{Probability measure}
  \index{Event}
  \index{Random variable}
  A \emph{probability space} $(\Omega, \mathcal A, \Prob)$ denotes
    the set of possible outcomes, a set of events, and a map from an element of $\mathcal A$ to a real value in $[0,1]$.
    $\mathcal A$ is a $\sigma$-algebra. As elements of $\mathcal A$ are sets, we can apply set operations on them.
    Let $\Omega$ be a finite set.
    \emph{Probability measure} $\Prob$ satisfies the following conditions:
    \begin{align}
      &\Prob[A] \geq 0 \text{ for } A \in \mathcal A \label{def:prob-1} \\
      &\Prob[A \cup B] = \Prob[A] + \Prob[B] \text{ for } A, B \in \mathcal A \text{ and } A \cap B = \emptyset \label{def:prob-2} \\
      &\Prob[\Omega] = 1 \label{def:prob-3}
    \end{align}
    These axioms are due to Kolmogorov~\cite{kolmogorov1950foundations}.
    Property~\ref{def:prob-2} implies linearity of the probability measure (for mutually exclusive events $A$):
    \begin{align}
      \Prob\left[\bigcup_{A \in \mathcal A} A\right] &= \sum_{A \in \mathcal A} \Prob\left[A\right] \label{eq:prob-linear}
    \end{align}
  For $R$ \emph{taking the value $k$}, we write,
  \[ \Prob[R=k] \coloneqq \Prob\left(\set{\omega \in \Omega \mid R(\omega) = k}\right) \]
  An \emph{event} is an element $A$ of $\mathcal A$, hence a subset $A$ of $\Omega$.
  A \emph{$\mathbb Z$-valued random variable $R$} is a map from $\Omega$ to $\mathbb Z$
  such that $R^{-1}(z) \in \mathcal A$ for any $z \in \mathbb Z$.
  A \emph{$\mathbb R$-valued random variable $R$} is a map from $\Omega$ to $\mathbb R$
  such that $R^{-1}((r, s]) \in \mathcal A$ for any real numbers $r < s$.
\end{definition}
\begin{example}
  \label{ex:cointoss}
  A coin toss has two possible outcomes, heads (\textit h) or tails (\textit t).
  We consider two coin tosses.
  Then $\Omega = \set{(h, h), (h, t), (t, h), (t, t)}$ and $\mathcal A$ is the powerset of $\Omega$ (i.e. set of all subsets).
  Let $\Prob[A] = \card{A} / \card{\Omega}$, the size of set $A$ divided by $4$.
  Let $R$ be our random variable mapping to $\mathbb Z$ defined as result of the first coin toss (1 represents head, 0 represents tails).
  Then $\Prob[R=1] = \Prob[\setdef{\omega \in \Omega}{R(\omega) = 1}] = \Prob[\set{(h, h), (h, t)}] = \frac24$.
\end{example}

In the following, we will declare random variables, but won't specify the group explicitly.
Either it is obvious from context (because of the numbers we use) or our statements work for both groups.
In that sense, a \emph{random variable} is either {$\mathbb Z$-valued} or {$\mathbb R$}-valued here.

%\subsection{Average Value}
%\label{sec:bp-av}
%%
%\begin{definition}
%  \index{Average value}
%  \index{Population mean}
%  \index{Sample mean}
%  Let $\Omega$ be a finite set and $R$ be a random variable.
%  The \emph{average value} $\avg{R}$ of random variable $R$ is defined as,
%  \begin{align}
%    \avg{R} &\coloneqq \frac{1}{\card{\Omega}} \sum_{a \in \Omega} R(a)
%  \end{align}
%  If all outcomes of the sample space $\Omega$ are considered,
%  we call $\avg{R}$ the \emph{population mean} (denoted $\mu$), otherwise \emph{sample mean}.
%\end{definition}

\subsection{Expected Value}
\label{sec:bp-ev}
%
\begin{definition}
  \index{Expected value}
  \index{Mean value}
  The \emph{expected value} $\E$ (also \emph{mean value}) of a random variable $R$ is defined by:
  \begin{align}
    \E[R]      &\coloneqq \mu
%              = \sum_{a \in \Omega} \Prob[a] \cdot R(\set{a})
%              = \sum_{z \in E} \Prob[R = z] \cdot z \\
               = \sum_{\omega \in \Omega} \Prob[\set{\omega}] \cdot R(\omega) \qquad
    \E_{A}[R]  = \sum_{a \in A} \Prob[\set{a}] \cdot R(a)
  \end{align}
  %where $\Prob[R = z]$ is the probability of the random variable $R$ taking the value $z$.
  %In other words, $\Prob[R = z] \coloneqq \Prob[A_z]$ with $A_z \coloneqq \setdef{e \in \Omega}{R(e) = z}$.
  Let $R$ be a map of $\Omega$ to $E$.
  Applying a function to an expected value means applying the function to the element of $E$.
  Let $f$ be any function with domain $E$, then
  \[
    \E[f(R)]      = \sum_{\omega \in \Omega} \Prob[\set{\omega}] \cdot f(R(\omega)) \qquad
    \E_{A}[f(R)]  = \sum_{a \in A} \Prob[\set{a}] \cdot f(R(a))
  \]
\end{definition}

Let $R$ and $S$ be two random variables. Let $c \in \mathbb R$.
The following properties are satisfied:
\begin{align}
  \E[c]     &= \sum_{\omega \in \Omega} \Prob[\set{\omega}] \cdot c = 1 \cdot c = c \\
  \E[R + c] &= \sum_{\omega \in \Omega} \left(\Prob[\set{\omega}] \cdot (R(\omega) + c)\right) \notag\\
            &= \sum_{\omega \in \Omega} \left(\Prob[\set{\omega}] \cdot R(\omega)\right) + \sum_{\omega \in \Omega} \left(\Prob[\set{\omega}] \cdot c\right) \notag\\
            &= \E[R] + c \cdot \sum_{\omega \in \Omega} \Prob[\set{\omega}] \notag\\
            &= \E[R] + c \cdot 1 = \E[R] + c \\
  \E[R + S] &= \sum_{\omega \in (\Omega_R \cup \Omega_S)}
              \begin{cases}
                \sum_{\omega \in \Omega_R} \Prob[\set{\omega}] \cdot R(\omega) & \text{if } \omega \in \Omega_R \\
                \sum_{\omega \in \Omega_S} \Prob[\set{\omega}] \cdot R(\omega) & \text{if } \omega \in \Omega_S
              \end{cases}\notag\\
            &= \sum_{\omega \in \Omega_R} \Prob[\set{\omega}] \cdot R(\omega) + \sum_{\omega \in \Omega_S} \Prob[\set{\omega}] \cdot R(\omega) \notag\\
            &= \E[R] + \E[S] \label{eq:ERS-RpS} \\
  \E[c \cdot R]
            &= \sum_{\omega \in \Omega} \Prob[\set{\omega}] \cdot (c \cdot R(\omega)) \notag\\
            &= c \cdot \sum_{\omega \in \Omega} \Prob[\set{\omega}] \cdot R(\omega) \notag\\
            &= c \cdot \E[R]
\end{align}
It immediately follows, that this holds true for $\E_A$ for any $A \in \mathcal A$.

\subsection{Equivalence of the continuous probability model}
\label{sec:bp-continuous}
%
\begin{definition}
  \index{Probability Density Function}
  Let $R$ be an $\mathbb R$-valued random variable and
  $f$ be a continuous function defined in $(-\infty, \infty) \subseteq \mathbb R$.
  Let $f$ satisfy the following properties:
  \[
    \Prob[R \leq y] \coloneqq \int\limits_{-\infty}^y f(x) \, dx \hspace{28pt}
    \Prob[z \leq R \leq y] \coloneqq \int\limits_z^y f(x) \, dx \hspace{28pt}
    \Prob[z \leq R] \coloneqq \int\limits_z^\infty f(x) \, dx
  \]
  This establishes a relation between function $f$ and random variable $R$.
  Because $\Prob[R \leq y]$ satisfies properties~\ref{def:prob-1} and \ref{def:prob-3} of probability measures,
  $f$ also satisfies:
  \begin{align}
    f(x) &\geq 0 \qquad \forall x \in (-\infty, \infty) \label{prop:pdf1} \\
       1 &= \int_{-\infty}^{\infty} f(x) \, dx \label{prop:pdf2}
  \end{align}
  $f$ is called a \emph{Probability Density Function} (PDF).
\end{definition}
\begin{example}
  \[ f(x \; | \; \mu, \sigma^2) = \frac{1}{\sqrt{2\pi\sigma^2} } \; e^{ -\frac{(x-\mu)^2}{2\sigma^2} } \]
  The normal distribution function, depending on parameters $\mu$ and $\sigma^2$, is one example for a probability density function.
  The function is introduced in Section~\ref{sec:bp-norm-dist} in detail.
\end{example}

We have seen that the continuous model follows equivalent properties like the discrete model.
This is also true for the expected value.

\begin{definition}
  \index{Expected value}
  The \emph{expected value} of random variable~$R$ %in the continuous model
  is defined as,
  \begin{align}
    \E[R] &\coloneqq \int_{-\infty}^\infty \left(\Prob[R=x] \cdot x\right) \, dx
  \end{align}
  Since intuitively $\int_{-\infty}^\infty \Prob[R=x] \, dx = \int_{\mathbb R} f(x) \, dx$,
  \[ \E[R] = \int_{\mathbb R} x \cdot f(x) \, dx \]
\end{definition}
%
\begin{proof}
  Let $R$ and $S$ be two random variables. Let $c \in \mathbb R$.
  The expected value satisfies:
  %
  \begin{align}
    \E[c]     &\coloneqq c \label{eq:Ec} \\
    \E[R + c] &\coloneqq \int_{\mathbb R} \Prob[R=x] \cdot (x + c) \, dx \notag\\
              &= \int_{\mathbb R} \left(\Prob[R=x] \cdot x + \Prob[R=x] \cdot c\right) \, dx \notag\\
              &= \int_{\mathbb R} \Prob[R=x] \cdot x \, dx + c \cdot \int_{\mathbb R} \Prob[R=x] \, dx \notag\\
              &= \E[R] + c \cdot 1 = \E[R] + c \\
    \E[R + S] &\coloneqq \int_{\mathbb R} \left(\Prob[R=x] \cdot x + \Prob[S=x] \cdot x\right) \, dx \notag\\
              &= \int_{\mathbb R} \Prob[R=x] \cdot x \, dx + \int_{\mathbb R} \Prob[S=x] \cdot x \, dx \notag\\
              &= \E[R] + \E[S] \label{eq:Evv} \\
    \E[c \cdot X] &\coloneqq \int_{\mathbb R} \Prob[R=x] \cdot (x \cdot c) \, dx \notag\\
              &= c \cdot \int_{\mathbb R} \Prob[R=x] \cdot x \, dx
              = c \cdot \E[X] \label{eq:Ecv}
  \end{align}
\end{proof}
Because $\Prob$ and $\E$ provide the same properties in the discrete and continuous case,
we often do not distinguish between these cases. The statements hold true for $\mathbb Z$-valued
as well as $\mathbb R$-valued random variables.

\subsection{Variance and standard deviation}
\label{sec:bp-var-sd}
%
\begin{definition}
  \index{Variance}
  \emph{Variance} quantifies how strong values are spread out from $\E[R]$: %the expected value $\E[R]$:
  \[ \sigma^2 \coloneqq \E\left[(R - \E[R])^2\right] \]
\end{definition}
Considering the entire population, the variance can also quantify over the population mean $\mu$:
\[ \sigma^2 = \Var[R] \coloneqq \E[(R - \mu)^2] \]
In the discrete case, this is equivalent to,
\[ \Var[R] = \E\left[\sum_{\omega \in \Omega} \Prob[\set{\omega}] \cdot \left(R(\omega) - \mu\right)^2\right] \]
and in the continuous case, we have:
\[ \Var[R] = \E\left[\int_{\mathbb R} \Prob[R=x] \cdot \left(x + \mu\right)^2 \, dx\right] \]
\begin{definition}
  \index{Standard deviation}
  The \emph{standard deviation} is defined as its second root:
  \[ \sd = \sqrt{\Var[R]} \]
\end{definition}

\subsection{Covariance}
\label{sec:covar}
%
\index{Covariance}
\emph{Covariance} measures the joint variability of two given random variables.
It is defined as:
\begin{align}
  \Cov[X,Y]
    &= \E[(X - \E[X])(Y - \E[Y])] \label{eq:cov}\\
    &= \E[XY - Y\cdot\E[X] - X\cdot\E[Y] + \E[X]\E[Y]] \notag\\
    &= \E[XY] - \E[Y]\cdot\E[X] - \E[X]\cdot\E[Y] + \E[X]\E[Y] \notag\\
    &= \E[XY] - \E[X]\cdot\E[Y] \label{eq:cov-exy}
\end{align}
%
If $X$ and $Y$ are independent (Section~\ref{sec:bp-ev-indep}), then the covariance is zero.
\begin{align}
  \Cov[X,Y] &= \E[XY] - \E[X] \cdot \E[Y] = \E[X] \cdot \E[Y] - \E[X] \cdot \E[Y] = 0 \label{eq:CovXY0}
\end{align}
We will exploit the following properties:
\begin{align}
  \Cov[X,Y] &= \E[(X - \E[X])(Y - \E[Y])] = \E[(Y - \E[Y])(X - \E[X])] = \Cov[Y,X] \\
  \Cov[X,X] &= \E[(X - \E[X])^2] = \Var[X]
\end{align}
%
Let $X$ be a set of $n$ independent variables $X_{1 \leq i \leq n}$.
Then it holds that:
\begin{align}
  \Var\left[\sum_{i=1}^n X_i\right]
    &= \E\left[\left(\sum_{i=1}^n X_i - \E\left[\sum_{i=1}^n X_i\right]\right)^2\right] \notag \\
    &= \E\left[\left(\sum_{i=1}^n \left(X_i - \E\left[X_i\right]\right)\right)^2\right] \notag \\
    &= \E\left[\sum_{i=1}^n \left(X_i - \E[X_i]\right) \cdot \sum_{j=1}^n \left(X_j - \E[X_j]\right)\right] \notag \\
    &= \E\left[\sum_{j=1}^n \left(\sum_{i=1}^n (X_i - \E[X_i])\right) (X_j - \E[X_j]) \right] \notag \\
    &= \E\left[\sum_{i,j \in [1,n]} \left(X_i - \E[X_i]\right) \left(X_j - \E[X_j]\right)\right] \notag \\
    &= \sum_{i,j \in [1,n]} \E[(X_i - \E[X_i])(X_j - \E[X_j])] \notag \\
    &= \sum_{i,j \in [1,n]} \Cov[X_i, X_j] \notag \\
    &= \sum_{i \in [1,n]} \Cov[X_i, X_i] + \sum_{\substack{i \neq j \\ i,j \in [1,n]}} \Cov[X_i, X_j] \notag \\
    &= \sum_{i=1}^n \Var[X_i] + \sum_{\substack{i \neq j \\ i,j \in [1,n]}} \Cov[X_i, X_j] \notag \\
    &= \sum_{i=1}^n \Var[X_i] + 2 \cdot \sum_{1 \leq i < j \leq n} \Cov[X_i, X_j] \label{eq:sumVarCov}
\end{align}

\subsection{Law of Large Numbers}
\label{sec:bp-lln}
%
The Law of Large Numbers stresses the practical importance of the expected value.

To prove this theorem, we use Chebyshev's Inequality, the Weak Law of Large numbers,
Borel-Cantelli Lemma, and the Strong Law of Large Numbers.
Depending on your requirements of certainty, either the \emph{Weak} Law of Large numbers or
the \emph{Strong} Law of Large Numbers might be considered as Law of Large Numbers.
Our proof structure is based on Craig A. Tracy's~\cite{cnfgen}\footnote{
  Please recognize that there is a small typographical error on page~3.
  \enquote{$S_n(\omega) = 1$ for every $n$} should be \enquote{$X_n(\omega) = 1$ for every $n$}.
}.

\subsubsection{Chebyshev's Inequality}
\label{sec:bp-chebyshev}
%
Consider the continuous case.
Let $R$ be a random variable, $f$ be a PDF over $R$,
$a \in \mathbb R_{\geq 0}$, $p \in \mathbb N$ and let $\E[R^p]$ be defined as follows:
\begin{align*}
  \E[R^p]
          = \int\limits_{\mathbb R} x^p \cdot f(x) \, dx
          \geq \int\limits_{x \geq a} x^p \cdot f(x) \, dx
          \geq a^p \int\limits_{x \geq a} f(x) \, dx
          = a^p \cdot \Prob[R \geq a]
\end{align*}
The discrete case follows immediately.
This concludes the correctness of the following theorem:
\begin{theorem}[Chebyshev's Inequality Theorem]\label{thm:chebyshev}
  Let $R$ be a random variable, $a \in \mathbb R_{\geq 0}$, and $p \in \mathbb N$ is arbitrary.
  Assume $\E[R^p] < \infty$.
  Then it holds that,
  \[ \Prob[R \geq a] \leq \frac{1}{a^p} \E[R^p] \]
\end{theorem}

\subsubsection{Weak Law of Large Numbers}
\label{sec:bp-weak-law}
%
The next theorem is called Weak Law of Large Numbers.
\begin{theorem}[Weak Law of Large Numbers, Bernoulli's Theorem]\label{thm:weak-lln}
  Let $R_i$ be a sequence of independent and identically distributed random variables
  (see section~\ref{sec:bp-iid} for a definition of i.i.d.)
  with common mean $\mean$ and variance $\var$. Let
  \[
      S_n \coloneqq \sum_{i=1}^n R_i \hspace{50pt}
      T_n \coloneqq \frac{S_n}{n} - \mean
  \]
  Then for any $\varepsilon > 0$,
  \[ \lim_{n\to\infty} \Prob[T_n \geq \varepsilon] = 0 \]
\end{theorem}

\begin{proof}
  First, we determine the expected values.
  \begin{align}
      \E[S_n] &= \E\left[\sum_{i=1}^n R_i\right] = \sum_{i=1}^n \E[R_i] = \sum_{i=1}^n \mu = n \cdot \mu & \eqref{eq:Evv} \label{eq:ES_n} \\
%      \E[T_n] &= \E\left[\frac{1}{n}\left(R_1 + R_2 + \ldots + R_n\right) - \mu\right] \notag\\
%              &= \frac1n \left(\E[R_1] + \E[R_2] + \ldots + \E[R_n]\right) - \E[\mu] & \eqref{eq:Ecv} \notag\\
%              &= \frac{n \cdot \mu}{n} - \mu = 0 \label{eq:tn0}
      \E[T_n] &= \E\left[\frac{S_n}{n} - \mu\right] = \frac{\E\left[S_n\right]}{\E\left[n\right]} - \E\left[\mu\right] & \eqref{eq:Evv} \notag\\
              &= \frac{n \cdot \mu}{n} - \mu = 0  & \eqref{eq:ES_n}, \eqref{eq:Ecv} \label{eq:tn0}
  \end{align}
  We also need a result regarding the variance.
  In the continuous and discrete case, it holds that $a^2 \cdot \Var[X] = \Var[a \cdot X]$:
  \begin{align}
    a^2 \cdot \Var[X] &= a^2 \cdot \int (x - \mu)^2 \cdot f(x) \, dx = \int (a \cdot (x - \mu))^2 \cdot f(x) \, dx \notag \\
    a^2 \cdot \Var[X] &= a^2 \cdot \sum_{i=1}^n p_i \cdot (x_i - \mu)^2 = \sum_{i=1}^n p_i \cdot a^2 \cdot (x_i - \mu)^2 = \sum_{i=1}^n p_i \cdot (a \cdot (x_i - \mu))^2 \label{eq:a2V}
  \end{align}
  The relation $\E[X^2] = \Var[X] + \E[X]^2$ holds as well,
  \begin{align}
    \Var[X] &= \E[(X - \mu)^2] = \E[X^2] - \E[2X\mu] + \E[\mu^2] \notag & \eqref{eq:Evv} \\
            &= \E[X^2] - 2 \cdot \E\left[X \cdot \E[X]\right] + \E\left[\E[X]^2\right] & \eqref{eq:Ecv} \notag\\
            &= \E[X^2] - 2 \cdot \E[X]^2 + \E[X]^2 & \eqref{eq:Ecv} \notag\\
            &= \E[X^2] - \E[X]^2 \label{eq:VXisEX2-EX2}
  \end{align}
  We use this result to prove $\Var[T_n] = \frac{\sigma^2}{n}$.
  \begin{align}
    \Var[T_n] &= \E[T_n^2] - \E[T_n]^2 = \E[T_n^2] - 0^2 = \E[T_n^2] & \eqref{eq:VXisEX2-EX2} \eqref{eq:tn0} \label{eq:EVT_n}\\
              &= \E\left[\left(\frac{S_n}{n} - \mu\right)^2\right] = \E\left[\left(\frac{S_n}{n}\right)^2 - 2\frac{S_n}{n}\mu + \mu^2\right] \notag \\
              &= \E\left[\left(\frac{S_n}{n}\right)^2\right] - 2\cdot\E\left[\frac{\mu}{n} S_n\right] + \E\left[\mu^2\right] & \eqref{eq:Evv}\eqref{eq:Ecv} \notag\\
              &= \Var\left[\frac{S_n}{n}\right] + \E\left[\frac{S_n}{n}\right]^2 - 2\mu \cdot \E\left[\frac{S_n}{n}\right] + \mu^2 & \eqref{eq:VXisEX2-EX2}\eqref{eq:Ec} \notag \\
              &= \frac1{n^2} \cdot \Var\left[S_n\right] + \left(\frac1{n} \cdot \E\left[S_n\right]\right)^2 - \frac{2\mu}{n} \cdot (n \cdot \mu) + \mu^2 & \eqref{eq:Ecv}\eqref{eq:a2V} \notag \\
              &= \frac1{n^2} \cdot \left(\Var\left[S_n\right] + \E[S_n]^2\right) - 2\mu^2 + \mu^2 \notag \\
              &= \frac{1}{n^2} \left(\Var\left[\sum_{i=1}^n R_i\right] + \E[S_n]^2\right) - \mu^2 \notag\\
              &= \frac{1}{n^2} \left(\sum_{i=1}^n \Var[R_i] + 2\sum_{\substack{i<j \\ i,j=1}}^n \Cov[R_i,R_j] + \E[S_n]^2\right) - \mu^2 & \eqref{eq:sumVarCov} \notag\\
              &= \frac{1}{n^2} \left(\sum_{i=1}^n \sigma^2 + 0 + (n \cdot \mu)^2\right) - \mu^2 & \eqref{eq:ES_n} & \eqref{eq:CovXY0} \notag\\
              &= \frac{1}{n^2} \left(n \cdot \sigma^2 + n^2 \cdot \mu^2\right) - \mu^2 = \frac{\sigma^2}{n} + \mu^2 - \mu^2 = \frac{\sigma^2}{n}
  \end{align}
\end{proof}
%
%% REMARK: Yes, this is a quote from Wikipedia. Yes, it is true. But it is too short, because we didn't show that the covariance is zero and whether they are uncorrelated or not. So I prefer the approach above
%\index{Bienaym\'e formula}
% This proves the so-called Bienaym\'e formula.
% A shorter approach uses the definition of the mean:
% \[ \Var[\avg{T_n}] = \Var\left(\frac1n \sum_{i=1}^n X_i\right) = \frac1{n^2} \sum_{i=1}^n \Var[X_i] = \frac{\sigma^2}{n} \]
%
%Furthermore, the following equation holds:
%\begin{align}
%    \Var[T_n] &= \E\left[(T_n - \E[T_n])^2\right] = \E\left[(T_n - 0)^2\right] = \E\left[T_n^2\right] \label{eq:EVT_n}
%\end{align}
%
Now we can apply Chebyshev's Inequality Theorem ($R = T_n$, $a = \varepsilon \in \mathbb R, p = 2$):
\begin{align}
  \Prob[\abs{T_n} \geq \varepsilon]
    &\leq \frac{1}{\varepsilon^2} \E\left[\abs{T_n}^2\right]
    = \frac{1}{\varepsilon^2} \Var\left[\abs{T_n}\right]
    = \frac{1}{\varepsilon^2} \frac{\sigma^2}{n}
    & \eqref{eq:EVT_n} \label{eq:cheby-applied}
\end{align}
For any $\varepsilon > 0$ with $n \to \infty$, it holds that
\[ \Prob[\abs{T_n} \geq \varepsilon] \to 0 \]
\[
    \Leftrightarrow \forall \varepsilon > 0:
    \lim_{n\to\infty} \Prob[\abs{T_n} \geq \varepsilon] = 0
\]

This concludes the proof of the Weak Law of Large Numbers.
In order to finish our proof of the Strong Law of Large numbers,
we will use the Borel-Cantelli Lemma, an important result of measure theory.

\subsubsection{Borel-Cantelli Lemma}
\label{sec:bp-borel-cantelli}
%
\begin{lemma}[Borel-Cantelli Lemma]\label{lemma:bcl}
  Let $R_i$ with $1 \leq i < \infty$ be a sequence of events.
  Assume the sum of these probabilities is finite, then it holds that:
  \begin{align}
    \sum_{i=1}^\infty \Prob[R_i] < \infty
    &\implies
    \Prob\left(\limsup\limits_{i\to\infty} R_i\right) = 0
  \end{align}
  So the probability, that the occuring event is an event which occurs infinitely often, is 0.
\end{lemma}

\begin{proof}
  Please consider, that the limsup is defined as,
  \begin{align}
    \limsup\limits_{i\to\infty} R_i &\coloneqq \bigcap_{j=1}^\infty \bigcup_{i \geq j}^\infty R_i \label{eq:limsup-char}
  \end{align}

  The condition requires, that $\sum_{i=1}^\infty \Prob[R_i] < \infty$.
  This statement is equivalent to
  \begin{align}
    \inf_{j \geq 1} \sum_{i=j}^\infty \Prob[R_i] &= 0  \label{eq:inf0}
  \end{align}

  We can make our final conclusion to prove the Borel-Cantelli Lemma:
  \begin{align}
    \Prob\left[\limsup_{i\to\infty} R_i\right]
      &= \Prob\left[\bigcap_{j=1}^\infty \bigcup_{i=j}^\infty R_i\right] & \eqref{eq:limsup-char} \notag\\
      &\leq \inf_{j \geq 1} \Prob\left[\bigcup_{i=j}^\infty R_i\right] & \eqref{eq:prob-linear}\notag\\
      &\leq \inf_{j \geq 1} \sum_{i=j}^\infty \Prob\left[R_i\right] \notag\\
      &= 0 & \eqref{eq:inf0}
  \end{align}
  The result of an intersection of elements creates a set, which
  is an actual subset in any of these sets. However, the infimum
  is not necessarily an element of the set. Hence, an inequality
  is introduced in the second line.
\end{proof}

\subsubsection{Strong Law of Large Numbers}
\label{sec:bp-strong-law}
%
\begin{theorem}[Strong Law of Large Numbers]\label{thm:slln}
  Assume the definitions of Theorem~\ref{thm:weak-lln}.
  Therefore, $R_1, R_2, \ldots$ is an infinite sequence of independent random variables
  with a common distribution ($\mu = \E[R_j]$, $\sigma^2 = \Var[R_j]$). $S_n$ and $T_n$
  are defined. Now consider event $\mathcal E$:
  \[
    \mathcal E = \set{
      \omega \in \Omega: \lim_{n\to\infty} \frac{S_n(\omega)}{n} = \mu
    }
  \]
  Then it holds that
  \[ \Prob[\mathcal E] = 1 \]
\end{theorem}

\begin{proof}
  The following proof assumes $\sigma^2 = \E[R_j^2] < \infty$ and $\E[R_j^4] < \infty$.
  This restriction makes our proof easier, but the less restricted case $\E[R_j] < \infty$ suffices as assumption
  (but this is not proven in this thesis).

  Without loss of generality we assume $\mu = 0$. \\
  If $\mu = 0$ is not satisfied, we consider $P_j \coloneqq R_j - \mu$ instead.

  First of all, we want to give a brief outline of the proof. If it holds that,
  \[ \lim_{n\to\infty} \frac{S_n(\omega)}{n} \neq 0 \]
  then $\exists \varepsilon \in \mathbb R$ with $\varepsilon > 0$ such that for infinitely many $n$
  \[ \frac{S_n(\omega)}{n} > \varepsilon \]
  So to prove the theorem, we will prove that for every $\varepsilon > 0$,
  \[ \Prob[S_n > n \cdot \varepsilon \text{ infinitely often}] = 0 \]
  In the following, this reveals that
  \[ \Prob[\mathcal E] = \Prob\left[\frac{S_n}{n} = 0\right] = 1 \]
  proving Theorem~\ref{thm:slln}. Hence condition $\frac{S_n}{n} = 0$ holds with probability $1$.

  Now we will actually carry out the proof. We define
  \[ A_n = \set{\omega \in \Omega: S_n \geq n \cdot \varepsilon} \]
  and look at $\Prob[A_n]$ using the Chebyshev inequality (Theorem~\ref{thm:chebyshev}) with $p=4$ and $a=(n\cdot\varepsilon)$:
  \[ \Prob[S_n \geq (n\cdot\varepsilon)] \leq \frac{1}{(n\cdot\varepsilon)^4} \E[S_n^4] \]
  We must determine $\E[S_n^4]$ which equals to
  \begin{align*}
    \E\left[\sum_{1 \leq i,j,k,l}^n R_i R_j R_k R_l\right]
      &= \E\left[\sum_{1\leq i}^n \sum_{1\leq j}^n \sum_{1\leq k}^n \sum_{1\leq l}^n R_i R_j R_k R_l\right] \notag\\
%      &= \E\left[(R_1^4 + \ldots + R_1 R_n^3) + (R_2^4 + \ldots) + \ldots + (R_n^4 + \ldots) \right]
      &= \E\left[(R_1^4 + R_1^3 R_2 + \ldots + R_1^4 R_n) + \ldots + (R_n R_1^3 + \ldots + R_n^4)\right]
  \end{align*}
  Here the expected value of an addition of multiplication of random variables is asked for.
  As far as the random variables are i.i.d.,
  we equivalently look for the addition of multiplication of the expected value of a random variable
  (compare with Equation~\ref{eq:ERS-RpS} and Section~\ref{sec:bp-ev-indep}).
  Because $\E[R_i] = 0$, we can remove all terms containing $R_j$ of degree 1 for any $j$.
  These are terms of the structure (assuming $i, j, k$ and $l$ distinct),
  \[
    \E[R_i^3 R_j], \,
    \E[R_i^2 R_j R_k], \,
    \E[R_i R_j R_k R_l]
  \]
  The non-zero terms are $\E[R_i^4]$ and $\E[R_i^2 R_j^2] = \left(\E[R_i^2]\right)^2 = \sigma^4$.
  Now, we need to quantify the occurences of these non-zero terms.
  $\E[R_i^4]$ occurs $n$ times.
  Terms $\E[R_i^2 R_j^2]$ occur $3n \cdot (n-1)$ times,
    as there are $\frac{(n-1) \cdot n}{2}$ ways to choose 2 indices
    and 6 ways to find $R_i^2 R_j^2$.
  In conclusion, we determined,
  \[
    \E\left[S_n^4\right]
      = n \cdot \E\left[R_1^4\right] + 3n \cdot (n - 1) \cdot \sigma^4
      = n \cdot \left( \E\left[R_1^4\right] + 3n \cdot \sigma^4 - 3 \sigma^4 \right)
  \]
  In this expression, $n$ is our only constant occuring with polynomial degree $2$. So for any $n$ sufficiently large, there exists $C \in \mathbb R$ such that
  \begin{align}
    3 \sigma^4 n^2 + \left(\E\left[R_1^4\right] - 3 \sigma^4\right) \cdot n &\leq C \cdot n^2 \label{eq:3sigman2}
  \end{align}
  \[ \Rightarrow \E\left(S_n^4\right) \leq C n^2 \]

  We return back to Chebyshev's inequality
  \[
      \Prob[S_n \geq (n\cdot\varepsilon)]
      \leq \frac{1}{(n\cdot\varepsilon)^4} \E[S_n^4]
      \leq \frac{C \cdot n^2}{\varepsilon^4 \cdot n^2 \cdot n^2}
  \]
  Let $n_0$ be the smallest $n$ such that Equation~\eqref{eq:3sigman2} is satisfied.
  Then it follows that,
  \[
      \sum_{n \geq n_0} \Prob\left[S_n \geq n \cdot \varepsilon\right]
      \leq \sum_{n \geq n_0} \frac{C}{\varepsilon^4 n^2} < \infty
  \]
  With this approach, we skip a finite number of elements of the sum. This does not affect its convergence or divergence.
  Therefore, the conditions to apply the Borel-Cantelli Lemma (Lemma~\ref{lemma:bcl}) are satisfied.
  For every $\varepsilon > 0$ it holds that,
  \[ \Prob\left[S_n \geq n \varepsilon \text{ infinitely often}\right] = 0 \]
\end{proof}

\subsection{Marginal distribution}
\label{sec:bp-marginalization}
%
\begin{definition}
  \index{Marginal probability}
  \index{Marginalization}
  Let $\Omega$ be finite.
  Let $R$ and $S$ be two, discrete random variables. Then we define,
  \[
    \Prob[R=r, S=s] \coloneqq \Prob[\setdef{\omega \in \Omega}{R(\omega) = r \land S(\omega) = s}]
  \]
  This definition enables us to define the \emph{marginal distribution of $R$ and $S$}.
  We also say, we \emph{marginalize $S$ out of $R$}:
  \[ \Prob[R=r] = \sum_{s \in S} \Prob[R=r, S=s] \]
\end{definition}
\begin{example}
  Consider our coin tossing example again.
  Let $R$ be our random variable mapping to $\mathbb Z$ defined as result of the first coin toss
  (1 represents head, 0 represents tails).
  $\mathbb Z$-valued random variable $S$ is defined as result of the second coin toss.
  Then
  \[
    \Prob[R=1, S=0] = \Prob[\setdef{\omega \in \Omega}{R(\omega) = 1 \land S(\omega) = 0}]
                    = \Prob[\set{(h, t)}] = \frac14
  \]
  Marginalization applies, if we query $\Prob[R=1]$:
  \[
    \Prob[R=1] = \Prob[R=1, S=0] + \Prob[R=1, S=1] = \Prob[\set{(h, t)}] + \Prob[\set{(h, h)}] = \frac14 + \frac14 = \frac24
  \]
\end{example}

\subsection{Joint distribution}
\label{sec:bp-joint-distribution}
%
\begin{definition}
  Let $R_i$ be discrete random variables in $1 \leq i \leq n$.
  Joint distribution is given by the \emph{chain rule of probability}:
  \begin{align*}
    \Prob[R_1=r_1, \ldots, R_n=r_n]
        &= \Prob[R_1=r_1] \cdot \Prob[R_2=r_2|R_1=r_1] \cdots \\
        &\cdot \Prob[R_n=r_n|R_1=r_1,R_2=r_2,\ldots,R_{n-1}=r_{n-1}]
  \end{align*}
  If independence (see Section~\ref{sec:bp-indep}) of the random variables is given, it holds that
  \[ \Prob[R_i=r_i|R_1=r_1,\ldots,R_{i-1}=r_{i-1}] = \Prob[R_i=r_i] \qquad \forall i=1,\ldots,n \]
  \[ \Rightarrow \Prob[R_1=r_1, \ldots, R_n=r_n] = \prod_{i=1}^n \Prob[R_i=r_i] \]
\end{definition}
\begin{example}
  In our coin tossing example, we have that:
  \[ \Prob[R=1, S=0] = \Prob[\set{(h, t), (h, h)}] \cdot \Prob[\set{(t, t), (h, t)}] = \frac24 \cdot \frac24 = \frac14 \]
  In the marginalization example, we used the same query and used a different approach to get the same result.
\end{example}

\subsection{Independence}
\label{sec:bp-indep}
%
\index{Independence}
\index{Mutual independence}
\index{Conditional independence}
\begin{definition}
  We assume $\Omega$ is a finite set and consider
  the probability space $(\Omega, \mathcal A, \Prob)$.
  Two $\mathbb Z$-valued random variables $R$ and $S$ are called \emph{(mutually) independent} if
  \[ \Prob[R=r, S=s] = \Prob[R=r] \cdot \Prob[S=s] \]
  for any $r, s \in {\mathbb Z}$ (see, e.g.,~\cite[p.~27]{ito},~\cite[p.~143]{grindstead}).
\end{definition}

Let $A$ be an element of the $\sigma$-algebra ${\mathcal A}$.
We denote by $1_A$ the indicator function of $A$ defined as
\[
  1_A(a) = \begin{cases}
    1 & \text{ if } a \in A \\
    0 & \text{ if } a \not\in A
  \end{cases}
\]
The indicator function $1_A$ defines a ${\mathbb Z}$-valued random variable.
Let $A$ and $B$ be elements of the $\sigma$-algebra ${\mathcal A}$.
The sets (events) $A$ and $B$ are called independent
when the random variables $1_A$ and $1_B$ are independent.

It is important to distinguish between \emph{mutual independence} and \emph{pairwise independence}.
Pairwise independence of events is a weaker condition.

\begin{example}
  Consider our previous coin tossing example (Example~\ref{ex:cointoss}).
  Let us denote coin tosses $ij$, where $i$ and $j$ are the results of the first and second coin toss respectively.
  Define the random variable $R$ as
  \[ R(00)=0, \quad R(01)=0, \quad R(10)=1, \quad R(11)=1 \]
  and define the random variable $S$ (intuitively the result of the second toss) as
  \[ S(00)=0, \quad S(01)=1, \quad S(10)=0, \quad S(11)=1 \]
  Then we have
  \[
    \Prob[R=0, S=0] =
    \Prob[R=0, S=1] =
    \Prob[R=1, S=0] =
    \Prob[R=1, S=1] = 1/4
  \]
  Since $\Prob[R=i] = \frac12$ and $\Prob[S=j] = \frac12$,
  we can see $\Prob[R=i, S=j] = \Prob[R=i] \cdot \Prob[S=j]$.
  Therefore $R$ and $S$ are independent random variables.
  The elements of $\sigma$-algebra
  $A = \set{01, 11} = R^{-1}(1)$ and $B = \set{10, 11} = S^{-1}(1)$
  are independent.
  Intuitively speaking, the event getting $1$ at the first toss
  and the event getting $1$ at the second toss
  are independent.
\end{example}

\subsection{Expected value of independent random variables}
\label{sec:bp-ev-indep}
%
Let $R$ and $S$ be two random variables. We prove $\E[R \cdot S] = \E[R] \cdot \E[S]$
for the discrete and continuous case:
%
\begin{align*}
  \E[R \cdot S] &= \int_{-\infty}^{\infty} \Prob[R=x] \cdot \Prob[S=y] \cdot R(x) \cdot S(y) \, dx \, dy \\
                &= \left(\int_{-\infty}^{\infty} \Prob[R=x] \cdot R(x) \, dx\right) \left(\int_{-\infty}^{\infty} \Prob[S=y] \cdot S(y) \, dy\right) \\
                &= \E[R] \cdot \E[S] \\
  \E[R \cdot S] &= \sum_{a \in \mathcal A_R} \sum_{b \in \mathcal A_S} \left(\Prob[R=a] \cdot \Prob[S=b] \cdot R(a) \cdot S(b)\right) \\
                &= \left(\sum_{a \in \mathcal A_R} \Prob[R=a] \cdot R(a)\right) \cdot \left(\sum_{b \in \mathcal A_S} \Prob[S=b] \cdot S(b)\right) \\
                &= \E[R] \cdot \E[S]
\end{align*}

\subsection{Conditional independence}
\label{sec:bp-cond-indep}
%
Let $R$, $S$, and $T$ be $\mathbb Z$-valued random variables.
The random variables $R$ and $S$ are conditionally independent under the given random variable $T$
if
\[
  \ProbCond{R=r, S=s}{T=t} =
  \ProbCond{R=r}{T=t} \cdot \ProbCond{S=s}{T=t}
\]
for any $r, s, t \in \mathbb Z$ (see, e.g.,~\cite[3.1]{dawid}).
Let
\[
  A_r = R^{-1}(r) \subseteq \Omega \qquad
  B_s = S^{-1}(s) \subseteq \Omega \qquad
  C_t = T^{-1}(t) \subseteq \Omega
\]
Then we can consider $\Prob[A_r]$, $\Prob[B_s]$, and $\Prob[C_t]$ as probability distribution functions associated to $R$, $S$, and $T$ respectively.
We note that we have
\begin{align*}
  \ProbCond{R=r, S=s}{T=t} &= \frac{\Prob[A_r \cap B_s \cap C_t]}{\Prob[C_t]} \\
  \ProbCond{R=r}{T=t} &= \frac{\Prob[A_r \cap C_t]}{\Prob[C_t]} \\
  \ProbCond{S=r}{T=t} &= \frac{\Prob[B_r \cap C_t]}{\Prob[C_t]}
\end{align*}
by the definition of the conditional probability.

\subsection{Bayes' Theorem}
\label{sec:bp-bayes}
%
\begin{theorem}[Bayes' Theorem]
  Let $A$ and $B$ be two events. Let $\Prob[B] \neq 0$. Then:
  \[ \Prob[A|B] = \frac{\Prob[B|A] \cdot \Prob[A]}{\Prob[B]} \]
\end{theorem}
\begin{proof}
  Due to the chain of probability, the following statement holds:
  \[ \Prob[A,B] = \Prob[B|A] \cdot \Prob[A] = \Prob[A|B] \cdot \Prob[B] \]
  Bayes' theorem follows immediately:
  \[ \frac{\Prob[B|A] \cdot \Prob[A]}{\Prob[B]} = \Prob[A|B] \]
\end{proof}

Bayes' Theorem is fundamental to theory we will cover in the following.
$\Prob[A]$ is called \emph{prior probability} and $\Prob[A|B]$ is called \emph{posterior probability} in the Bayesian interpretation.
The names derive from the fact, that $\Prob[A]$ is known beforehand in most applications and $\Prob[A|B]$ is the degree of belief in $A$ after $B$ happened.

\section{Probability distributions}
\label{sec:bp-dist}
%
Probability distributions are templates for probability density functions
satisfying the criteria mentioned in Section~\ref{sec:bp-continuous}.
They are parameterized by one or more variables.
Besides the continuous distributions presented here, discrete distributions also exist.

\subsection{Normal distribution}
\label{sec:bp-norm-dist}
%
\index{Normal distribution}
\index{Gaussian distribution}
\index{Bell curve}
The \emph{Normal distribution} (also \emph{Gaussian distribution} or informally \emph{Bell curve}) is a continuous
probability distribution parameterized by $\mu$ (the mean of the distribution) and $\sigma^2$ (variance).
\[ f(x \,|\, \mu, \sigma^2) = \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-\frac{(x - \mu)^2}{2 \sigma^2}} \]
The distribution is visualized with 4 configurations in Figure~\ref{fig:norm-dist}.

\begin{figure}[p]
  \begin{center}
    \includegraphics[width=0.7\textwidth]{img/normal_distribution.pdf}
    \caption{
      Four different normal distributions with $\mu=0$ and $\sigma^2 = 0.2$ [blue],
      $0.5$ [red], $1.0$ [green], and $5.0$ [yellow]
    }
    \label{fig:norm-dist}
  \end{center}
\end{figure}

\subsection{Beta distribution}
\label{sec:beta-dist}
%
\index{Beta distribution}
The \emph{Beta distribution} is parameterized by $a$ and $b > 0$.
\[ f(x \,|\, a, b) = \frac{1}{B(a, b)} x^{a - 1} (1 - x)^{b-1} \]
where $B(a, b) = \int_0^1 x^{a-1} (1 - x)^{b-1} \, dx$, the so-called beta function.
The distribution is visualized with 4 configurations in Figure~\ref{fig:beta-dist}.

\begin{figure}[p]
  \begin{center}
    \includegraphics[width=0.7\textwidth]{img/beta_distribution.pdf}
    \caption{
      Four different beta distributions:
      $(a=0.5, b=1.5)$ [blue], $(1, 2)$ [red], $(2.5, 4)$ [green], and $(5, 5)$ [yellow]
    }
    \label{fig:beta-dist}
  \end{center}
\end{figure}

\subsection{Independent and identically distributed}
\label{sec:bp-iid}
%
Let $R_i$ be a set of $n$ random variables. $R_i$ are called \emph{independent and
identically distributed} (i.i.d.) if they are mutually independent and each
variable follows the same probability distribution.

\begin{example}
  Three board game players possess two fair dice each.
  Every player is asked to throw them and tell the sum of these two thrown dice.

  All three die sums share the same distribution.
  Sum $2$ is thrown with a chance of $\frac{1}{36}$.
  Sum $7$ is thrown with a chance of $\frac{6}{36}$ (etc.).
  The result of one player does not influence the result of the other player.
  They are mutually independent.
\end{example}

\section{Graphical models}
\label{sec:gm}
%
\index{Bayesian network}
\index{Graphical model}
In order to specify a probabilistic model, you have to provide the following definitions:
\begin{itemize}
  \item A set of random variables and their co-domain ($E$ is either $\mathbb Z$ or $\mathbb R$)
  \item The random variables' dependencies (according to Section~\ref{sec:bp-indep})
  \item You can assign probabilities (conditional probabilities or unconditioned probabilities)
\end{itemize}
%
A graphical model explicitly defines the dependencies using a graph.
Particularly, we looked at \emph{Bayesian networks}, which use a directed acyclic graph (DAG).

Given a Bayesian network, we can make queries (ask for determination of a probability).
Using all the rules about probability theory we learned before, we can derive new knowledge
and might be able to answer the query.

Graphical models furthermore employ two interesting properties.

\subsection{Joint probability of the network}
\label{sec:gm-joint-prob}
%
The joint probability of the Bayesian network with random variables $R_1, R_2, \ldots, R_n$ is given with
\[ \Prob[R_1, \ldots, R_n] = \prod_{i=1}^n \ProbCond{R_i}{\operatorname{ParentNodes}(R_i)} \]
where $\operatorname{ParentNodes}(R_i)$ denotes all the nodes $R_i$ depends on.
This property can be helpful to answer queries.

\subsection[d-separation]{$d$-separation}
\label{sec:gm-d-separation}
%

\subsection{Tool: Bayesian inference}

\TODO{Show symmetry, decomposition, weak union and contraction, via Dawid et al.}

\section{Example: Polynomial curve fitting problem}
\label{sec:bp-curve-fitting}
%
\subsection{The problem}
%
\TODO{visualization}

In the following, we introduce the curve fitting problem similar to Bishop~\cite[p.~4~ff.]{Bishop}.
The problem is defined as follows:

\begin{problem}[Polynomial curve fitting problem]
  Consider a polynomial of arbitrary degree.

  \begin{description}
  \item{Given}
  $x = (x_1, \ldots, x_n)^N$ as a vector of $N$ x-values and
  $t = (t_1, \ldots, t_n)^N$ as the corresponding y-values drawn from the polynomial.
  Furthermore let $E(w)$ be an error function for given polynomial coefficients $w$.

  \item{Find}
  a polynomial with coefficients $w$ which approximates values $t$ minimizing $E(w)$.
  \end{description}
\end{problem}

The degree of the polynomial is purposely unknown.
\emph{Model selection} is a branch of Machine Learning dedicated to finding appropriate models for given problems.
So for polynomial degree choice for our curve fitting problem, we refer to research literature in Model Selection. \TODO{provide useful references for Curve Fitting}
\index{Error function}
An \emph{error function} in this context is any function using $w$, $x$ and $t$ to determine a numeric value representing how much $w$ deviates from $t$ at $x$.
Popular error functions include
\begin{align}
  E(w) &= \frac12 \sum_{n=1}^N \left(y(x_n, w) - t_n\right)^2 \tag{Mean squared error, MSE} \\
  E(w) &= \sqrt{\frac{1}{N} \sum_{n=1}^N (y(x_n, w) - t_n)^2} \tag{Root mean square, RMS} \\
  E(w) &= \frac1N \sum_{n=1}^N (y(x_n, w) - t_n)              \tag{Mean signed deviation, MSD}
\end{align}
% TODO: error function does not need to be derivable, right? But in our design, we derive it to find the gradient to minimize it. So probably it is not required because of backpropagation, but by gradient descent.

\begin{figure}[!h]
  \begin{center}
    \begin{subfigure}[b]{0.45\textwidth}
      \includegraphics[width=\textwidth]{img/interpolation-linear.pdf}
      \caption{Linear interpolation}
      \label{fig:linear-interpolation}
    \end{subfigure}%
    \begin{subfigure}[b]{0.45\textwidth}
      \includegraphics[width=\textwidth]{img/interpolation-nonlinear.pdf}
      \caption{Nonlinear interpolation}
      \label{fig:nonlinear-interpolation}
    \end{subfigure}
    \caption{
      In Figure~\ref{fig:linear-interpolation}, y-values $t_1$ and $t_2$ can be interpolated
      easily using a linear approach. But with more $y$-values (3, in the case of Figure~\ref{fig:nonlinear-interpolation}),
      the problem often requires non-linear solutions.
    }
    \label{fig:interpolation}
  \end{center}
\end{figure}

\subsection{Overfitting}
\label{sec:bp-overfitting}
%
\index{Training data}
\index{Validation data}
\index{Test data}
\index{Overfitting}
Machine Learning distinguishes between a \emph{training} and \emph{validation} dataset as input.
It uses the training set to learn which output is desired for some given input.
Therefore all elements of the training set are labelled such that the error in the output can be quantified.
\emph{Overfitting} describes the situation, when the learning algorithm approximates the output with little error,
but input from the validation set (which contains different inputs) is computed with high error.
So the algorithm perfectly adapted itself to recognize the training data, but performs badly for any other input.

\TODO{visualization}

\subsection{Regularization as countermeasure}

In order to restrict the neural network from overfitting, you penalize it for using
large weights.

\[ E(w) = \frac12 \sum_{n=1}^N (y(x_n, w) - t_n)^2 + \frac{\lambda}{2} \abs{w}^2 \]

Here, the term $\frac{\lambda}{2} \abs{w}^2$ was added where $\abs{w}^2$ denotes the
matrix multiplication
\[ \abs{w}^2 = w^T \cdot w = w_0^2 + w_1^2 + \ldots + w_M^2 \]

% introduce gaussian distribution to every data point

We now model the problem from a probabilistic view:

\subsection{Maximum Likelihood Estimator}
%
% In particular, explain a theoretical reason of adding the term $|w|^2$ for
% the MLE problem in terms of the Baysian theorem.
%
% The Bayesian prior is exponentially to |w|^2. This is a common criticism of the application
% of MLE to Bayesian theory. There is always the prior as assumption, which in this case has computational implications.
% Computational experiments are required to verify the use.
%
The Maximum Likelihood Estimator (MLE) is a technique to estimate the parameters of a probability distribution.
It maximizes the likelihood that the given data actually occurs.

\TODO{Illustrate that the Curve Fitting problem is considered Bayesian here}

\begin{theorem}
  Consider input data $x$, mean $\mu$ and variance $\sigma^2$:
  \[ \ln{\Prob[x | \mu, \sigma^2]} = -\frac1{2\sigma^2} \sum_{n=1}^N (x_n - \mu)^2 - \frac{N}{2} \ln{\sigma^2} - \frac{N}{2} \ln(2\pi) \]
  Then
  $\mu_{\text{ML}} = \frac{1}{N} \cdot \sum_{n=1}^N x_n$ for maximized $\mu$ and \\
    $\sigma_{\text{ML}} = \frac{1}{N}\cdot \sum_{n=1}^N (x_n - \mu_{\text{ML}})^2$ for maximized $\sigma^2$
\end{theorem}

So we want to determine the 2 parameters of a Gaussian distribution, namely $\mu$ and $\sigma^2$, in the maximum likelihood case.
We begin with $\mu$:

\begin{proof}
\begin{enumerate}
  \item Derive $\ln{\Prob[x| \mu, \sigma^2]}$ for $\mu$
    \begin{align*}
      \frac\partial{\partial \mu} \ln{\Prob[x | \mu, \sigma^2]}
      &= \frac\partial{\partial \mu} \left(-\frac{1}{2\sigma^2} \cdot \sum_{n=1}^N (x_n - \mu)^2 - \frac{N}{2} \ln{\sigma^2} - \frac{N}{2} \ln(2\pi)\right) \\
      &= \frac\partial{\partial \mu} \left(-\frac{1}{2\sigma^2} \cdot \sum_{n=1}^N (x_n^2 - 2 x_n \mu + \mu^2) - \frac{N}{2} \ln{\sigma^2} - \frac{N}{2} \ln(2\pi)\right) \\
      &= -\frac{1}{2\sigma^2} \cdot \sum_{n=1}^N (-2x_n + 2\mu) \\
      &= -\frac1{\sigma^2} \cdot \sum_{n=1}^N (\mu - x_n)
    \end{align*}
  \item Set result zero
    \[ 0 = -\frac{1}{\sigma^2} \cdot \sum_{n=1}^N (\mu - x_n) = \sum_{n=1}^N (\mu - x_n) = N \cdot \mu - \sum_{n=1}^N x_n \]
    \[ \implies \mu_{\text{ML}} = \frac1N \cdot \sum_{n=1}^N x_n \qquad \text{commonly called \enquote{sample mean}} \]
\end{enumerate}
\end{proof}

We continue with $\sigma^2$ and use the same approach:

\begin{proof}
  \begin{enumerate}
  \item Derive $\ln{\Prob[x | \mu, \sigma^2]}$ for $\sigma^2$
    \begin{align*}
      \frac{\partial}{\partial \sigma^2} \ln{\Prob[x | \mu, \sigma^2]}
      &= \frac{\partial}{\partial \sigma^2} \left(-\frac{1}{2\sigma^2} \cdot \sum_{n=1}^N (x_n - \mu)^2 - \frac{N}2 \ln{\sigma^2} - \frac{N}2 \ln(2\pi)\right) \\
      &= \frac{1}{2\sigma^4} \cdot \sum_{n=1}^N (x_n - \mu)^2 - \frac{N}{2} \cdot \frac{1}{\sigma^2} \\
      &= \frac{1}{2\sigma^2} \left(\frac{1}{\sigma^2} \cdot \sum_{n=1}^N (x_n - \mu)^2 - N\right)
    \end{align*}
  \item Set result zero
    \begin{align*}
      0 &= \frac{1}{2\sigma^2} \left(\frac{1}{\sigma^2} \cdot \sum_{n=1}^N (x_n - \mu)^2 - N\right) \\
      N \cdot \sigma^2 &= \sum_{n=1}^N (x_n - \mu)^2 \\
      \sigma^2_{\text{ML}} &= \frac{1}{N} \cdot \sum_{n=1}^N (x_n - \mu)^2 \qquad \text{commonly called \enquote{sample variance}}
    \end{align*}
  \end{enumerate}
\end{proof}

And now we derive the precision parameter $\beta$ in the maximum likelihood case:

\begin{theorem}
  Given
  \[ \ln{\Prob[t | x, w, \beta]} = -\frac{\beta}{2} \cdot \sum_{n-1}^N \left(y(x_n,w) - t_n\right)^2 + \frac{N}2 \ln{\beta} - \frac{N}{2} \ln(2\pi) \]
  then find
  \[ \frac{1}{\beta_{\text{ML}}} = \frac{1}{N} \cdot \sum_{n=1}^N (y(x_n, w_{\text{ML}}) - t_n)^2 \] by maximizing $\beta$
\end{theorem}

\begin{proof}
\begin{enumerate}
  \item Derive $\ln{\Prob[t | x,w,\beta]}$ with $\beta$
    \begin{align*}
      \frac{\partial}{\partial \beta} \ln{\Prob[t | x,w,\beta]}
      &= \frac{\partial}{\partial \beta} \left(-\frac{\beta}{2} \sum_{n=1}^N (y(x_n,w) - t_n)^2 + \frac{N}{2} \ln\beta - \frac{N}{2} \ln(2\pi)\right) \\
      &= -\frac12 \cdot \sum_{n=1}^N \left(y(x_n,w) - t_n\right)^2 + \frac{N}{2} \cdot \frac1\beta
    \end{align*}
  \item Set result zero
    \begin{align*}
      0 &= -\frac{1}{2} \cdot \sum_{n=1}^N \left(y(x_n,w) - t_n\right)^2 + \frac{N}{2\beta} \\
      \frac{N}{\beta} &= \sum_{n=1}^N \left(y(x_n,w) - t_n\right)^2 \\
      \frac{1}{\beta_{\text{ML}}} &= \frac{1}{N} \cdot \sum_{n=1}^N (y(x_n,w) - t_n)^2
    \end{align*}
\end{enumerate}
\end{proof}

The maximum of the logarithm of an expression corresponds to the minimum of the negative logarithm of the same expression. \TODO{so why do we minimize and not maximize?}

\begin{align}
  -\log{\Prob[\omega|x,t,\alpha,\beta]}
    &\propto -\log\left[\Prob[t|x,\omega,\beta] \cdot \Prob[\omega|\alpha]\right] \\
\intertext{due to proportionality, $\exists c \in \mathbb R$ such that} \\
    &= -\log{\Prob[t|x,\omega,\beta]} - \log\Prob[\omega|\alpha] - \log{c} \\
\intertext{insert formula Bishop 1.62} \\
    &= \frac\beta2 \sum_{n=1}^N \left(y(x_n,\omega)-t_n\right)^2 - \frac{N}{2} \ln\beta + \frac{N}{2} \ln{2\pi} - \log\left(\left(\frac{\alpha}{2\pi}\right)^{\frac{M+1}{2}} \cdot \exp\left(-\frac{\alpha}{2} \omega^\transpose \omega\right)\right) - \log{c} \\
    &= \frac\beta2 \sum_{n=1}^N \left(y(x_n,\omega)-t_n\right)^2 - \frac{N}{2} \ln\beta + \frac{N}{2} \ln{2\pi} - \frac{M+1}{2} \log\alpha + \frac{M+1}{2} \log{2\pi} + \frac{\alpha}{2} \omega^\transpose \omega - \log{c}
\end{align}

Let $f$ be any function with a minimum. Then $\argmin_{\omega} f(\omega) = \argmin_{\omega} c \cdot f(\omega) + a$ for any $c, a \in \mathbb R$.
This applies also to our case:

\begin{align}
  \argmin_{\omega} -\log{\Prob[\omega|x,t,\alpha,\beta]}
    &= \argmin_{\omega} \left(\frac\beta2 \cdot \sum_{n=1}^N \left(y(x_n, \omega) - t_n\right)^2 + \frac\alpha2 \omega^\transpose \omega\right) \\
    &= \argmin_{\omega} \beta\left(\frac12 \sum_{n=1}^N \left(y(x_n, \omega) - t_n\right)^2 + \frac{\frac\alpha\beta}{2} \omega^\transpose \omega^\transpose\right) \\
    &= \argmin_{\omega} \frac12 \sum_{n=1}^N \left(y(x_n, \omega) - t_n\right)^2 + \frac{\frac\alpha\beta}{2} \omega^\transpose \omega^\transpose \\
\end{align}

Hence, the coefficients maximizing the probability that the coefficients correspond to our model parameters ($x, t, \alpha, \beta$) are given in the last line.
Considering we determine the best coefficients by minimizing the error function, it is justified to consider these coefficients as optimum. Let $\lambda = \frac{\alpha}{\beta}$, then \dots

\[ \tilde{E}(\omega) = \frac12 \sum_{n=1}^N \left(y(x_n, \omega) - t_n^2\right)^2 + \frac{\lambda}{2} \|\omega\|^2 \]


\subsection{Lagrange's theorem}

Lagrange's theorem states that for any finite group \(G\), the order of every subgroup is a divisor of the order of \(G\).

Consider subset \(H\). We know that all cosets are disjoint, and that the union of all cosets is \(G\).

As cosets are the same size, we know that:

\(|G|=m|H|\), where \(m\) is the number of cosets.

This means that if a group has order \(10\), a subgroup must have order \(1\), \(2\) \(5\) or \(10\).


\subsection{Big \(O\) and little-\(o\) notation}

\subsubsection{Big \(O\) notation}

In big \(O\) notation we are interested in t he size of a function as it getes larger. We ignore constant multiples.

\(cx\in O(x)\)

And addition of constants.

\(cx+b\in O(x)\)

If there are two terms and one is larger, we keep the largest.

\(x+x^2\in O(x^2)\)

More generally we write:

\(f(x)\in O(g(x))\)

\subsubsection{Little-\(o\) notation}


\section*{Problem 1: Background Material [20pts] (Devendra \& Hyun-Ah)}

\subsection*{1.1 Probability review (Devendra)}

Consider a naive classifier which stochastically assigns labels to data points; specifically, for each label $l$, it assigns that label with probability proportional to the number of times that label appears in the training dataset. Let the size of training set be $N$, total number of classes be $C$ and $n_i$ be number of datapoints of class $i$ ($\sum_i^C{n_i} = N$), then the probability of labelling a datapoint with class $i$ is $n_i/N$.

1) Consider a training set with $N=100$, $C=2$, $n_1 = 50$, $n_2=50$ and a test set with, $N=100$, $C= 2$, $n_1 = 30$, $n_2 = 70$.\\
a) What is the expected accuracy of the classifier on the training set?\\
b) What is the expected accuracy of the classifier on the test set?\\

2) Now consider a training set with $N=100$, $C=3$, $n_1 = 50$, $n_2=20$, $n_3=30$ and a test set with, $N=100$, $C= 3$, $n_1 = 20$, $n_2 = 20$, $n_3 = 60$.\\
a) What is the expected accuracy of the classifier on the training set?\\
b) What is the expected accuracy of the classifier on the test set?\\

This kind of analysis can be used to calculate a baseline accuracy for various Machine Learning algorithms.

\subsection* {Problem 1.2: Bayes Rule [10 pts] (Hyun-Ah) }

Smith owns a retail store for selling phones. The phones are manufactured at three different factories: $A, B, C$. Factory A, B, and C produces $20\%, 30\%,$ and $50\%$ of the phone being sold at Smith's store. The probabilities of the defective phones from stores $A, B,$ and $C$ are $2\%, 1\%,$ and $0.05\%$, respectively. The total number of phones being sold at Smith's store is $10000$.
One day, a customer walks up to Smith's store, and ask for a refund for a defective phone.

a) What is the probability of a phone being defective?

b) What is the probability that this defective phone is manufactured at factory $A$?

c) What is the probability that this defective phone is manufactured at factory $B$?

d) What is the probability that this defective phone is manufactured at factory $C$?

\newpage


\chapter{True Statements in Propositional Logic}
\label{chapter:propositional-truth}
\marginurl{%
  Proofs Using Truth Tables:\\\noindent
  Introduction to Mathematical Logic \#2
}{youtu.be/DOAxnmScpPc}

Typical theorem in mathematics have the following template:
``if some statements are true, then another statement is also true''.
In propositional logic statements are described using propositional formulas.
So our goal is to present a way to describe proofs of results that looks like:
if $\phi_1$, \dots, $\phi_k$ are true, then $\psi$ is also true.

This section discusses the method which is based on truth tables (we discussed
them before in \Cref{chapter:predicates}).

We start from an example similar to
the proof gaven in the beginning of the \Cref{chapter:proofs}. Assume that we
know that if $x$ is a real number such that $x < -2$ or $x > 2$, then $x^2 > 4$.
We can derive that if $\lnot (x^2 > 4)$, then $\lnot (x < -2)$ and
$\lnot (x > 2)$.

In order to emphasize the logical structure of the argument let us denote the
statement $x > 2$ by $p$, the statement $x < -2$ by $q$, and the statement
$x^2 > 4$ by $r$. In this case the argument is as follows:
if $(p \lor q) \limplies r$ is true, then
$\lnot r \limplies (\lnot p \land \lnot q)$ is true as well.

The simplest way to explain why this argument is true is to use
a truth table.
\begin{center}
  \begin{tabular}{l l l l l}
    \toprule
    $p$ & $q$ & $r$ & $(p \lor q) \limplies r$ &
    $\lnot r \limplies (\lnot p \land \lnot q)$ \\
    \midrule
    T & T & T & T & T \\
    T & T & F & F & F \\
    T & F & T & T & T \\
    T & F & F & F & F \\
    F & T & T & T & T \\
    F & T & F & F & F \\
    F & F & T & T & T \\
    F & F & F & T & T \\
    \bottomrule
  \end{tabular}
\end{center}
Note that each line where $(p \lor q) \limplies r$ is true has
$\lnot r \limplies (\lnot p \land \lnot q)$ true as well. So we proved that
the argument is indeed correct.

We may also note that we showed that
\[
  \bigl((p \lor q) \limplies r\bigr) \iff
  \bigl(\lnot r \limplies (\lnot p \land \lnot q) \bigr)
\]
is always true (we say that this propositional formula is a
\emph{tautology}). A generalization of this saying the if $p \limplies q$ is
true, then $\lnot q \limplies \lnot p$ is also true is called the
\emph{contraposition} argument.

Let us now consider another argument. If we know that Joe was a good boy and we
know that if Joe is a good boy, then Santa gives a present to Joe. We may
conclude that Santa gives a present to Joe. We can similarly to the previous
example write this argument using variables and connectives.
If we know that $p$ and $p \limplies q$, we may conclude that $q$ is true.
\begin{exercise}
  Show that $(p \land (p \limplies q)) \limplies q$ is a tautology.
\end{exercise}
Such an argument is called \emph{modus ponens}.

A notion connected to being a tautology is the notion of being satisfiable.
We say that a formula (a set of formulas) is \emph{satisfiable} iff there
is a substitution to the variables such that the value of the formula is true
(the values of all the formulas are true). Note that a formula is not
satisfiable (the formula is \emph{unsatisfiable}) iff its negation is
a tautology. Therefore, using truth tables one may check whether a formula
is satisfiable or not.\footnote{%
  Note that the procedure is awfully not efficient since if the formula
  uses $n$ variables we need to do $2^n$ operations. Unfortunately,
  we do not know anything that always works better since
  satisfiability problem (the problem of determining whether a given formula is
  satisfiable or not) is NP complete.
}

\section{Semantic Implication}
As we mentioned at the beginning of the chapter, most of the
statements in mathematics are in the form
``if some statements are true, then some statement is also true'';
this type of statements can be described using the notion of semantic
implication. We say that a set $\Sigma$ of propositional formulas with
variables from a set $V$ \emph{semantically implies} a propositional formula
$\phi$ with variables from the set $V$ (we denote it by $\Sigma \models \phi$)
iff whenever all the formulas from $\Sigma$ are true under some propositional
assignement to $V$, the formula $\phi$ is also true under this propositional
assignement; i.e.,
$\Sigma \models \phi$ iff for any $\rho : V \to \set{\ltrue, \lfalse}$,
$\substitute{\phi}{\rho} = \ltrue$ provided that
$\substitute{\psi}{\rho} = \ltrue$ for all $\psi \in \Sigma$.
(Note that the set $\Sigma$ may be infinite.)

We explained that if we have a finite set $\Sigma$, then it is possible to check
whether a formula $\phi$ is semantically implied by $\Sigma$. Let us try to find
out whether we can do the same for infinite sets $\Sigma$.

Partial answer to this question is given by the following theorem.
\begin{theorem}[compactness theorem]
  A set $\Sigma$ of propositional formulas is satisfiable iff every finite
  subset is satisfiable.
\end{theorem}
\begin{proof}
  We say that a set is \emph{finitely satisfiable} if every finite subset
  is satisfiable.

  Let us enumerate all the propositional formulas $\alpha_1$, $\alpha_2$, \dots.
  We define a family of sets $\Delta_1$, \dots, $\Delta_n$, \dots
  such that $\Delta_1 = \Sigma$ and
  \[
    \Delta_{n + 1} =
    \begin{cases}
      \Delta_n \cup \set{\alpha_{n + 1}} & \text{if }
        \Delta_n \cup \set{\alpha_{n + 1}} \text{is finitely satisfiable,} \\
      \Delta_n \cup \set{\lnot \alpha_{n + 1}} & \text{otherwise.}
    \end{cases}
  \]
  Note that all the $\Delta_n$ are finitely satisfiable.

  Let $\Delta = \cup_{n \in \N} \Delta_n$. It is clear that $\Delta$ is
  finitely satisfiable and for any propositional formula $\alpha$,
  either $\alpha$ or $\lnot \alpha$ belongs to $\Delta$.

  Let us consider a substitution $v_1$, \dots, $v_n$, \dots to the variables
  $x_1$, \dots, $x_n$, \dots such that $v_i = \ltrue$ iff the formula $x_i$
  belongs to $\Delta$. We may note that this substitution satisfies any
  formula $\phi \in \Delta$.
\end{proof}

Using this theorem, we can show that any implication of an infinite set is
actually an implication of a finite subset of it.
\begin{corollary}
  Let $\Sigma$ be a set of propositional formulas over the variables
  $x_1$, $x_2$, \dots, $x_n$, \dots, and $\phi$ be a propositional formula
  over the same set. If $\Sigma \models \phi$, then there is a finite
  $\Sigma' \subseteq \Sigma$ such that $\Sigma' \models \phi$.
\end{corollary}
\begin{proof}
  Note that $\Sigma \not\models \phi$ iff $\Sigma \cup \set{\phi}$ is
  satisfiable.

  Let us now assume that for any finite $\Sigma' \subseteq \Sigma$,
  $\Sigma' \not\models \phi$. This implies that $\Sigma' \cup \set{\phi}$
  is satisfiable for all finite $\Sigma'$. Therefore, $\Sigma \cup \set{\phi}$
  is satisfiable, which is a contradiction to the assumption that
  $\Sigma \models \phi$.
\end{proof}

Therefore if we wish to check whether a formula $\phi$ is semantically implied
by $\Sigma$, we just need to brute-force all the finite subsets of $\Sigma$ and
check whether they semantically imply $\phi$. By the previous argument, if
$\phi$ is implied by $\Sigma$, this procedure reports ``yes'' at some point,
and in the opposite case it will work infinitely long.

\begin{chapterendexercises}
  \exercise Show that $((A \land B) \lor C) \limplies (A \lor C)$ is a
    tautology.
  \exercise Show that $((A \limplies B) \land (\lnot A \limplies C)) \limplies
    (B \lor C)$ is a tautology.
  \exercise Show that
    $((W \limplies X) \land (Y \limplies Z)) \limplies
      ((W \lor Y) \limplies (X \lor Z))$
     is a tautology.
\end{chapterendexercises}