\section*{Data Clustering \& Mixture Models}
\subsection*{K-Means}
$\mathbf{Z} \in \{0,1\}^{N\times K}$ (if point $i$ assigned to cluster $j$) \\
\textbf{Target:} $\min_{\mathbf{U}, \mathbf{Z}} J(\mathbf{U}, \mathbf{Z}) = \|\mathbf{X} - \mathbf{U} \mathbf{Z}\|_F^2$\\
$= \sum_{n=1}^N \sum_{k=1}^K \mathbf{z}_{k,n} \|\mathbf{x}_n - \mathbf{u}_k\|_2^2$\\
1. \textbf{Initiate:} choose $K$ centroids $\mathbf{U} = [\mathbf{u}_1, \ldots, \mathbf{u}_K]$\\
2. \textbf{Cluster Assign:} assign data points to closest cluster  $z_{ij}^* = 1 \text{ if } j=\argmin_k||\mathbf{x}_i - \mathbf{u}_k||^2 \text{ else } 0$\\
3. \textbf{Update centroids}: $\mathbf{u}_k = \frac{\sum_{n=1}^N z_{k,n} \mathbf{x}_n}{\sum_{n=1}^N z_{k,n}}$. Repeat 2\\
Stop if $||\mathbf{Z} - \mathbf{Z_{new}}||_F^2 = 0$.
Guaranteed to converge to local optimum.
Computational cost: $O(k\cdot n \cdot d)$
\subsection*{Gaussian Mixture Models (GMM)}
$p(x;\mu;\Sigma)=\frac{1}{|\Sigma|^{\frac{1}{2}}(2\pi)^{\frac{D}{2}}}\mathit{exp}[-\frac{1}{2}(x-\mu)^T \Sigma^{-1} (x-\mu)]$ \\
$\int \mathcal{N}(z;\mu, \Sigma) \log(\mathcal{N}(z;0,I)) dz = E_z[\mathcal{N}(z;0, I)]$\\
$=-D/2 \log(2\pi)-1/2 \sum_{i=1}^D(\mu_i^2 + \sigma_i^2)$ \\
For GMM let $\boldsymbol{\theta}_k = (\boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k)$; $p_{\theta_k}(\mathbf{x}) = \mathcal{N}(\mathbf{x} | \boldsymbol{\mu}_k, \Sigma_k)$\\
\textbf{Mixture Models:} $p_\theta(\mathbf{x}) = \sum_{k=1}^K \pi_k p_{\theta_k}(\mathbf{x})$\\
\textbf{Generate:}
sample cluster $j \sim Categorical(\pi)$, \\
sample data from $j$-th cluster: $x \sim \mathcal{N}(\mu_j, \Sigma_j)$\\
\textbf{Assignment variable (generative model):} \\
$z_{ij} \in \{0, 1\}$, $\sum_{j=1}^k z_{ij} = 1$\\
$\operatorname{Pr}(z_k = 1) = \pi_k \Leftrightarrow p(\mathbf{z}) = \prod_{k=1}^K \pi_k^{z_k}$\\
\textbf{Complete data distribution:}\\
$p_\theta(\mathbf{x}, \mathbf{z}) = \prod_{k=1}^K \left( \boldsymbol{\pi}_k p_{\theta_k}(\mathbf{x})\right)^{z_k}$\\
\textbf{Posterior Probabilities:} $\\\operatorname{Pr}(z_k = 1 | \mathbf{x}) = \frac{\operatorname{Pr}(z_k = 1) p(\mathbf{x} | z_k = 1)}{\sum_{l=1}^K \operatorname{Pr}(z_l = 1) p(\mathbf{x} | z_l = 1)} = \frac{\boldsymbol{\pi}_k p_{\theta_k}(\mathbf{x})}{\sum_{l=1}^K \boldsymbol{\pi}_l p_{\theta_l}(\mathbf{x})}$\\
$\text{posterior } p(A|B)=\frac{\text{prior } p(A)\ \times \ \text{likelihood } p(B|A)}{\text{evidence } p(B)}$\\
\textbf{Likelihood of observed data $\mathbf{X}$:}\\
$p_\theta(\mathbf{X}) = \prod_{n=1}^N p_\theta(\mathbf{x}_n) = \prod_{n=1}^N \left(\sum_{k=1}^K \pi_k p_{\theta_k}(\mathbf{x}_n)\right)$
\textbf{Max. Likelihood Estimation (MLE):}\\
$\argmax_\theta\sum_{n=1}^N \log \left( \sum_{k=1}^K \pi_k p_{\theta_k}(\mathbf{x}_n)\right)$ (mult. 1)\\
$\ge \sum_{n=1}^N \sum_{k=1}^K{q_{nk}[\log p_{\theta_k}(\mathbf{x}_n) + \log \pi_k - \log q_{nk}]}$\\
with $\sum_{k=1}^K{q_{nk}} = 1$ by Jensen Inequality.
\subsection*{Expectation-Maximization (EM) for GMM}
\textbf{E-Step:}
$Pr[z_{j}=1|\mathbf{x}_i] = q_{ij}=\frac{\pi_j p(\mathbf{x}_i;\theta_j)}{\sum_{l=1}^K \pi_l p(\mathbf{x}_i;\theta_l)}$ \\
\textit{Derivation:} Can maximize independent of $i$. Take derivative of lower bound w.r.t. $q_{ij}$ with Lagrangian: $\lambda(\sum_j q_{ij} -1)$ to get: $\log \pi_j +\log p(x_i|\mu_j, \Sigma_j)-\log q_{ij} - 1 + \lambda = 0 \to q_{ij}=\pi_j p(x_i|\mu_j, \Sigma_j) e^{\lambda-1}$. Now use $\sum_j q_{ij}=\sum_j \pi_j p(x_i|\mu_j, \Sigma_j)e^{\lambda-1}=1 \to e^{\lambda -1}= 1/(\sum_j pi_j p(x_i|\mu_j, \Sigma_j))$ and plug in.
\\
\textbf{M-Step:}
$\boldsymbol{\mu}_j^{*} := \frac{\sum_{i=1}^N q_{ij} \mathbf{x}_i}{\sum_{i=1}^N q_{ij}}$
$, \pi_j^* := \frac{1}{N} \sum_{i=1}^N q_{ij}$\\
$\Sigma_k^{*} = \frac{\sum_{i=1}^N q_{ij} (\mathbf{x}_i - \boldsymbol{\mu}_j)(\mathbf{x}_i - \boldsymbol{\mu}_j)^\top}{\sum_{i=1}^N q_{ij}}$\\
\textit{Derivations:}  Take derivative of lower bound w.r.t. $\pi_k$ with Lagrangian: $\lambda (\sum_k \pi_k - 1)$ to get $\sum_i q_{ik} (1 / \pi_k) + \lambda = 0 \to \pi_k = (\sum_i q_{ik}) / \lambda$. Then use $\sum_k \pi_k=1 \to (\sum_k \sum_i q_{ik})/\lambda = N / \lambda = 1 \to \lambda = 1 / N$ and plug in. For $\mathbf{\mu}_k$: $-\sum_i q_{ik} \Sigma_k^{-1}(\mathbf{x}_i - \mathbf{\mu}_i)=\Sigma_k^{-1}\sum_i q_{ik}(\mathbf{x}_i - \mathbf{\mu}_k)$ (note: $\Sigma$ is symmetric \& invertible so $Ax=0$ iff $x=0$), so $\sum_i q_{ik}(\mathbf{x}_i - \mathbf{\mu}_k) = 0 \to \mathbf{\mu}_k = (\sum_i q_{ik}\mathbf{x_i}) / (\sum_i q_{ik})$
\\
Guaranteed to converge to local optimum. \\
\textbf{Comparison to K-Means}\\
Soft assignments (not hard), learn cov. matrix (not spherical clusters), slow (not fast), more (not less) iterations, use K-means as initialization (use sample coveriance as matrix, use fraction of datapoints as mixing weights). K-means as a special case of GMM with covariances $\Sigma_j = \sigma^2 I$. in the limit of $\sigma \rightarrow 0$, recover K-means (hard assignments). \\
\textbf{Model Order Selection (AIC / BIC for GMM)}\\
Trade-off between data fit (i.e. likelihood $p(\mathbf{X} | \theta)$) and complexity (i.e. \# of free parameters $\kappa(\cdot)$). Compare for different parameters and take smallest:\\
$\operatorname{AIC}(\theta | \mathbf{X}) = -\log p_\theta(\mathbf{X}) + \kappa(\theta)$\\
$\operatorname{BIC}(\theta | \mathbf{X}) = -\log p_\theta(\mathbf{X}) + \frac{1}{2} \kappa(\theta) \log N$ \\
(penalizes complexity more)\\
\textbf{Example:} \#free params, fixed cov. matrix: $\kappa(\theta) = K \cdot D + (K - 1)$ ($K$: \# clusters, $D$: $\mathsf{dim}\text{(data)}=\mathsf{dim}(\mu_i)$), full cov. matrix: $\kappa(\theta) = K(D + \frac{D(D+1)}{2}) + (K - 1)$.


\subsection{Velocities}
Suppose $f(t)$ is a position function of an object, representing the displacement of the object from the origin at time $t$.
In terms of derivatives, the \dfont{velocity of an object is:}
$$v(a)=f'(a)$$
The change of velocity with respect to time is called the \dfont{acceleration} and can be found as follows:
$$a(t)=v'(t)=f''(t).$$
Acceleration is the derivative of the velocity function and the second derivative of the position function.

\begin{example}{Position, Velocity and Acceleration}{PositionFunction}
Suppose the position function of an object is $f(t)=t^2~metres$ at $t$ seconds.
Find the velocity and acceleration of the object at time $t=1s$.
\end{example}

\begin{solution}
By the definition of velocity and acceleration we need to compute $f'(t)$ and $f''(t)$.
Using the definition of derivative, we have,
$$f'(t)=\lim_{h\to 0}\frac{(t+h)^2-t^2}{h}=\lim_{h\to 0}\frac{2th+h^2}{h}=\lim_{h\to 0}(2t+h)=2t.$$
Therefore, $v(t)=f'(t)=2t$.
Thus, the velocity at time $t=1$ is $v(1)=2~m/s$.
We now have that the acceleration at time $t$ is:
$$a(t)=f''(t)=\lim_{h\to 0}\frac{2(t+h)-2t}{h}=\lim_{h\to 0}\frac{2h}{h}=2.$$
Therefore, $a(t)=2$.
Substituting $t=1$ into the function $a(t)$ gives $a(1)=2~m/s^2$.
\end{solution}


\chapter{Exponential Decay}

In a previous chapter, we saw that an investment of $P$ getting
compound interest with an annual interest rate of $r$, grows
exponentially. At the end of year $t$, your balance would be

$$P\left(1 + r\right)^t$$

Because $r$ is positive, this number grows as time passes.  You get a
nice exponential growth curve that looks something like this:

\includegraphics[width=0.7\textwidth]{exponential_growth.png}

This is \$30 invested with 10\% annual interest rate. So the formula
for the balance after $t$ years would be

$$(30)(1.1)^t$$

What if $r$ were negative? This would be \textit{exponential decay}.

\section{Radioactive Decay}

Until around 1970, there were companies making watches whose faces and
hands were coated with radioactive paint. The paint usually contained
radium. When a radium atom decays, it gives off some energy, loses two
protons and two neutrons, and becomes becomes a different element
(radon). Some of the energy given off is visible light. Thus, these
watches glow in the dark.\index{radioactive decay}

How many of the radium atoms in the paint decay each century? About 4.24\%.

Notice the quantity of atoms lost is proportional to the number of
atoms you have. This is exponential decay. If we assume that we start
with a million radium atoms, the number of atoms decreases over time like this:

\includegraphics[width=0.7\textwidth]{radium_decay.png}

\begin{itemize}
\item We start off with 1,000,000 atoms.
\item At 16 centuries, we have only 500,000 (half as many) left.
\item 16 centuries after that, we have only 250,000 (half again) left.
\item 16 centures after that, we have only 125,000 (half again) left.
\end{itemize}

A nuclear chemist would say that radium has a \textit{half-life} of
1,600 years. Note that this means that if you bought a watch with
glowing hands in 1960, it will be glowing half as brightly in the year
3560.\index{half-life}

How do we calculate the amount of radium left at the end of century
$t$? If you start with $P$ atoms, at the end of the $t$-th century you
will have

$$P\left(1 - 0.0424\right)^t$$

This is exponential decay.\index{exponential decay}

\section{Model Exponential Decay}

Let's say you get hired to be run a company with 480,000
employees. Each year $1/8$ of your employees leave the company for
some reason (retirement, quitting, etc.). For some reason, you never
hire any new employees.

Make a spreadsheet that indicates how many of the original 480,000
employees will still be around at the end of each year for the next 12.  Then make a
bar graph from that data.


\subsection{Special orthogonal groups \(SO(n, F)\)}

The special orthogonal group, \(SO(n,F)\), is the subgroup of the orthogonal group where \(|M|=1\).

As a result it includes only the rotation operators, not the flip operators.

\(SO(3)\) is rotations in 3d space.

\(SO(2)\) is rotations in 2d space.

\subsubsection{Determinant of the orthogonal group}

The orthogonal group has determinants of \(-1\) or \(1\).

\(O^T=O^{-1}\)

\(\det (O^T)=\det (O^{-1})\)

\(\det O=\dfrac{1}{\det O}\)

\(\det O=\pm 1\)


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS671: Machine Learning
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 2}

Let $\mathcal{X}$ be a set of examples.
Suppose that the hypotheses space consists of \textit{all} functions $h : \mathcal{X} \longrightarrow \{-\infty, \infty \}$.
Prove that any unobserved example satisfies exactly half of hypotheses in the current version space, regardless of which training examples had been observed.

\subsection*{Solution}

Prove is given by induction on the number $n$ of attributes.

\begin{itemize}
\item \textit{Base case} ($n = 1$)\\
When we have only one attribute, the set of hypotheses $\mathcal{H}$ would only have two elements $h_1$ and $h_2$ where $h_1$ is $x_1$ and $h_2$ is $\bar{x_1}$.
As set of possible unobserved examples has only two elements $x_1$ and $\bar{x_1}$, we argue that $x_1$ is either positive or negative but not both at the same time.
If $x_1$ is positive, it only satisfies hypothesis $h_1$.
If it is negative, it only satisfies $h_2$.
Same argument can be made for $\bar{x_1}$.
Thus exactly half of hypotheses in version space would be satisfied.

\item \textit{Induction Step}\\
When there are $k$ attributes, set of hypotheses space would have $2^k$ elements $h_i$ where $1 \leq i \leq 2^k$.
We define set of unobserved examples in this case as $\mathcal{S}$.
$\mathcal{S}$ would have $2^k$ elements as well.
We form the inductive hypothesis as any unobserved example $c$ would satisfy exactly half of the possible hypotheses in current version space.
Our objective now is to show that the statement would still hold true if we add one more attribute.

By adding one more attribute $x_{k+1}$, number of hypotheses in set of hypotheses space as well as number of elements in set of possible unobserved examples $\mathcal{S}^{\prime}$ would change to $2^{k+1}$.
We can write $\mathcal{S}^{\prime}$ as given by Equation \ref{eq21}.

\begin{equation}\label{eq21}
\mathcal{S}^{\prime} = \{c^{\prime} | c^{\prime} \in \{cx_{k+1}, c\overline{x_{k+1}}, \overline{c}x_{k+1}, \overline{c}\overline{x_{k+1}}\}, c \in \mathcal{S} \}
\end{equation}

Now we can argue, if $x_{k+1}$ is positive only hypotheses that are satisfied by examples of the form $cx_{k+1}$ or $\overline{c}x_{k+1}$ are satisfied and if $x_{k+1}$ is negative, only hypotheses that are satisfied by examples of the form $c\overline{x_{k+1}}$ or $\overline{c}\overline{x_{k+1}}$ are satisfied.
Thus any concept $c$ would satisfy exactly half of hypotheses in $\mathcal{S}^{\prime}$.

\end{itemize}


\subsection{Measure space}

In a metric space, the structure was defining a value for each two elements of the set.

In a measure space, the structure defines a value of subsets of the set.

A measure space includes the set \(X\), subsets of the set, \(\Sigma \), and a function \(\mu \) which maps from \(\Sigma \) to \(\mathbb{R}\).

\subsubsection{Sigma algebra}

Requirement for \(\Sigma \).


\chapter{Vector spaces}

In Chapter~\ref{cha:vectors-rn}, we considered $\R^n$, the set of
$n$-dimensional column vectors. We now introduce a more general
concept of ``vector'', as an element of an abstract vector
space. Basically, vectors are entities that can be added and
scaled. While some vectors look like lists of numbers (for example,
column vectors, row vectors), other kinds of vectors don't look like
lists of numbers at all (for example, functions, polynomials). Part of
the power of linear algebra comes from our ability to find vector
spaces in many unexpected places.

Much of the content of this chapter will be a repetition of things we
have already seen in Chapter~\ref{cha:vectors-rn} in the context of
$\R^n$. For example, we will be talking about linear combinations,
linear independence, spanning sets, bases, subspaces, linear
transformations, and so on. We initially introduced these concepts in
the context of the vector space $\R^n$, so that they would be easier
to understand. We will now see that they in fact apply to \textit{all}
vector spaces.


\subsubsection{Discrete Distributions}

The discrete distributions packages contains the following.

\subsubsection*{Binomial}

The binomial distribution (class name Binomial) has the parameters $p$ for the prior proportion of successes and $n$ for the total number of trials and calculates the probability of $y$ successes
$$
P(y; n; p) = \sum_{i=1}^n {n \choose y_i} p^y_i (1-p)^{n-y_i}
$$
The properties of the binomial are:\\\\
Mean: $\mu = np$\\
Variance: $\sigma^2 = np \left( 1-p \right) = npq$

\subsubsection*{Geometric Distribution}
The geometric distribution (class name Geometric) with 1 parameter for probability p and $y \ge 1$.
The variable represents the nth trial where the success occurs (for instance if y=2 then the trial was successful on the 2nd attempt). The parameter p represents the probability of success. The probability  function calculates the probability of success at the nth trial.
$$
P(y;p) = p(1-p)^{y-1}
$$
The simple properties of the distribution are:\\\\
Mean: $\mu = \frac{1}{p}$\\
Variance: $\sigma^2 = \frac{1-p}{p^2}$

\subsubsection*{Hyper Geometric Distribution}
The Hypergeometric (class name HyperGeometric) distribution represents the probability of choosing y number of events of the same kind from a subset of r like items within a population of all N possible items (of different kinds) for the sample of size n containing the mixed items. The constraints are such that $r \le n \le N$ and $y \le r \le n$. The parameters are $y,r,n,N$.
The probability distribution is defined as follows.
$$
P(y; r,n,N) = \frac{ {r \choose y } {{N - r} \choose {n - y} } } {  {N \choose n} }
$$
The simple properties of the distribution are:\\\\
Mean: $\mu = \frac{nr}{N}$\\
Variance: $\sigma^2 = n \left( \frac{r}{N} \right) \left( \frac{N - r}{N} \right) \left( \frac{N - n}{N-1}\right)$

\subsubsection*{Negative Binomial Distribution}
The Negative Binomial Distribution (class name NegativeBinomial) provides the probability of the nth success or potentially nth failure of a bernoulli trial. The parameters are r representing the (r -1) initial trials that where the successful and y the total number of trials before the next success r occurs. The distribution is calculated as follows:
$$
P(y;r) = {y - 1 \choose r - 1}p^rq^{y-r}
$$
where $y = r, r + 1, ...$\\\\
The simple properties of the distribution are:\\\\
Mean: $\mu = \frac{r}{p}$\\
Variance: $\sigma^2 = \frac{r(1-p)}{p^2}$

\subsubsection*{The Poisson Distribution}
The Poisson Distribution (class name Poisson) provides the probability of an event occurring a certain number of times within an interval. It is commonly used to model a number of events occuring in a certain period of time. We can use the two parameters lambda and y to represent the number of events, and period of time respectively.
The distribution is defined as:
$$
P(y;\lambda) = \frac{ \lambda^y e^{-\lambda} }{ y! }
$$
where $y \ge 0$ and $\lambda > 0$\\
The simple properties of the distribution are:\\\\
Mean: $\mu = \lambda$\\
Variance: $\sigma^2 = \lambda$


\chapter{Metric Spaces}
Up until this point, everything we've discussed has risen out of just the original vector space $V$; we created the dual space, vectors, covectors, tensor product spaces, and basis transformations based only on the original definition of $V$, its scalar field $\mathbb{F}$, and its basis $\vec{e}_\mu$.
Now, we're going to add a new object into the mix.
This will be the \emph{inner product} or \emph{metric}, denoted $\rho: V \times V \to \mathbb{F}$.
Vector spaces which are equipped with this function are known as \emph{metric spaces} or \emph{inner product spaces}.
We want this function to satisfy some key properties.
\begin{enumerate}
    \item $\rho$ must map an ordered pair of vectors to the scalar field of $V$.
    \item $\rho$ must be linear in both arguments; then $\rho(\lambda\vec{v},\vec{u}) = \lambda \rho(\vec{v},\vec{u})$ and $\rho(\vec{v} + \vec{u},\vec{w}) = \rho(\vec{v},\vec{w}) + \rho(\vec{u},\vec{w})$.
    \item[3*.] $\rho$ must be symmetric; $\rho(\vec{v},\vec
    u) = \rho(\vec{u},\vec{v})$.
    \item[4*.] $\rho(\vec{v},\vec{v}) \geq 0$, and $\rho(\vec{v},\vec{v}) = 0$ if and only if $\vec{v} = \vec{0}$. This is known as a \emph{positive-definite} form. If this rule does not hold, it is a \emph{indefinite} form.
    If $\rho$ is positive-definite, then $V$ is a \emph{Reimannian} metric space; if it is indefinite, $V$ is a \emph{pseudo-Reimannian} metric space.
\end{enumerate}
These last two conditions are marked with asterisks because the definition of $\rho$ when used in general relativity differs from the definition of $\rho$ when discussing pure mathematics.
In general relativity, we insist that $\rho$ be symmetric, but we don't require that $\rho(\vec{v},\vec{v})$ be greater than or equal to 0, so General Relativity is based on a pseudo-Remannian metric space.
This function is a specific function, not a set of all possible functions like the dual space is.
By adding this specific function, we promote $V$ to become a metric space.

There are a few notational conventions for an inner product.
\begin{itemize}
    \item A named function, like $\rho(\cdot,\cdot)$.
    \item Using ordered pairs, like $(\cdot, \cdot)$.
    \item The Bra-Ket notation of quantum mechanics, $\langle \cdot \mid \cdot \rangle$.
    \item The braket notation which we have so far used to mean mappings, $\langle \cdot, \cdot \rangle$.
    \item The elementary vector notation of the dot product, where $\vec{v}\cdot\vec{u}$ is the inner product of $\vec{v}$ and $\vec{u}$.
\end{itemize}
Throughout this lesson, we will use the ordered pair notation of an inner product.

\subsection{Creating the Inner Product}
We define the inner product by what defining what happens to the basis vectors of $V$.
Since the metric is linear, we can simply factor out the scalars and use these definitions to calculate the inner product of any two vectors.
In special relativity, we use the \emph{Minkowski metric}\footnote{Some books take the reverse convention, where $\eta_{00} = 1$ and $\eta_{11}, \eta_{22}, \eta_{33} = -1$. This choice between $(-,+,+,+)$ and $(+,-,-,-)$ is known as the \emph{metric signature}.} $\eta$, where
\[ (\vec{e}_\mu,\vec{e}_\nu) = \eta_{\mu\nu},\quad \eta_{\mu\nu} =
    \begin{cases}
        -1 & \mu = \nu = 0 \\
        1 & \mu = \nu \in \{1,2,3\} \\
        0 & \mu \not= \nu
    \end{cases}.
\]
This can be represented by a $4\times4$ matrix with each element having the value of $\eta_{r,c}$, like
\[
	\eta =
    \begin{bmatrix}
        -1 & 0 & 0 & 0 \\
         0 & 1 & 0 & 0 \\
         0 & 0 & 1 & 0 \\
         0 & 0 & 0 & 1
    \end{bmatrix}.
\]
In general relativity, however, we don't have the luxury of picking such nice values.
Instead, we look to what values nature seems to prefer.


\section{Vectors and Polytopes}
\label{polytopes-sect}

We will often need to refer to sets of natural numbers. We follow the
notation $[n]=\{ i\in\naturals\ |\ 1\leq i\leq n \}$.

We denote the transpose of a matrix $A$ as $A\T$.
Vectors will be considered as column vectors, so $u\T v$ is
the scalar product of $u,v\in\reals^d$. A vector for which all components
are $0$'s will be denoted as $\0$. A vector for which all
components are $1$'s will be denoted as $\1$. The $i$-th component of
the {\em $i$-th unit vector} $e_i$ is equal to 1, whereas
all its other components are 0.
An inequality of the form $u\geq v$ is intended to hold for every component.

An {\em affine combination} of points in an Euclidean space
$z_i\in\reals^d$, where $i\in[m]$,
is $\sum_{i\in [m]} \lambda_i z_i$ where $\lambda_i\in\reals$ such that
$\sum_{i\in [m]} \lambda_i = 1$.
The points $z_i$ are {\em affinely independent} if none of them
is an affine combination of the others.
A {\em convex combination} of the points $z_i$ is such an
affine combination with $\lambda_i\geq 0$ for all $i\in [m]$.
The {\em convex hull} of a set $Z$ of points is the set of all
its convex combinations. We denote it as $\conv(Z)$, so
\[
\conv \{z_i\ \mid\ i\in [m] \}=\{ \sum_i \lambda_i z_i\ \mid\ %
\lambda_i\geq 0
\text{ for } i\in [m],~
\sum_i \lambda_i=1 \}~.
\]
A set of points $Z$ is {\em convex} if $Z=\conv(Z)$. It has
{\em dimension} $d$ if it contains exactly $d + 1$ affinely independent
points. The convex hull of $d+1$ points of dimension
$d$ is called a {\em $d$-simplex}. The {\em standard $d$-simplex} is
the convex hull of the first $d+1$ unit vectors, that is,
 $\Delta_d=\conv\{ e_i\ \mid\ i\in[d+1] \}$.

A {\em polytope} is the convex hull of a finite set of points, not
necessarily affinely independent.
Its {\em dimension} is its dimension as a convex set.
A {\em polyhedron} is the intersection of finitely many closed halfspaces
$\{ x\in \reals^d\ \mid\ a\T x\leq a_0\}$. It can be shown that a bounded
polyhedron is a polytope.
A {\em vertex} of a
$d$-dimensional polytope $P=\conv(Z)$ is a
point $z\in Z$ that is not the convex combination of other
points in $P$.
% you have not defined convex hulls of infinite sets
% such that $\conv(Z\setminus \{ z \})\neq P$;
An {\em edge} of P is a 1-dimensional line segment that has two vertices
as endpoints.
A {\em facet} of $P$ is a set of dimension $d-1$ that is the convex hull of
a set of vertices that lie on a hyperplane of the form
$\{ x\in \reals^d\ \mid\ a\T x=a_0\}$ so that $a\T u<a_0$ for all other
vertices
$u$ of $P$.
A $d$-dimensional polytope $P$ is {\em simplicial} if it is the convex hull
of a set of at least $d+1$ points $v\in \reals^d$ such that no
$d+1$ of them are on a common hyperplane. This is equivalent to requiring
that every facet of $P$ is a $d$-simplex. A $d$-dimensional polytope
$P$ is {\em simple}
if every point of $P$ lies on at most $d$ facets. Notice that the points
lying on exactly $d$ facets are exactly the vertices.
Consider a polytope $P$ which has $\0$ in its
interior and is given by $n$ inequalities with normal
vectors $c_i$ for $i\in[n]$ according to
\[
P=\{ x\in\reals^d\ \mid\ c_i\T x\leq 1,\ i\in [n]~\}.
\]
Then the {\em polar} of the polytope $P$ is denoted by $P^\Delta$, where
\[
P^\Delta = \conv\{ c_i\ \mid\ i\in [n]~\}.
\]
If $P$ is a polytope and contains the origin $\0$, the
polar of its polar is $P$ itself, $P^{\Delta\Delta}=P$.
Furthermore, if $P$ is simplicial then $P^\Delta$
is simple, and vice versa.
For details see Ziegler~\cite{ziegler}.


\subsection{Bezout's identity}

For any two non-zero natural numbers \(a\) and \(b\) we can select natural numbers \(x\) and \(y\) such that

\(ax+by=c\)

The value of \(c\) is always a multiple of the greatest common denominator of \(a\) and \(b\).

In addition, there exist \(x\) and \(y\) such that \(c\) is the greatest common denominator itself. This is the smallest positve value of c..

Let's take two numbers of the form \(ax+by\):

\(d=as+bt\)

\(n=ax+by\)

Where \(n>d\). And \(d\) is the smallest non-zero natural number form.

We know from Euclidian division above that for any numbers \(i\) and \(j\) there is the form \(i=jq+r\).

So there are values for \(q\) and \(r\) for \(n=dq+r\).

If \(r\) is always zero that means that all values of \(ax+by\) are multiples of the smallest value.

\(n=dq+r\) so \(r=n-dq\).

\(r=ax+by-(as+bt)q\)

\(r=a(x-sq)+b(y-tq)\)

This is also of the form \(ax+by\). Recall that \(r\) is the remainder for the division of \(d\) and \(n\), and that \(d=ax+by\) is the smallest positive value.

\(r\) cannot be above or equal to \(d\) due to the rules of euclidian division and so it must be \(0\).

As a result we know that all solutions to \(ax+by\) are multiples of the smallest value.

As every possible \(ax+by\) is a multiple of \(d\), \(d\) must be a common divisor to both numbers. This is because \(a.0+b.1\) and \(a.1+b.0\) are also solutions, and \(d\) is their divisor.

So we know that the smallest positive solution is a common mutliple of both numbers.

We now need to show that that \(d\) is the largest common denominator. Consider a common denominator \(c\).

\(a=pc\)

\(b=qc\)

And as before:

\(d=ax+by\)

So:

\(d=pcx+qcy\)

\(d=c(px+qy)\)

So \(d\ge c\)


\section{Linear Support Vector Regression}

In the case of regression the goal is to predict a real-valued output for $y'$ so that our training data is of the form:

\begin{equation}
	\{(x_i,y_i), x\in\Re^m, y_i\in\Re, i=1, \dots, n\} \label{eq:svr_data}
\end{equation}

The regression SVM use a loss function that not allocating a penalty if the predicted value $y'_i$ is less than a distance $\epsilon$ away from the actual value $y_i$, i.e., if $|y_i-y'_i| \leq \epsilon$, where $y'_i = w^T x_i + b$. The region bound by $y'_i\pm\epsilon \ \forall_i$ is called an $\epsilon$-insensitive tube. The output variables which are outside the tube are given one of two slack variable penalties depending on whether they lie above, $\xi^+$, or below, $\xi^-$, the tube, provided $\xi^+ \geq 0$ and $\xi^- \geq 0 \ \forall_i$:

\begin{equation} \label{eq:svr_consts}
	\begin{aligned}
		& y_i\leq y'_i+\epsilon+\xi^+ \ \forall_i \\
    	& y_i\geq y'_i-\epsilon-\xi^- \ \forall_i \\
    	& \xi_i^+, \xi_i^- \geq 0 \ \forall_i
	\end{aligned}
\end{equation}

The objective function for SVR can then be written as:

\begin{equation} \label{eq:quad_svr_obj}
    \begin{aligned}
        \min_{w,b,\xi^+,\xi^-} \quad & \frac{1}{2} \| w \|^2 + C \sum_{i=1}^n (\xi_i^+ + \xi_i^-) \\
            \text{subject to} \quad & y_i - w^T x_i - b \leq \epsilon + \xi_i^+ \ \forall_i \\ & w^T x_i + b - y_i \leq \epsilon + \xi_i^- \ \forall_i \\ & \xi_i^+, \xi_i^- \geq 0 \ \forall_i
    \end{aligned}
\end{equation}

\begin{figure}[h!]
	\centering
  	\includegraphics[scale=0.6]{img/linear_dual_svr_hyperplane}
  	\caption{Linear SVR hyperplane}
  	\label{fig:linear_dual_svr_hyperplane}
\end{figure}

\subsection{Epsilon-insensitive loss}

The \emph{epsilon-insensitive} loss is defined as:

\begin{equation} \label{eq:eps_loss1}
	\mathcal{L}_\epsilon =
	\begin{cases}
		0 & \text{if} \ |y - (w^T x + b)| \leq \epsilon \\
		|y - (w^T x + b)| - \epsilon & \text{otherwise} \\
	\end{cases}
\end{equation}

or, equivalently:

\begin{equation} \label{eq:eps_loss2}
	\mathcal{L}_\epsilon = \max(0, |y - (w^T x + b)| - \epsilon)
\end{equation}

As the \emph{hinge} loss, also the \emph{epsilon-insensitive} loss is a nondifferentiable convex function due to its nonsmoothness in $\pm\epsilon$, but has a subgradient wrt $w$ that is given by:

\begin{equation} \label{eq:eps_loss_der}
	\frac{\partial \mathcal{L_\epsilon}}{\partial w}=
		\begin{cases}
            (y - (w^T x + b)) x & \text{if} \ |y - (w^T x + b)| > \epsilon \\
            0 & \text{otherwise} \\
        \end{cases}
\end{equation}

\subsubsection{Primal formulation}

The general primal unconstrained formulation takes the same form of~\eqref{eq:primal_svc}.

The quadratic optimization problem~\eqref{eq:quad_svr_obj} can be equivalently formulated as:

\begin{equation} \label{eq:svr_eps}
	\min_{w,b} \frac{1}{2} \| w \|^2 + C \sum_{i=1}^n \max(0, |y_i - (w^T x_i + b)| - \epsilon)
\end{equation}

where we make use of the \emph{epsilon-insensitive} loss~\eqref{eq:eps_loss1} or~\eqref{eq:eps_loss2}.

The above formulation penalizes slacks $\xi$ linearly and is called $\mathcal{L}_1$-SVR.

\begin{figure}[h!]
	\centering
  	\includegraphics[scale=0.4]{img/svr_eps_loss}
  	\caption{SVR Epsilon-insensitive loss with different optimization steps}
  	\label{fig:svr_eps_loss}
\end{figure}

\subsubsection{Wolfe Dual formulation}

To reformulate the~\eqref{eq:quad_svr_obj} as a \emph{Wolfe dual}, we introduce the Lagrange multipliers $\alpha_i^+ \geq 0, \alpha_i^- \geq 0, \mu_i^+ \geq 0, \mu_i^- \geq 0 \ \forall_i$:

\begin{equation} \label{eq:svr_wolfe_dual}
	\begin{aligned}
    	\max_{\alpha^+,\alpha^-,\mu^+,\mu^-} \min_{w,b,\xi^+,\xi^-} \mathcal{W}(w,b,\xi^+,\xi^-,\alpha^+,\alpha^-,\mu^+,\mu^-) = \frac{1}{2} \| w \|^2 + C \sum_{i=1}^n (\xi_i^+ + \xi_i^-)-\sum_{i=1}^n (\mu_i^+ \xi_i^+ + \mu_i^- \xi_i^-) \\ -\sum_{i=1}^n \alpha_i^+(\epsilon+\xi_i^+ + y'_i-y_i)-\sum_{i=1}^n \alpha_i^-(\epsilon+\xi_i^- - y'_i+y_i)
	\end{aligned}
\end{equation}

Substituting for $y_i$, differentiating wrt $w, b, \xi^+$, $\xi^-$ and setting the derivatives to $0$ gives:

\begin{equation} \label{eq:svr_wolfe_der_w}
	\frac{\partial \mathcal{W}}{\partial w}=w-\sum_{i=1}^n (\alpha_i^+ - \alpha_i^-) x_i \Rightarrow w=\sum_{i=1}^n (\alpha_i^+ - \alpha_i^-) x_i
\end{equation}

\begin{equation} \label{eq:svr_wolfe_der_b}
	\frac{\partial \mathcal{W}}{\partial b}=-\sum_{i=1}^n (\alpha_i^+ - \alpha_i^-)\Rightarrow \sum_{i=1}^n (\alpha_i^+ - \alpha_i^-)=0
\end{equation}

\begin{equation}\label{eq:svr_wolfe_der_xip}
	\frac{\partial \mathcal{W}}{\partial\xi_i^+}=0\Rightarrow C=\alpha_i^+ + \mu_i^+
\end{equation}

\begin{equation} \label{eq:svr_wolfe_der_xim}
	\frac{\partial \mathcal{W}}{\partial\xi_i^-}=0\Rightarrow C=\alpha_i^- + \mu_i^-
\end{equation}

Substituting~\eqref{eq:svr_wolfe_der_w} and~\eqref{eq:svr_wolfe_der_b} in, we now need to maximize $\mathcal{W}$ wrt $\alpha_i^+$ and $\alpha_i^-$, where $\alpha_i^+ \geq 0,\ \alpha_i^- \geq 0 \ \forall_i$:

\begin{equation} \label{eq:svr_max_wolfe_dual}
    \max_{\alpha^+,\alpha^-} \mathcal{W}(\alpha^+,\alpha^-) = \sum_{i=1}^n y_i(\alpha_i^+ - \alpha_i^-)-\epsilon\sum_{i=1}^n (\alpha_i^+ + \alpha_i^-)-\frac{1}{2}\sum_{i,j}(\alpha_i^+ - \alpha_i^-)\langle x_i, x_j \rangle(\alpha_j ^+ - \alpha_j ^-)
\end{equation}

Using $\mu_i^+ \geq 0$ and $\mu_i^- \geq 0$ together with~\eqref{eq:svr_wolfe_der_w} and~\eqref{eq:svr_wolfe_der_b} means that $\alpha_i^+ \leq C$ and $\alpha_i^- \leq C$. We therefore need to find:

\begin{equation} \label{eq:svr_min_wolfe_dual}
    \begin{aligned}
        \min_{\alpha^+,\alpha^-} \quad & \frac{1}{2}(\alpha^+ - \alpha^-)^TK(\alpha^+ - \alpha^-)+\epsilon q^T(\alpha^+ + \alpha^-)-y^T(\alpha^+ - \alpha^-) \\
            \text{subject to} \quad & 0\leq\alpha_i^+,\alpha_i^- \leq C \ \forall_i \\ & q^T(\alpha^+ - \alpha^-)=0
    \end{aligned}
\end{equation}

where $q^T = [1, \dots, 1]$.

We can write the~\eqref{eq:svr_min_wolfe_dual} in a standard quadratic form as:

\begin{equation}
    \begin{aligned} \label{eq:svr_min_qp_wolfe_dual}
        \min_{\alpha} \quad & \frac{1}{2}\alpha^T Q\alpha-q^T\alpha \\
            \text{subject to} \quad & 0\leq\alpha_i\leq C \ \forall_i \\ & e^T\alpha=0
    \end{aligned}
\end{equation}

where the Hessian matrix $Q$ is
$
\begin{bmatrix}
K & -K\\
-K & K
\end{bmatrix}$
, $q$ is
$
\begin{bmatrix}
-y\\
y
\end{bmatrix}$ + $\epsilon$
, and $e$ is
$
\begin{bmatrix}
1\\
-1
\end{bmatrix}$.

Each new predictions $y'$ can be found using:

\begin{equation} \label{eq:svr_pred}
    y'= \sum_{i=1}^n (\alpha_i^+ - \alpha_i^-)\langle x_i, x' \rangle+b
\end{equation}

A set $S$ of support vectors $x_s$ can be created by finding the indices $i$ where $0\leq\alpha\leq C$ and $\xi_i^+=0$ or $\xi_i^-=0$.

This gives us:

\begin{equation} \label{eq:svr_b}
    b=y_s-\epsilon-\sum_{m\in S}(\alpha_m^+ -\alpha_m^-) \langle x_m, x_s \rangle
\end{equation}

As before it is better to average over all the indices $i$ in $S$:

\begin{equation} \label{eq:svr_b_avg}
    b=\frac{1}{N_s}\sum_{s\in S}y_s-\epsilon-\sum_{m \in S}(\alpha_m^+ - \alpha_m^-)\langle x_m, x_s \rangle
\end{equation}

From~\eqref{eq:svr_min_qp_wolfe_dual} we can notice that the equality constraint $e^T \alpha = 0$ arises form the stationarity condition $\partial_{{b}} \mathcal{W}=0$. So, again, for simplicity, we can again consider the bias term $b$ embedded into the weight vector. We report below the box-constrained dual formulation~\cite{hsu2002simple} that arises from the primal~\eqref{eq:primal_svc_hinge1} or~\eqref{eq:primal_svc_hinge2} where the bias term $b$ is embedded into the weight vector $w$:

\begin{equation} \label{eq:svr_min_bcqp_wolf_dual}
    \begin{aligned}
        \min_{\alpha} \quad & \frac{1}{2} \alpha^T (Q + ee^T)\alpha+q^T\alpha \\
            \text{subject to} \quad & 0\leq\alpha_i\leq C \ \forall_i
    \end{aligned}
\end{equation}

\subsubsection{Lagrangian Dual formulation}

In order to relax the constraints in the \emph{Wolfe dual} formulation~\eqref{eq:svr_min_wolfe_dual} we define the problem as a \emph{Lagrangian dual} relaxation by embedding them into objective function, so we need to allocate the Lagrangian multipliers $\mu \geq 0, \lambda_+ \geq 0$, $\lambda_- \geq 0$:

\begin{equation} \label{eq:svr_lagrangian_dual}
	\begin{aligned}
		    \max_{\mu,\lambda_+,\lambda_-} \min_{\alpha} \mathcal{L}(\alpha,\mu,\lambda_+,\lambda_-) &= \frac{1}{2} \alpha^T Q\alpha+q^T\alpha - \mu^T (e^T \alpha) - \lambda_+^T (u - \alpha) - \lambda_-^T \alpha \\
    &= \frac{1}{2} \alpha^T Q\alpha + (q - \mu e + \lambda_+ - \lambda_-)^T \alpha - \lambda_+^T u
	\end{aligned}
\end{equation}

where the upper bound $u^T = [C, \dots, C]$.

Taking the derivative of the Lagrangian $\mathcal{L}$ wrt $\alpha$ and settings it to 0 gives:

\begin{equation} \label{eq:svr_lagrangian_der_a}
	\frac{\partial \mathcal{L}}{\partial \alpha}=0\Rightarrow Q \alpha + (q - \mu e + \lambda_+ - \lambda_-) = 0
\end{equation}

With $\alpha$ optimal solution of the linear system:

\begin{equation} \label{eq:svr_lagrangian_sol}
    Q \alpha = - (q - \mu e + \lambda_+ - \lambda_-)
\end{equation}

the gradient wrt $\mu$, $\lambda_+$ and $\lambda_-$ are:

\begin{equation} \label{eq:svr_lagrangian_der_mu}
	\frac{\partial \mathcal{L}}{\partial \mu}=-e \alpha
\end{equation}

\begin{equation} \label{eq:svr_lagrangian_der_lp}
	\frac{\partial \mathcal{L}}{\partial \lambda_+}=\alpha - u
\end{equation}

\begin{equation} \label{eq:svr_lagrangian_der_lm}
    \frac{\partial \mathcal{L}}{\partial \lambda_-}=-\alpha
\end{equation}

If the Hessian matrix Q is not positive definite, i.e., the Lagrangian function is not strictly convex since it will be linear along the eigenvectors correspondent to the null eigenvalues and so it will be unbounded below, the Lagrangian dual relaxation will be nondifferentiable, so it will have infinite solutions and for each of them it will have a different subgradient. In order to compute an approximation of the gradient, we will choose $\alpha$ in such a way as the one that minimizes the norm of the residual:

\begin{equation} \label{eq:svr_lagrangian_krylov_sol}
	\begin{aligned}
		\min_{\alpha_n \in K_n(Q, b)} \quad & \| Q \alpha_n - b \| \\
		\text{where} \quad & b = - (q - \mu e + \lambda_+ - \lambda_-)
	\end{aligned}
\end{equation}

Since we are dealing with a symmetric but indefinite linear system we will choose a well-known Krylov method that performs the Lanczos iterate, i.e., symmetric Arnoldi iterate, called \emph{minres}, i.e., symmetric \emph{gmres}, to compute the vector $\alpha_n$ that minimizes the norm of the residual $r_n = Q \alpha_n - b$ among all vectors in $K_n(Q, b) = span(b, Qb, Q^2b, \dots, Q^{n-1}b)$.

\bigskip

From~\eqref{eq:svr_min_qp_wolfe_dual} we can notice that the equality constraint $e^T \alpha = 0$ arises form the stationarity condition $\partial_{{b}} \mathcal{W}=0$. So, again, for simplicity, we can again consider the bias term $b$ embedded into the weight vector. In this way the dimensionality of~\eqref{eq:svr_lagrangian_dual} is reduced of 1/3 by removing the multipliers $\mu$ which was allocated to control the equality constraint $e^T \alpha=0$, so we will end up solving exactly the problem~\eqref{eq:svr_min_bcqp_wolf_dual}.

\begin{equation} \label{eq:svr_bcqp_lagrangian_dual}
	\begin{aligned}
    	\max_{\lambda_+,\lambda_-} \min_{\alpha} \mathcal{L}(\alpha,\lambda_+,\lambda_-) &= \frac{1}{2} \alpha^T (Q + ee^T)\alpha+q^T\alpha - \lambda_+^T (u - \alpha) - \lambda_-^T \alpha \\
    &= \frac{1}{2} \alpha^T (Q + ee^T)\alpha + (q + \lambda_+ - \lambda_-)^T \alpha - \lambda_+^T u
	\end{aligned}
\end{equation}

where, again, the upper bound $u^T = [C, \dots, C]$.

Now, taking the derivative of the Lagrangian $\mathcal{L}$ wrt $\alpha$ and settings it to 0 gives:

\begin{equation} \label{eq:svr_bcqp_lagrangian_der_a}
	\frac{\partial \mathcal{L}}{\partial \alpha}=0\Rightarrow (Q + ee^T) \alpha + (q + \lambda_+ - \lambda_-) = 0
\end{equation}

With $\alpha$ optimal solution of the linear system:

\begin{equation} \label{eq:svr_bcqp_lagrangian_sol}
    (Q + ee^T) \alpha = - (q + \lambda_+ - \lambda_-)
\end{equation}

the gradient wrt $\lambda_+$ and $\lambda_-$ are:

\begin{equation} \label{eq:svr_bcqp_lagrangian_der_lp}
	\frac{\partial \mathcal{L}}{\partial \lambda_+}=\alpha - u
\end{equation}

\begin{equation} \label{eq:svr_bcqp_lagrangian_der_lm}
    \frac{\partial \mathcal{L}}{\partial \lambda_-}=-\alpha
\end{equation}

\subsection{Squared Epsilon-insensitive loss}

The \emph{squared epsilon-insensitive} loss is defined as:

\begin{equation} \label{eq:squared_eps_loss1}
	\mathcal{L}_\epsilon^2 =
	\begin{cases}
		0 & \text{if} \ |y - (w^T x + b)| \leq \epsilon \\
		(|y - (w^T x + b)| - \epsilon)^2 & \text{otherwise} \\
	\end{cases}
\end{equation}

or, equivalently:

\begin{equation} \label{eq:squared_eps_loss2}
	\mathcal{L}_\epsilon^2 = \max(0, |y - (w^T x + b)| - \epsilon)^2
\end{equation}

As the \emph{squared hinge} loss, also the \emph{squared epsilon-insensitive} loss is a strictly convex function and it has a gradient wrt $w$ that is given by:

\begin{equation} \label{eq:squared_eps_loss_der}
	\frac{\partial \mathcal{L}_\epsilon^2}{\partial w}=
		\begin{cases}
            2 ((y - (w^T x + b)) x) & \text{if} \ |y - (w^T x + b)| > \epsilon \\
            0 & \text{otherwise} \\
        \end{cases}
\end{equation}

\subsubsection{Primal formulation}

To provide a continuously differentiable function the optimization problem~\eqref{eq:svr_eps} can be formulated as:

\begin{equation} \label{eq:svr_squared_eps}
    \min_{w,b} \frac{1}{2} \| w \|^2 + C \sum_{i=1}^n \max(0, |y_i - (w^T x_i + b)| - \epsilon)^2
\end{equation}

where we make use of the \emph{squared epsilon-insensitive} loss that quadratically penalized slacks $\xi$ and is called $\mathcal{L}_2$-SVR.

\begin{figure}[h!]
	\centering
  	\includegraphics[scale=0.4]{img/svr_squared_eps_loss}
  	\caption{SVC Squared Epsilon-insensitive loss with different optimization steps}
  	\label{fig:svr_squared_eps_loss}
\end{figure}


\documentclass[../research.tex]{subfile}

\subsection{Quotients}
\label{sec:2.1}

  \begin{definition}[Binary relation]
    Let $X$ be a non-empty set. A \textit{binary relation}, $\sim$, on $X$ is a map
  $$\cdot\sim\cdot:X\times X\to\{\verb%True, False%\}.$$
  \end{definition}

  \begin{definition}[Equivalence relation]
    Let $\sim$ be a binary operation on a non-empty set $X$. We say $\sim$ is an
    \textit{equivalence relation} if it is
    \begin{enumerate}
      \item Reflexive: $x\sim x \quad\forall x\in X$
      \item Symmetric: $x\sim y$ $\Leftrightarrow$ $y\sim x\quad\forall x,y\in X$
      \item Transitive: $x\sim y$, $y\sim z$ $\Rightarrow$ $x\sim z\quad\forall x,y,z\in X$
    \end{enumerate}
  \end{definition}

  \begin{prop}
    Let $\sim$ be an equivalence operation on a non-empty set $X$. Given an $x\in X$, we define
    the \textit{equivalence class} of $x$, $[x]$, to be the set of elements in $X$ equivalent to $x$
    under $\sim$. In other words:
    $$[x]=\{y\in X:y\sim x\}.$$
    Then the equivalence classes partition $X$.
  \end{prop}

  \begin{proof}
    Since every element $x_i\in X$, $i\in I$, is equivalent to itself, $x_i\in[x_i]$, so
    $$X = \bigcup_{i\in I}{[x_i]}.$$
    So equivalence classes cover $X$.\\

    Then if $x\nsim y$, $y\nsim x$, so $y\notin[x]$. And if $z\in[y]$, $z\sim y$ so $z\nsim x$,
    and so $z\notin [x]$. Hence
    $$x\nsim y \Leftrightarrow [x]\cap[y]=\varnothing.$$
    So equivalence classes partition $X$.
  \end{proof}

  \begin{definition}[Quotient set]
    Let $X$ be a non-empty set, and $\sim$ an equivalence relation on $X$. We define the \textit{quotient set},
    or \textit{quotient space}, $\X$ to be the set of equivalence classes of $X$. I.e.
    $$\X=\{[x]:x\in X\}.$$
    When $X$ has an equivalence relation, $\sim\,$, we call $(X,\sim)$ a \textit{setoid}. Another way
    of thinking about quotient sets is by turning $\sim$ into equality, and thus turning
    the setoid $(X,\sim)$ into a quotient set $\X$.
  \end{definition}

  \begin{example}[Quotient group]
    Let $(G,+)$ be a group, and $N$ a normal subgroup of $G$, where we write $xy$ for $x+y$ $\forall x,y\in G$. We define an equivalence relation
    $\sim$ on $G$ to be such that
    $$x\sim y\text{ if and only if }Nx=Ny.$$
    Then equivalence classes $[x]$ are the cosets of $N$ in $G$, $Nx=\{nx:n\in N\}$, so the quotient set
    $$\G=\{Nx:x\in G\}.$$
    Equivalently, two elements of $G$ are equivalent if they lie in the same coset.
    Then we define the \textit{quotient group}
    $$\frac{G}{N}=(\G,+)$$
    where the binary operation $\cdot+\cdot:\frac{G}{N}\times\frac{G}{N}\to\frac{G}{N}$ is defined by
    $$(Nx)(Ny)=N(xy).$$
  \end{example}

  \begin{prop}
    $\frac{G}{N}$ is indeed a group under $+$.
  \end{prop}

  \begin{proof}
    We first check that $+$ is well-defined on $\frac{G}{N}$. \\

    Suppose that $Nx=Nx'$, and $Ny=Ny'$.
    So $xx'^{-1}\in N$ and $yy'^{-1}\in N$. Then $xx'^{-1}yy'^{-1}\in N$ this implies that $xy(x'y')^{-1}\in N$,
    since $N$ is a normal subgroup.\\

    Then $Nxy=Nx'y'$, which implies that $(Nx)(Ny)=(Nx')(Ny')$.\\

    So $+$ is well-defined. We now check the group axioms:
    \begin{enumerate}
      \item \textit{Closure}: Since $G$ is closed under $+$, $\frac{G}{N}$ is clearly closed under $+$.
      \item \textit{Associativity}: \begin{align*}
                                      \forall x,y,z\in G, (Nx)((Ny)(Nz)) &= (Nx)(N(yz)) \\
                                                                              &= Nx(yz) \\
                                                                              &= N(xy)z \\
                                                                              &= (Nxy)(Nz) \\
                                                                              &= ((Nx)(Ny))(Nz)
                                    \end{align*}
      \item \textit{Identity}: $N$ is the identity. To see this, observe that $N(Nx)=Nx=(Nx)N$.
      \item \textit{Inverses}: Given $Nx$, $(Nx)^{-1}=Nx^{-1}$. To see this, observe that $(Nx)(Nx^{-1})=N=(Nx^{-1})(Nx)$.
    \end{enumerate}
  \end{proof}

  \begin{example}[Quotient ring]
    Let $(R,+,\times)$ be a ring, and $I$ an ideal of $R$. We can define an equivalence relation,
    $\sim$, on $R$ by
    $$r\sim s\Longleftrightarrow I+r=I+s.$$
    Then $(R/\sim,+,\times)$ forms a ring under $+$ and $\times$, called the \textit{quotient ring},
    $R/I$. \\

    We can construct the field of complex numbers, $\mathbb{C}$, by taking the following quotient: \\

    Consider $I:=(x^2+1)\R[x]$ as an ideal of $\R[x]$. We define an equivalence relation $\sim$ on $\R[x]$
    by
    $$f(x)\sim g(x)\Longleftrightarrow I+f(x)=I+g(x)\quad,\forall f(x),g(x)\in\R[x].$$
    I claim that the quotient ring $\R[x]\,/\sim$,
    $$\frac{\R[x]}{I}=\{a(I+x)+(I+b):a,b\in\R\}$$.
    \begin{proof}
      Fix a coset $I+p(x)$, $p(x)\in\R[x]$. Since $\R$ is a field, $\R[x]$ is a Euclidean domain, so
      $\exists q(x),r(x)\in\R[x]$, with $deg(r(x))<deg(x^2+1)=2$, or else $r(x)=0$, such that
      $$p(x)=q(x)(x^2+1)+r(x).$$
      Then we write the quotient ring as
      \begin{align*}
        \frac{\R[x]}{I} &=\{I+p(x):p(x)\in\R[x]\} \\
                        &=\{I+q(x)(x^2+1)+r(x):deg(r)<2\} \\
                        &=\{I+r(x):deg(r)<2\} \\
                        &=\{I+(ax+b):a,b\in\R\} \\
                        &=\{a(I+x)+(I+b):a,b\in\R\}
      \end{align*}
    \end{proof}
    Note that $(I+x)^2=I+x^2=(I+x^2)+(I+1)-(I+1)=-(I+1)$. This is clearly isomorphic to $\mathbb{C}$,
    since $i^2=-1$ when we identify $i$ with $I+x$.
  \end{example}

  \begin{example}[Quotient topology]
    Let $(X,\tau_x)$ be a topological space, $Y$ a set, and $f:X\to Y$ a surjective function. Then a subset
    $U\subset Y$ is open in $Y$ if and only if $f^{-1}(U)$ is open in $X$. In other words, this
    topology, called the \textit{quotient topology}, is the finest topology we can define on $Y$ such
    that $f$ is continuous, called the \textit{final topology} on $Y$ with respect to $f$. \\

    The key example here is given by defining an equivalence relation, $\sim$ on $X$, and then
    taking $Y=\X$. We then define the \textit{natural map}, $p:X\to\X$, by $p(x)=[x]$ $\forall x\in X$. \\

    Then $(\X,\tau)$ is a topological space, where the quotient topology, $\tau$, is defined by
    $$\tau=\{U\in \X:p^{-1}(U)\in\tau_X\}.$$
    Note that $\tau$ is the final topology on $\X$ with respect to $p$.
  \end{example}

  \begin{definition}[Separateness conditions]
    \verb%%
    \begin{enumerate}
      \item Hausdorff: A topological space $X$ is called \textit{Hausdorff} iff $\forall x,y\in X,\,
            \exists U_x,U_y\text{ such that }U_x\cap U_y=\varnothing$, where $U_x,U_y$ are open
            neighbourhoods of $x,y$ respectively. \\

            This asserts that every two points in $X$ can be `separated' by open sets - open
            neighbourhoods that are disjoint. Conversely, $X$ is non-Hausdorff if there exists a pair of points
            $x,y\in X$ such for every open neighbourhood $U_x$ of $x$ and $U_y$ of $y$, $U_x\cap U_y
            \neq\varnothing$. \\

            For example, if $X=\{x,y,z\}$ and $\tau=\{\varnothing,\{x\},\{x,y\},\{x,z\},X\}$, then $X$ is
            non-Hausdorff since the open neighbourhoods of $x$ are $\{x\},\{x,y\},\{x,z\}$, and the only open
            neighbourhoods of $y,z$  $\{x,y\},\{x,z\}$ respectively, which are all clearly non-disjoint.
      \item Regular: A topological space $X$ is called \textit{regular} iff for any $x\in X$ and any
            closed set $V\subset X$, there exists an open neighbourhood $U_x$ of $x$ and an open
            superset $U_V$ of $V$, such that $U_x\cap U_V=\varnothing$.
      \item Normal: A topological space $X$ is called \textit{normal} iff for any pair of closed sets
            $V_1,V_2\subset X$ there exist open supersets $U_1,U_2$ of $V_1,V_2$ such that $U_1\cap U_2=\varnothing$.
    \end{enumerate}
  \end{definition}

  \begin{example}
    Let $X$ be a non-Hausdroff topological space. Then we can turn $X$ into a Hausdorff space by
    considering the equivalence relation $\sim$ on $X$ defined by
    $$x\sim y\Longleftrightarrow U_x\cap U_y\neq\varnothing$$
    for all open neighbourhoods $U_x,U_y$ of $x,y$ respectively. \\

    Then the quotient space $\X$ is Hausdorff when given the quotient topology.
  \end{example}

  \begin{proof}
    So $U\subset\X$ is open if and only if $p^{-1}(U)$ is open, where $p:X\to\X$ maps $x\in X$ to
    the set of points which cannot be separated from $x$ by open sets, denoted $[x]$. \\

    So we fix distinct $[x],[y]\in\X$. Now, $p^{-1}(\{[x]\})$ is
    open in $X$, so $\{[x]\}$ is open in $\X$. Similarly, $\{[y]\}$ is open in $\X$. Finally, note
    that $\{[x]\}\cap\{[y]\}=\varnothing$, and so $\X$ is Hausdorff.
  \end{proof}

  \begin{definition}[Quotient map]
    We say $q:X\to Y$ between topological spaces $X$ and $Y$ is a \textit{quotient map}
    if it is surjective, and $U\subset Y$ is open in $Y$ if and only if $q^{-1}(U)\subset X$
    is open in $X$.
  \end{definition}

  \begin{example}
    The natural map $p:X\to \X$ is a quotient map.
  \end{example}

  \begin{proof}
    $p$ is surjective, since $\forall[x]\in \X$, $p(x)=[x]$. The second property follows from the
    definition of $p$.
  \end{proof}

  \begin{prop}
    Let $q:X\to Y$ be a quotient map. Then $f:Y\to Z$ is continuous if and only if
    $f\circ q:X\to Z$ is continuous.
  \end{prop}

  \begin{proof}
    If $f$ is continuous, then $f\circ q$ is continuous by composition. Now assume
    $f\circ q$ to be continuous, and let $U\subset Z$ be open in $Z$. Then
    $(f\circ q)^{-1}(U)=q^{-1}(f^{-1}(U))$ is open in $X$, so $f^{-1}(U)$ is open in $Y$.
  \end{proof}

  \begin{prop}[Universal property of quotients]
    If $f:X\to Y$ is continuous and constant on equivalence classes of $X$. I.e.
    $f|_{[x]}$ is constant $\forall[x]\in\X.$\\

    Then there exists a unique continuous map $g:\X\to Y$ such that
    $$f=g\circ p.$$
    We say that $f$ descends to the quotient.
  \end{prop}

  \begin{proof}
    So given $x_i\in X$, $f(x)=c_i$ $\forall x\in[x_i]$. Then we define $g:\X\to Y$ by
    $g([x_i])=c_i$. Clearly this choice of $g$ is unique. We first check this is well-defined:\\

    Suppose $[x_i]=[y_i]$. Then if $y\in[y_i]$, $y\in[x_i]$, so $g([y_i])=c_i$. Now,
    $g\circ p(x_i)=g(p(x_i))=g([x_i])=c_i=f(x_i)$. Hence $g\circ p=f$.\\

    We now check $g$ is continuous. Since $p$ is a quotient map and $g\circ p$ is continuous,
    $g$ is continuous by the previous proposition.
  \end{proof}

\subsection{Group Actions}
\label{sec:2.2}

  \begin{definition}[Automorphism group]
    Given a mathematical structure $X$, an \textit{automorphism} is an isomorphism, $\phi$, from $X$ to itself. Explicitly,
    $$\phi:X\to X.$$
    Coloquially, automorphisms are invertible mappings that preserve structure. In fact, they form a group under composition,
    called the \textit{automorphism group} of $X$, which we denote $\Aut{X}$.
  \end{definition}

  \begin{example}
    \verb%%
    \begin{enumerate}
      \item If $X$ is a set with $n$ elements, then
            $$\Aut{X}=S_n,$$
            the group of permutations of the elements of $X$.
      \item If $V$ is an $n$-dimensional vector space over a field $F$, then
            $$\Aut{V}=\text{GL}_n(F),$$
            the group of invertible $n\times n$ matrices over $F$, or equivalently the group of invertible linear
            transformations from $V$ to itself.
      \item If $(X,\tau)$ is a topological space, then
            $$\Aut{X}=\Homeo{X},$$
            the group of homeomorphisms of $X$. Note that homeomorphism groups of homeomorphic spaces
            are isomorphic as groups. \\
    \end{enumerate}
  \end{example}

  Given a set $X$, we can think of $\Aut{X}$ as \textit{acting} on $X$ in an invertible way.
  Explicitly, we describe the \textit{group action} of $\Aut{X}$ on $X$ by the function:
  $$\Aut{X}\times X\to X:(\sigma,x)\mapsto \sigma\cdot x,\quad\forall\sigma\in\Aut{x},$$
  such that the following hold $\forall x\in X$, where $\sigma\cdot x$ refers to outcome of acting
  on $x\in X$ by an automorphism $\sigma\in\Aut{X}$, and the identity in $\Aut{X}$ is denoted $e$:
  \begin{enumerate}
    \item \textit{Identity}: $e\cdot x=x$
    \item \textit{Compatibility}: $(\sigma\rho)\cdot x=\sigma\cdot(\rho\cdot x),\quad\forall
                                    \sigma,\rho\in\Aut{X}$
  \end{enumerate}

  In fact, this is a special case of an arbitrary group $G$ acting on $X$. We specify the action
  by describing a homomorphism
  $$\varphi:G\to\Aut{X}.$$
  Which amounts to assigning an automorphism of $X$ to each element of $G$ in such a way that
  \begin{enumerate}
    \item The identity automorphism in $\Aut{X}$ is assigned to the identity in $G$,
    \item The composition of the two automorphisms assigned to two elements $g,h\in G$ is assigned to
          the product $gh$.
  \end{enumerate}
  Since $\varphi(G)$ is a subgroup of $\Aut{X}$, we define the action of $G$ on $X$ by
  the group action of the homomorphic image of $G$, in $\Aut{X}$, on $X$. So $g\cdot x=\varphi(g)x$,
  where $\varphi(g)x$ is the result of applying the automorphism $\varphi(g)\in\Aut{X}$ to $x\in X$.
  Specifically:

  \begin{definition}[Group action]
    Let $G$ be a group and $X$ a set. The \textit{(left) group action}, is a function
    $$G\times X\to X:(g,x)\mapsto g\cdot x,$$
    such that the following hold $\forall x\in X$, where we $e$ is the identity in $G$:
    \begin{enumerate}
      \item \textit{Identity}: $e\cdot x=x$
      \item \textit{Compatibility}: $(gh)\cdot x=g\cdot(h\cdot x)\quad\forall g,h\in G$
    \end{enumerate}
    Here $G$ is called a \textit{transformation group}, $X$ is called a \textit{$G$-set}.
  \end{definition}

  \begin{definition}[Orbit]
    Consider a group $G$ acting on a set $X$. We define the \textit{orbit}, $G\cdot x$,
    of $x\in X$ to be the set of elements in $X$ to which $x$ can be mapped to by the elements of $G$. Explicitly:
    $$G\cdot x=\{g\cdot x:g\in G\}.$$
  \end{definition}

  We can define an equivalence relation $\sim$ on $X$ by
  $$x\sim y\text{ if and only if }G\cdot x=G\cdot y,\quad x,y\in X.$$
  Equivalently, $x\sim y$ if and only if $\exists g\in G$ such that $g\cdot x=y$ (and so $g^{-1}\cdot y=x$).
  Then the orbits of $X$, under the action of $G$, partition $X$. The quotient set induced by $\sim$ is
  $$\X=\{G\cdot x:x\in X\}.$$
  This is the set of all orbits of $X$ under the action of $G$, called the \textit{quotient} of the action and denoted $\frac{X}{G}$.

  \begin{definition}[Stabiliser subgroup]
    Given $g\in G$ and $x\in X$ such that $g\cdot x=x$, we say that $g$ fixes $x$, in other words
    $g$ acts as the identity on $x$. So for all $x\in X$, we define the \textit{stabiliser subgroup},
    $G_x\subset G$, of $G$ with respect to $x$ as the set of elements in $G$ that fix $x$. Explicitly, given $x$,
    $$G_x=\{g\in G:g\cdot x=x\}.$$
  \end{definition}

  \begin{prop}
    $G_x$ is indeed a subgroup of $G$.
  \end{prop}

%  \begin{prop}
%    Let $\psi:G\to\text{Aut}(X)$ be a homomorphism. Then
%    $$\text{Ker}(\psi)=\bigcap_{x\in X}{G_x}.$$
%  \end{prop}

  $G_x$ is generally not a normal subgroup of $G$. However, given an $x\in X$, we can define an
  equivalent relation, $\sim$, on $G$ by
  $$g\sim h\Longleftrightarrow G_xg=G_xh,\quad g,h\in G.$$
  We can then form the quotient set $\G$ to be the set of cosets of $G_x$ in $G$, denoted $G/G_x$.
  Explicitly, given $x\in X$,
  $$\frac{G}{G_x}=\{G_xg:g\in G\}.$$

  \begin{lemma}
    Given $x,y\in X$, and $g\in G$ such that $y=g\cdot x$,
    the two stabiliser groups $G_x,G_y$ are related by
    $$G_y=g^{-1}G_x g.$$
  \end{lemma}

  \begin{proof}
    Fix $x,y\in X$, and let $g\in G$ be such that $y=g\cdot x$, and fix $h\in G_y$. Then
    \begin{align*}
      h\cdot y= y &\Longleftrightarrow h\cdot(g\cdot x)=g\cdot x \\
                  &\Longleftrightarrow (hg)\cdot x=g\cdot x \\
                  &\Longleftrightarrow g^{-1}\cdot((hg)\cdot x)=g^{-1}\cdot(g\cdot x) \\
                  &\Longleftrightarrow (g^{-1}hg)\cdot x=(g^{-1}g)\cdot x = x \\
                  &\Longleftrightarrow g^{-1}hg\in G_x \\
                  &\Longleftrightarrow h\in g^{-1}G_xg
    \end{align*}
  \end{proof}

  \begin{theorem}[Orbit-stabiliser theorem]
    For a fixed $x\in X$, there exists a bijection $\psi:G/G_x\to G\cdot x.$
  \end{theorem}

  \begin{proof}
    We define a map $\psi:G/G_x\to G\cdot x$ by $\psi(G_xg)=g\cdot x$, $\forall g\in G$.
    First, we show this an injective, well-defined function: Fix $x\in X$, and let $G_xg=G_xh$. Then
    \begin{align*}
      G_xg^{-1}h=G_x &\Longleftrightarrow g^{-1}h\in G_x \\
                     &\Longleftrightarrow (g^{-1}h)\cdot x=g^{-1}\cdot(h\cdot x)=x \\
                     &\Longleftrightarrow h\cdot x=g\cdot x \\
                     &\Longleftrightarrow \psi(G_xg)=\psi(G_xh)
    \end{align*}
    Now we show that $\psi$ is surjective: Fix $x\in X$, and $g\cdot x\in G\cdot x$.
    $$G_xg=\{hg:h\cdot x=x, \,\forall h\in G\}$$
  \end{proof}

  \begin{cor}[Burnside's lemma]
    The number of orbits of $G$ in $X$, $|X/G|$, is given by
    $$|X/G|=\frac{1}{|G|}\sum_{g\in G}{|X^g|},$$
    where $X^g\subset X$ is the set of elements in $X$ fixed by $g$.
  \end{cor}

  In fact, if $G$ is finite, then by Lagrange's theorem,
  $$|G|=|G\cdot x||G_x|.$$

  \begin{definition}[Topological group]
    A \textit{topological group} is a non-empty set $G$ which is both a group and topological space,
    in which the group operations of product:
    $$G\times G\to G:(x,y)\mapsto xy,\quad x,y\in G$$
    and taking inverses:
    $$G\to G:x\mapsto x^{-1},\quad x\in G$$
    are continuous, where $G\times G$ is given the product topology.

    For example, we can turn every finite group into a topological group by giving it the discrete
    topology. Returning to our idea of group actions, since the automorphism groups of a topological
    space $X$ is $\Homeo{X}$, we can think of $\Homeo{X}$ as acting of $X$

    Let $X$ be a topological space, and $G$ a topological group. Then we say the action of $G$ on
    $X$,
    $$\phi:G\times X\to X:(g,x)\mapsto g\cdot x,$$
    is \textit{continuous} if $\phi^{-1}(U)$ is open in $G\times X$, with respect to the product
    topology, whenever $U$ is open in $X$.

    We then call $X$ a \textit{$G$-space}. \\

    Another way of thinking about a continuous group action is if we define a homomorphism
    $$\varphi:G\to\Homeo{X},$$
    then the action of an element $g\in G$ on $x\in X$ is $g\cdot x:=\varphi(g)x$.

      %\begin{definition}[Homomorphism of topological groups]
  %  Let $G,H$ be topological groups. Then a \textit{homomorphism of topological groups} is a
  %  structure-preserving map (i.e. a group homomorphism), $\phi:G\to H$, which is continuous. Explicitly,
  %  \begin{enumerate}
  %    \item $\phi(xy)=\phi(x)\phi(y),\quad x,y\in G$
  %    \item $\phi^{-1}(U)$ is open in $G$ whenever $U$ is an open subset of $H$.
  %  \end{enumerate}
%
  %  An \textit{isomorphism} is a bijective group homomorphism which is also a homeomorphism of the
  %  underlying topological spaces.
  %\end{definition}
  \end{definition}

\subsection{Initial and Final Topologies}
\label{sec:2.3}

  Given a set $X$, the indiscrete topology is the coursest topology we can give to $X$.
  However, for a family of topological spaces $Y_i$ indexed by $i\in I$, the functions
  $$f_i:X\to Y_i$$
  will almost never be continuous. The initial topology is the coursest topology such that
  they are.

  \begin{definition}[Initial topology]
    Given a family of topological spaces $Y_i$ indexed by $i\in I$, the \textit{initial topology}
    with respect to the functions
    $$f_i:X\to Y_i$$
    is that in which open sets $U\subset X$ are those for which $f_i(U)$ is open in $Y_i$ $\forall i\in I$.
    If $X$ already has a topology, then these functions are continuous if and only if the existing
    topology is finer than the initial topology. Thus, the initial topology is the coursest topology that
    can be defined on $X$ such that these functions are continuous. \\

    Explicitly, it is the topology generated by taking all finite intersections and arbitrary unions of
    sets of the form $f_i^{-1}(U_i)$, where $U_i$ is open in $Y_i$. Consider the following examples:

    \begin{enumerate}
      \item Let $A\subset X$. The subspace topology $\tau_A=\{A\cap U:U\text{ is open in }X\}$
            is the initial topology on $A$ with respect to the canonical inclusion map
            $$i:A\hookrightarrow X.$$
      \item Given a family of topological spaces $X_i$ indexed by $I$, let $X=\prod_{i\in I}{X_i}.$
            the product topology on $X$ is the initial topology on $X$ with respect to
            the canonical projections
            $$p_i:X\to X_i\quad\forall i\in I.$$
    \end{enumerate}

    If $f$ is injective, the initial topology can be identified with the \textit{subspace} topology of
    $X$, when $X$ is viewed as a subset of $Y_i$.
  \end{definition}

  \begin{theorem}[Characteristic property of initial topologies]
    Let $X,Y,Z_i$ be topological spaces, with $Z_i$ indexed by $i\in I$ and $Y$ given the initial
    topology with respect to the functions
    $$g_i:Y\to Z_i.$$

    Let $f:X\to Y$. Then $f$ is continuous if and only if $g_i\circ f$ is continuous $\forall i\in I$. \\

    In other words, the following diagram is commutative for all $i\in I$: \\
    \begin{center}
      \begin{tikzcd}
        X \arrow{r}{f} \arrow[swap]{dr}{g_i\circ f} & Y \arrow{d}{g_i} \\
         & Z_i
      \end{tikzcd}
    \end{center}
  \end{theorem}

  \begin{proof}
    Assume first that $f$ is continuous. Then since $g_i$ is continuous $\forall i\in I$, $g_i\circ f$ is continuous. \\

    Now assume $g_i\circ f$ is continuous $\forall i\in I$. Let $U\subset Y$ be open in $Y$. Then $U$ is a finite intersection
    and arbitrary union of open sets of the form $g_i^{-1}(V_i)$, where each $V_i$ is open in $Z_i$. \\

    Then $f^{-1}(U)$ is a finite intersection and arbitrary union of open sets of the form $f^{-1}(g_i^{-1}(V_i))=
    (g_i\circ f)^{-1}(V_i)$. This is equal to $(g_i\circ f)^{-1}(W_i)$, where $W_i$ is a finite intersection and arbitrary union
    of the open sets $V_i$. \\

    Then since each $W_i$ is open, and each $g_i\circ f$ is continuous, $(g_i\circ f)^{-1}(W_i)=f^{-1}(g_i^{-1}(W_i))$ is open in $X$.
    $g_i^{-1}(W_i)$ is a finite intersection and arbitrary union of $g_i^{-1}(V_i)$, which is open in $Y$. Hence $f^{-1}(U)$ is open in $X$.
  \end{proof}

  \begin{definition}[Final topology]
    Given a family of topological spaces $X_i$ indexed by $i\in I$, the \textit{final topology}
    with respect to the functions
    $$f_i:X_i\to Y$$
    is that in which open sets $A\subset Y$ are those for which $f_i^{-1}(U)$ is open in $X_i$ $\forall
    i\in I$. If $Y$ already has a topology, the $f_i$ are continuous precisely when the existing
    topology is \textit{courser} than the final topology on $Y$. Thus, the final topology is the finest
    topology that can be given to $Y$ such that these functions are continuous. \\

    If each $f_i$ is surjective, we call this topology the \textit{quotient topology} under the equivalence
    relation $\sim$ defined by
    $$x\sim y\Longleftrightarrow f_i^{-1}(\{x\})=f_i^{-1}(\{y\})\quad\forall x,y\in X_i,\,i\in I.$$
  \end{definition}

  \begin{theorem}[Universal property of final topologies]
    Let $X_i,Y,Z$ be topological spaces, with $X_i$ indexed by $i\in I$ and $Y$ given the final
    topology with respect to the functions
    $$f_i:X_i\to Y$$

    Then $g:Y\to Z$ is continuous if and only if $g\circ f_i$ is continuous $\forall i\in I$. \\

    In other words, the following diagram is commutative $\forall i\in I$: \\
    \begin{center}
      \begin{tikzcd}
        X_i \arrow{r}{f_i} \arrow[swap]{dr}{g\circ f_i} & Y \arrow{d}{g} \\
         & Z
      \end{tikzcd}
    \end{center}
  \end{theorem}

\subsection{Review of point-set topology}
\label{sec:2.4}

  \begin{prop}
    Any compact subset of a Hausdroff space is closed.
  \end{prop}

  \begin{prop}
    Any closed subset of a compact space is compact.
  \end{prop}

  \begin{prop}
    Let $f:X\to Y$ be injective and continuous. Then if $Y$ is Hausdorff, $X$ is Hausdorff.
  \end{prop}

  \begin{prop}
    Let $X$ be a compact topological space, and $Y$ a Hausdorff space.
    Let $f:X\to Y$ be surjective and continuous. Then $f$ is a quotient map.
  \end{prop}

  \begin{proof}
    Let $U\subset Y$ be non-empty. Since $f$ is surjective, $f^{-1}(U)$ is non-empty.
    Suppose $f^{-1}(U)$ is open in $X$. Then since $X\backslash f^{-1}(U)$ is closed and $X$ is compact,
    $X\backslash f^{-1}(U)$ is compact. Then since $f$ is continuous, $f(X\backslash f^{-1}(U))$
    is compact. \\

    Then since $Y$ is Hausdorff, $f(X\backslash f^{-1}(U))=Y\backslash f(f^{-1}(U))=Y\backslash U$ is closed,
    and hence $U$ is open, so $f$ is a quotient map.
  \end{proof}

\subsection{Construction of topological spaces}
\label{sec:2.5}

  \begin{definition}[Disjoint union]
    Consider a family of non-empty sets $X_i$ indexed by $I$. Let $X_i^*=\{(x,i):x\in X_i\}.$
    Then we define the \textit{disjoint union} of the $X_i$ as
    $$\coprod_{i\in I}{X_i}=\bigcup_{i\in I}{X_i^*}.$$
  \end{definition}

  \begin{definition}[Disjoint union topology]
    Let $X=\coprod_{i\in I}{X_i}$. For each $i\in I$, we define
    $$\phi_i:X_i\to X:x\mapsto(x,i)$$
    to be the \textit{canonical injection}. Then we say $U$ is open in $X$ if and only if
    $\phi_i^{-1}(U)$ is open in $X_i$, $\forall i\in I$. \\

    Equivalently, $U$ is open in $X$ if and only if $U\cap X_i$ is open in $X_i$, $\forall i\in I$. \\

    Note that this topology is the finest topology on $X$ such that all of the canonical injections
    $\phi_i$ are continuous. In other words, it is the final topology on $X$ with respect to the $\phi_i$.
  \end{definition}

  We use this to construct some topological spaces.

  \begin{enumerate}
    \item Let $A$ be a subset of a topological space $X$, and $q:A\to Y$ be a quotient map.
          We define an equivalence relation $\sim$ on $A$ by
          $$x\sim y \Leftrightarrow q(x)=q(y),\quad\forall x,y\in A.$$
          Then we define $X\cup_q Y=\X.$ \\

          If $Y$ is a singlet set, then we call this \textit{collapsing}, and
          denote it $X/A$.
    \item Instead let $f:A\to Y$ be continuous, and define $\sim$ on $Y$ by
          $x\sim f(x)\quad\forall x\in A$. Then we define
          $$X\cup_q Y=X\sqcup Y\,/\sim.$$
  \end{enumerate}

  \begin{definition}[Attaching map]
  \end{definition}

  \begin{definition}[Adjunction space]
  \end{definition}

  \begin{lemma}[Gluing lemma]
  \end{lemma}


\chapter{Appendix}\label{ch:mathematical-background}
\section{Stability theory}\label{dynamical-systems}
A dynamical system is a system that evolves in time. Stability theory is concerned with finding and classifying equilibriums. These are states of the system that do not change with time. Many dynamical systems can be represented through differential equations. If the system of differential equations has $n$ variables, choosing any set of values for the variables defines a point $x\in\mathbb{R}^n$. As the differential equations describe some change in time, moving time forwards sees this point moves through the space $\mathbb{R}^n$. The path made by including all points passed through in a period of time is called a \textit{trajectory}.\\
\\
A \textit{fixed point}, $\bf{x}$, is a point at which, if there is no perturbation, the system will stay forever. It is a time-independent solution to the system. However, there are many different types of fixed point. The two basic classes are stable and unstable fixed points. Informally, a fixed point $x$ is stable if a small perturbation from the point will bring it back to $x$. Similarly, a fixed point is unstable if a small perturbation from the point $x$ makes the system move away from $x$.\\
\\
%Let's get a bit more specific with this. Three key concepts are attraction, liapanov stability and asymptotic stability.
Let $\bf{x}\in\mathbb{R}^n$ and $\bf{f}:\mathbb{R}^n\rightarrow\mathbb{R}^n$ such that $\bf{\dot x}=\bf{f}(\bf{x})$. This is a compact form of
\begin{equation}\label{eq:compact-form}
\frac{d}{dt}
\begin{bmatrix}
 x_1 \\
\vdots\\
x_n
\end{bmatrix}
=
\begin{bmatrix}
f_1(x_1,...,x_n) \\
\vdots\\
f_n(x_1,...,x_n)
\end{bmatrix}
\end{equation}
We say that $\bf{x}^*$ is a \textit{fixed point} of a dynamical system if $\bf{f}(\bf{x}^*)=\bf(0)$. Geometrically this means that this point would not move. $\bf{x}^*$ is also called an equilibrium.\\
\\
%Let $\bf{x}^*$ be a fixed point. We say that a point is \textit{attracting} if all trajectories starting near $x^*$ approach $x^*$ as time goes to infinity. Formally this means that
%\begin{equation*}
%\lim_{t\rightarrow\infty}\norm{\bf{x}(t)-\bf{x}^*(t)} = 0
%\end{equation*}
%A point is \textit{stable} (or Liapanov stable) if all trajectories that start close to $x^*$ stay close to it for all time. Formally this means
%\begin{equation*} \forall\epsilon>0,\exists\delta>0\text{ s.t. }\norm{x(0)-x^*}<\delta\implies\norm{x(t)-x^*}<\epsilon,\forall t>0
%\end{equation*}
%A fixed point that is not (Liapanov) stable is \textit{unstable}\cite{Braun:1993kx}.\\
%\\
%More strongly, a fixed point is \textit{asymptotically stable} if it is both attracting and Liapanov stable. The difference between Liapanov and asymptotic stability is that a small perturbation from a (Liapanov) stable point will stay close. However, a small perturbation from an asymptotically stable point will return to the fixed point. Thus a pendulum without friction is Liapanov stable but a pendulum with friction is also asymptotically stable.
%https://www.cds.caltech.edu/~murray/courses/cds101/fa02/faq/02-10-23_lyapexact.html
We say $\bf{x}(t)$ is \textit{stable} if it is a solution of Eq. (\ref{eq:compact-form}) with initial condition $\bf{x}(t=0)=\bf{x_0}$ and $\forall \epsilon>0,\exists \delta>0$ such that if $\bf{x'}(t)$ is another solution of Eq. (\ref{eq:compact-form}) with $\bf{x'}(t=0)=\bf{x_0}'$ and $\norm{\bf{x_0'}-\bf{x_0}}<\delta$ then $\norm{\bf{x'}(t)-\bf{x}(t)}<\epsilon$ for all $t\geq0$\cite{Braun:1993kx}. Finally, a solution of Eq. (\ref{eq:compact-form}) that is not stable is \textit{unstable}\cite{Braun:1993kx}.
\subsubsection{Jacobian}
The stability of a point $\mathbf{x}$ can be investigated by considering the Jacobian matrix at that point. The Jacobian is the generalisation of a derivative, providing the best linear approximation of a function at a differentiable point.\\
\\
%Suppose we have a triple integral in the set of variables $x, y, z$:
%\begin{equation*}
%\int\int\int f(x,y,z)\,dx\,dy\,dz
%\end{equation*}
%Let $r, s, t$ be another set of variables, related to $x, y, z$ by the equations
%\begin{equation*}
%r=r(x,y,z), \hspace{1cm} s=s(x,y,z), \hspace{1cm} t=t(x,y,z)
%\end{equation*}
The general Jacobian of a function $\bf{f}:\mathbb{R}^n\rightarrow\mathbb{R}^m$ that takes elements $\bf{x}\in\mathbb{R}^n$ and outputs the vector $\bf{f}(\bf{x})\in\mathbb{R}^m$ is given by:
\begin{equation*}
{\displaystyle \mathbf {J} ={\begin{bmatrix}{\dfrac {\partial \mathbf {f} }{\partial x_{1}}}&\cdots &{\dfrac {\partial \mathbf {f} }{\partial x_{n}}}\end{bmatrix}}={\begin{bmatrix}{\dfrac {\partial f_{1}}{\partial x_{1}}}&\cdots &{\dfrac {\partial f_{1}}{\partial x_{n}}}\\\vdots &\ddots &\vdots \\{\dfrac {\partial f_{m}}{\partial x_{1}}}&\cdots &{\dfrac {\partial f_{m}}{\partial x_{n}}}\end{bmatrix}}}
\end{equation*}
More specifically, if $\bf{f}:\mathbb{R}^3\rightarrow\mathbb{R}^3$ is a function taking $(r,s,t)$ to $(x,y,z)$ then the Jacobian is:
%={\partial(x,y,z) \over \partial(r,s,t)} =
%{\displaystyle \mathbf {J} ={\begin{bmatrix}{\dfrac {\partial \mathbf {f} }{\partial r}}&\cdots &{\dfrac {\partial \mathbf {f} }{\partial t}}\end{bmatrix}}
\begin{equation*}
J
={\begin{bmatrix}{\partial x \over \partial r} & {\partial x\over \partial s} & {\partial x \over \partial t} \cr
		{\partial y \over \partial r} & {\partial y\over \partial s} & {\partial y \over \partial t} \cr
		{\partial z \over \partial r} & {\partial z\over \partial s} & {\partial z \over \partial t}\end{bmatrix}}
\end{equation*}

If the Jacobian at a fixed point has eigenvalues all with negative real part then the point is
%asymptotically
stable. If it has at one or more eigenvalues with a positive real part it is unstable\cite{Izhikevich:2007}.
\section{Code}
All of the code I have written for this project (around 3000 lines) is open-source and can be accessed, downloaded, modified and reused at \url{https://github.com/joekroese/networks-and-revolution}. Included are some key excerpts.
\subsection{Compartmental model of revolution}
This code runs the simulation described in Section \ref{sec:rev-compartment} and creates graphs such as Fig. \ref{fig:rev-traj-default} and Fig. \ref{fig:rev-traj-diff-gamma}.
\lstinputlisting[language=python]{../Code/illustrative/rev/run.py}
\subsection{Revolution on network model}
The following code runs the agent-based model of revolution as described in Section \ref{sec:abm-rev}.
\lstinputlisting[language=python]{../Code/illustrative/abm-rev-zealot/run-slim.py}
\lstinputlisting[language=python]{../Code/illustrative/abm-rev-zealot/model.py}
\subsection{Evolutionary Prisoner's Dilemma on a torus}
Creates the live visualisation of the evolutionary prisoner's dilemma on a torus as seen in Section \ref{p-d-torus}.
\lstinputlisting[language=java]{../Code/illustrative/evo_prisoners_dilemma/evo_prisoners_dilemma.pde}
\lstinputlisting[language=java]{../Code/illustrative/evo_prisoners_dilemma/Game.pde}
\lstinputlisting[language=java]{../Code/illustrative/evo_prisoners_dilemma/Cell.pde}


% Author: Jonathan Siegel
% This chapter aims to introduce the logistic regression, a model which
% is the basis of machine learning classification problems.

\section{The Logistic Regression}
\subsection{Soft-max}
Now we introduce the softmax: Given $z=\left( \begin{array}{ccc}
	z_1\\
	\vdots \\
	z_k
\end{array} \right)\in \mathbb R^k$, let

\begin{equation}
	\label{softmax}
	q_j=\frac{e^{z_j}}{\sum_{i=1}^ke^{z_i}}, \quad j=1:k.
\end{equation}
We note that
$$
q_j\ge 0, \quad \sum_{j=1}^kq_j=1
$$
which can be viewed as probability distribution.
We can write \eqref{softmax} as a more compact form:
$$
q = (\mathbbm{1}^Te^z)^{-1}e^z
$$

The softmax is a good measure when the classification is not
distinctive.  For example, in the MNIST, we may not be able to
distinct some handwritings for ``$0$'' versus ``$6$", ``$2$'' versus
``$7$'', ``$3$'' versus ``$5$''.  In these cases, the softmax is
expected to give relative large components corresponding to these
numbers.
\subsection{2-class case} Let $A_1$ and $A_2$ are two linearly
separable sets and $(w,b)$ is as in Lemma~\ref{lem:2class}. We apply
the soft-max to the linear classifying vector function
\begin{equation}
  \label{eq:1}
{1\over 2}
\begin{pmatrix}
  wx+b\\
-(wx+b)
\end{pmatrix}
\end{equation}
we easily obtain the following classifier:
\begin{equation}
  \label{f-sigmoidal}
f(x)=
\begin{pmatrix}
s(wx+b)\\
s(-(wx+b)
\end{pmatrix}
\end{equation}
 where $s$ is the so-called sigmoidal function defined by
\begin{equation}
\label{sigmoidal}
s(x)=\frac{e^x}{e^x+e^{-x}}
\end{equation}
We define $s(x)$ in this way so that $f(x)$ is a probability distribution on
$A_1$ and $A_2$.
\begin{remark} We note that, in the above discussions, we have
  encountered with $4$ special functions, namely
\begin{enumerate}
\item the Heaviside function given by \eqref{heaviside};
\item the linear-step function given by
  \eqref{linear-step};
\item the rectified linear unit function given by \eqref{ReLU}
\item the sigmoidal function given by \eqref{sigmoidal}.
\end{enumerate}
These functions are examples of the so-called activation functions
used in machine learning.  The Heaviside function was probably the
most natural and introduced most early in the machine learning
literature, but its lack of continuity makes the relevant model hards
to train using gradient based training algorithm.  Both the
linear-step and the sigmoid functions are smoothed variants of the
Heaviside function, while the former is only Liptschitz and the latter
is infinitely differential.  We note that
\begin{equation}
  \label{approx-Heaviside}
\lim_{\epsilon\to 0}\sigma(x/\epsilon) =
\lim_{\epsilon\to 0}s(x/\epsilon) =h_0(x), \quad\forall x\in
\mathbb R^1.
\end{equation}
The ReLU function \eqref{ReLU} emerges to be the most popular
activation used in the modern machine learning literature.  The
popularity of ReLU may attribute to the following two properties:
\begin{enumerate}
\item it generates the linear step function $\sigma$
  \eqref{linear-step} as in \eqref{sigma-ReLU};
\item it generates the identity mapping:
  \begin{equation}
    \label{ReLU-identity}
x=ReLU(x)-ReLU(-x)
  \end{equation}
\end{enumerate}

\end{remark}
The following lemma is a trivial consequence of the definition of linear separability
\begin{lemma}
	If $A_1$, $A_2\subset\mathbb{R}^n$ are linearly separable,
        then there exist $\theta^* = (w^*,b^*)\in\mathbb{R}^{1\times
          (n+1)}$ such that
\begin{equation}\label{ftheta}
f(\theta^*;x)\cdot e_i>f(\theta^*;x)\cdot e_j, \mbox{ if }\ x\in A_i,\ i=1,2, j\neq i.
\end{equation}
\end{lemma}

Define
$$
y(x)=e_i,\quad x\in A_i.
$$
and consider the loss function
\begin{equation}\label{logit_loss}
 L(\theta) = \displaystyle\sum_{i = 1}^N -\log\left(f(\theta;x_i)\cdot y(x_i)\right)
\end{equation}

\begin{lemma}\label{lemma116}
 If $A_1$ and $A_2$ are linearly separable, then
 \begin{equation}
  \inf_{\theta} L(\theta) = 0
 \end{equation}
 Moreover, this infimum is not achieved.
\end{lemma}
\begin{proof}
 Since $A_1$ and $A_2$ are linearly separable, there exists a $\theta = (w,b)$ such that
 \begin{equation}
  wx+b
  \begin{cases}
   > 0 & x\in A_1 \\
   < 0 & x\in A_2
  \end{cases}
 \end{equation}

 This means that for each $i$,
 $$-\log\left(f(\theta;x_i)\cdot y(x_i)\right) = \log(e^{c_i} + e^{-c_i}) - c_i$$
 for some $c_i > 0$. Now we note that
 $$-\log\left(f(K\theta;x_i)\cdot y(x_i)\right) = \log(e^{Kc_i} + e^{-Kc_i}) - Kc_i$$
 Thus we see that
 \begin{equation}
  \lim_{K\rightarrow \infty} L(K\theta) = 0
 \end{equation}
 so $\inf_{\theta} L(\theta) \leq 0$. It is a simple exercise to see that $L(\theta) > 0$
 for any $\theta$, which completes the proof.

\end{proof}

\begin{lemma}\label{lemma146}
Suppose $A_1,A_2$ are two linearly separable sets and let $(\theta_i)_{i = 1}^\infty$ be
a minimizing sequence for the loss functions $L$ in (\ref{logit_loss}), i.e.
\begin{equation}
 \lim_{i\rightarrow \infty} L(\theta_i) = \inf_{\theta} L(\theta)
\end{equation}
 Then there exists an $N$ such that for all $n > N$, (\ref{ftheta}) holds for $\theta_n$.
\end{lemma}
\begin{proof}
 We have from Lemma \ref{lemma116} that $\inf_{\theta} L(\theta) = 0$. So let $N$ be chosen so that
 $L(\theta_n) < \log(2)$ for all $n > N$.

 We claim that (\ref{ftheta}) holds for $\theta_n$ as long
 as $n > N$.

 To prove this we simply note that if (\ref{ftheta}) fails to hold for $\theta_n$, then
 there exists a $j$ such that $f(\theta;x_j)\cdot y(x_j) < \frac{1}{2}$. Thus
 \begin{equation}
  L(\theta_n) = -\log\left(f(\theta;x_j)\cdot y(x_j)\right) + \displaystyle\sum_{i\neq j}-\log\left(f(\theta;x_i)\cdot y(x_i)\right) > \log(2)
 \end{equation}
 as $f(\theta;x_j)\cdot y(x_j) < \frac{1}{2}$ and $f(\theta;x_i)\cdot y(x_i) < 1$ for all $i$.

 Since this contradicts our choice of $N$ above, we have that (\ref{ftheta}) holds for $\theta_n$ as desired.
\end{proof}

Next, we examine what happens when we add a regularization term to the loss function in (\ref{logit_loss}), i.e. we consider
the loss function
\begin{equation}\label{regularized_logit_loss}
 L(\theta,\lambda) = \displaystyle\sum_{i = 1}^N -\log\left(f(\theta;x_i)\cdot y(x_i)\right) + \lambda \|\theta\|_2^2
\end{equation}
Note that this loss is strongly convex and thus has a unique minimizer. We have the following lemma.
\begin{lemma}
 Suppose $A_1,A_2$ are two linearly separable sets and let $\theta^*$ be the unique minimizer of the
 regularized loss function in (\ref{regularized_logit_loss}), i.e.
 \begin{equation}
  \theta^* = \argmin_\theta L(\theta, \lambda)
 \end{equation}
 Then for sufficiently small $\lambda > 0$, (\ref{ftheta}) holds for $\theta^*$.
\end{lemma}
\begin{proof}
 Lemma \ref{lemma116} implies that there exists a $\theta^\prime$ such that $L(\theta^\prime) < \frac{1}{2}\log(2)$ (note
 that $L$ here is the unregularized loss in (\ref{logit_loss})). Set
 $$\lambda < \frac{\log(2)}{2\|\theta^\prime\|_2^2}$$
 Then we have
 \begin{equation}
  L(\theta^\prime, \lambda) = L(\theta) + \lambda \|\theta\|_2^2 < \frac{1}{2}\log(2) + \frac{1}{2}\log(2) = \log(2)
 \end{equation}
 This means that $L(\theta^*,\lambda) < \log(2)$. But the same argument as in the proof of Lemma \ref{lemma146} shows that
 if (\ref{ftheta}) fails for $\theta^*$, then
 $$L(\theta^*, \lambda) \geq L(\theta^*) > \log(2)$$
 This contradiction shows that (\ref{ftheta}) holds for $\theta^*$ as desired.

\end{proof}

We can easily generalize those consequences to multicalass situations.
\begin{lemma}{\label{fade}}
	Assume that the sets $A_1,\cdots,A_k$ are linearly separable. Then we must have
	\begin{equation}
	\inf_{\theta} L(\theta,\alpha) \rightarrow 0, \alpha \rightarrow 0.
	\end{equation}
\end{lemma}
\begin{proof}
	Easy to observe that if $\theta = (W,b)\in \Theta$, then $K\theta \in \Theta$ for all $K>0$. And
	\begin{equation}
	\lim\limits_{K\rightarrow +\infty} \displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i}
	\left(\log(\mathbbm{1}^T\exp(KW x + Kb)) - (KW x + Kb)_i\right) = 0.
	\end{equation}
	Given $\epsilon>0$ arbitrarily. Then there must exist a $\theta^{\epsilon} = (W^{\epsilon},b^{\epsilon}) \in \Theta$ such that
	\begin{equation}
	\displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i}
	\left(\log(\mathbbm{1}^T\exp(W^{\epsilon} x + b^{\epsilon})) - (W^{\epsilon} x + b^{\epsilon})_i\right) \leq \epsilon/2.
	\end{equation}
	So for all $0 < \alpha \leq \frac{\epsilon}{2\max_i \|w^{\epsilon}_i\|}$, we have
	\begin{equation}
	|\inf_{\theta} L(\theta,\alpha)| = \inf_{\theta} L(\theta,\alpha) \leq L(\theta^{\epsilon},\alpha) \leq \frac{\epsilon}{2} + \frac{\epsilon}{2} = \epsilon
	\end{equation}
	So we obtain
	\[
	\inf_{\theta} L(\theta,\alpha) \rightarrow 0, \alpha \rightarrow 0.
	\]
\end{proof}

\begin{lemma}
	Suppose that the sets $A_i,\cdots,A_k$ are linearly separable. We claim that when $\alpha$ is sufficiently small, we must have
	\begin{equation}
	\theta_\alpha = \argmin_{\theta} L(\theta,\alpha) \in \Theta.
	\end{equation}
\end{lemma}

\begin{proof}
	Notice that for all $\theta \notin \Theta$ and $\alpha>0$, we have
	\begin{equation}
	L(\theta,\alpha) \geq \displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i}
	\left(\log(\mathbbm{1}^T\exp(W x + b)) - (W x + b)_i\right) > \log 2.
	\end{equation}
	But according to lemma (\ref{fade}), we know there exists a $\delta>0$ such that
	\begin{equation}
	\inf_{\theta} L(\theta,\alpha) < \log 2,~~\forall 0< \alpha < \delta.
	\end{equation}
	So when $0< \alpha < \delta$, we must have
	\[
	\theta_\alpha = \argmin_{\theta} L(\theta,\alpha) \in \Theta.
	\]
\end{proof}

\begin{remark}
$\theta^*$ is a desirable weight, i.e. $f(\theta^*;x)$ is a classifier, if $\{A_i\}_i$ are all finite.
\end{remark}

\subsection{Description of the model}
The logistic regression model takes as input a vector of real valued features
and outputs a probability distribution over a set of labels. So the feature, or input space is
$\mathcal{X} = \mathbb{R}^n$, the label set is $\mathcal{Y} = \{l_1,...,l_k\}$, and the output of the model
lies in the set $\mathcal{Y}^* = \mathcal{P}(\mathcal{Y})$. In what follows, we will denote the
input features by a vector $x\in \mathcal{X} = \mathbb{R}^n$.

\subsubsection{Special Case: Two Labels}
We consider first the simple case where there are only two possible labels, i.e. $\mathcal{Y} = \{l_1,l_2\}$.
In this case, a probability distribution on the labels is uniquely determined by the probability of the first label
$l_1$. Thus, in this case we can view the output of the logistic regression as a single number $p_1\in [0,1]$, which
represents the probability of the first label.

The model consists of the composition of an affine linear function which the logistic function, i.e.
\begin{equation}
 p_i(x~|~W,b) \sim \exp(W_{i}x + b_i)
\end{equation}
Here
$W_{i}$ denotes the $i$-th row of $W$. Explicitly normalizing this probability distribution gives
$$p_i(x~|~W,b) = \left(\displaystyle\sum_{j = 1}^2\exp(W_{j}x + b_j)\right)^{-1}\exp(W_{i}x + b_i)$$
%where $$l(z) = \frac{\exp(z)}{1 + \exp(z)}$$ is the logistic function.
The parameters of the model are $b\in \mathbb{R}^2$
and $W\in \mathbb{R}^{2\times n}$, which specify an affine linear function from the parameter space to $\mathbb{R}$.

\subsection*{Loss functions for two labels case}
In the case of a logistic regression with two labels, we have
\begin{equation}
p_1(x~|~W,b)  = \frac{\exp(W_{1}x + b_1)}{\exp(W_{1}x + b_1)+\exp(W_{2}x + b_2)}.
\end{equation}
Then the likelihood is defined as follows
\begin{equation}
\begin{split}
p(\mathcal{D}~|~W, b) &=
\prod_{j = 1}^n
\left(
\left(p_1(x~|~W,b)\right)^{(y_j)_1}\cdot(1-p_1\left(x~|~W,b)\right)^{(y_j)_2}
\right)
\\
&=
\prod_{j = 1}^n
\left(
\left(
\frac{\exp(W_{1}x + b_1)}{\exp(W_{1}x + b_1)+\exp(W_{2}x + b_2)}
\right)^{(y_j)_1}
\cdot
\left(
\frac{\exp(W_{2}x + b_2)}{\exp(W_{1}x + b_1)+\exp(W_{2}x + b_2)}
\right)^{(y_j)_2}
\right),
\end{split}
\end{equation}
where $y_j$ is the label of the $j$-th data point. Finally, we take the negative log likelihood as the loss function

\begin{equation}\label{LossTwoLabels}
\begin{aligned}
&-\log(p(\mathcal{D}~|~W, b)) \\
&=
-\displaystyle\sum_{j=1}^n
\left(
{(y_j)_1}
\log\left(
\frac{\exp(W_{1}x + b_1)}{\exp(W_{1}x + b_1)+\exp(W_{2}x + b_2)}
\right)
+
(y_j)_2\log\left(
\frac{\exp(W_{2}x + b_2)}{\exp(W_{1}x + b_1)+\exp(W_{2}x + b_2)}
\right)
\right)
\\
&=
-\displaystyle\sum_{j=1}^n \left(
(y_j)_1(W_{1}x + b_1)+(y_j)_2(W_{2}x + b_2)-\log\left(\exp(W_{1}x + b_1)+\exp(W_{2}x + b_2)\right)
\right).
\end{aligned}
\end{equation}

\subsubsection{General Case: Multiple Labels}
We now consider the more general case where the number of labels can be greater than $2$, i.e. $\mathcal{Y} = \{l_1,...,l_k\}$
with $k > 2$. The output of the logistic regression model will be a probability distribution over $\mathcal{Y}$, i.e. for each
input $x\in \mathbb{R}^n$ and index $i = 1,...,k$ the model gives a probability $p_i(x~|~W,b)$, such that the
sum of the probabilities over $i$ is $1$.

\begin{itemize}
\item
If given $(W,b)$,
\begin{equation}
p_i(x~|~W,b) = {\rm Pr}(Y=i | X = x),
\end{equation}
means the probability of the random variable $Y =i $ with condition $ X = x$ where
$X$ is the random variable corresponding to the data space
and $Y$ is the random variable corresponding to the label space.

\item Here
\begin{equation}
p_i(x~|~W,b) = {\rm Pr}( X = x, Y = i | W ,b),
\end{equation}
means the probability of the random variable $(X,Y) = (x,i)$ with condition $W ,b$ are given.
Here $X$ is the random variable corresponding to the data space
and $Y$ is the random variable corresponding to the label space.
Following the notation in the next paragraphs, this setup seems more reasonable.

\end{itemize}

The model consists of an affine linear map composed with the softmax function, i.e.
\begin{equation}
 p_i(x~|~W,b) \sim \exp(W_{i}x + b_i)
\end{equation}
where the parameters of the model are $b\in \mathbb{R}^k$ and $W\in \mathbb{R}^{k\times n}$. Here
$W_{i}$ denotes the $i$-th row of $W$. Explicitly normalizing this probability distribution gives
$$p_i(x~|~W,b) = \left(\displaystyle\sum_{j = 1}^k\exp(W_{j}x + b_j)\right)^{-1}\exp(W_{i}x + b_i)$$

A simpler way of saying this is that the output of the model is given by
\begin{equation}
 \mathcal{Y}^* = \mathcal{P}(\mathcal{Y})\ni h(x~|~W,b) = l(W x + b)
\end{equation}
here $W x$ simply denotes matrix-vector multiplication and the map $l$ is the softmax function
$l:\mathbb{R}^k\rightarrow \mathcal{P}(\mathcal{Y})$, defined by
$$l(y)_i = \frac{\exp(y_i)}{\exp(y_1) + \cdots + \exp(y_k)}$$
where $l(y)_i$ denotes the probability of label $l_i$ under the distribution $l(y)$.

\subsubsection{Remarks}
\begin{itemize}
 \item Notice that adding any vector to the rows of $W$ or adding any constant to each entry of $b$ doesn't
 change the model. We are only fitting the parameters up to these symmetries. Essentially, what is important are the
 differences between each pair of the rows of $W$ and each pair of entries of $b$. We can remove this redundancy in
 the parameter space by assuming that the first row of $W$ and the first entry of $b$ are $0$. Whether this is
 useful depends on which algorithm we are using to fit the parameters.
 \item The success of the logistic regression depends on how good the features (the entries of $x$) are at linearly distinguishing
 the different classes. In machine learning practice, these features must be engineered based on some domain knowledge.
 Many neural network models attempt to learn the features in addition to the coefficients.
 \item Another way of thinking about the logistic regression is as follows. Given two labels, $l_i$ and $l_j$, define the
 odds of $l_i$ over $l_j$ to be the ratio $o_{i,j} = p(l_i) / p(l_j)$ (i.e. the ratio of the respective probabilities).
 The logistic regression models the logarithm of the
 odds, $\log(o_{i,j})$, as a linear function of the features for all pairs $(i,j)$.
 \item Somewhat counterintuitively, if the training data is perfectly linearly separable, then the negative log-likelihood
 objective has no minimizers because scaling up the parameters $W$ and $b$ always produces a smaller loss. This means that
 some form of regularization is necessary if the data are well-separated by linear functions.
\end{itemize}

\subsection{Loss functions}
Given a collection of labelled data points $\mathcal{D} = (x_1,y_1),...,(x_n,y_n)$ with $x_i\in \mathcal{X} = \mathbb{R}^n$ and
$y_i \in \mathcal{Y} = \{l_1,...,l_n\}$, we want to fit parameters $W^*,b^*$ from the data $\mathcal{D}$. There are
many different ways of doing this (a large part of statistics is concerned with what it means to fit parameters from data),
but we will only describe two standard approaches, the maximum likelihood and maximum posterior estimators.
\subsubsection{Maximum Likelihood Estimator}
The maximum likelihood estimator finds the parameter values which maximize the likelihood $p(\mathcal{D}~|~W, b)$
of the observed data, i.e. we set
\begin{equation}
 \theta^*=(W^*,b^*) = \argmax_{W, b} p(\mathcal{D}~|~W, b)
\end{equation}
Very often, the above optimization problem is replaced by the equivalent minimization of the negative log likelihood,
\begin{equation}
 \theta^*= (W^*,b^*) = \argmin_{W, b} -\log(p(\mathcal{D}~|~W, b))
\end{equation}
the reason for considering the log likelihood is that the objective now becomes a sum over the data points
$(x_1,y_1),...,(x_n,y_n)$ instead of a product. Specifically, in the case of a logistic regression, we have
\begin{equation}
 p(\mathcal{D}~|~W, b) = \prod_{j = 1}^n p_{i_j}(x_j ~|~W, b ) =
\prod_{j = 1}^n
\frac{\exp(W_{i_j}x_j + b_{i_j})}
{\displaystyle\sum_{i=1}^k\exp(W_{i}x_j + b_i)} =
\displaystyle\prod_{j = 1}^n
 \frac{\exp(W x_j + b)_{i_j}}
 {\mathbbm{1}^T\exp(W x_j + b)}
\end{equation}
where $i_j$ denote the index such that $y_j = l_{i_j}$ (i.e. $i_j$ is the label of the $j$-th data point), $\exp$ is applied
to a vector entrywise, and $\mathbbm{1}$ denotes the vector of ones. Then the negative log likelihood becomes
\begin{equation}
 -\log(p(\mathcal{D}~|~W, b)) = \displaystyle\sum_{j=1}^n \left(\log(\mathbbm{1}^T\exp(W x_j + b)) -
 (W x_j + b)_{i_j}\right) = \sum_{j=1}^n Cross-Entropy(p(x_j ~|~ W,b), y_j).
\end{equation}

\subsubsection{Maximum Posterior Estimator}
The maximum posterior estimator is very similar to the maximum likelihood estimator, but approaches the problem from a
Bayesian point of view. Equivalently, we can think of the maximum posterior estimator as a regularized maximum likelihood estimator.

In the Bayesian setting we fix a prior distribution over the parameters, $p(W,b)$, which represents domain
knowledge that we have concerning the values of the parameters. The posterior distribution (or distribution
given the data), $p(W,b~|~\mathcal{D})$, is determined by Bayes' rule
\begin{equation}
 p(W,b~|~\mathcal{D}) \sim p(W,b)p(\mathcal{D}~|~W,b)
\end{equation}

Because the posterior distribution is difficult to deal with in general, various approximate approaches exist, for example
sampling from the posterior distribution using MCMC or approximating the posterior distribution by a parametrized class
of distributions (known as variational Bayes'). The approach we consider here is the simpler elementary approach of
choosing the parameter values which maximize the posterior distribution, or equivalently minimize the negative log of the
posterior
\begin{equation}
 (W^*,b^*) = \argmin_{W, b} -\log(p(W, b~|~D))
\end{equation}
Plugging Bayes' formula into this yields
\begin{equation}
 (W^*,b^*) = \argmin_{W, b} -\log(p(W,b)) - \log(p(W, b~|~D))
\end{equation}
and we see that this becomes a `regularized' maximum likelihood estimate with regularizer $-\log(p(W,b))$,
which depends on the assumed prior. Often, the prior is a Gaussian centered at $0$, which results in a quadratic regularizer
of the form $\alpha (\|W\|_2^2 + \|b\|_2^2)$.

Taking into account the form of the likelihood function for the
logistic regression, we get the following problem
\begin{equation}
 (W^*,b^*) = \argmin_{W, b} \alpha (\|W\|_2^2 + \|b\|_2^2) + \displaystyle\sum_{j=1}^n \left(\log(\mathbbm{1}^T\exp(W x_j + b)) -
 (W x_j + b)_{i_j}\right)
\end{equation}

We now consider the logistic regression with a different type of regularizer.
\begin{equation}
 (W^*,b^*) = \argmin_{W, b} \alpha (\max_i \|w_i\|_2) + \displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i}
 \left(\log(\mathbbm{1}^T\exp(W x_j + b)) - (W x_j + b)_i\right)
\end{equation}

\begin{theorem}
 Suppose that the sets $A_i$ are linearly separable. Let
 $$\theta_\alpha = \argmin_{W, b} \alpha (\max_i \|w_i\|_2) + \displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i}
 \left(\log(\mathbbm{1}^T\exp(W x + b)) - (W x + b)_i\right)
 $$
 and
 $$\theta^* = \argmin_{\substack{w_ix + b_i \geq w_jx + b_j + 1\\x\in A_i,~j\neq i}} \max_i \|w_i\|_2$$
 Then, up to a rescaling, $\theta_\alpha \rightarrow \theta^*$ as $\alpha\rightarrow 0$, i.e.
 \begin{equation}
  \lim_{\alpha\rightarrow 0} \frac{\theta_\alpha}{\max_i \|w^\alpha_i\|_2} = \frac{\theta^*}{\max_i \|w^*_i\|_2}
 \end{equation}

\end{theorem}
\begin{proof}
 The key is to consider the auxiliary problem
 \begin{equation}
  \theta_M = \argmin_{\substack{W, b\\\max_i \|w_i\|_2 = M}}\displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i}
 \left(\log(\mathbbm{1}^T\exp(W x_j + b)) - (W x_j + b)_i\right)
 \end{equation}
 The linear separability of the sets $A_i$ implies that $\theta_\alpha \rightarrow \infty$ as $\alpha\rightarrow 0$.
 This means that
 \begin{equation}
  \lim_{\alpha\rightarrow 0} \frac{\theta_\alpha}{\max_i \|w^\alpha_i\|_2} = \lim_{M\rightarrow \infty} \frac{\theta_M}{M}
 \end{equation}
 So we only need to consider the problem
 \begin{equation}
  \frac{\theta_M}{M} = \argmin_{\substack{W, b\\\max_i \|w_i\|_2 = 1}}\displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i}
 \left(\log(\mathbbm{1}^T\exp(M(W x_j + b))) - (M(W x_j + b))_i\right)
 \end{equation}
 Additionally, by rewriting the optimization for $\theta^*$, we obtain
 \begin{equation}
  \frac{\theta^*}{\max_i \|w^*_i\|_2} = \argmax_{\substack{W, b\\\max_i \|w_i\|_2 = 1}} \min_{i\neq j,~x\in A_i} [(w_ix + b_i) - (w_jx + b_j)]
 \end{equation}
 It now suffices to show that given $\epsilon > 0$, there exists an $M(\epsilon)$ such that given $\theta, \theta^\prime$ with $\max_i \|w_i\|_2 = \max_i \|w^\prime_i\|_2 = 1$ and
 $$\min_{i\neq j,~x\in A_i} [(w^\prime_ix + b^\prime_i) - (w^\prime_jx + b^\prime_j)]
 > \min_{i\neq j,~x\in A_i} [(w_ix + b_i) - (w_jx + b_j)] + \epsilon
 $$
 we have
 \begin{equation}
 \begin{split}
  &\displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i}
 \left(\log(\mathbbm{1}^T\exp(M(\epsilon)(W^\prime x_j + b^\prime))) - (M(\epsilon)(W^\prime x_j + b^\prime))_i\right) > \\
 &\displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i}
 \left(\log(\mathbbm{1}^T\exp(M(\epsilon)(W x_j + b))) - (M(\epsilon)(W x_j + b))_i\right)
 \end{split}
 \end{equation}
 To see why this is the case, it helps to rewrite the above objective as
 \begin{equation}
  \displaystyle\sum_{i=1}^k \displaystyle\sum_{x\in A_i} \log\left(\displaystyle\sum_{j = 1}^k \exp(-M[(w_ix+b_i) - (w_jx+b_j)])\right)
 \end{equation}
 It is now clear that for sufficiently large $M$, this term is dominated by the terms which minimize $[(w_ix + b_i) - (w_jx + b_j)]$
 and thus for large $M$, this objective behaves as
 \begin{equation}
  C(\theta) \exp(-M\min_{i\neq j,~x\in A_i} [(w_ix + b_i) - (w_jx + b_j)])
 \end{equation}
 where $C(\theta)$ depends on how many indices and data points
 achieve the minimum $\min_{i\neq j,~x\in A_i} [(w_ix + b_i) - (w_jx + b_j)]$. This clearly implies that claim and completes the
 proof.

\end{proof}