\section{Linearly separable sets and margin}
Given dataset $(\bm x_i, y_i)$, $i = 1,2,\cdots,N$, $y$ is the label of $\bm x$ where $y\in \{1,2,\cdots,k\}$ or $\{e_1,e_2,\cdots,e_k\}$.

\begin{definition}[Unbiasedly Linearly Separable]
	A collection of subsets $A_1,...,A_k\subset \mathbb{R}^d$ are unbiased
	linearly separable if there exists $\bm \theta$ where
	\begin{equation}
	\label{theta}
	\bm\theta=
	\begin{pmatrix}
	\theta_1\\
	\vdots\\
	\theta_k
	\end{pmatrix}
	\in \mathbb{R}^{k\times d},
	\end{equation}
	such that for each $1\le i\le k$ and $ j \neq i$
	\begin{equation}
	\label{eq:3}
	\theta_i x > \theta_j x,\ \forall x\in A_i.
	\end{equation}
	or
	\begin{equation}
	\label{eq:3}
	(e_i-e_j)\cdot\bm\theta x > 0,\ \forall x\in A_i,
	\end{equation}
\end{definition}

\begin{definition}[Linearly Separable]
	A collection of subsets $A_1,...,A_k\subset \mathbb{R}^d$ are
	linearly separable if there exists $\bm \theta = (W,b)$ where
	\begin{equation}
	\label{Wb}
	W=
	\begin{pmatrix}
	w_1\\
	\vdots\\
	w_k
	\end{pmatrix}
	\in \mathbb{R}^{k\times d},
	b=
	\begin{pmatrix}
	b_1\\
	\vdots\\
	b_k
	\end{pmatrix}
	\in \mathbb{R}^{k},
	\end{equation}
	such that for each $1\le i\le k$ and $ j \neq i$
	\begin{equation}
	\label{eq:3}
	w_ix+b_i > w_jx+b_j,\ \forall x\in A_i.
	\end{equation}
	or
	\begin{equation}
	\label{eq:3}
	(e_i-e_j)\cdot(Wx+b) > 0,\ \forall x\in A_i,
	\end{equation}

\end{definition}

\begin{lemma}{\label{Interplation}}
	Define
	\begin{equation}
	\label{Gammai}
	\Gamma_i(\bm\theta) = \{x\in\mathbb R^n: (Wx+b)_i > (Wx+b)_j,\ \forall j \neq i\}
	\end{equation}
	Then for any set collection $A_1,\cdots,A_k$, $\bm\theta$ can separate $A_1,\cdots,A_k$ iff for each $i$,
	\begin{equation}
	\label{AiGamma}
	A_i \subset \Gamma_i(W,b)
	\end{equation}
\end{lemma}
We note that  each $\Gamma_i(W,b)  $ is a polygon whose boundary consists of hyperplanes
\begin{equation}
\label{Hij}
H_{ij}=\{(w_i-w_j)\cdot x+(b_i-b_j) = 0\}, \quad \forall j\neq i.
\end{equation}

Denote $\tilde x =
\begin{pmatrix}
x\\
1
\end{pmatrix}
$, we have $\bm\theta \tilde{x} = Wx+b$. For $x\in A_i$, then we have

\begin{lemma}
	A collection of subsets $A_1,...,A_k\subset \mathbb{R}^d$ are
	linearly separable iff $\tilde A_1,\cdots,\tilde{A}_k$ are unbiasedly linearly separable.
\end{lemma}

Denote
\begin{align}
&m_j^i(x,\bm\theta) = (e_i - e_j)\cdot (Wx+b) = (e_i - e_j)\cdot (\bm\theta \tilde{x}),\\
&\bm m^i(x,\bm\theta) = (m^i_1(x,\bm\theta),\cdots,m^i_{i-1}(x,\bm\theta),m^i_{i+1}(x,\bm\theta),\cdots,m^i_{k}(x,\bm\theta))
\end{align}
easy to observe that $x$ is correctly classified by $\bm\theta$ if and only if $m_j^i(x,\bm\theta) > 0$ for all $j\neq i$.

Define $\rho: \mathbb{R}^{k\times (d+1)} \rightarrow \mathbb{R}$ which can be chosen as $\rho(\bm\theta) = \|\bm\theta\|$ or  $\rho(\bm\theta) = \|W\|$. Notice that when $\rho(\bm\theta) = 0$, $\bm\theta$ can not classify any linearly separable set collections. So we only focus on those $\bm\theta$ where $\rho(\bm\theta) > 0$.

\begin{definition}[margin]
	The \textbf{soft margin} $m(\bm\theta)$ of a linearly separable subset collection $A_1,...,A_k\subset \mathbb{R}^d$ with respect to $\bm\theta = (W,b)$ and the \textbf{feasible separating parameter set} $\bm\Theta_0$ is defined as
	\begin{equation}
	m(\bm\theta) =  \min_{i} \min_{x\in \tilde{A}_i} \min_{j\neq i} m_j^i(x,\bm\theta)
	\end{equation}
	where $\tilde x =
	\begin{pmatrix}
	x\\
	1
	\end{pmatrix}$,  $\tilde A_i = \{\tilde{x}: x\in A_i\}$, and $\bm \theta \tilde{x} = Wx + b$, and
	\begin{equation}
	\bm\Theta_0 = \{\bm\theta: m(\bm\theta) > 0, \forall j\neq i, x\in A_i, i = 1,\cdots,k.\}
	\end{equation}
	And we can define the \textbf{max margin} $m^*$ and the \textbf{max margin parameter set} $\bm\Theta^*$ w.r.t $\rho(\cdot)$ as
	\begin{equation}
	m^* = \max_{\rho(\bm \theta)>0}\ \frac{m(\bm\theta)}{\rho(\bm\theta)} = \max_{\rho(\bm \theta)= 1}\ m(\bm\theta)
	\end{equation}

	\begin{equation}
	\bm\Theta^* = \mathop{\rm argmax}_{\rho(\bm \theta)= 1}\ m(\bm\theta).
	\end{equation}
\end{definition}

\begin{remark}
	$m(\bm\theta)>0$ implies $\rho(\bm\theta)>0$. Because if $\rho(\bm\theta)$ = 0, namely, $\bm\theta = 0$ or $\bm W = 0$, we have $m(\bm\theta) \leq \min (b_1-b_2,b_2-b_1) \leq 0$.
\end{remark}

Notice that the min of a famlily of concave functions is still concave, so $m(\bm\theta)$ is a concave function w.r.t $\bm\theta$. More precisely, $m(\bm\theta)$ is a concave homogeneous piecewise linear function w.r.t $\bm\theta$. \\
Because $m(\cdot)$ is continuous and $\{\bm\theta\ | \ \rho(\bm\theta) = 1\}$ is compact in a finite-dimensional space, we can easily obtain the following lemma:

\begin{lemma}
	If $\rho(\cdot) = \|\cdot\|$ is a norm on $\mathbb{R}^{k\times (d+1)} $ then $\bm\Theta^*$ must be nonempty.
\end{lemma}

\begin{lemma}
	If $\rho(\cdot) = \|\cdot\|$ is a strictly convex norm on $\mathbb{R}^{k\times (d+1)} $, then $\bm\Theta^*$ is a singleton set.
\end{lemma}

\begin{proof}
	suppose not, we can find $\bm\theta_1\neq\bm\theta_2$ such that $\|\bm\theta_1\| = \|\bm\theta_2\| = 1$ and $m(\bm\theta_1) = m(\bm\theta_2) = m^*$. \\
	\indent Take $\bm\theta = \frac{\bm\theta_1+\bm\theta_2}{2}$, we have $\|\bm\theta\| < 1$ and $m(\bm\theta) \geq \frac{m(\bm\theta_1)+m(\bm\theta_2)}{2} = m^*$ because of the concaveness of $m(\cdot)$. So we obtain
	\[
	m(\frac{\bm\theta}{\|\bm\theta\|}) > m(\bm\theta) \geq m^*,
	\]
	which leads to a contradiction to the definition of $m^*$.
\end{proof}

\begin{lemma}
	Given any $\bm\theta \in \mathbb{R}^{k\times (d+1)}$, we have
	\[
	m(\bm\theta + \bm{1}\alpha^T) = m(\bm\theta),
	\]
	for any $\alpha \in \mathbb{R}^{d+1}$.
\end{lemma}
\begin{proof}
	We only need to notice that
	\begin{equation}
		m^i_j(x,\bm\theta + \bm{1}\alpha^T) = (\theta_i + \alpha)\tilde{x} - (\theta_j + \alpha)\tilde{x} = (\theta_i - \theta_j)\tilde{x} = m^i_j(x,\bm\theta).
	\end{equation}
	So
	\begin{equation}
		m(\bm\theta + \bm{1}\alpha^T) = \min_{i} \min_{x\in \tilde{A}_i} \min_{j\neq i} m_j^i(x,\bm\theta + \bm{1}\alpha^T) = \min_{i} \min_{x\in \tilde{A}_i} \min_{j\neq i} m_j^i(x,\bm\theta) = m(\bm\theta).
	\end{equation}
\end{proof}

\begin{corollary}
	\begin{equation}
		\mathop{\rm argmax}_{\|W\| = 1, b_1 = 0} m(\bm\theta) \subset \mathop{\rm argmax}_{\|W\| = 1} m(\bm\theta).
	\end{equation}
\end{corollary}
\begin{proof}
	The case of $\mathop{\rm argmax}_{\|W\| = 1, b_1 = 0} m(\bm\theta) = \emptyset$ is trivial. We only consider the situation of $\mathop{\rm argmax}_{\|W\| = 1, b_1 = 0} m(\bm\theta) \neq \emptyset$. For any $\bm\theta^*\in\mathop{\rm argmax}_{\|W\| = 1, b_1 = 0} m(\bm\theta) = \emptyset$ and any $\bm\theta\in \{\bm\theta: \|W\| = 1\}$ we have
	\begin{equation}
		m(\bm\theta^*) \geq m(\bm\theta + \bm{1}^T\alpha) = m(\bm\theta)
	\end{equation}
	where $\alpha = (\bm{0},-b_1)^T$, which implies $\mathop{\rm argmax}_{\|W\| = 1, b_1 = 0} m(\bm\theta) \subset \mathop{\rm argmax}_{\|W\| = 1} m(\bm\theta).$
\end{proof}

\begin{lemma}
	If $\rho(\cdot)$ satisfies $\rho(\bm\theta) = \|W\|$ where $\|\cdot\|$ is a norm on $\mathbb{R}^{k\times d}$,  then $\bm\Theta^*$ is nonempty.
\end{lemma}

\begin{proof}
	Take $x_i\in A_i,\ i = 1,\dots,k$, then we have
	\begin{align}
	(e_j-e_l)\cdot(W x_j) + b_j - b_l > 0,\\
	(e_l-e_j)\cdot(W x_l) + b_l - b_j > 0,
	\end{align}
	which implies
	\begin{equation}
	|b_j-b_l|\leq \|e_j-e_l\|(\|Wx_l\|+\|Wx_j\|)\leq  (\sum_{i=1}^{k}\|e_i\|)(\sum_{i=1}^{k} \|x_i\|),
	\end{equation}
	for all $i \neq j$, $ (W,b)\in \bm\Theta_0$.\\
	According to the last corollary, we only need to prove that $\mathop{\rm argmax}_{\|W\| = 1, b_1 = 0} m(\bm\theta) \neq \emptyset.$ Because $b_1 = 0$, we fix $l = 1$, let $j \neq 1$, and we obtain
	\begin{equation}
		|b_j|\leq (\sum_{i=1}^{k}\|e_i\|)(\sum_{i=1}^{k} \|x_i\|),
	\end{equation}
	for all $j \neq 1$. so $\overline{\{\bm\theta: \|W\| = 1, b_1 = 0\}\cap \bm\Theta_0}$ is closed and bounded, thus compact, which implies $\mathop{\rm argmax}_{\|W\| = 1, b_1 = 0} m(\bm\theta) \neq \emptyset.$
\end{proof}

\section{Relation between margin and geometric margin}
Suppose that $X$ is a banach space with norm $\|\cdot\|$, $f$ is a bounded linear functional on $X$. Denote the kernel of $f$ as $\ker f$, then we can induce a quotient space $X/\ker f = \{x + \ker f : x\in X\}$, and we denote $x+\ker f $ as $[x]$. According to the quotient space theory in functional analysis, we can define a norm $\|\cdot\|_q$ on $X/\ker f$ as
\begin{equation}
	\|[x]\|_q = \inf_{y\in \ker f} \|x-y\| = d(x, \ker f),
\end{equation}
where $d$ is a distance induced by norm $\|\cdot\|$ on $X$, and $X/\ker f$ is a banach space. Also we can define a linear functional $f_q$ on $X/\ker f$ such that
\begin{equation}
	f_q([x]) = f(x),
\end{equation}
easy to check that $f_q$ is bounded and $\|f_q\| = \|f\|$. Because $\ker f$ is a maximal linear subspace of $X$, so $X/\ker f$ is a one-dimensional linear space. Thus we have
\begin{equation}
	|f_q([x])| = \|f_q\|_{dual} \|[x]\|_q,
\end{equation}
or namely,
\begin{equation}
	|f(x)| = \|f\|_{dual} \inf_{y\in \ker f} \|x-y\| = d(x,\ker f).
\end{equation}
Suppose that $x_0$ satisfy $f(x_0) + b = 0$, then
\begin{equation}
	|f(x) + b| = \|f(x-x_0)\| = \|f\|_{dual} \inf_{y\in \ker f} \|x-x_0-y\| = \|f\|_{dual} d(x, x_0+\ker f)
\end{equation}
So $\frac{|f(x)+b|}{\|f\|_{dual}}$ equals to the distance between $x$ and a hyperplane $\{x: f(x) + b = 0\}$ in a sense of norm $\|\cdot\|$ on $X$. Specially, if $X$ is a Hilbert space, then there must exist a $w \in X$ such that $f(x) = <x,w>$. and we have
\begin{equation}
	\frac{|<x,w>+b|}{\|w\|_{dual}} = \inf_{y\in\{x: <x,w> + b = 0\}} \|x-y\|.
\end{equation}

\begin{definition}[Geometric margin]
	Suppose that $k$ subsets of $\mathbb{R}^n$, $A_1,...,A_k$ are linearly separable by $\bm\theta = (W,b)$,
	the \textbf{geometric margin} of separation with respect to a metric
	$d$ is given by
	\begin{equation}
	\widehat{m}(\bm\theta) = \min_{i} \min_{x\in A_i} d(x,\partial\Gamma_i(\bm\theta)).
	\end{equation}
	where
	\begin{equation}
	d(x,\partial\Gamma_i(\bm\theta)) = \min_{y\in \partial\Gamma_i(\bm\theta) } d(x,y).
	\end{equation}
\end{definition}

\begin{lemma}
	If $d$ is a metric induced by a norm $\|\cdot\|$, we have
	\begin{equation}
	\widehat{m}(\bm\theta) =  \min_{i} \min_{x\in A_i} \min_{j\neq i} \frac{m_j^i(x,\bm\theta)}{\|w_i-w_j\|_{dual}}.
	\end{equation}
\end{lemma}

\begin{lemma}
	If $\rho(\bm\theta) = \|W\|$, there must exist a positive number $\alpha$ which relies on the choice of vector and matrix norm such that
	\begin{equation}
		m(\bm\theta) \leq \alpha \widehat{m}(\bm\theta), \ \forall \bm\theta\in \bm\Theta_0,
	\end{equation}
	or namely, geometric margin can always dominate margin.
\end{lemma}
\begin{proof}
	Notice that for any classifiable $\bm\theta$ (which means $\bm\theta\in\bm\Theta_0$), we have
	\begin{equation}
		\widehat{m}(\bm\theta) \geq \frac{1}{2} \min_{i} \min_{x\in A_i} \min_{j\neq i} \frac{m_j^i(x,\bm\theta)}{\max_i \|w_i\|} = \frac{\|W\|}{2\max_i \|w_i\|} m(\bm\theta)
	\end{equation}
	Because $2\max_i \|w_i\|$ is also a matrix norm of $W$, so there must exist a positive number $\alpha$ such that
	\begin{equation}
		\frac{\|W\|}{2\max_i \|w_i\|} \geq \frac{1}{\alpha},\ \forall W\neq 0.
	\end{equation}
	thus we obtain $m(\bm\theta) \leq \alpha \widehat{m}(\bm\theta), \ \forall \bm\theta\in \bm\Theta_0$.
\end{proof}

Next, we will discuss the equivalence between margin maximization and geometric margin maximization in binary classification ($k = 2$). To simplify the statement, we may just denote $m(\bm\theta)$ and $\tilde{m}(\bm\theta)$ as $m(w_1,w_2,b)$ and $\widehat{m}(w_1,w_2,b)$. An easy observation is that
\begin{equation}
	\widehat{m}(w_1+w,w_2+w,b) = \widehat{m}(w_1,w_2,b),\ \forall w\in \mathbb{R}^{1\times d}.
\end{equation}
Specailly, if we take $w = -\frac{w_1+w_2}{2}$, we have
\begin{equation}
\widehat{m}(w_1,w_2,b) = \widehat{m}(\frac{w_1-w_2}{2},\frac{w_2-w_1}{2},b).
\end{equation}
So without loss of classifiers, we may restrict $W$ to have the form $W = \begin{pmatrix}
w\\
-w
\end{pmatrix}$ when maximizing geometric margin.

\begin{lemma}
	If $k = 2$ and $\rho(\bm\theta) = \|W\| = \max_i \|w_i\|$, then $\mathop{\rm argmax}_{\bm\theta\in \bm\Theta_0} m(\bm\theta)$ and  $\mathop{\rm argmax}_{\bm\theta\in \bm\Theta_0} \widehat{m}(\bm\theta)$ contains the same classifiers.
\end{lemma}
\begin{proof}
Notice that
\begin{equation}
	\max_i \|w_i +w\| \geq \frac{1}{2}\|w_1-w_2\|, \forall w\in \mathbb{R}^{1\times d},
\end{equation}
and the equality holds when $w = -\frac{w_1+w_2}{2}$, which implies
\begin{equation}
	m(w_1,w_2,b) \leq m(\frac{w_1-w_2}{2},\frac{w_2-w_1}{2},b).
\end{equation}
So both margin maximization and geometric margin maximization can be restricted on $\{\bm\theta: W = \begin{pmatrix}
w\\-w
\end{pmatrix}\}$ without loss of classifiers. Notice that
\[
m(w,-w,b) = 2\widehat{m}(w,-w,b),
\]
so $\mathop{\rm argmax}_{\bm\theta\in \bm\Theta_0} m(\bm\theta)$ and  $\mathop{\rm argmax}_{\bm\theta\in \bm\Theta_0} \widehat{m}(\bm\theta)$ contains the same classifiers.
\end{proof}

\begin{lemma}
	If $k = 2$ and $\rho(\bm\theta) = \|W\| = \|w_1\| + \|w_2\|$, then $\mathop{\rm argmax}_{\bm\theta\in \bm\Theta_0} m(\bm\theta)$ and  $\mathop{\rm argmax}_{\bm\theta\in \bm\Theta_0} \widehat{m}(\bm\theta)$ contains the same classifiers.
\end{lemma}

\begin{proof}
	We only need to notice that
	\begin{equation}
	\|w_1+w\| + \|w_2+w\| \geq \|w_1-w_2\|, \forall w\in \mathbb{R}^{1\times d},
	\end{equation}
	and the equality holds when $w = -\frac{w_1+w_2}{2}$. The rest are the same to the proof of the last lemma .

\end{proof}

\begin{lemma}
	If $k = 2$ and $\rho(\bm\theta) = \|W\| = (\|w_1\|^p + \|w_2\|^p)^{\frac{1}{p}}$ where $p\geq 1$, then $\mathop{\rm argmax}_{\bm\theta\in \bm\Theta_0} m(\bm\theta)$ and  $\mathop{\rm argmax}_{\bm\theta\in \bm\Theta_0} \widehat{m}(\bm\theta)$ contains the same classifiers.
\end{lemma}

\section{Margin Maximizing Loss Functions}

Denote the extended real number system as $\bar{\mathbb{R}} = \mathbb{R}\cup \{-\infty,+\infty\}$ and we use $\bm m = (m_1,m_2,\cdots,m_{k-1})$ to represent an elemnt of $\bar{\mathbb{R}}^k$. Consider functions $l: \bar{\mathbb{R}}^{k-1} \rightarrow \mathbb{R}$ with the following assumptions:
\begin{itemize}
	\item $l(\cdot)$ is commutative and decreasing in each coordinate, and is continuous on $\bar{\mathbb{R}}^{k-1}$.
	\item There exists a $T>0$ in $\bar{\mathbb{R}} $ such that
	\begin{equation}
	l(\bm m)
	\begin{cases}
	> 0,\ \ \min_i m_i < T,\\
	= 0,\ \ \min_i m_i \geq T.
	\end{cases}
	\end{equation}
	Notice that when $T = \infty$, $l(\bm m) = 0$ if and only if $m_i = \infty$ for all $i = 1,2,\cdots,k-1$.
	\item If $T = \infty$, we assume for any $\bm u, \bm v \in \bar{\mathbb{R}}^{k-1}$ where $\min_i u_i < \min_i v_i$ must satisfy
	\begin{equation}
	\lim\limits_{t\rightarrow \infty}\frac{l(t\bm u)}{l(t\bm v)} = \infty.
	\end{equation}
	Specially, we have
	\begin{equation*}
	\lim\limits_{t\rightarrow \infty}\frac{l(tu,\infty,\cdots,\infty)}{l(tv,tv,\cdots,tv)} = \infty,\ \ \forall u<v.
	\end{equation*}

\end{itemize}

Denote that $R: \mathbb{R}^{+}\rightarrow \mathbb{R}^{+}$ which is a strictly increasing function such that $R(0) = 0$ and $R(x) \rightarrow +\infty$ as $x\rightarrow +\infty$. The final loss function has the following fomulation
\begin{equation}
L(\bm\theta,\lambda) = \sum_{i =  1}^{k} \sum_{x\in A_i} l(\bm m^i(x,\bm  \theta)) + \lambda R(\rho(\bm\theta))
\end{equation}
And we denote the optimal parameter set of the above problem as
\begin{equation}
\bm\Theta(\lambda) = \mathop{\rm argmin}_{\bm\theta} L(\bm\theta,\lambda)
\end{equation}

\begin{lemma}
	If $\rho: \bm\theta \mapsto \|\bm\theta\|$ where $\|\cdot\|$ is a norm on $\mathbb{R}^{k\times (d+1)} $, then $\bm\Theta(\lambda)$ must be nonempty for any $\lambda > 0$.
\end{lemma}

\begin{lemma}
	If $\rho: \bm\theta \mapsto \|W\|$ where $\|\cdot\|$ is a norm on $\mathbb{R}^{k\times d} $, then $\bm\Theta(\lambda)$ must be nonempty for sufficiently small $\lambda > 0$.
\end{lemma}

Here we show the loss function of logistic regression and SVM as examples.

Logistic regression($T = \infty$):\\
\[
\sum_{i =  1}^{k} \sum_{x\in A_i} \log(1+\sum_{j\neq i} e^{(\theta_j - \theta_i) x}) + \lambda R(\rho(\bm\theta))
\]

where
\[
l(\bm m) = \log(1 + \sum_{j}^{k-1} e^{-m_j})
\]

\indent Support vector machine($T = 1$):\\
\[
\sum_{i =  1}^{k} \sum_{x\in A_i}\sum_{j\neq i} {\rm ReLU}(1 - (\theta_i - \theta_j) x) + \lambda R(\rho(\bm\theta))
\]
where
\[
l(\bm m) = \sum_{j = 1}^{k-1} {\rm ReLU}(1 - m_j)
\]
%\begin{lemma}
%	If $l(\bm m(x,\bm\theta))$ is convex w.r.t $\bm\theta$ for any fixed $x$, $R(\cdot)$ is convex and $\rho(\cdot)$ is a strictly convex norm, then $L(\bm\theta,\lambda)$ is strictly convex w.r.t $\bm\theta$ and $\bm\Theta(\lambda)$ contains the unique element for any $\lambda > 0$.
%\end{lemma}

From now on, for any given $\lambda > 0$, we fix a $\bm\theta(\lambda)\in \bm\Theta(\lambda)$. We'll prove that this kind of loss functions has the margin maxmizing property.

\begin{lemma}

	\begin{equation}
	\lim\limits_{\lambda\rightarrow 0} L(\bm\theta(\lambda),\lambda) = 0.
	\end{equation}
\end{lemma}

\begin{proof}
    It's sufficient to show that $\mathop{\rm limsup}_{\lambda \rightarrow 0} L(\bm\theta(\lambda),\lambda) = 0$.\\
    Given any $\epsilon>0$, there must exist a $\bm\theta_{\epsilon}\in\bm\Theta_0$ such that
    \[
    \sum_{i =  1}^{k} \sum_{x\in A_i} l(\bm m^i(x,\bm  \theta_{\epsilon})) < \epsilon.
    \]
    So
    \begin{equation}
    	\mathop{\rm limsup}_{\lambda \rightarrow 0} L(\bm\theta(\lambda),\lambda) \leq \mathop{\rm limsup}_{\lambda \rightarrow 0} L(\bm\theta_\epsilon,\lambda) \leq \epsilon.
    \end{equation}
    Because of the arbitrariness of $\epsilon$, we have
    \[
    \mathop{\rm limsup}_{\lambda \rightarrow 0} L(\bm\theta(\lambda),\lambda) = 0.
    \]
\end{proof}

\begin{corollary}
	For sufficiently small $\lambda$, we must have $\bm\Theta(\lambda) \subset \bm\Theta_0 $, or namely,
	\[
	m(\bm\theta) > 0,\ \rho(\bm\theta) >0,
	\]
	for any $\bm\theta \in \bm\Theta(\lambda)$,
\end{corollary}

In order to make the problem more intuitive, we can transfer the problem to a new coordinate system similarily to the polar coordinate system. Define
\begin{equation}
\bm S = \{\bm\omega\in \mathbb{R}^{k\times (d+1)}: \rho(\bm \omega) = 1\}
\end{equation}
Given any $\bm\theta\in \{\bm\theta: \rho(\bm\theta) > 0\}$, we can define its $\rho$-polar coordinates $(r,\omega) \in [0,+\infty) \times \bm S$ as
\begin{equation}
r = \rho(\bm\theta),\ \bm\omega = \frac{\bm\theta}{\rho(\bm\theta)}.
\end{equation}

Notice that this coordinate transformation is a bijection from $\{\bm\theta: \rho(\bm\theta) > 0\}$ to  $(0,+\infty)\times \bm S$.
Correspondingly, we can define a new loss function in this new coordinate as
\begin{equation}
 \mathcal L(r,\bm\omega,\lambda) = L(r\bm\omega, \lambda) = \sum_{i =  1}^{k} \sum_{x\in A_i} l(r\bm m^i(x,\bm\omega)) + \lambda R(r)
\end{equation}
And define the minima set of this new loss as
\begin{equation}
\bm Z(\lambda) = \mathop{\rm argmin}_{r, \bm\omega} \mathcal L(r,\bm\omega,\lambda)
\end{equation}

Given any $\lambda>0$, we fix a $ (r(\lambda),\omega(\lambda))\in \bm Z(\lambda)$. Similarily, we have the following lemma.
\begin{lemma}
	\begin{equation}
	\lim\limits_{\lambda\rightarrow 0} \mathcal L(r(\lambda),\bm\omega(\lambda),\lambda) = 0.
	\end{equation}
\end{lemma}

\begin{corollary}\label{Maxmargin3}
	For sufficiently small $\lambda >0$, we must have
	\[
	r> 0,\ m(\bm\omega) > 0,
	\]
	for any $(r,\bm\omega) \in \bm Z(\lambda)$.
\end{corollary}

\begin{lemma}
	For sufficiently small $\lambda >0$, the coordinate transformation restricted on $\bm\Theta(\lambda)$ is a bijection from $\bm\Theta(\lambda)$ to $\bm Z(\lambda)$.
\end{lemma}

So to study the convergence of $\frac{\bm\theta(\lambda)}{\|\bm\theta(\lambda)\|}$, we only need to figure out the convergence of $\bm\omega(\lambda)$ as $\lambda\rightarrow 0$.

\subsection{Case 1: $T<\infty$}

We first discuss the case of $T<\infty$.

\begin{lemma}
	\begin{equation}
		r(\lambda) \leq \frac{T}{m^*}, \forall \lambda>0.
	\end{equation}
\end{lemma}

\begin{proof}
	Find any $\bm\omega^*\in\bm\Theta^*$, we have
	\[
	\lambda R(r(\lambda)) \leq \mathcal{L}(r(\lambda),\bm\omega(\lambda),\lambda)\leq\mathcal{L}(\frac{T}{m^*},\bm\omega^*,\lambda) = \lambda R(\frac{T}{m^*}).
	\]
	which implies
	\[
	r(\lambda) \leq \frac{T}{m^*}.
	\]

\end{proof}

\begin{theorem}
	If $T<\infty$, any convergence point of $\bm\omega(\lambda)$ must be a margin maximizing parameter in $\bm\Theta^*$.
\end{theorem}

\begin{proof}
	Suppose that there is a sequence $\lambda_n \searrow 0$ such that  $\bm\omega(\lambda_n)\rightarrow \overline{\bm\omega}$ and $m(\overline{\bm\omega})< m^*$. So there exsits a $\epsilon>0$ and $K\in \mathbb{N}$ such that
	\[
	m(\bm\omega(\lambda_n)) < m^* - \epsilon, \ \forall n>K.
	\]
	Then we have
	\begin{align}
	&\mathcal{L}(r(\lambda_n),\bm\omega(\lambda_n),\lambda_n) \\
	\geq\ &\sum_{i =  1}^{k} \sum_{x\in A_i} l(r(\lambda_n)\bm m^i(x,\bm\omega(\lambda_n)))\\
	\geq\ & l(\frac{m^*-\epsilon}{m^*}T,\infty,\cdots,\infty) > 0,\ \forall n>K,
	\end{align}
	which leads to a contradiction with $\lim\limits_{\lambda\rightarrow 0} \mathcal L(r(\lambda),\bm\omega(\lambda),\lambda) = 0$.
\end{proof}

\subsection{Case 2: $T=\infty$}
Next, we discuss the case of $T=\infty$.

\begin{lemma}\label{Maxmargin7}
	If $T = \infty$, we have
	\begin{equation}
	\lim\limits_{\lambda\rightarrow 0}\ r(\lambda) = \infty.
	\end{equation}
\end{lemma}

\begin{proof}
	Suppose that there is an $M>0$ and a sequence $\lambda_n \searrow 0$ such that  $r(\lambda_n) \leq M$. Then we have
	\[
	\mathcal{L}(r(\lambda_n),\bm\omega(\lambda_n),\lambda_n) \geq l(Mm^*,\infty,\cdots,\infty) > 0,\ \forall n\in\mathbb{N},
	\]
	which is contradictory to $\lim\limits_{\lambda\rightarrow 0} \mathcal L(r(\lambda),\bm\omega(\lambda),\lambda) = 0$.
\end{proof}

\begin{theorem}
	If $T = \infty$, any convergence point of $\bm\omega(\lambda)$ must be a margin maximizing parameter in $\bm\Theta^*$.
\end{theorem}

\begin{proof}
	Suppose that there is a sequence $\lambda_n \searrow 0$ such that  $\bm\omega(\lambda_n)\rightarrow \widehat{\bm\omega}$ and $m(\widehat{\bm\omega})< m^*$. So there exsits a $\epsilon>0$ and $K_1\in \mathbb{N}$ such that
	\[
	m(\bm\omega(\lambda_n)) < m^* - \epsilon, \ \forall n>K_1.
	\]
	Take
	\[
	\bm u = (m^*-\epsilon,\infty,\cdots,\infty),\ \bm v = (m^*, m^*,\cdots,m^*),
	\]
	then there exists a $K_2\in\mathbb{N}$ such that
	\[
	l(r(\lambda_n) \bm u) > N l(r(\lambda_n) \bm v), \forall n>K_2.
	\]
	Let $K = \max\{K_1,K_2\}$. Notice that
	\[
	\mathcal{L}(r(\lambda_n),\bm\omega(\lambda_n),\lambda_n) \geq  l(r(\lambda_n)\bm u) + \lambda_n \rho(r(\lambda_n)),
	\]
	\[
	\mathcal{L}(r(\lambda_n),\bm\omega^*,\lambda_n) \leq N l(r(\lambda_n) \bm v) + \lambda_n \rho(r(\lambda_n)).
	\]
	Thus given any $\bm\omega^*\in \bm\Theta^*$, we have
	\[
	\mathcal{L}(r(\lambda_n),\bm\omega^*,\lambda_n) < \mathcal{L}(r(\lambda_n),\bm\omega(\lambda_n),\lambda_n),\ \forall n > K,
	\]
	which is contradictory to $(r(\lambda_n),\bm\omega(\lambda_n))\in \bm Z(\lambda)$.
\end{proof}

\begin{corollary}
	If $\bm\Theta^*$ only contains the unique element $\bm\theta^*$, then $\frac{\bm\theta(\lambda)}{\rho(\bm\theta(\lambda))}$ must converge to $\bm\theta^*$ as $\lambda\rightarrow 0$.
\end{corollary}

\begin{corollary}
	If $\rho(\cdot)$ is a strictly convex norm, then $\frac{\bm\theta(\lambda)}{\rho(\bm\theta(\lambda))}$ must converge to $\bm\theta^*$ as $\lambda\rightarrow 0$.
\end{corollary}

\newpage
\section{Regularization by maximizing margins $k=2$}
The classic SVM, without kernels, is based on the belief that
bigger margin will give better generalization.

The question is
\begin{itemize}
\item What is the right metric to define margin?
\end{itemize}

We make the following observations:
\begin{enumerate}
\item  The regularization norm for the loss function should be dual to the
norm  for the data space
\item The norm for the data space ideally should reflect the features
  of the image, such as

\begin{enumerate}
\item rotation invariant (continuity)
\item translation invariant (continuity)
\item deformation invariant (continuity)
\end{enumerate}

\item One possibility is to use the following metrix
$$
d(\phi(x), \phi(y) )
$$
where $\phi(x)$ is some CNN function transferred from other data
\end{enumerate}

In LR, we have
$$
L(x)=(\Theta, x)
$$

\subsection{Conclusion}
\begin{itemize}
\item The commonly used regularization such as $\ell^2$ norm should
  not be helpful to increase genearlization accuracy, since its dual norm is not
  a good metric to measure the dat, but such a regularization may be
  useful for noisy.
\end{itemize}

\newpage

\section{Some comments on the metric of images}
$\ell^2$ norm might not be a proper norm for us to do classification in the digital image space, because if you rotate the index of an image, the new image you get might have larger distance to the original than some images from other labels. That means defferent classes of images are not separated very well as we imagine, but are mixed together and not linearly separable. \\
\indent What kind of 'metric' is reasonable for image classification? At least, it should satisfy some invariance like translation-invariance and rotation-invariance in index space. Theoretically, we can define a 'metric' using the notion of quotient space. Abstractly, We regard a digital image as a function from $\mathbb{R}^2$ to $\mathbb{R}$. And we can assume the whole digital image space to be a $L^p(\mathbb{R}^2)$ space. And given $Q\in \mathbb{R}^{2\times 2}$, $b\in ]\mathbb{R}^2$, we define an operation $A(Q,b): L^p(\mathbb{R}^2)\circlearrowleft$ such that
\[
A(Q,b)\circ f(x) = f(Qx+b).
\]
When we take $Q$ is an orthogonal matrix, it's a rotation or reflection in the domain, while $A(Q,b)$ ia a bijectiction in $L^p$ space which keeps the value of norm.
\begin{definition}{equivalence of images}
	We say two images $f,g \in L^p(\mathbb{R}^2)$ are equivalent if there is an orthogonal matrix $Q$ and a vector $b$ such that $A(Q,b)\circ f = g$, denoted as $f \sim g$.
\end{definition}
Thus we can naturally induce a quotient space by the notion of image equivalence. We define a equivalence class from the quotient space as $\bar{f} = \{g: g\sim f\}$. But unfortunately, the natural way to define plus opration in the quotient space dosen't work in this situation. If we define $\bar{f}+\bar{g} = \bar{f+g}$, then we will find this plus opration rely on the representitive you choose, so is not well-defined. But we can still define a metric in this quotient space as
\begin{equation}
	d(\bar{f},\bar{g}) = \inf_{s\in \bar{f},t\in \bar{g}} \|s-t\|_{L^p}.
\end{equation}
Although this metric satisfy those invariance we expect, the quotient space is not linear at all, so this metric can not be a norm which means it's hard to use in the real situation. \\
\indent Another way to achieve those invariance is to find some feature mapping $F$ to map those equivalent points to be very close to each other in feature space and then we use the norm in feature space to determine the metric. For example, we define $d(f,g) = \|F(f)-F(g)\|$. How to achieve those invariance via this $F$? Roughly speaking, it's concerned with your data and model. First, your model should have the capacity to get a proper function $F$ with those expected property. Second, your image data should contain those rotation or translation from the same image to help your model build such invariance properties. For example, data augmentation can  do such things.\\
\indent So an interesting question is, do those deep learning models have the ability to achieve or approximately achieve those invariance?


% !TeX root = ./apxthy.tex

\section{Splines}
%
\label{sec:splines}
%
In this very short chapter we will briefly introduce and explore some
consequences of piecewise polynomial approximation (as opposed to global
polynomial approximation as in \S~\ref{sec:poly}). The basic results will be
very easy to obtain. For lack of time we will skip the more interesting
algorithmic aspects, in particular B-Splines (we will briefly define them and
show some examples, but we won't go into the implementation details, at least
not this year).

\subsection{Motivation}
%
\label{sec:splines:motivation}
%
Let us motivate the idea of splines as follows: consider
the function $f(x) = \sqrt{x}$ on $[0, 1]$. After rescaling to $[-1,1]$ we can
approximate it with polynomials to obtain the convergence rate (cf. Jackson's
Theorem \ref{eq:poly:jackson1})
\[
    \inf_{p \in \Poly_N} \|f - p\|_{L^\infty(0,1)} \lesssim N^{-1/2}.
\]
This is a very slow rate of convergence, purely caused by the singularity at
$x = 0$. But in $[1/2, 1]$ $f$ is analytic and on that interval we would
expect
\[
    \inf_{p \in \Poly_N} \|f - p\|_{L^\infty(1/2,1)} \lesssim \rho^{-N},
\]
for some $\rho > 1$. We can then prescribe a second polynomial on $[1/4, 1/2]$,
and so forth, thus obtaining a piecewise polynomial approximation. The
subintervals $[1/2,1], [1/4, 1/2], \dots$ are called a mesh and the flexibility
in choosing these sub-intervals can lead to very strong results. We will later
see that in this particular case we obtain almost exponential convergence.

\subsection{Splines for $C^j$ functions}
%
\label{sec:splines:Cj}
%
To work with splines we will need to construct polynomial approximations on
arbitrary sub-intervals $[a,b] \subset \R$. The Chebyshev nodes on $[a,b]$ are
simply the rescaled nodes
\[
    x_j^{[a,b]}  = a + \frac{(x_j+1)(b-a)}{2},
\]
where $x_j$ are the Chebyshev nodes on $[-1,1]$. The resulting
interpolation operator is denoted by $I_N^{[a,b]}$.

We can now quantify the effect of domain size with the following lemma.

\begin{lemma}
    Let $f \in C^{p-1,1}([a,b])$ where $a < b$, and $N \leq p$, then
    \[
        \|f - I_N^{[a,b]} f \|_{L^\infty(a,b)}
        \leq \frac{c^N \log N}{N!} \b(b-a\big)^N \| f^{(N)} \|_{L^\infty(a,b)},
    \]
    where $c$ is a generic constant.
\end{lemma}
\begin{proof}
    Let $g(y) = f(\xi(y))$ where $\xi(y) = a + (b-a)(1+y)/2$, i.e.,
    \[
        \xi : [-1,1] \to [a, b]
    \]
    is affine and bijective. Then according to Jackson's theorem (the sharp
    version; cf. Exercise~\ref{exr:poly:convergence}),
    \[
        \|f - I_N^{[a,b]} f\|_{L^\infty(a,b)} =
        \| g - I_N g \|_{L^\infty(-1,1)}
        \leq  \frac{c_1^N\log N}{N!} \| g^{(N)} \|_{L^\infty(-1,1)}.
    \]
    Next, since $\xi$ is affine it is easy to show that
    \[
        g'(y) = f'(\xi(y)) \xi'(y) = f'(\xi(y)) \smfrac{b-a}{2},
    \]
    and hence
    \[
        g^{(j)}(y) = f^{(j)}(\xi(y)) \B(\smfrac{b-a}{2}\B)^j.
    \]
    Combining this with the interpolation error estimate for $g$
    yields the stated result.
\end{proof}

Thus we see that we now have two parameters to control the approximation error:
the polynomial degree $N$ and the interval lengths $(b-a)$. This extra freedom
is what can make splines a powerful alternative to polynomials.

\begin{definition}
    Let $y_0 < y_1 < \dots < y_M$ be a partition of an interval $[y_0, y_M]$,
    then we define the space of splines (piecewise polynomials) of degree $N$ on
    that partition to be
    \[
        \Spl_N(\{y_i\}) := \b\{ s : [y_0, y_M] \to \R, \quad
            s|_{[y_{m-1}, y_m]} \in \Poly_N \text{ for all }
            m = 1, \dots, M \b\}
    \]
    Splines are of course $C^\infty$ in each interval $[y_{j-1}, y_j]$, but
    sometimes it is also interesting to require that splines have a certain
    regularity on the entire interval $[y_0, y_M]$. We therefore define
    \[
        \Spl_N^p(\{y_i\}) := \Spl_N(\{y_i\}) \cap C^p([y_0, y_M]).
    \]
    It is worth nothing that $s \in \Spl_N^p$ implies in fact that $s \in
    C^{p,1}$.
\end{definition}

\begin{remark}
    It is of course also possible to define splines with varying polynomial
    degree, i.e. in each subinterval $[y_{j-1}, y_j]$ we might impose a degree
    $N_j$. This has advantages for some applications but we will not consider it
    here.
\end{remark}

It takes a bit more work to construct splines of regularity $p = 1$ or higher,
but $\Spl_N^0$ splines are obtained by simply taking Chebyshev interpolants on
each sub-interval. We call the resulting interpolant $I_{N,M}$,
\[
    I_{N,M} f(x) := I_N^{[y_{m-1},y_m]} f(x)    \qquad \text{for }
    x \in [y_{m-1}, y_m].
\]
We then obtain the following basic approximation
error estimates.

\begin{theorem} \label{th:splines:convergence_Cj}
    Let $f \in C^p([a,b])$ and $a = y_0 < \dots < y_M = b$ a partition of $[a,
    b]$, and let $h_m := y_m - y_{m-1})$ be the mesh size, and $N \leq p$, then
    \[
        \| f - I_{N,M} f \|_{L^\infty(a,b)}
        \leq  C_N \max_{m = 1, \dots, M} h_m^N
        \| f^{(N)} \|_{L^\infty(y_{m-1}, y_m)},
    \]
    where $C_N = \frac{c^N \log N}{N!}$.
    In particular, if the partition is uniform,
    $y_m = a + h m$ where $h = (b-a)/M$ then
    \[
        \| f - I_{N,M} f \|_{L^\infty(a,b)}
        \leq C_N h^N \|f^{(N)}\|_{L^\infty(a,b)}.
    \]
\end{theorem}
\begin{proof}
    Left as an exercise.
\end{proof}

\subsection{Splines for functions with singularities}
%
\label{sec:splines:sing}
%
We will demonstrate how splines can be used to effectively resolve singular
behaviour using the example from the beginning of this chapter,
\[
    f(x) = \sqrt{x} \qquad \text{on } x \in [0, 1]
\]
A possible analytic continuation is given by
\[
    f(r e^{i \varphi}) = \sqrt{r} e^{i \varphi / 2},
\]
which is analytic in $\C \setminus (-\infty, 0]$. Moreover, we have $|f(z)| =
\sqrt{|z|}$ which will make it easy to estimate $\|f\|_{L^\infty(E_\rho)}$ where
$E_\rho$ will be some suitable Bernstein ellipsi.

Our strategy will be to use a partition
\[
    0, 2^{-M}, 2^{-M+1}, \dots, 2^{-1}, 1.
\]
Since $f$ is analytic in each subinterval $[2^{-m}, 2^{-m+1}]$ we will be able
to use the exponential convergence rates from
Theorem~\ref{th:poly:err_analytic}.

Let us therefore consider $f$ on $[2^{-m}, 2^{-m+1}]$. We rescale
\[
    g(y) = f\b(2^{-m} + 2^{-m-1}(1+y)\b),
\]
then the singularity $x = 0$ maps to $y = -3$, hence $g$ in analytic in $\Re z >
-3$. In particular taking $\rho = 4$ we have $a = \smfrac12(\rho+\rho^{-1}) < 3$
and
\begin{align*}
    \|g\|_{L^\infty(E_\rho)} &\leq g(a) \leq f(2^{-m} + 2^{-m-1}(1+a)) \\
    &\leq  f(2^{-m} + 2^{-m+1}) \\
    &\leq \sqrt{2^{-m+2}} \\
    &= 2^{-m/2+1}.
\end{align*}
Thus, we obtain
\[
    \| f - I_N^{[2^{-m},2^{-m+1}]} f\|_{L^\infty(2^{-m},2^{-m+1})}
    =
    \| f - I_N g \|_{L^\infty(-1,1)}
    \leq C 4^{-N} 2^{-m/2}
\]
To make our life a little easier we can just estimate
\[
    \| f - I_N^{[2^{-m},2^{-m+1}]} f\|_{L^\infty(2^{-m},2^{-m+1})}
    \leq
    C 4^{-N} \qquad \text{for } m = M, M-1, \dots, 1;
\]
that is,
\[
    \| f - I_{N,M} f \|_{L^\infty(2^{-M}, 1)} \leq
    C N^{-4}.
\]

Finally, we address the first interval $[0, 2^{-M}]$. We rescale again
as before, but now the singularity becomes part of the domain $[-1,1]$, i.e.,
$g \in C^{0,1/2}([-1,1])$ and no better. Jackson's theorem therefore tells
us the
\[
    \| g - I_N g \|_{L^\infty(0, 2^{-M})}
    \leq
    C \omega_g(N^{-1}) = C N^{-1/2}.
\]
But the constant matters here! Specifically, we can show that
\[
    \omega_g(r) = c 2^{-M/2} \sqrt{r},
\]
that is, we even have
\[
     \|f - I_N^{[0, 2^{-M}]} f \|_{L^\infty(0, 2^{-M})}
     \leq C 2^{-M/2} N^{-1/2}.
\]
Let us again make our life a little easier and ignore the $N^{-1/2}$ term, then
we want to balance $2^{-M/2} = 4^{-N}$; that is,
\[
    M = 4 N.
\]
With this choice, we finally obtain
\[
    \| f - I_{N,M} f \|_{L^\infty(0, 1)} \leq C 4^{-N}.
\]

To conclude we convert this into a cost estimate. The cost of evaluating
$I_{N,M} f$ at a single point in space is the same as evaluating a
polynomial of degree $N$, that is
\[
    {\rm COST-EVAL}(I_{N,M} f) = O(N)
\]
and in particular, we obtain the very nice exponential convergence
result
\[
    \| f - I_{N,M} f \|_{L^\infty(0, 1)} \leq C \rho^{-{\rm COST-EVAL}},
\]
for some $\rho > 0$. The cost to ``build and store'' $I_{N,M} f$ is the cost of
evaluating $f$ at $M \cdot N$ points, i.e., $O(N^2)$ so this cost is a little
higher, but still very attractive.

This example is intended to demonstrate the power of adapting the spline grid to
the features of the function to be approximated. Automating this process is of
great interest but goes beyond the scope of this module.

\begin{remark}
    We can do slightly better by balancing the two terms in
    \[
        \| f - I_N^{[2^{-m},2^{-m+1}]} f\|_{L^\infty(2^{-m},2^{-m+1})}
        \leq
        C 4^{-N_m} 2^{-m/2}
        = C 4^{- N_m - m/4},
    \]
    i.e., choosing $N_m +  m/4 = N = {\rm const}$. But one can easily
    check that this only gives an improvement in some constants, but
    not qualitatively.
\end{remark}

\subsection{Exercises}

\begin{exercise} \label{exr:splines:}
    Prove Theorem~\ref{th:splines:convergence_Cj}
\end{exercise}

\begin{exercise}
    \begin{enumerate} \ilist
    \item Suppose you are given a function $f \in C^{p-1,1}([-1,1])$. For
    simplicity, assume even that in each subinterval $[a,b] \subset
    [-1,1]$ the regularity of $f$ is no better than $C^{p-1,1}$. Assume
    you discretise $[-1,1]$ with a uniform grid. How would you optimally
    balance the grid spacing $h$ against the polynomial degree $N$?
    (i.e. minimise the error against the number of function evaluations
    you need to specify the approximant)

    \item Now suppose that $f \in A([-1,1])$; how would you balance $h$
    against $N$ now?

    \item For the following functions compare the performance of
    global polynomial versus $\Spl_N^0$ approximation on a uniform grid:
    \begin{itemize}
        \item $f(x) = |x|$
        \item $f(x) = |x+\pi|$
        \item $f(x) = |\sin(x/2)|$
        \item $f(x) = (1+25 x^2)^{-1}$
        \item $f(x) = x \sin(1/x)$
    \end{itemize}
    \end{enumerate}
\end{exercise}

\begin{exercise}
    For the following functions $f : [-1,1] \to \R$, design a spline
    approximation with quasi-optimal rate of convergence in
    $\|\cdot\|_{L^\infty(-1,1)}$ in terms of evaluation cost.
    \begin{itemize}
        \item $f(x) = |x|$
        \item $f(x) = |\sin(x/2)|$
        \item $f(x) = (1+25 x^2)^{-1}$
        \item $f(x) = x \sin(1/x)$
    \end{itemize}
\end{exercise}

\begin{exercise}[Linear Splines] Show that for we can write continuous linear
    spline interpolations, i.e. $s \in \Spl_1^0(\{y_m\})$ in terms of a nodal
    basis,
    \[
        s(y) = \sum_{m = 0}^M f(y_m) \phi_m(y),
    \]
    where $\phi_m$ are ``hat-functions'' that you should specify
    explicitly.
\end{exercise}

\begin{exercise}[Hermite Interpolation with Cubic Splines]
    Let $y_0 < \dots < y_M$ be a grid and let $f_m, f_m'$ be
    function and derivative values at those grid points. Show that there
    exists a unique cubic spline $s \in \Spl_3^1(\{y_m\})$ such that
    \[
        s(y_m) = f_m, \quad \text{and} \quad
        s'(y_m) = f_m' \quad \text{for } m = 0, \dots, M.
    \]
    {\it HINT: in each interval $[y_{m}, y_{m+1}]$ write $s(x) = f_{m} + f_{m}'
        (x-x_{m}) + a_m (x-x_m)^2 + b_m (x-x_m)^3$ and show that there exist
        unique $a_m, b_m$ such that $s(x_{m+1}) = f_{m+1}, s'(x_{m+1}) =
        f_{m+1}'$. You may wish to derive explicit expressions for
        $a_m, b_m$ in preparation for the next exercise.}
\end{exercise}

\begin{exercise}[B-Splines]
    Depending on regularity requirements of an application it is
    sometimes advantageous to require higher regularity of the approximant,
    i.e., we should consider $\Spl_N^p$, $p > 0$. The case $\Spl_N^{N-1}$
    turns out to be particularly natural; these are alled the B-splines. And
    amongst those, the cubic splines enjoy particular polularity.

    \begin{enumerate} \ilist
        \item Suppose for the moment that $s \in \Spl_3^2(\{y_m\})$ with
        $s(y_m) = f_m$ where $f_m$ are some nodal values. Prove that,
        for {\em any} $g \in C^2[a,b]$ with $g(y_m) = f_m$,
        \[
            \int_{a}^b |s''(x)|^2 \,dx \leq \int_a^b |g''(x)|^2 \,dx,
        \]
        {\em provided} that $s$ satisfies a condition at the end-points
        $a = y_0, b = y_M$, which you should derive.

        Thus, $s''$ with this end-point condition minimises curvature amongst
        all $C^2$ functions satisfying the nodal interpolation conditions.
        These splines are therefore called natural splines.

        {\it HINT: } Consider $\int_a^b |s''|^2 + 2 s'' (g''-s'') + |s'' - g''|^2 \, dx$
        and show that the middle term vanishes if the correct end-point
        condition is applied.

        \item Given $(f_m)_{m = 0}^M \in \R^{M+1}$, prove that there exists a
        unique $s \in \Spl_3^2(\{y_m\})$ satisfying the nodal interpolation
        conditions $s(y_m) = f_m$ and the end-point conditions found in part
        (ii). For the sake of simplicity you may wish to assume that the nodes
        are equispaced, i.e. $y_m = y_0 + h m$.

        {\it HINT: Prescribe artificial derivative values $f_m'$, then derive a
        tridiagonal linear system for $(f_m')_{m=0}^M$ and show that it has a
        unique solution. Note that this system can be solved in $O(M)$ time.}
        \qedhere
    \end{enumerate}
\end{exercise}


\chapter{Beginning Combinatorics}

Discrete probabilities problems often include some counting. For
example, we figured out that there were 36 different ways the two dice
, but all of them summed to some number 2 through 12. How
many different ways could three 8-sided dice come up? We would need to
count them, right? As the numbers get big we will need some tricks so
we don't need to write them all down and count them one-by-one.

The branch of mathematics that focuses on tricks for counting is
called \textit{combinatorics}.\index{combinatorics}
% KA: https://www.khanacademy.org/computing/pixar/crowds/crowds-1/v/combinatorics1

How can we be sure that there were 36 different configurations for the
two 6-sided dice? The first die could have come up as any one of six
numbers. For each of those, the second could have come up with any one
of six numbers. Thus, the number of possibilities is $ 6 \times 6 =
36.$

How many different configurations for 3 8-sided dice?  $8 \times 8
\times 8 = 8^3 = 512$.

What about seven dice, each with 20 sides? There would be $20^7=1,280,000,000$
configurations. See, aren't you glad we don't need to write them all
down?

Now, let's say that six people (Anne, Brock, Carl, Dev, Edgar, and Fred) are
going to run a race. You have to make a plaque that says who won first
place, who won second place, and who won third. If you want to get all
the possible plaques created beforehand, and just pull the right one
out as soon as the race ends, how many plaques would you need to get
engraved?

In this case, once someone has been given first place, they can't win
second or third place. Thus, any of the 6 people can come in first,
but once you have engraved that person's name on the plaque, there are
only 5 people whose name can appear in second place. Once you have
engraved that name, there are only 4 people whose name can appear in
third place. Thus, you would get $6 \times 5 \times 4 = 120$ plaques
engraved.
% ADD: This situation is a little confusing given if they're doing the plaques before hand, they wouldn't know who came in each place

What if the plaque includes all 6 places?  Then you would need $6 \times 5
\times 4 \times 3 \times 2 \times 1 = 720$ plaques engraved.  We use
this process often enough that we gave it a name.  We say ``I need 6
factorial plaques engraved.''  When we write a factorial, we use an
exclamation point:\index{factorial}
% ADD: Same issue here

$$6! = 6 \times 5 \times 4 \times 3 \times 2 \times 1 = 720$$

We use the word ``permutation'' to mean a particular ordering.
This rule says $n$ items can be ordered in $n!$ ways. Thus
mathematicians actually say ``If you have a list of $n$ items then we
can generate $n!$ different permutations of those items''.

In Python, there is a \pyfunction{factorial} function in the math library:
\begin{Verbatim}[commandchars=\\\{\}]
> \textbf{python3}
>>> \textbf{import math}
>>> \textbf{math.factorial(6)}
720
\end{Verbatim}

Handy, right? Now you don't need to write a loop to calculate factorials.

Remember when we only wanted the first three names on the plaque? We can do that problem using factorials:

$$6 \times 5 \times 4 = \frac{6 \times 5 \times 4 \times 3 \times 2 \times 1}{3 \times 2 \times 1} = \frac{6!}{3!}$$

This formulation makes it easy to figure out on any calculator with a ``!'' button.

The rule on this is to fill $m$ positions from $n$ items, it can be done this many ways:

$$\frac{n!}{(n-m)!}$$
% KA: https://www.khanacademy.org/computing/pixar/crowds/crowds2/v/combinatorics8

\subsection{Choose}

Let's say that there are 12 kids in a classroom, and you need a team
of 4 to wipe down the desks. How many different possible teams are
there? You know that if you were giving out four different positions
(Like the race gave out 1st, 2nd, and 3rd), the answer would be $12
\times 11 \times 10 \time 9$ or $12! / (12 - 4)!$.
% ADD: This is the probability that one person would be chosen

However, once we pick the 4 people, we don't care what order they are
in, right?  In this problem, the team ``Anne, Brad, Carl, and Don'' is
the same as the team ``Carl, Don, Brad, and Anne''.

Thus, the quantity $12! / (12 - 4)!$ is many times too large because
it counts each permutation separately. To get the right number, we
just divide this by the number of possible permuations for a group of
four people: $4!$

That gets us our answer: How many different teams of four can be chosen from 12 people?

$$\frac{12!}{(12-4)! 4!}= 495$$
% ADD: Needs a bit more explanation for claretiy, might just be my understanding
In combinatorics, we use this quantity a lot, so we have given it a name: \textit{choose}\index{choose function}

We have also given it a notation. ``12 choose 4'' is written like this:

$${12 \choose 4}$$
% KA Binomial Therom: https://www.khanacademy.org/math/precalculus/x9e81a4f98389efdf:series/x9e81a4f98389efdf:binomial/v/binomial-theorem

Python has the \pyfunction{math.comb} function:

\begin{Verbatim}[commandchars=\\\{\}]
> \textbf{python3}
>>> \textbf{import math}
>>> \textbf{comb(12, 4)}
495
\end{Verbatim}