%\theoremheaderfont{\scshape}
%\theorembodyfont{\slshape}
%\excludeversion{mycomments}
%\excludeversion{mycomments}
%\numberwithin{equation}{section}
%%% change between excludeversion and includeversion for including own working comments
%  environments "mycomments-env" allows for comments which are not printed.
% To have "mycomments" printed, switch between
% \includeversion{mycomments}
% \excludeversion{mycomments}
% by commenting out one of these. Normally,
% \excludeversion is active and comments are not printed,
% but can be seen on the screen.
%\includeversion{mycomments}

\documentclass[11pt,twoside]{article}%
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{theorem}
\usepackage{makeidx}
\usepackage{version}%
\setcounter{MaxMatrixCols}{30}
%TCIDATA{OutputFilter=latex2.dll}
%TCIDATA{Version=5.50.0.2953}
%TCIDATA{CSTFile=LaTeX article (bright).cst}
%TCIDATA{Created=Tue Feb 04 17:45:04 2003}
%TCIDATA{LastRevised=Wednesday, March 30, 2011 17:32:17}
%TCIDATA{<META NAME="GraphicsSave" CONTENT="32">}
%TCIDATA{<META NAME="SaveForMode" CONTENT="1">}
%TCIDATA{BibliographyScheme=Manual}
%TCIDATA{<META NAME="DocumentShell" CONTENT="Journal Articles\Standard LaTeX Article">}
%BeginMSIPreambleData
\providecommand{\U}[1]{\protect\rule{.1in}{.1in}}
%EndMSIPreambleData
\pagestyle{headings}
\theoremstyle{change}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{acknowledgement}[theorem]{Acknowledgement}
\newtheorem{algorithm}[theorem]{Algorithm}
\newtheorem{axiom}[theorem]{Axiom}
\newtheorem{case}[theorem]{Case}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{conclusion}[theorem]{Conclusion}
\newtheorem{condition}[theorem]{Condition}
\newtheorem{conjecture}[theorem]{Conjecture}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{criterion}[theorem]{Criterion}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{exercise}[theorem]{Exercise}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{notation}[theorem]{Notation}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{solution}[theorem]{Solution}
\newtheorem{summary}[theorem]{Summary}
\theorembodyfont{\upshape}
\newtheorem{example}[theorem]{Example}
\newenvironment{proof}[1][Proof]{\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\newenvironment{remark-notnumbered}[1][Remark]{\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\newenvironment{quote-env}{\begin{quote}\sffamily }{\end{quote}}
\newenvironment{mycomments-env}[1][Mycomments]{\textbf{#1.} \begin{quote-env} }{ \end{quote-env}  \ \rule{0.5em}{0.5em}}
\excludeversion{mycomments}
\textheight20truecm \textwidth 16,5truecm
\oddsidemargin0truecm
\evensidemargin0truecm
\newcommand {\abst}{{\vskip0.3cm}\noindent}
\parindent0cm
\setcounter{secnumdepth}{2}
\makeindex
\def\ds{\displaystyle}
\def\n{\noindent}
\def\wt{\widetilde}
\def\ul{\underline}
\def\ov{\overline}
\def\Var{\text{Var}}
\def\a{\Cal A}
\def\e{\Cal E}
\def\f{\Cal F}
\def\j{\Cal J}
\def\p{\Cal P}
\def\r{\Cal R}
\def\t{\Cal T}
\def\x{\Cal X}
\def\y{\Cal Y}
\begin{document}

\title{Topics in Statistical Learning Theory}
\author{Michael Nussbaum\\{\small Department of Mathematics, Cornell University}}
\maketitle

\begin{abstract}
These are course notes for MATH 7740, Statistical Learning Theory.

\end{abstract}
\tableofcontents

\setcounter{tocdepth}{2} \newpage

\textbf{Primary References }(a full list of references is at the end of these
notes)\textbf{:}

\begin{description}
\item[{[HTF]}] Hastie, T, Tibshirani, R. J. H. Friedman, R.H., \textsl{The
Elements of Statistical Learning (Data Mining, Inference and Prediction)}%
\textit{, }Second Edition, Springer, 2009.

\item[{[DGL]}] Devroye, L, Gyorfi, L., Lugosi, G., \textsl{A Probabilistic
Theory of Pattern Recognition}, Springer 1997

\item[{[Vap1]}] Vapnik, V. , \textsl{Statistical Learning Theory}, Wiley, 1998.

\item[{[Vap2]}] Vapnik, V. \textsl{The Nature of Statistical Learning Theory},
2nd ed, Springer 1999

\item[{[ScS]}] Sch\"{o}lkopf, B. and Smola, A. \textsl{Learning with Kernels:
Support Vector Machines, Regularization, Optimization, and Beyond} (Adaptive
Computation and Machine Learning). MIT Press, 2002

\item[{[RW]}] Rasmussen, C. E. and Williams, C. K. I., \textsl{Gaussian
Processes for Machine Learning} (Adaptive Computation and Machine Learning).
MIT Press 2006
\end{description}

\section{The Learning Problem}

\subsection{Supervised Learning as Classification}

\textbf{Example: handwritten digit recognition.} In 1990, a U.S. Postal
Service database contained 7300 training patterns from handwritten ZIP codes
on envelopes from U.S. postal mail. Each image is a segment from a five digit
ZIP code, isolating a single digit. The images are 16 x 16 eight-bit grayscale
maps (with each pixel ranging in intensity from 1 to 255). Some sample images
are shown in Figure 12 in the textbook.

The images have been normalized to have approximately the same size and
orientation. The task is to predict, from the $16\times16$ matrix of pixel
intensities, the identity of each image $\left\{  0,1,...,9\right\}  $ quickly
and accurately. If it is accurate enough, the resulting algorithm would be
used as part of an automatic sorting procedure for envelopes. This is a
classification problem for which the error rate needs to be kept very low to
avoid misdirection of mail.

Thus each pattern is a matrix of dimension $16\times16$, or equivalently a
vector of values in $16\cdot16=256$ dimensional space. The values are the
intensities of each pixel, with possible values $1$ to $255$. $\ $This dataset
can be written $X_{1},\ldots,X_{n}$ where $X_{i}$ is in $\mathbb{R}^{d}$ for
$d=256$. But this is a "training set", meaning that each $X_{i}$ has already
been "recognized" (apparently by eye), thus there are also given values
$Y_{1},\ldots,Y_{n}$ where each $Y_{i}\in\left\{  0,1,...,9\right\}  $.

The training set together may be written $T^{(n)}=\left(  \left(  X_{i}%
,Y_{i}\right)  ,i=1,\ldots,n\right)  $; a machine learning algorithm to be
developed has to use the training set to \textbf{predict} the value of $Y_{i}$
for a future or incoming $X_{i}.$ In other words, the algorithm will
\textbf{classify} all possible $X_{i}$ into one of $10$ possible categories
(digits). Apparently prediction and classification are here used synonymously.
Below we will make this notion precise, for an idealized setting where $Y_{i}$
takes only two values $0$ and $1.$

The problem is called \textit{supervised learning; }$\ $the\ "student" is
identified with the algorithm and the "teacher" is the human eye which gave
the correct digit $Y_{i}$ for each $X_{i}$ in the training set. The U.S.
Postal Service database also contained an additional test set $\left(  \left(
X_{i},Y_{i}\right)  ,i=1,\ldots,n\right)  $of $n=2000$ \ patterns and
corresponding digits. These are used to test the performance of the learning
algorithm, e.g. to estimate its error rate. The division of data into training
and test set is arbitrary. Training and test sets are routinely used in
machine learning to "train" and test classifiers like neural networks, support
vector machines (SVM) and others.

\subsection{Unsupervised Learning}

\textbf{Example: human tumor microarray data.} Data are a $6830\times64$
matrix of real numbers, each representing an expression level for a gene (row)
and sample (column). Thus, one column (called a "sample" in this context) is a
vector of $6830$ gene expressions (ranging from $-6$ to $6$) from one
particular type of human tumor (breast cancer, melanoma and so on). Let
$x_{i}$, $i=1,\ldots,N$ be these colums ($N=64$) in $\mathbb{R}^{d}$ for
$d=6830$. Each $x_{i}$ carries a label, designating the type of cancer it came
from. In an example of \textit{unsupervised learning} one performs a cluster
analysis, to find groups (clusters) within \ the $x_{i}$ (ignoring the
labels). Afterwards, the clustering obtained is compared with the labels, and
one obtains a possible grouping of types of cancer, with respect to genetic similarity.

Cluster analysis is an example of unsupervised learning since the analysis is
performed without any "outcomes" $Y_{i}$ indicating membership in a group. (In
the human tumor microarray data, such $Y_{i}$ are in fact present and are
given by the cancer labels, but in the initial analysis they are not included.
In this example, this reflects a choice how to perform an initial data analysis).

\subsubsection{$K$-means clustering}

(\textit{follows [HTF], 460-62}) This algorithms requires a number of clusters
$K$ chosen. It is usually performed for various $K$ and it is then judged
which $K$ gives the best compromise between data fit and complexity (i.e.
number $K$ of clusters).

Data are $x_{i}$, $i=1,\ldots,N$ from $\mathbb{R}^{d}$. \ An underlying
probabilistic assumption is that these are i.i.d random vectors from a density
$p$ on $\mathbb{R}^{d}$ which has multiple peaks, each corresponding to a
"probability cluster". This may correspond to the density $p$ being a mixture
$p=\sum_{i=1}^{K}\lambda_{i}p_{i}$ where $p_{i}$ are (unimodal) densities and
$\sum_{i=1}^{K}\lambda_{i}=1$. In the example, one may assume that each type
of cancer appears in the sample with probability $\lambda_{i} $, and the type
itself has a random gene expression vector with density $p_{i}$. The total
sample $x_{i}$, $i=1,\ldots,N$ may then be considered i.i.d. with density $p$.
However, the clustering algorithms does not make use of a probability
assumption about the sample. It can be described as follows:

\begin{itemize}
\item for each data point $x_{i}$, the closest cluster center (in Euclidean
distance) is identified;

\item each cluster center is replaced by the coordinatewise average of all
data points that are closest to it.
\end{itemize}

To describe it in more detail, a prespecified number of clusters $K<N$ is
postulated and each one is labeled by an integer $k\in\left\{  1,\ldots
,K\right\}  $. Each observation is assigned to one and only one cluster. These
assignments can be characterized by a many-to-one mapping, or encoder
$k=C(i)$, that assigns the $i$-th observation to the $k$-th cluster. One seeks
the particular encoder $C^{\ast}(i)$ that achieves the required goal (details
below) , based on the dissimilarities $d(x_{i},x_{i^{\prime}})$ between every
pair of observations. The most basic choice is squared Euclidean distance
\[
d(x_{i},x_{i^{\prime}})=\left\Vert x_{i}-x_{i^{\prime}}\right\Vert ^{2}.
\]
Generally, the encoder $C(i)$ is explicitly delineated by giving its value
(cluster assignment) for each observation i. Thus, the \textquotedblleft
parameters\textquotedblright\ of the procedure are the individual cluster
assignments for each of the $N$ observations. These are adjusted so as to
minimize a \textquotedblleft loss\textquotedblright\ function that
characterizes the degree to which the clustering goal is not met.

Since the goal is to assign close points to the same cluster, a natural loss (
or \textquotedblleft energy\textquotedblright\ ) function would be%
\begin{equation}
W(C)=\frac{1}{2}\sum_{k=1}^{K}\sum_{i:C(i)=k}\sum_{i^{\prime}:C(i^{\prime}%
)=k}d(x_{i},x_{i^{\prime}})\label{within-point-scatter}%
\end{equation}
This criterion characterizes the extent to which observations assigned to the
same cluster tend to be close to one another. It is sometimes referred to as
the \textquotedblleft within cluster\textquotedblright\ point scatter since%
\[
T=\frac{1}{2}\sum_{i=1}^{N}\sum_{i^{\prime}=1}^{N}d_{ii^{\prime}}=\frac{1}%
{2}\sum_{k=1}^{K}\sum_{i:C(i)=k}\left(  \sum_{i^{\prime}:C(i^{\prime}%
)=k}d_{ii^{\prime}}+\sum_{i^{\prime}:C(i^{\prime})\neq k}d_{ii^{\prime}%
}\right)
\]
or
\[
T=W(C)+B(C)
\]
where $d_{ii^{\prime}}=d(x_{i},x_{i^{\prime}})$. Here $T$ is the total point
scatter, which is a constant given the data, independent of cluster
assignment. The quantity%
\[
B(C)=\frac{1}{2}\sum_{k=1}^{K}\sum_{i:C(i)=k}\sum_{i^{\prime}:C(i^{\prime
})\neq k}d(x_{i},x_{i^{\prime}})
\]
is the between-cluster point scatter. This will tend to be large when
observations assigned to different clusters are far apart. Thus one has%
\[
W(C)=T-B(C)
\]
and minimizing $W(C)$ is equivalent to maximizing $B(C)$.

Cluster analysis by combinatorial optimization is straightforward in
principle. One simply minimizes $W$ or equivalently maximizes $B$ over all
possible assignments of the $N$ data points to $K$ clusters. Unfortunately,
such optimization by complete enumeration is feasible only for very small data
sets. The number of distinct assignments is
\[
S(N,K)=\frac{1}{K!}\sum_{k=1}^{K}\left(  -1\right)  ^{K-k}\left(
%TCIMACRO{\QATOP{K}{k}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{K}{k}%
%EndExpansion
\right)  k^{N}%
\]

\begin{description}
\item[Comment.] {\small A "cluster" must have at least one element;}
{\small the above formula gives the number of those assigments. To see this,
consider first the number of assigments to one of }$K${\small \ groups
labelled }$1,\ldots,K${\small \ when groups are allowed to have }%
$0${\small \ elements. There are }$K^{N}${\small \ such possible assigments
(each data point is assigned to exactly one of the groups), and }$K^{N}%
/K!${\small \ is the number of assigments when group labels don't matter. To
account for the restriction that each group has at least one element, subtract
from }$K!${\small \ the number of assignments where at least one group has
}$0${\small \ elements. If one group has }$0${\small \ elements then
}$(K-1)^{N}${\small \ assigments remain, and summing over labels we get
}$K(K-1)^{N}${\small . Thus the "corrected" number of assigments is }$\frac
{1}{K!}\left(  K^{N}-K(K-1)^{N}\right)  ${\small . But this correction needs
to be corrected; indeed it counts twice those assignments where two groups
have zero elements. We have to add these back, so we get }$\frac{1}{K!}\left(
K^{N}-K(K-1)^{N}+\left(
%TCIMACRO{\QATOP{K}{K-2}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{K}{K-2}%
%EndExpansion
\right)  \left(  K-2\right)  ^{N}\right)  ${\small . Continuing in this
manner, we obtain the formula.}
\end{description}

For example, $S(10,4)$ $=34,105$ which is quite feasible. But $S(N,K)$ grows
very rapidly with increasing values of its arguments. Already $S(19,4)\simeq
10^{10}$ and most clustering problems involve much larger data sets than
$N=19$. For this reason, practical clustering algorithms are able to examine
only a very small fraction of all possible encoders $k=C(i)$. The goal is to
identify a small subset that is likely to contain the optimal one, or at least
a good suboptimal partition.

Such feasible strategies are based on iterative greedy descent. An initial
partition is specified. At each iterative step, the cluster assignments are
changed in such a way that the value of the criterion is improved from its
previous value. When the prescription is unable to provide an improvement, the
algorithm terminates with the current assignments as its solution.

In the $K$-means algorithm, the within-point scatter
(\ref{within-point-scatter}) can be written as
\begin{equation}
W(C)=\frac{1}{2}\sum_{k=1}^{K}\sum_{i:C(i)=k}\sum_{i^{\prime}:C(i^{\prime}%
)=k}\left\Vert x_{i}-x_{i^{\prime}}\right\Vert ^{2}=\sum_{k=1}^{K}N_{k}%
\sum_{i:C(i)=k}\left\Vert x_{i}-\bar{x}_{k}\right\Vert ^{2}%
\label{within-point-scatter-2}%
\end{equation}
where $\bar{x}_{k}=\left(  \bar{x}_{1k},\ldots,\bar{x}_{dk}\right)  $ is the
mean vector associated with the $k$-th cluster, and $N_{k}$ is the cardinality
of each cluster. Indeed, we have for given $k$ and $C(i)=k$, $C(i^{\prime})=k$%
\[
\left\Vert x_{i}-x_{i^{\prime}}\right\Vert ^{2}=\left\Vert x_{i}-\bar{x}%
_{k}+\bar{x}_{k}-x_{i^{\prime}}\right\Vert ^{2}=\left\Vert x_{i}-\bar{x}%
_{k}\right\Vert ^{2}+\left\langle x_{i}-\bar{x}_{k},\bar{x}_{k}-x_{i^{\prime}%
}\right\rangle +\left\Vert \bar{x}_{k}-x_{i^{\prime}}\right\Vert ^{2},
\]%
\begin{align*}
\sum_{i^{\prime}:C(i^{\prime})=k}\left\langle x_{i}-\bar{x}_{k},\bar{x}%
_{k}-x_{i^{\prime}}\right\rangle  & =\left\langle x_{i}-\bar{x}_{k}%
,\sum_{i^{\prime}:C(i^{\prime})=k}\left(  \bar{x}_{k}-x_{i^{\prime}}\right)
\right\rangle \\
& =\left\langle x_{i}-\bar{x}_{k},N_{k}\bar{x}_{k}-N_{k}\bar{x}_{k}%
\right\rangle =0
\end{align*}
hence
\begin{align*}
\sum_{i:C(i)=k}\sum_{i^{\prime}:C(i^{\prime})=k}\left\Vert x_{i}-x_{i^{\prime
}}\right\Vert ^{2}  & =N_{k}\sum_{i:C(i)=k}\left\Vert x_{i}-\bar{x}%
_{k}\right\Vert ^{2}+N_{k}\sum_{i^{\prime}:C(i^{\prime})=k}\left\Vert
x_{i^{\prime}}-\bar{x}_{k}\right\Vert ^{2}\\
& =2N_{k}\sum_{i:C(i)=k}\left\Vert x_{i}-\bar{x}_{k}\right\Vert ^{2}%
\end{align*}
which implies (\ref{within-point-scatter-2}). Thus, the criterion is minimized
by assigning the $N$ observations to the K clusters in such a way that within
each cluster the average dissimilarity of the observations from the cluster
mean, as defined by the points in that cluster, is minimized.

An iterative descent algorithm for minimizing (\ref{within-point-scatter-2})
over $C$ can be obtained by noting that for any set of observations $S$%
\begin{equation}
\bar{x}_{S}=\text{\textrm{argmin}}_{m}\sum_{i\in S}\left\Vert x_{i}%
-m\right\Vert ^{2}.\label{means-of-clusters}%
\end{equation}
Hence we can obtain a minimum by solving the enlarged optimization problem%
\begin{equation}
\min_{C,\left\{  m_{k},k=1,\ldots,K\right\}  }\sum_{k=1}^{K}N_{k}%
\sum_{i:C(i)=k}\left\Vert x_{i}-m_{k}\right\Vert ^{2}%
\label{total-clust-variance}%
\end{equation}
This can be minimized by an alternating optimization procedure as follows:

\textbf{Algorithm. }

\begin{enumerate}
\item For a given cluster assignment $C$, the total cluster variance
(\ref{total-clust-variance}) is minimized with respect to $\left\{
m_{k},k=1,\ldots,K\right\}  $ yielding the means of the currently assigned
clusters (\ref{means-of-clusters}).

\item Given a current set of means $\left\{  m_{k},k=1,\ldots,K\right\}  $,
(\ref{total-clust-variance}) is minimized by assigning each observation to the
closest (current) cluster mean. That \ is,%
\[
C(i)=\text{\textrm{argmin}}_{1\leq k\leq K}\left\Vert x_{i}-m_{k}\right\Vert
^{2}.
\]

\item Steps 1 and 2 are iterated until the assignments do not change.
\end{enumerate}

One should start the algorithm with many different random choices for the
starting means and compare results. .

\section{The formal classification problem}

\subsection{Introduction}

Suppose we observe a random variable $X$ with values in $\mathbb{R}^{d}$. The
distribution of $X$ is given by one of two possible densities: $f_{0}$,
$f_{1}$, and we have to decide which is the true one. The densities $f_{0}$,
$f_{1}$ may be such that they have disjoint support $S_{0}$, $S_{1}$
respectively, or the support may be overlapping or even identical. In any
case, each density is identified with a "class" or a category, and the
decision: which density is the true one (where $X$ came from), is called the
\textbf{problem of classification.} Thus a \textbf{classifier }$h$ is a
measurable function: $h:\mathbb{R}^{d}\mapsto\left\{  0,1\right\}  $, and
$"h(X)=1"$ is interpreted as the decision "$X$ came from density $1$". In fact
$h$ is the same as a nonrandomized test function with critical region
$\mathbf{1}_{\left\{  h(X)=1\right\}  }$.

This problem is just a special case of parameter estimation (or testing), when
the parameter space is the set $\left\{  0,1\right\}  $. It becomes different
however when the two densities (or "classes") are unknown, and instead there
are only preliminary data available: $X_{1},\ldots,X_{n}$ with information
where each $X_{i}$ came from: density $0$ or density $1$. This information can
be written as a pair $\left(  X_{i},Y_{i}\right)  $ where $Y_{i}$ takes values
in the set $\left\{  0,1\right\}  $. The data $\left(  X_{i},Y_{i}\right)  $,
$i=1,\ldots,n$ are called the \textbf{training set.} From the data we have to
"learn" the actual shape of the two densities $f_{0} $, $f_{1}$, and construct
a decision function $\hat{h}$ accordingly. The decision function or classifier
$\hat{h}$ is then supposed to act on a "new" incoming $X$ and classify it as
accurately as possible. Thus $\hat{h}$ depends on the training set
$T^{(n)}=\left(  \left(  X_{i},Y_{i}\right)  ,i=1,\ldots,n\right)  $ and
addditionally has argument $X$: thus $\hat{h}\left(  T^{(n)},X\right)  =1$
means that the classifier constructed on the training set $T^{(n)}$ classifies
the new $X$ as coming from density $f_{1}$.

\bigskip

\textbf{A priori distributions. }So far we have assumed that each $X_{i}$ or
$X$ comes from one of the two densities $f_{0}$, $f_{1}$ in a nonrandom way (a
special case of a "parametric model" $X\sim\left\{  P_{\vartheta},\vartheta
\in\Theta\right\}  $ where $X$ is supposed to "come" from one of the
probability measures $P_{\vartheta}$). In classification theory one assumes
however that there is an a priori distribution on the index: $\pi$ is the
probability $\pi=P\left(  Y=1\right)  $ and $1-\pi=P\left(  Y=0\right)  $.
Thus the index $0$ or $1$ is assumed random as well; giving rise to a joint
distribution of $\left(  X,Y\right)  $. The joint distribution is given by,
for any measurable $A\subset\mathbb{R}^{d}$, $i\in\left\{  0,1\right\}  $
\begin{align*}
P\left(  X\in A,Y=i\right)   & =P\left(  X\in A|Y=i\right)  P\left(
Y=i\right) \\
& =P\left(  Y=i\right)  \int_{A}f_{i}(x)dx
\end{align*}
where $P\left(  Y=1\right)  =\pi$ and $P\left(  Y=0\right)  =1-\pi$. The
training set $T^{(n)}=\left(  \left(  X_{i},Y_{i}\right)  ,i=1,\ldots
,n\right)  $ is assumed to consist of i.i.d. random variables having this
joint distribution.

\bigskip

\textbf{Error probabilities. } Classifiers $h$ should be optimal with respect
to the error probability. Assume first that our classifier $h$ does not depend
on the training set (possibly it uses knowledge of $f_{0}$,$f_{1}$ and may be
theoretical only, not available in reality). Initially, if there were no prior
probability $\pi,$ we could consider the error probability of first and second
kinds:
\[
P\left(  h(X)=1|Y=0\right)  \text{ and }P\left(  h(X)=0|Y=1\right)  \text{.}%
\]
When the class index $Y$ is also random it is natural to consider the total
error probability:%
\begin{align}
P\left(  h(X)\neq Y\right)   & =P\left(  h(X)=1|Y=0\right)  \left(
1-\pi\right)  +P\left(  h(X)=0|Y=1\right)  \pi\label{err-probab}\\
& =\left(  1-\pi\right)  \int_{\left\{  h(x)=1\right\}  }f_{0}(x)dx+\pi
\int_{\left\{  h(x)=0\right\}  }f_{1}(x)dx.\nonumber
\end{align}

\subsection{Bayesian classification\label{sec:bayes-classif}}

The theoretical decision rule $h^{\ast}$ which minimizes the above error
probability is called the \textbf{Bayes rule} or Bayesian classifier. It is
not based on any training set, but presupposes knowledge of the joint
distribution of $\left(  X,Y\right)  $. More specifically, it can be written
in terms of the "true" \textbf{regression function}
\begin{align}
r(x)  & =E\left(  Y|X=x\right)  =P\left(  Y=1|X=x\right) \\
& =\frac{\pi f_{1}(x)}{\pi f_{1}(x)+\left(  1-\pi\right)  f_{0}(x)}%
\label{shape-of-posterior}%
\end{align}
such that
\begin{equation}
h^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $r(x)>1/2$\\
$0$ otherwise
\end{tabular}
\right. \label{bayes-rule-def}%
\end{equation}
To comment, note that initially (a priori) $Y$ is a Bernoulli r.v. with
$P\left(  Y=1\right)  =\pi$. From the joint distribution of $(X,Y)$ we then
can find a conditional distribution of $Y$ given $X=x$. This may also be
called a \textbf{posterior distribution} of $Y$ (after $X$ has been observed).
Since $Y$ takes values in $\left\{  0,1\right\}  $, the posterior distribution
is also Bernoulli, but now depending on $x$. Since for a Bernoulli, the
expectation is the "success probability", we have
\[
E\left(  Y|X=x\right)  =P\left(  Y=1|X=x\right)  .
\]
The formula (\ref{shape-of-posterior}) can easily be obtained from the formula
for conditional densities. For this, we have to define a density $p_{Y}$ of
$Y$, with respect to counting measure on $\{0,1\}$ as
\[
p_{Y}(0)=1-\pi\text{, }p_{Y}(1)=\pi;
\]
indeed a probability function for a discrete random variable can also be seen
as a density. We then understand $f_{i}(x)$ as a conditional density of $X$
given $Y=y$:%
\[
f_{y}(x)=p_{X}(x|y);
\]
then
\begin{align*}
P\left(  Y=y|X=x\right)   & =p_{Y}(y|x)=\frac{p_{Y}(y)p_{X}(x|y)}{p_{X}(x)}\\
& =\frac{p_{Y}(y)p_{X}(x|y)}{p_{X}(x|0)p_{Y}(0)+p_{X}(x|1)p_{Y}(1)}%
\end{align*}
and for $y=1$ we obtain
\[
P\left(  Y=1|X=x\right)  =\frac{f_{1}(x)\pi}{f_{0}(x)\left(  1-\pi\right)
+f_{1}(x)\pi}%
\]
which confirms our claim (\ref{shape-of-posterior}).

So far we have only defined a "Bayes rule" $h^{\ast}(x)$ in
(\ref{bayes-rule-def}); before we prove the acutal optimality statement, let
us note several equivalent forms:
\begin{align*}
h^{\ast}(x)  & =\left\{
\begin{tabular}
[c]{l}%
$1$ if $r(x)>1/2$\\
$0$ otherwise
\end{tabular}
\right.  =\left\{
\begin{tabular}
[c]{l}%
$1$ if $P\left(  Y=1|X=x\right)  >P\left(  Y=0|X=x\right)  $\\
$0$ otherwise
\end{tabular}
\right. \\
& =\left\{
\begin{tabular}
[c]{l}%
$1$ if $f_{1}(x)\pi>f_{0}(x)\left(  1-\pi\right)  $\\
$0$ otherwise
\end{tabular}
\right.  .
\end{align*}
In the case $\pi=1/2$ it coincides with the maximum likelihood rule:
\[
h^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $f_{1}(x)>f_{0}(x)$\\
$0$ otherwise
\end{tabular}
\right.
\]
Our performance criterion for any classification rule is the total error
probability $P\left(  h(X)\neq Y\right)  $ given by (\ref{err-probab}). To
express it in a different way, consider a statistical "loss function":
$l(h(x),y)$ taken to be $0$ for a correct decision $h(x)$, and $1$ for an
incorrect one. Thus
\[
l(h(x),y)=\left\{
\begin{tabular}
[c]{l}%
$0$ if $h(x)=y$\\
$1$ otherwise
\end{tabular}
\right.
\]
or written as an indicator function $l(h(x),y)=\mathbf{1}_{\left\{  h(x)\neq
y\right\}  }$. We then have
\[
P\left(  h(X)\neq Y\right)  =E\mathbf{1}_{\left\{  h(X)\neq Y\right\}
}=El(h(X),Y)
\]
i.e. the error probability is written as an expected loss, or as the
\textbf{\ risk} of a decision rule.

\begin{theorem}
\label{theor-bayes-rule}The Bayes rule $h^{\ast}$ is optimal, that is, if h is
any other classification rule then
\[
P\left(  h^{\ast}(X)\neq Y\right)  \leq P\left(  h(X)\neq Y\right)  .
\]

\end{theorem}

\textbf{Comment.} We are comparing $h^{\ast}$ with other "nonpractical"
classification rules which (possibly) utilize knowledge of $f_{0}$,$f_{1}$ and
$\pi$. Intuitively, the "practical" rules where this knowledge is substituted
by availability of training data $T$ cannot be better. After the proof below
we give a more formal reasoning on this.

\begin{proof}
This statement is exactly the same as the Bayes optimality of the maximizer of
the posterior density in Bayesian inference. Assume we have densities
$f_{\vartheta}(x)$ for observations $X$ and an a priori density $\pi
(\vartheta)$ for $\vartheta.$ (If the parameter space is finite, i.e.
$\{0,1\}$ then we take the density to be the probability function.). The total
risk for any decision rule $h(x)$ with loss
\[
l(h(x),\vartheta)=\left\{
\begin{tabular}
[c]{l}%
$0$ if $h(x)=\vartheta$\\
$1$ otherwise$.$%
\end{tabular}
\right.
\]
(that is the zero-one loss: $0$ loss for a correct decision, $1$ for an
incorrect one) is (writing $f(x)=\int f_{\vartheta}(x)\pi(\vartheta
)d\vartheta$ for the marginal density of $X$)
\begin{align}
EE_{\vartheta}l(h(X),\vartheta)  & =\int\int l(h(x),\vartheta)f_{\vartheta
}(x)dx\pi(\vartheta)d\vartheta\label{total risk}\\
& =\int\int l(h(x),\vartheta)f_{\vartheta}(x)\pi(\vartheta)dxd\vartheta
\nonumber\\
& =\int\int l(h(x),\vartheta)\frac{f_{\vartheta}(x)\pi(\vartheta)}%
{f(x)}d\vartheta f(x)dx\nonumber\\
& =\int\int l(h(x),\vartheta)f(\vartheta|x)d\vartheta f(x)dx\nonumber
\end{align}
Thus for every $x$ we need to minimize in $h$
\[
\int_{\Theta}l(h(x),\vartheta)f(\vartheta|x)d\vartheta
\]
that is, the a posteriori loss. What we can choose here is $h(x)$, that is for
the given $x$ we assign a value $h\in\left\{  0,1\right\}  $. Since
$\vartheta$ takes values in $\left\{  0,1\right\}  $, the last integral is in
fact
\begin{align*}
& l(h,0)P\left(  \vartheta=0|x\right)  +l(h,1)P\left(  \vartheta=1|x\right) \\
& =l(h,0)P\left(  \vartheta=0|x\right)  +\left(  1-l(h,0)\right)  P\left(
\vartheta=1|x\right)  .
\end{align*}
since always $l(h,1)=1-l(h,0)$. To minimize this in $h,$ we should select $h$
such that the greater of the two probabilities gets factor $0$ and the other
gets $1.$ This gives%
\[
h(x)=\left\{
\begin{tabular}
[c]{l}%
$0$ if $P\left(  \vartheta=0|x\right)  >P\left(  \vartheta=1|x\right)  $\\
$1$ otherwise$.$%
\end{tabular}
\right.
\]
This is the same as taking $h(x)$ to be the maximizer in $\vartheta$ of
$P\left(  \vartheta|x\right)  $ (which is also $f(\vartheta|x)$), that is the
maximizer of the posterior probability (or density). We have shown that this
rule minimizes the total risk (\ref{total risk}).
\end{proof}

\bigskip

Recall that the Bayes rule presupposes knowledge of $f_{0}$ and $f_{1}$ (even
if we assume $\pi$ known, e.g. $\pi=1/2$), it it usually not available in
practice. The "practical" rules $\hat{h}$ which we will consider depend also
on a training set $T^{(n)}=T$, they can be written $\hat{h}\left(  T,X\right)
$. But their absolute error probability $P\left(  \hat{h}\left(  T,X\right)
\neq Y\right)  ,$ \textit{when }$P$\textit{\ refers also to randomness in }%
$T$, cannot be better than $P\left(  h^{\ast}(X)\neq Y\right)  $.

Let us give a formal argument for that (almost obvious) claim. Recall that $T
$ and $\left(  X,Y\right)  $ are assumed to be independent. Introduce
$X^{\ast}:=\left(  T,X\right)  $ and consider the classification (or
prediction) problem with respect to the joint distribution $\left(  X^{\ast
},Y\right)  $. Now we are trying to predict $Y$ from $X^{\ast}=\left(
T,X\right)  $ but $Y$ is independent of $T$. Using the Bayes rule in this
context allows to dispense with $T$ for the prediction.

Formally, we now admit classification rules $h(x^{\ast})$ which are functions
of $x^{\ast}$. The Bayes rule in this framework is
\begin{equation}
h^{\ast}(x^{\ast})=\left\{
\begin{tabular}
[c]{l}%
$0$ if $P\left(  Y=0|x^{\ast}\right)  >P\left(  Y=1|x^{\ast}\right)  $\\
$1$ otherwise$.$%
\end{tabular}
\right. \label{bayes-rule-extend}%
\end{equation}
We will show that this coincides with the previous Bayes rule, i.e. $h^{\ast
}(x^{\ast})=h^{\ast}(t,x)$ does not depend on $t$. For this, it suffices to
show that $P\left(  Y=1|x^{\ast}\right)  =P\left(  Y=1|t,x\right)  $ does not
depend on $t.$

For this purpose, let $p_{T}(t)$ be the density of $T$ with respect to an
appropriate dominating measure (since $T=\left(  \left(  X_{1},Y_{1}\right)
,\ldots,\left(  X_{n},Y_{n}\right)  \right)  $, this one should be the
$n$-fold product of $\nu$, if $\nu$ is the product of Lebesgue measure with
counting measure on $\left\{  0,1\right\}  $). Since $T$ is independent of
both $X$ and $Y$, the joint density of $\left(  X^{\ast},Y\right)  $ is (for
$x^{\ast}=(x,t)$
\[
p_{Y}(y)p_{X^{\ast}}(x^{\ast}|y)=p_{Y}(y)p_{X}(x|y)p_{T}(t)
\]
and hence
\begin{align*}
P\left(  Y=y|X^{\ast}=x^{\ast}\right)   & =p_{Y}(y|x^{\ast})=\frac
{p_{Y}(y)p_{X^{\ast}}(x^{\ast}|y)}{p_{X^{\ast}}(x^{\ast})}\\
& =\frac{p_{Y}(y)p_{X}(x|y)p_{T}(t)}{p_{X}(x)p_{T}(t)}=\frac{p_{Y}%
(y)p_{X}(x|y)}{p_{X}(x)}\\
& =p_{Y}(y|x)=P\left(  Y=y|X=x\right)  .
\end{align*}

\subsubsection{The $k$ classes case}

Let us consider the case where there are possibly more than two classes, let
us say $K$ classes. The formal setup for Bayesian classification is very
similar to the above. Suppose $(X,Y)$ are random variables with a joint
distribution such that $X=(X_{1},\ldots,X_{d})$ takes values in a set
$\mathcal{X}\subset\mathbb{R}^{d}$ and $Y$ takes values in a finite set
$\mathcal{Y}=\left\{  y_{1},\ldots,y_{K}\right\}  $. The $y_{j}$ can be any
labels but for simplicity we assume them to be $1,\ldots,K$. A classification
rule $h$ is a function $h:\mathcal{X\mapsto}\left\{  1,\ldots,K\right\}  $.
The error rate of $h$ is
\[
L(h)=P\left(  \left\{  h(X)\neq Y\right\}  \right)
\]
where $P$ refers to the joint distribution of $(X,Y)$. Denote $p_{k}=P\left(
Y=k\right)  $ and assume that there are conditional densities of $X$ given
$Y=k$, denoted $f_{k}(x)$.

Define the Bayes rule again as
\begin{align*}
h^{\ast}(x)  & =\text{argmax}_{k}P\left(  Y=k|X=x\right) \\
& =\text{argmax}_{k}p_{k}f_{k}(x)
\end{align*}
where
\[
P\left(  Y=k|X=x\right)  =\frac{p_{k}f_{k}(x)}{\sum_{r=1}^{K}p_{r}f_{r}(x)}%
\]
where argmax$_{k}$ means "the value of $k$ that maximizes that expression".
This classifier is also known as the "MAP" (maximum a posteriori) estimator.
Indeed, in Bayesian terminology $P\left(  Y=k|X=x\right)  $ is the a
posteriori probability of $Y=k$ (after $X=x$ has been observed).

\begin{proposition}
The statement of Theorem \ref{theor-bayes-rule} is true in the $k$-class case.
\end{proposition}

\begin{proof}
It follows verbatim the previous proof, where we write again $\vartheta$ for
$Y$ with a priori density $\pi(\vartheta)$ (or probability function) for
$\vartheta$ given by $p_{k}=P\left(  Y=k\right)  =\pi(k)$ if $\vartheta=k$.
Again we take a $0-1$ loss $l(h(x),\vartheta)=l(h(x),\vartheta)=\mathbf{1}%
_{\left\{  h(x)\neq\vartheta\right\}  }$. As above, for every $x$ we then need
to minimize in $h$
\[
\int_{\Theta}l(h(x),\vartheta)f(\vartheta|x)d\vartheta=\sum_{k=1}%
^{K}l(h,k)P\left(  Y=k|x\right)  .
\]
To minimize this in $h,$ we should select $h$ such that the maximal
probability $P\left(  Y=k|x\right)  $ gets factor $0$ and the others all get
factor $1.$ Thus for the maximal probability $P\left(  Y=k|x\right)  $ we have
$l(h,k)=0$ which means $h=k$. This gives%
\[
h^{\ast}(x)=\text{argmax}_{k}P\left(  Y=k|X=x\right)
\]
as claimed.
\end{proof}

\section{The perceptron algorithm}

Suppose again that $X$ is an $\mathbb{R}^{d}$-valued random vector and $Y$ is
a r.v. with two possible values, and $\left(  X,Y\right)  $ have a joint
probability distribution. For convenience we will assume now that these values
are $\{-1,1\}$. Consider a linear classification rule:
\[
\phi(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $a^{\top}x+a_{0}\geq0$\\
$-1$ otherwise$.$%
\end{tabular}
\right.  =\mathrm{sgn}\left(  a^{\top}x+a_{0}\right)
\]
where $a\in\mathbb{R}^{d}$ and $a_{0}$ is a scalar. (Here $\mathrm{sgn}\left(
t\right)  $ is the sign of a scalar $t$; we take it to be $1$ if $t\geq0$ and
$-1$ if $t<0$.) In the history of artificial intelligence and neural network
research, linear classifiers of this type were called \textbf{perceptrons.}
(Fisher's linear discrimination analysis (LDA, 1936) is also given by a linear
classifier). In this section we present a training algorithm for a perceptron,
invented in 1957 at the Cornell Aeronautical Laboratory by Frank Rosenblatt.
The algorithm created a great deal of interest when it was first introduced.
It is guaranteed to converge if there exists a hyperplane that correctly
classifies the training data.

The error probability of the rule is
\[
L\left(  \phi\right)  =P\left(  \phi(X)\neq Y\right)
\]
Let a hyperplane in $\mathbb{R}^{d}$ be given by $\left(  w,b\right)  $ where
$w\in\mathbb{R}^{d}$ and $b\in\mathbb{R}$, by the equation
\begin{equation}
\left\langle w,x\right\rangle +b=0\text{, }x\in\mathbb{R}^{d}%
\label{hyperplane-descr}%
\end{equation}
where $\left\langle w,x\right\rangle =w^{\top}x$ is the scalar product. Note
that for any scalar $\lambda$, the pairs $\left(  w,b\right)  $ and $\left(
\lambda w,b\lambda\right)  $ define the same hyperplane. Under the assumption
$\left\Vert w\right\Vert =1$ the correspondence $\left(  w,b\right)  $ to
hyperplanes is one-to-one.

Suppose we have a training set $\left(  x_{1},y_{1}\right)  ,\ldots,\left(
x_{n},y_{n}\right)  $. It is called \textbf{linearly separable} if there is a
hyperplane given by $\left(  w,b\right)  $ such that all $x_{i}$ with
$y_{i}=1$ are on one side of the hyperplane, while all $x_{i}$ with $y_{i}=-1$
are on the other side. Let us make that notion precise.

\begin{definition}
Let $\left(  x_{i},y_{i}\right)  $ be an "example" (point in the training set
$\left(  x_{1},y_{1}\right)  ,\ldots,\left(  x_{n},y_{n}\right)  $). The
functional margin of $\left(  x_{i},y_{i}\right)  $ with respect to $\left(
w,b\right)  $ is given by
\[
\gamma_{i}=y_{i}\left(  \left\langle w,x_{i}\right\rangle +b\right)  .
\]

\end{definition}

With this definition, we say a training set is linarly separable if there
exists a pair $\left(  w,b\right)  $ defining a hyperplane such that all
$\gamma_{i}$, $i=1,\ldots,n$ have the same sign. Note that if we find a
hyperplane such that all $\gamma_{i}\leq0$, we can simply take $\left(
-w,-b\right)  $, and for this hyperplane we have all $\gamma_{i}\geq0$.

\begin{definition}
The geometric margin of $\left(  x_{i},y_{i}\right)  $ with respect to
$\left(  w,b\right)  $ is
\[
\tilde{\gamma}_{i}=y_{i}\left(  \left\langle \frac{w}{\left\Vert w\right\Vert
},x_{i}\right\rangle +\frac{b}{\left\Vert w\right\Vert }\right)  .
\]

\end{definition}

\bigskip Here we have just $\tilde{\gamma}_{i}=\gamma_{i}/\left\Vert
w\right\Vert $. Note that the equation%
\[
\left\langle \frac{w}{\left\Vert w\right\Vert },x\right\rangle +\frac
{b}{\left\Vert w\right\Vert }=0\text{, }x\in\mathbb{R}^{d}%
\]
describes the same hyperplane as (\ref{hyperplane-descr}), but is a "canonical
form" based on the unit vector $w/\left\Vert w\right\Vert $. Note that
$\left\vert \tilde{\gamma}_{i}\right\vert $ measures the distance of $x_{i}$
from the hyperplane given by $\left(  w,b\right)  $ (picture here).

\begin{definition}
A hyperplane given by $\left(  w,b\right)  $ \textbf{separates} the training
set $S=\left\{  \left(  x_{1},y_{1}\right)  ,\ldots,\left(  x_{n}%
,y_{n}\right)  \right\}  $ if $\min_{1\leq i\leq n}\gamma_{i}>0$. In this case
the geometric margin of $\left(  w,b\right)  $ with respect to $S$ is defined
as $\min_{1\leq i\leq n}\tilde{\gamma}_{i}$.
\end{definition}

A training set $S$ is linearly separable if there exists $\left(  w,b\right)
$ which separates $S$.

It is clear that the geometric margin of all separating hyperplanes is bounded
from above. A hyperplane $\left(  w,b\right)  $ which attains the largest
possible value is called a \textit{maximal margin hyperplane}. Its geometric
margin is a characteristic of the training set $S$; it is called the
\textit{margin of the} (linearly separable) \textit{training set} $S$.

Let $\eta>0$ be some number, to be used as a "tuning parameter" in the algorithm.

\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_

\textbf{Algorithm (Perceptron). }

$w_{0}\leftarrow\mathbf{0}\in\mathbb{R}^{d}$; $b_{0}\leftarrow0,$
$k\leftarrow0$\newline$R\leftarrow\max_{1\leq i\leq n}\left\Vert
x_{i}\right\Vert $\newline%

\begin{tabular}
[c]{llll}%
\textbf{repeat} &  &  & \\
& \textbf{for} $i=1$ \textbf{to} $n$ &  & \\
&  & \textbf{if} & $y_{i}\left(  \left\langle w_{k},x_{i}\right\rangle
+b_{k}\right)  \leq0$ \textbf{then}\\
&  &  & $w_{k+1}\leftarrow w_{k}+\eta y_{i}x_{i}$\\
&  &  & $b_{k+1}\leftarrow b_{k}+\eta y_{i}R^{2}$\\
&  &  & $k\leftarrow k+1$\\
&  & \textbf{endif} & \\
& \textbf{endfor} &  & \\
&  &  & \\
&  &  &
\end{tabular}
$_{{}}$\newline\textbf{until} no mistakes made within the \textbf{for}
loops\newline\textbf{return} $\left(  w_{k},b_{k}\right)  $ where $k$ is the
number of mistakes

\bigskip\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_

A "mistake" occurs if $y_{i}\left(  \left\langle w_{k},x_{i}\right\rangle
+b_{k}\right)  \leq0$, i.e. a misclassification occurs (or a limiting case
where $x_{i}$ is exactly on the current hyperplane, $\gamma_{i}=0$). Note that
the starting value $(w,b)=(\mathbf{0},0)$ is not a hyperplane but it defines a
trival linear classifier where $\phi(x)=1$ for all $x$.

\pagebreak

\begin{theorem}
(Novikoff, 1962). Let $S$ be a nontrivial training set (not all $y_{i}$ have
the same sign), linearly separable. Let $R=\max_{1\leq i\leq n}\left\Vert
x_{i}\right\Vert $. Suppose there exist $(w_{\mathrm{opt}},b_{\mathrm{opt}})$
such that $\left\Vert w_{\mathrm{opt}}\right\Vert =1$ and
\[
y_{i}\left(  \left\langle w_{\mathrm{opt}},x_{i}\right\rangle +b_{\mathrm{opt}%
}\right)  \geq\gamma\text{, }i=1,\ldots,n.
\]
Then the perceptron algorithm stops after $k_{S}$ steps, where
\[
k_{S}\leq\left(  \frac{2R}{\gamma}\right)  ^{2}.
\]

\end{theorem}

To comment, note that a large value of $\gamma$ will cause the algorithm to
stop earlier, while a large bound $R$ has the opposite effect.

\begin{proof}
Augment the input vectors by an extra coordinate with value $R$. Denote the
new vectors
\[
\hat{x}_{i}=\left(
%TCIMACRO{\QATOP{x_{i}}{R}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{x_{i}}{R}%
%EndExpansion
\right)  .
\]
Similarly, add an extra coordinate to the weight vector $w$ by setting
\[
\hat{w}=\left(
%TCIMACRO{\QATOP{w}{b/R}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{w}{b/R}%
%EndExpansion
\right)  .
\]
The starting values are $\hat{w}_{0}=\mathbf{0\in}\mathbb{R}^{d+1}$ and we
denote the updates $\hat{w}_{t}.$

Let $\hat{w}_{t-1}$ be the augmented weight vector prior to the $t$-th
mistake. The $t$-th update is performed when
\[
y_{i}\left\langle \hat{w}_{t-1},\hat{x}_{i}\right\rangle =y_{i}\left(
\left\langle w_{t-1},x_{i}\right\rangle +b_{t-1}\right)  \leq0.
\]
The update is
\begin{align}
\hat{w}_{t}  & =\left(
%TCIMACRO{\QATOP{w_{t}}{b_{t}/R}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{w_{t}}{b_{t}/R}%
%EndExpansion
\right)  =\left(
%TCIMACRO{\QATOP{w_{t-1}}{b_{t-1}/R}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{w_{t-1}}{b_{t-1}/R}%
%EndExpansion
\right)  +\eta y_{i}\left(
%TCIMACRO{\QATOP{x_{i}}{R}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{x_{i}}{R}%
%EndExpansion
\right) \nonumber\\
& =\hat{w}_{t-1}+\eta y_{i}\hat{x}_{i}\label{one-star-handwr}%
\end{align}
since $b_{t}=b_{t-1}+\eta y_{i}R^{2}$, hence $b_{t}/R=b_{t-1}/R+\eta y_{i}R.$

Now note
\begin{align*}
\left\langle \hat{w}_{t},\hat{w}_{\mathrm{opt}}\right\rangle  & =\left\langle
\hat{w}_{t-1},\hat{w}_{\mathrm{opt}}\right\rangle +\eta y_{i}\left\langle
\hat{x}_{i},\hat{w}_{\mathrm{opt}}\right\rangle \\
& \geq\left\langle \hat{w}_{t-1},\hat{w}_{\mathrm{opt}}\right\rangle
+\eta\gamma.
\end{align*}
By induction we obtain
\begin{equation}
\left\langle \hat{w}_{t},\hat{w}_{\mathrm{opt}}\right\rangle \geq t\eta
\gamma.\label{inequ-induc}%
\end{equation}
Furthermore, from (\ref{one-star-handwr}) we obtain
\begin{align*}
\left\Vert \hat{w}_{t}\right\Vert ^{2}  & =\left\Vert \hat{w}_{t-1}\right\Vert
^{2}+2\eta y_{i}\left\langle \hat{w}_{t-1},\hat{x}_{i}\right\rangle +\eta
^{2}\left\Vert \hat{x}_{i}\right\Vert ^{2}\\
& \leq\left\Vert \hat{w}_{t-1}\right\Vert ^{2}+\eta^{2}\left\Vert \hat{x}%
_{i}\right\Vert ^{2}\\
& \leq\left\Vert \hat{w}_{t-1}\right\Vert ^{2}+2\eta^{2}R^{2}%
\end{align*}
where we used $\eta y_{i}\left\langle \hat{w}_{t-1},\hat{x}_{i}\right\rangle
\leq0$ for the first inequality (a "mistake" occurred \ in the $t$-th step),
and $\left\Vert \hat{x}_{i}\right\Vert ^{2}=\left\Vert x_{i}\right\Vert
^{2}+R^{2}\leq2R^{2}$ for the second. By induction again
\begin{equation}
\left\Vert \hat{w}_{t}\right\Vert ^{2}\leq2t\eta^{2}R^{2}%
.\label{threestar-handrw}%
\end{equation}
Now use Cauchy-Schwartz on (\ref{inequ-induc}) to obtain
\[
\left\Vert \hat{w}_{t}\right\Vert \left\Vert \hat{w}_{\mathrm{opt}}\right\Vert
\geq t\eta\gamma
\]
and write (\ref{threestar-handrw}) as
\[
\left\Vert \hat{w}_{t}\right\Vert \leq\eta R\sqrt{2t}.
\]
Combine the last two inequalities to obtain%
\[
t\eta\gamma\leq\left\Vert \hat{w}_{\mathrm{opt}}\right\Vert \eta R\sqrt{2t}%
\]
which implies
\[
t^{2}\gamma^{2}\leq\left\Vert \hat{w}_{\mathrm{opt}}\right\Vert ^{2}R^{2}2t
\]
and hence
\[
t\leq\left\Vert \hat{w}_{\mathrm{opt}}\right\Vert ^{2}\frac{2R^{2}}{\gamma
^{2}}.
\]
Furthermore
\[
\left\Vert \hat{w}_{\mathrm{opt}}\right\Vert ^{2}=\left\Vert w_{\mathrm{opt}%
}\right\Vert ^{2}+\frac{b_{\mathrm{opt}}^{2}}{R^{2}}.
\]
Now we have $b_{\mathrm{opt}}^{2}\leq R^{2}$ for nontrivial separation of $S$.
(Indeed, $\left\vert b_{\mathrm{opt}}\right\vert $ is the distance of the
optimal hyperplane from the origin; any point $x$ on the hyperplane $x$ has
$\left\Vert x\right\Vert \geq\left\vert b_{\mathrm{opt}}\right\vert $. Hence
(picture) there is a halfspace on one side of the hyperplane such that all
points $x$ in that halfspace also have $\left\Vert x\right\Vert \geq\left\vert
b_{\mathrm{opt}}\right\vert $, which implies $R\geq\left\vert b_{\mathrm{opt}%
}\right\vert $). Hence
\[
\left\Vert \hat{w}_{\mathrm{opt}}\right\Vert ^{2}\leq\left\Vert
w_{\mathrm{opt}}\right\Vert ^{2}+1\leq2
\]
which implies the assertion
\[
t\leq\left(  \frac{2R}{\gamma}\right)  ^{2}.
\]

\end{proof}

However, if the training set is not linearly separable, the above online
algorithm is not guaranteed to converge. Other training algorithms for linear
classifiers are possible: e.g., support vector machine and logistic regression.

\subsection{Intuitive background}

Suppose for any training set, we try to find a hyperplane minimizing
\begin{equation}
J_{n}(w,b)=-\sum_{\gamma_{i}<0}y_{i}\left(  \left\langle w,x_{i}\right\rangle
+b\right) \label{target-function}%
\end{equation}
under a restriction $\left\Vert w\right\Vert =1$. Since $\gamma_{i}%
=y_{i}\left(  \left\langle w,x_{i}\right\rangle +b\right)  )$ and $\gamma
_{i}<0$ means that $x_{i}$ is misclassified, we minimize the sum of the
distances of $x_{i}$ to the hyperplane for all misclassified $x_{i}$. Indeed,
as we saw, $\left\vert y_{i}\left(  \left\langle w,x_{i}\right\rangle
+b\right)  \right\vert $ is the distance of $x_{i}$ to the hyperplane given by
$\left(  w,b\right)  $. The criterion $J_{n}(w,b)$ depends on the training
set, and is defined for any training set $S=\left\{  \left(  x_{1}%
,y_{1}\right)  ,\ldots,\left(  x_{n},y_{n}\right)  \right\}  $, separable or not.

\textbf{The method of steepest descent (gradient method).} Given a hyperplane
$\hat{w}_{t}=\left(  w_{t},b_{t}\right)  $, we try to find an improved value
$\hat{w}_{t+1}=\hat{w}_{t}+\delta_{t}$ by adding a certain $\delta_{t}$ which
points in the direction of "steepest descent". Since $J(\hat{w}_{t}%
)=J_{n}(w,b)$ is differentiable with respect to the $d+1$ variables $\hat
{w}_{t},$ we can write for an increment vector $v$ and a small $\varepsilon$
\begin{equation}
J(\hat{w}_{t}+\varepsilon v)\approx J(\hat{w}_{t})+\varepsilon\left\langle
\nabla J(\hat{w}_{t}),v\right\rangle \label{increment}%
\end{equation}
where \ $\nabla J(\hat{w}_{t})$ is the gradient of the function $J(\cdot)$ at
$\hat{w}=\hat{w}_{t}$:%
\[
\nabla J(\hat{w})=\left(  \frac{\partial J}{\partial\hat{w}^{j}}\right)
_{j=1,\ldots,d+1}%
\]
where $\hat{w}^{j}$ denotes the components of the vector $\hat{w}$. Suppose we
want to find a direction $v$ (given by a unit vector $\left\Vert v\right\Vert
=1$) such that $J(\hat{w}_{t}+\varepsilon v)<J(\hat{w}_{t})$ and the change is
as large as possible. That means $\varepsilon\left\langle \nabla J(\hat
{w}),v\right\rangle $ should be negative and $\left\vert \left\langle \nabla
J(\hat{w}),v\right\rangle \right\vert $ should be maximal given $\left\Vert
v\right\Vert =1$. We have by Cauchy-Schwarz
\[
\left\vert \left\langle \nabla J(\hat{w}),v\right\rangle \right\vert
\leq\left\Vert \nabla J(\hat{w})\right\Vert \left\Vert v\right\Vert
=\left\Vert \nabla J(\hat{w})\right\Vert
\]
with equality if $v$ is a multiple of $\nabla J(\hat{w})$, i.e. $v=\nabla
J(\hat{w})/\left\Vert \nabla J(\hat{w})\right\Vert .$ Thus the direction of
steepest descent is given by $v=-\nabla J(\hat{w})/\left\Vert \nabla J(\hat
{w})\right\Vert $, and (\ref{increment}) becomes
\[
J(\hat{w}+\varepsilon v)\approx J(\hat{w})-\varepsilon\left\Vert \nabla
J(\hat{w})\right\Vert .
\]
This method (also called gradient method) is a commonly applied principle for
iterative minimization of functions; in each successive step an "improvement"
is found by taking a small step in the direction of steepest descent.

Applying this method to the target function (\ref{target-function}), which is
linear in $\hat{w}=\left(  w,b\right)  $ we find that the direction of
steepest descent is given by the unit vector $v=v_{0}/\left\Vert
v_{0}\right\Vert $ where
\[
v_{0}=-\nabla J(\hat{w})=\sum_{\gamma i<0}y_{i}\left(
%TCIMACRO{\QATOP{x_{i}}{1}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{x_{i}}{1}%
%EndExpansion
\right)  .
\]
That could give rise to a stepwise minimization procedure of
(\ref{target-function}), sometimes called the "batch perceptron algorithm".
However the algorithm we discussed is a modification, where each contribution
from a false classification is treated separately:
\[
J_{(i)}(w,b)=-y_{i}\left(  \left\langle w,x_{i}\right\rangle +b\right)
\]
with negative gradient
\begin{equation}
-\nabla J_{(i)}(\hat{w})=y_{i}\left(
%TCIMACRO{\QATOP{x_{i}}{1}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{x_{i}}{1}%
%EndExpansion
\right)  .\label{gradientx}%
\end{equation}
In each step in the "for $i=1$ to $n$" cycle, an improvement is found for one
component $J_{(i)}(w,b)$ of the target function, corresponding to a particular
misclassified $x_{i}$. The actual recipe $w_{k+1}\leftarrow w_{k}+\eta
y_{i}x_{i}$, $b_{k+1}\leftarrow b_{k}+\eta y_{i}R^{2}$ represent a further
modification ((\ref{gradientx}) would suggest $b_{k+1}\leftarrow b_{k}+\eta
y_{i}$); the stepsize $\eta$ is in some sense arbitrary.

\textbf{Remark.} The above discussion is somewhat flawed, since we treated
$\hat{w}=(w,b)$ as $d+1$ free variables, i.e. we neglected the restriction
$\left\Vert w\right\Vert =1$. However the minimization of $J(\hat{w})$ should
occur over the set $\left\{  (w,b):\left\Vert w\right\Vert =1\right\}  $;
indeed otherwise $(w,b)$ and $(\lambda w,\lambda b)$ represent the same
hyperplane, while $(\lambda w,\lambda b)$ yields a smaller target function. So
for a correct argument, all perturbations of a unit vector $w$ should only be
on the surface of the unit sphere, or (asymptotically) perpendicular to $w.$%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
(INCOMPLETE) Let us see what a perturbation $\hat{w}+\varepsilon v$ without
this restriction actually means, in terms of change of the hyperplane. Let
$v=\left(  v_{1},v_{2}\right)  $ where $v_{1}\in\mathbb{R}^{d}$ and $v_{2}%
\in\mathbb{R}$. After the perturbation $\hat{w}+\varepsilon v=\left(
w+\varepsilon v_{1},b+\varepsilon v_{2}\right)  $, we have a new hyperplane,
and to obtain its canonical representation $(\tilde{w},\tilde{b})$ we divide
by $\left\Vert w+\varepsilon v_{1}\right\Vert $:
\begin{align*}
\tilde{w}  & =\frac{w+\varepsilon v_{1}}{\left\Vert w+\varepsilon
v_{1}\right\Vert }\\
\tilde{b}  & =\frac{b+\varepsilon v_{2}}{\left\Vert w+\varepsilon
v_{1}\right\Vert }.
\end{align*}
The following "perturbation expansions" are for $\varepsilon\rightarrow0$ and
we neglect terms of higher order than $\varepsilon$. We have
\begin{align*}
\left\Vert w+\varepsilon v_{1}\right\Vert  & \sim\left\Vert w\right\Vert
+\varepsilon\left\langle w,v_{1}\right\rangle =1+\varepsilon\left\langle
w,v_{1}\right\rangle ,\\
\left\Vert w+\varepsilon v_{1}\right\Vert ^{-1}  & \sim1-\varepsilon
\left\langle w,v_{1}\right\rangle
\end{align*}
hence
\begin{align*}
\tilde{w}  & \sim\left(  w+\varepsilon v_{1}\right)  \left(  1-\varepsilon
\left\langle w,v_{1}\right\rangle \right)  \sim w+\varepsilon\left(
v_{1}-w\left\langle w,v_{1}\right\rangle \right)  ,\\
\tilde{b}  & \sim\left(  b+\varepsilon v_{2}\right)  \left(  1-\varepsilon
\left\langle w,v_{1}\right\rangle \right)  \sim b+\varepsilon\left(
v_{2}-b\left\langle w,v_{1}\right\rangle \right)  .
\end{align*}
Here $\tilde{v}_{1}=v_{1}-w\left\langle w,v_{1}\right\rangle $ is the
projection of $v_{1}$ onto the linear subspace of $\mathbb{R}^{d}$
perpendicular to $w$ (indeed check $\left\langle w,\tilde{v}_{1}\right\rangle
=0$). Let $\Pi$ be the linear projection operator onto that subspace; then we
have $\tilde{v}_{1}=\Pi v_{1}$. Set also $\tilde{v}_{2}=v_{2}-b\left\langle
w,v_{1}\right\rangle $. We found that the perturbation $\hat{w}+\varepsilon v$
is equivalent to a change of hyperplane (asymptotically, for small
$\varepsilon$)%
\[
\left(  \tilde{w},\tilde{b}\right)  \text{ where }\tilde{w}=w+\varepsilon
\tilde{v}_{1}\text{, }\tilde{b}=b+\varepsilon\tilde{v}_{2}%
\]
and indeed $\left\langle w,\tilde{v}_{1}\right\rangle =0$. In terms of the
target function, the change means
\begin{align*}
J_{(i)}(w,b)-J_{(i)}(\tilde{w},\tilde{b})  & =-y_{i}\left(  \left\langle
w,x_{i}\right\rangle +b\right)  +y_{i}\left(  \left\langle \tilde{w}%
,x_{i}\right\rangle +\tilde{b}\right) \\
& =\varepsilon y_{i}\left(  \left\langle \tilde{v}_{1},x_{i}\right\rangle
+\tilde{v}_{2}\right) \\
& =\varepsilon y_{i}\left(  \left\langle \Pi v_{1},x_{i}\right\rangle
+\tilde{v}_{2}\right)  .
\end{align*}
The "perceptron choice" of perturbation direction $v=(v_{1},v_{2})$ was
$v_{1}=y_{i}x_{i}$, $v_{2}=y_{i}$ (neglecting the requirement $\left\Vert
v\right\Vert =1$ here which can be absorbed into $\varepsilon$). Hence%
\[
\tilde{v}_{2}=v_{2}-b\left\langle w,v_{1}\right\rangle =y_{i}-b\left\langle
w,y_{i}x_{i}\right\rangle =y_{i}\left(  1-b\left\langle w,x_{i}\right\rangle
\right)  ,
\]%
\[
J_{(i)}(w,b)-J_{(i)}(\tilde{w},\tilde{b})=\varepsilon y_{i}^{2}\left(
\left\langle \Pi x_{i},x_{i}\right\rangle +\left(  1-b\left\langle
w,x_{i}\right\rangle \right)  \right)  .
\]
Here $\left\langle \Pi x_{i},x_{i}\right\rangle \geq0.$ ???? Assume also
$\left\Vert x_{i}\right\Vert \leq1$ then $\left\langle w,x_{i}\right\rangle $
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\subsection{Nonseparable training sets}

We will give an illustration that the perceptron algorithm fails for
nonseparable training sets. Consider two class-conditional laws on the real
line given as follows: $P_{1}$ is uniform on the set $\left\{  -1,2\right\}  $
and $P_{-1}$ is uniform on the set $\left\{  -2,1\right\}  $:
\begin{align*}
P_{1}(X  & =2)=P_{1}(X=-1)=1/2,\\
P_{-1}(X  & =1)=P_{-1}(X=-2)=1/2.
\end{align*}
and $P\left(  Y=1\right)  =P\left(  Y=-1\right)  =1/2$. Obviously $P_{1}$ and
$P_{-1}$ have disjoint support, so the Bayes risk is zero. A linear rule in
this case takes the form
\[
\phi(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $wx+b\geq0$\\
$-1$ otherwise$.$%
\end{tabular}
\right.  =\mathrm{sgn}\left(  wx+b\right)
\]
where $w=\pm1$ (a "threshold" rule). Since the present situation is symmetric
in the two classes, it suffices to consider the case $w=1$. There is a linear
rule with error probability $L\left(  \phi\right)  =P\left(  \phi(X)\neq
Y\right)  =1/4:$ take $\phi(x)=1$ if $x\geq1.5$, $\phi(x)=-1$ otherwise.
Indeed, if $Y=-1$ then $X=1$ or $X=-2$, i.e. $\phi(X)=-1$ always, and $X$ is
always classified correctly. If $Y=1$ then $X=-1$ or $X=2$, and $X$ is
misclassified only if $X=-1$, i.e. with probability $1/2$ conditionally on
$Y$. Hence the overall probability of misclassification is $1/4.$

Now consider the linear rule which minimizes the perceptron criterion
(\ref{target-function}), which now becomes
\[
J_{n}(b)=-\sum_{\gamma i<0}y_{i}\left(  x_{i}+b\right)
\]
Assume that $n$ is large, so that we practically minimize an expectation.
Since $\gamma_{i}=y_{i}\left(  x_{i}+b\right)  $, we can write
\[
J_{n}(b)=\sum_{i=1}^{n}\left(  -y_{i}\left(  x_{i}+b\right)  \right)  _{+}%
\]
where $\left(  z\right)  _{+}=z$ if $z\geq0,$ $\left(  z\right)  _{+}=0$
otherwise. Minimizing this is the same as minimizing
\[
\frac{1}{n}\sum_{i=1}^{n}\left(  -y_{i}\left(  x_{i}+b\right)  \right)
_{+}\approx E\left(  -Y\left(  X+b\right)  \right)  _{+}=:J(b)
\]
by the law of large numbers. Now $(X,Y)$ has a uniform distribution on the $4
$ pairs $\left(  2,1\right)  $, $\left(  -1,1\right)  $, $\left(  1,-1\right)
$, $\left(  -2,-1\right)  $, i.e. each of these $4$ points is taken with
probability $1/4$. Therefore
\begin{align*}
J(b)  & =\frac{1}{4}\left(  -\left(  2+b\right)  \right)  _{+}+\frac{1}%
{4}\left(  -\left(  -1+b\right)  \right)  _{+}+\frac{1}{4}\left(  \left(
1+b\right)  \right)  _{+}+\frac{1}{4}\left(  \left(  -2+b\right)  \right)
_{+}\\
& =\frac{1}{4}\left(  b-(-2)\right)  _{-}+\frac{1}{4}\left(  b-1\right)
_{-}+\frac{1}{4}\left(  b-(-1)\right)  _{+}+\frac{1}{4}\left(  b-2\right)
_{+}\\
& =\frac{1}{4}\left(  f_{1}(b)+f_{2}(b)+f_{3}(b)+f_{4}(b)\right)
\end{align*}
say, where we used notation $\left(  z\right)  _{-}=\left(  -z\right)  _{+}=z$
if $z\leq0,$ $\left(  z\right)  _{-}=0$ otherwise. Now note that inside the
interval $-2\leq b\leq2$, we have $f_{1}(b)+f_{4}(b)=0$ and outside the
interval, the sum is positive. Furthermore, inside the interval $-1\leq
b\leq1$ the we have $f_{2}(b)+f_{3}(b)=\left(  1-b\right)  +\left(
1+b\right)  =2$ and outside this interval we have $f_{2}(b)+f_{3}(b)>2$ (e.g.
for $b>1$ we have $f_{2}(b)+f_{3}(b)=f_{3}(b)=1+b>2$) so it is clear that
$J(b)$ is minimzed for any $b\in\left[  -1,1\right]  $, in particular $b=0$ is
a minimizer.

Now consider the error probability of the rule with $b=0$, i.e.
\[
\phi(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $x\geq0$\\
$-1$ otherwise$.$%
\end{tabular}
\right.  =\mathrm{sgn}\left(  x\right)
\]
Obviously it is
\begin{align*}
P\left(  \phi(X)\neq Y\right)   & =\frac{1}{2}P_{1}\left(  X\leq0\right)
=\frac{1}{2}P_{-1}\left(  X\geq0\right) \\
& =\frac{1}{2}\cdot\frac{1}{2}+\frac{1}{2}\cdot\frac{1}{2}=\frac{1}{2}.
\end{align*}
This is very bad compared to the best linear rule having error probability
$1/4$. In fact error probability $1/2$ is the worst possible in some sense; it
can be achieved without a training set, by flipping a coin and classifying $X$
according to the result.

\pagebreak

\section{Fisher's linear discriminant analysis}

Fisher's linear discriminant analysis \ (LDA, 1936) has been derived from a
certain empirical principle, involving sample means and covariances of the two
classes. As almost always in statistics, such a "second moment" approach is
connected to an intrinsic assumption that the data are normal.

Assume we have a traing set $\left(  X_{i},Y_{i}\right)  $, $i=1,\ldots,n$
where $Y_{i}$ is uniform on $\left\{  0,1\right\}  $ and $X|Y=i$ is
multivariate normal $N_{d}\left(  m_{i},\Sigma\right)  $, $i=0,1,$ with the
same covariance matrix $\Sigma$. Let us derive the Bayes rule for this case.
We know that for equal prior probabilities $\pi=1-\pi=1/2$, the Bayes rule
coincides with the maximum likelihood rule:
\[
h^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $f_{1}(x)>f_{0}(x)$\\
$0$ otherwise
\end{tabular}
\right.
\]
where $f_{i}$ is are the class-conditional densities, i. e. the densities of
$N_{d}\left(  m_{i},\Sigma\right)  $, $i=0,1.$ We have
\[
f_{i}(x)=\frac{1}{\sqrt{2\pi\det(\Sigma)}}\exp\left(  -\frac{1}{2}%
(x-m_{i})^{\top}\Sigma^{-1}(x-m_{i})\right)  .
\]
Hence the Bayes rule can be written
\[
h^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $r_{1}^{2}<r_{0}^{2}$\\
$0$ otherwise
\end{tabular}
\right.
\]
where
\[
r_{i}^{2}=(x-m_{i})^{\top}\Sigma^{-1}(x-m_{i})\text{, }i=0,1
\]
are the \textit{squared Mahalanobis distances} of $x$ from $m_{0}$ and from
$m_{1}$. Hence the rule decides $1$ if
\[
(x-m_{1})^{\top}\Sigma^{-1}(x-m_{1})<(x-m_{0})^{\top}\Sigma^{-1}(x-m_{0})
\]
which is equivalent to
\[
-2x^{\top}\Sigma^{-1}m_{1}+m_{1}{}^{\top}\Sigma^{-1}m_{1}<-2x^{\top}%
\Sigma^{-1}m_{0}+m_{0}{}^{\top}\Sigma^{-1}m_{0}%
\]
or to
\[
x^{\top}\Sigma^{-1}(m_{1}-m_{0})>\frac{1}{2}\left(  m_{1}{}^{\top}\Sigma
^{-1}m_{1}-m_{0}{}^{\top}\Sigma^{-1}m_{0}\right)
\]
Thus the \textbf{normal Bayes rule} is
\begin{equation}
h_{N}^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $x^{\top}\Sigma^{-1}(m_{1}-m_{0})>\frac{1}{2}\left(  m_{1}{}^{\top
}\Sigma^{-1}m_{1}-m_{0}{}^{\top}\Sigma^{-1}m_{0}\right)  $\\
$0$ otherwise
\end{tabular}
\right. \label{normal-bayes-rule}%
\end{equation}
Let us substitute $m_{i}$ by estimates
\begin{align*}
\hat{m}_{0}  & =\frac{1}{n_{0}}\sum_{j:Y_{j}=0}X_{j}\text{, }\hat{m}_{1}%
=\frac{1}{n_{1}}\sum_{j:Y_{j}=1}X_{j},\\
n_{0}  & =\sum_{j:Y_{j}=0}1\text{, }n_{1}=\sum_{j:Y_{j}=1}1
\end{align*}
and $\Sigma$ by an estimate
\[
\hat{\Sigma}=(n-2)^{-1}\left(  S_{0}+S_{1}\right)
\]
where $S_{i}$ are the class scatter matrices%
\[
S_{0}=\sum_{i:Y_{i}=0}\left(  X_{i}-\hat{m}_{0}\right)  \left(  X_{i}-\hat
{m}_{0}\right)  ^{\top},\;S_{1}=\sum_{i:Y_{i}=1}\left(  X_{i}-\hat{m}%
_{1}\right)  \left(  X_{i}-\hat{m}_{1}\right)  ^{\top}.
\]
The heuristics of the estimate $\hat{\Sigma}$ is obvious: both $\hat{\Sigma
}_{0}:=S_{0}/(n_{0}-1)$ and $\hat{\Sigma}_{1}:=S_{1}/(n_{1}-1)$ are unbiased
estimates of $\Sigma$, and $\hat{\Sigma}$ is the pooled estimate, i.e. an
unbiased linear combination:
\[
\hat{\Sigma}=\frac{n_{0}-1}{n-2}\hat{\Sigma}_{0}+\frac{n_{1}-1}{n-2}%
\hat{\Sigma}_{1}\text{. }%
\]
This gives \textbf{Fisher's rule}%
\begin{equation}
h_{F}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $x^{\top}\hat{\Sigma}^{-1}(\hat{m}_{1}-\hat{m}_{0})>\frac{1}{2}\left(
\hat{m}_{1}{}^{\top}\hat{\Sigma}^{-1}\hat{m}_{1}-\hat{m}_{0}{}^{\top}%
\hat{\Sigma}^{-1}\hat{m}_{0}\right)  $\\
$0$ otherwise.
\end{tabular}
\right. \label{Fisher-rule}%
\end{equation}

\subsection{Geometric interpretation}

For this we drop the "hats" over $\hat{m}_{i}$ and $\hat{\Sigma}$, i.e. we
refer to the normal Bayes rule (\ref{normal-bayes-rule}). The whole reasoning
is also valid for Fisher's rule, if the corresponding sample analogs are used.
Assume $\Sigma=I$, otherwise we transform the data $\tilde{X}_{i}%
=\Sigma^{-1/2}X_{i}$ $i=1,\ldots,n$. Define
\begin{align}
a  & =m_{1}-m_{0},\\
a_{0}  & =\frac{1}{2}\left(  \left\Vert m_{0}\right\Vert ^{2}-\left\Vert
m_{1}\right\Vert ^{2}\right)  ,\label{find-constant}%
\end{align}
then the normal Bayes rule uses a sample split
\begin{equation}
h_{N}^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $x^{\top}a+a_{0}>0$\\
$0$ otherwise.
\end{tabular}
\right. \label{normal-bayes-rule-a}%
\end{equation}
Thus it is a linear classifier, or a "perceptron" in an older terminology.
Fisher's rule is also a linear classifier or "perceptron" where $a,a_{0}$ are
estimated using the two sample means and the estimated covariance matrix from
the training set. Thus the difference to Rosenblatt's perceptron algorithm
lies only in how the $a,a_{0}$ are determined (the training method).

Let us describe another way of determining the constant $a_{0}$ according to
the "Principle" below. Recall that if $v,$ $w$ are vectors and $w$ is a unit
vector ($\left\Vert w\right\Vert =1$) then $v^{\top}w$ is the length of the
projection of $v$ onto the linear space spanned by $w$. Consider the unit
vector corresponding to the optimal $a$ above
\[
\bar{a}=\frac{a}{\left\Vert a\right\Vert }=\frac{m_{1}-m_{0}}{\left\Vert
m_{1}-m_{0}\right\Vert }.
\]
Also, make $m_{0}$ the origin of the vector space by subtracting it from $x$,
i.e. consider $x-m_{0}$. Now the vector $\bar{a}$ points from the origin in
the direction of $\ m_{1}$ (since $m_{0}+\left(  m_{1}-m_{0}\right)  =m_{1}$).

\bigskip

\textbf{Principle.} \textit{We want our classification rule such that if
}$\left(  x-m_{0}\right)  ^{\top}\bar{a}$\textit{\ is greater than one half
the length }$\left\Vert m_{1}-m_{0}\right\Vert $\textit{\ then we classify
}$1$\textit{, otherwise we classify }$0.$

\bigskip

This means we classify $1$ if
\[
\left(  x-m_{0}\right)  ^{\top}\bar{a}>\frac{1}{2}\left\Vert m_{1}%
-m_{0}\right\Vert
\]
or equivalently
\begin{align*}
\left(  x-m_{0}\right)  ^{\top}\left(  m_{1}-m_{0}\right)   & >\frac{1}%
{2}\left\Vert m_{1}-m_{0}\right\Vert ^{2}=\frac{1}{2}\left(  m_{1}%
-m_{0}\right)  ^{\top}\left(  m_{1}-m_{0}\right) \\
& =\frac{1}{2}m_{1}^{\top}\left(  m_{1}-m_{0}\right)  -\frac{1}{2}m_{0}^{\top
}\left(  m_{1}-m_{0}\right)  .
\end{align*}
This is equivalent to
\[
x^{\top}\left(  m_{1}-m_{0}\right)  >\frac{1}{2}\left(  m_{1}+m_{0}\right)
^{\top}\left(  m_{1}-m_{0}\right)  =\frac{1}{2}\left(  m_{1}^{\top}m_{1}%
-m_{0}^{\top}m_{0}\right)
\]
which gives the same choice as (\ref{find-constant}) for $a_{0}$:%
\[
a_{0}=-\frac{1}{2}\left(  m_{1}^{\top}m_{1}-m_{0}^{\top}m_{0}\right)  .
\]
Thus the rule derived from the "Principle" coincides with the normal Bayes
rule (\ref{normal-bayes-rule}) in the case $\Sigma=I$. Fisher's LDA rule
(\ref{Fisher-rule}) was originally obtained from the empirical version of this
idea: first $\hat{\Sigma}$ and $\hat{m}_{0}$, $\hat{m}_{1}$ are substituted
for $\Sigma$, $m_{0}$, $m_{1}$, and then the "Principle" is applied to
transformed data $\tilde{X}_{i}=\hat{\Sigma}^{-1/2}X_{i}$ $i=1,\ldots,n$.

\subsection{Linear inconsistency}

Since Fisher's LDA is based on the first two moments of the training set, and
thus on an underlying normality assumption, it is bound to encounter
difficulties for nonnormal distributions. We will find an example where
Fisher's rule is \textbf{linearly inconsistent}, i.e. for training set size
$n\rightarrow\infty$ does not asymptotically attain the best linear risk. We
cannot expect Fisher's rule to attain the Bayes risk in the general case,
since it is a linear rule and the Bayes rule may be nonlinear. However we will
show that even if the classes (distributions) can be separated by a linear
rule, Fisher's LDA may not asymptotically find that rule.

Define class-conditional distributions as follows. Consider three points
$v_{1}$,$v_{2}$,$v_{3}$ in $\mathbb{R}^{2}$ forming a equilateral triangle
with side length $l$, i.e we have
\[
\left\Vert v_{1}-v_{2}\right\Vert =\left\Vert v_{1}-v_{3}\right\Vert
=\left\Vert v_{2}-v_{3}\right\Vert =l.
\]
Let $P_{0}$ be the uniform distribution on the vertices, i.e.
\[
P_{0}\left(  X=v_{i}\right)  =\frac{1}{3}\text{, }i=1,2,3.
\]
Consider another equilateral triangle with vertices $w_{1}$,$w_{2}$,$w_{3}$ in
$\mathbb{R}^{2}$ and the same sidelength, such that the two triangles do not
intersect. Let $P_{1}$ be the uniform distribution on the vertices $w_{1}
$,$w_{2}$,$w_{3}$. We will show that a) $l$ can be chosen such that
$P_{0},P_{1}$ have unit covariance matrix, b) the vertices can be chosen such
that $P_{0},P_{1}$ are linearly separable, c) the vertices can also be chosen
such that Fisher's rule does not provide linear separation, i.e. it does not
find a separating hyperplane even for training set size $n\rightarrow\infty$.

\bigskip

\textbf{Description of a equilateral triangle.} We start with side length
$l=2$ and vertices $v_{1}=\left(  1,0\right)  $, $v_{2}=\left(  -1,0\right)
$, and $v_{3}=\left(  0,\sqrt{3}\right)  $. To check equal side length, use
Pythagoras: we find $\left\Vert v_{3}-v_{2}\right\Vert =$ $\sqrt{1^{2}+3}=2$.
To find the center of this triangle ($v_{0}$, say), compute the expected value
of $P_{0}$: if $X\sim P_{0}$ then
\begin{align*}
v_{0}  & =EX=\frac{1}{3}\sum_{i=1}^{3}v_{i}=\frac{1}{3}\left(
%TCIMACRO{\QATOP{1}{0}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{1}{0}%
%EndExpansion
\right)  +\frac{1}{3}\left(
%TCIMACRO{\QATOP{-1}{0}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{-1}{0}%
%EndExpansion
\right)  +\frac{1}{3}\left(
%TCIMACRO{\QATOP{0}{\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{0}{\sqrt{3}}%
%EndExpansion
\right) \\
& =\left(
%TCIMACRO{\QATOP{0}{1/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{0}{1/\sqrt{3}}%
%EndExpansion
\right)  .
\end{align*}
For the covariance matrix $\Sigma:=EXX^{\top}-\left(  EX\right)  \left(
EX\right)  ^{\top}$, note
\begin{align*}
EXX^{\top}  & =\\
& =\frac{1}{3}%
\begin{pmatrix}
1 & 0\\
0 & 0
\end{pmatrix}
+\frac{1}{3}%
\begin{pmatrix}
1 & 0\\
0 & 0
\end{pmatrix}
+\frac{1}{3}%
\begin{pmatrix}
0 & 0\\
0 & 3
\end{pmatrix}
\\
& =%
\begin{pmatrix}
2/3 & 0\\
0 & 1
\end{pmatrix}
,\\
\left(  EX\right)  \left(  EX\right)  ^{\top}  & =%
\begin{pmatrix}
0 & 0\\
0 & 1/3
\end{pmatrix}
,
\end{align*}
and hence%
\[
\Sigma:=%
\begin{pmatrix}
2/3 & 0\\
0 & 2/3
\end{pmatrix}
.
\]
Thus $\Sigma=(2/3)I$ where $I$ is the unit matrix, and we see that the r.v.
$\lambda X$ for $\lambda=\sqrt{3/2}$ has unit covariance matrix. Thus changing
the side length of the equilateral triangle to $l=2\lambda=\sqrt{6} $ gives a
unit covariance matrix for the uniform distribution on the vertices. For our
point about Fisher's rule, based on the geometry, the factor $\lambda$ is
irrelevant, and we will continue with $l=2$.

\textbf{Remark: }It is easily seen that instead of the the triangle with
vertices $\left\{  v_{1},v_{2},v_{3}\right\}  $, we could have taken $\left\{
v_{1},v_{2},-v_{3}\right\}  $ and obtained the same covariance matrix $\Sigma
$, but center point $-v_{0}$.

Now we will change the position of the triangle $\left\{  v_{1},v_{2}%
,v_{3}\right\}  $ by shifting the center (and thus all the vertices). Draw a
horizontal line though the center point $v_{0}$ and find the intersection of
this line with the left upper side of the triangle. i.e. with the line joining
$v_{2}$ and $v_{3}$. Call this point $\tilde{v}$; we will shift the triangle
so that $\tilde{v}$ comes to coincide with the origin. To find the x-component
of $\tilde{v}=\left(  \tilde{v}_{x},\tilde{v}_{y}\right)  $, note that
$\tilde{v}_{y}=1/\sqrt{3}$ and by similarity of triangles%
\[
\frac{\sqrt{3}}{1}=\frac{\tilde{v}_{y}}{1-\left\vert \tilde{v}_{x}\right\vert
}=\frac{1/\sqrt{3}}{1-\left\vert \tilde{v}_{x}\right\vert }%
\]
which gives $1-\left\vert \tilde{v}_{x}\right\vert =1/3$ and hence $\tilde
{v}_{x}=-2/3$. Hence
\[
\tilde{v}=\left(
%TCIMACRO{\QATOP{-2/3}{1/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{-2/3}{1/\sqrt{3}}%
%EndExpansion
\right)  .
\]
Now subtract the vector $\tilde{v}$ from all three vertices, to obtain a
shifted triangle with vertices
\begin{align*}
v_{1}^{\ast}  & =v_{1}-\tilde{v}=\left(
%TCIMACRO{\QATOP{1}{0}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{1}{0}%
%EndExpansion
\right)  -\left(
%TCIMACRO{\QATOP{-2/3}{1/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{-2/3}{1/\sqrt{3}}%
%EndExpansion
\right)  =\left(
%TCIMACRO{\QATOP{5/3}{-1/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{5/3}{-1/\sqrt{3}}%
%EndExpansion
\right)  ,\\
v_{2}^{\ast}  & =v_{2}-\tilde{v}=\left(
%TCIMACRO{\QATOP{-1}{0}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{-1}{0}%
%EndExpansion
\right)  -\left(
%TCIMACRO{\QATOP{-2/3}{1/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{-2/3}{1/\sqrt{3}}%
%EndExpansion
\right)  =\left(
%TCIMACRO{\QATOP{-1/3}{-1/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{-1/3}{-1/\sqrt{3}}%
%EndExpansion
\right) \\
v_{3}^{\ast}  & =v_{3}-\tilde{v}=\left(
%TCIMACRO{\QATOP{0}{\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{0}{\sqrt{3}}%
%EndExpansion
\right)  -\left(
%TCIMACRO{\QATOP{-2/3}{1/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{-2/3}{1/\sqrt{3}}%
%EndExpansion
\right)  =\left(
%TCIMACRO{\QATOP{2/3}{2/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{2/3}{2/\sqrt{3}}%
%EndExpansion
\right)
\end{align*}
\textit{Now define }$\tilde{P}_{0}$\textit{\ to be the uniform distribution on
}$\left\{  v_{1}^{\ast},v_{2}^{\ast},v_{3}^{\ast}\right\}  $. It follows from
the above that its covariance matrix is $\Sigma=(2/3)I$ (since the covariance
matrix does not change under translations by a constant vector $\tilde{v}$).

Define another equilateral triangle as follows: it has vertices $\left\{
-v_{1}^{\ast},-v_{2}^{\ast},-v_{3}^{\ast}\right\}  $. This can be visualized
as a double mirror image of $\left\{  v_{1}^{\ast},v_{2}^{\ast},v_{3}^{\ast
}\right\}  $: first mirrored at the x-axis and then mirrored at the y-axis.
Both triangles contain the origin in one of their sides, and in fact they have
a section in common: the segment joining $v_{1}^{\ast}$ and $-v_{1}^{\ast}$,
i.e. the set $\left\{  tv_{1}^{\ast},\;-1\leq t\leq1\right\}  .$ \textit{Now
define }$\tilde{P}_{1}$\textit{\ to be the uniform distribution on }$\left\{
-v_{1}^{\ast},-v_{2}^{\ast},-v_{3}^{\ast}\right\}  $. Clearly that has the
same covariance matrix $\Sigma$ as $\tilde{P}_{0}$, (since it can also be
obtained as a translation of the triangle given by $\left\{  v_{1}%
,v_{2},-v_{3}\right\}  $, \textit{\ }which gives the same $\Sigma$ according
to the remark above.

The distributions $\tilde{P}_{0}$, $\tilde{P}_{1}$ are separable (the Bayes
risk is zero) , since their supports do not intersect. However they are not
\textit{linearly separable}: there is no straight line separating them; in
fact the only candidate straight line $\left\{  tv_{1}^{\ast},\;t\in
\mathbb{R}\right\}  $ contains one vertex from each triangle.

However by "moving them apart" by an amount $\varepsilon$ makes them linearly
separable. Define the vector $u=\left(  1,0\right)  $ and consider the two
triangles $\left\{  v_{1}^{\ast}+\varepsilon u,v_{2}^{\ast}+\varepsilon
u,v_{3}^{\ast}+\varepsilon u\right\}  $ and $\left\{  -v_{1}^{\ast
}-\varepsilon u,-v_{2}^{\ast}-\varepsilon u,-v_{3}^{\ast}-\varepsilon
u\right\}  $. Let $P_{0}$, $P_{1}$ be the respective uniform distributions on
the vertices. If $\varepsilon>0$, the distributions are now linearly separable
(e.g. by the hyperplane $\left\{  tv_{1}^{\ast},\;t\in\mathbb{R}\right\}  $).
On the other hand, if $\varepsilon<1/3$ then they are \textit{not linearly
separable by the y-axis}, and not by any hyperplane parallel to it. This means
that Fisher's LDA fails for $P_{0}$, $P_{1}$, as we will discuss now

\bigskip

\textbf{Behaviour of Fisher's LDA for distributions }$P_{0}$\textbf{, }$P_{1}
$\textbf{.} In a large training set, we may identify Fisher's rule with the
normal Bayes rule (\ref{normal-bayes-rule}) for separating laws $N(m_{0}%
,\Sigma)$, $N(m_{1},\Sigma)$ . Recall that our actual distributions are not
normal now; data come from the two $P_{0}$\textbf{, }$P_{1}$ constructed above
with mean vectors $m_{0},m_{1}$ and covariance matrix $\Sigma$. When we use
Fishers' rule, the sample means $\hat{m}_{i}$ will converge to $m_{i}$ and
$\hat{\Sigma}$ will converge to $\Sigma$. So asymptotically we are using the
normal Bayes rule (\ref{normal-bayes-rule}) on actual distributions $P_{0}%
$\textbf{, }$P_{1}$. Since $\Sigma$ is a multiple of the unit matrix, we have
\begin{equation}
h_{N}^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $x^{\top}(m_{1}-m_{0})>c$\\
$0$ otherwise
\end{tabular}
\right. \label{rule-of-type}%
\end{equation}
where $c=\frac{1}{2}\left(  m_{1}{}^{\top}m_{1}-m_{0}{}^{\top}m_{0}\right)  $.
In our construction of $P_{0}$\textbf{, }$P_{1}$, the two mean points were
such that they both are on the x-axis:
\begin{align*}
m_{0}  & =v_{0}-\tilde{v}+\varepsilon u=\left(
%TCIMACRO{\QATOP{0}{1/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{0}{1/\sqrt{3}}%
%EndExpansion
\right)  -\left(
%TCIMACRO{\QATOP{-2/3}{1/\sqrt{3}}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{-2/3}{1/\sqrt{3}}%
%EndExpansion
\right)  +\varepsilon\left(
%TCIMACRO{\QATOP{1}{0}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{1}{0}%
%EndExpansion
\right)  =\left(
%TCIMACRO{\QATOP{2/3+\varepsilon}{0}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{2/3+\varepsilon}{0}%
%EndExpansion
\right)  ,\\
m_{1}  & =-v_{0}+\tilde{v}-\varepsilon u=\left(
%TCIMACRO{\QATOP{-2/3-\varepsilon}{0}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{-2/3-\varepsilon}{0}%
%EndExpansion
\right)  ,
\end{align*}
hence $m_{1}-m_{0}$ is a vector parallel to the (or on the) x-axis. Thus any
rule of type (\ref{rule-of-type}) projects the support of the two laws $P_{0}
$\textbf{, }$P_{1}$ on the x-axis and attempts to separate them by a cut.

Now projected on the x-axis, $P_{0}$ will be uniform on the three points
$\left\{  5/3+\varepsilon,-1/3+\varepsilon,2/3+\varepsilon\right\}  $ and
$P_{1}$ will be uniform on the three points $\left\{  -5/3-\varepsilon
,1/3-\varepsilon,-2/3-\varepsilon\right\}  $. More precisely, these are
projections of laws: call them $P_{x,0}$ and $P_{x,1}$. Though $P_{x,0}$ tends
to be more on the positive side and $P_{x,1}$ tends to be more on the negative
side, as long as $0<\varepsilon<1/3$, we will have $-1/3+\varepsilon
<0<1/3-\varepsilon$, and hence one point from the support of $P_{x,0}$ will be
negative and one point from the support of $P_{x,1}$ will be positive. Thus
$P_{x,0}$ and $P_{x,1}$ cannot be separated by a linear rule (a split): at
least one point is always misclassified, from either $P_{x,0}$ and $P_{x,1}$
(one may choose ) so the best linear error probability is $\left(  1/3\right)
\cdot\left(  1/2\right)  =1/6$. But Fisher's rule (\ref{rule-of-type}) uses
$c=0$ and hence has error probability $1/3$.

Recall that the original distributions $P_{0}$\textbf{, }$P_{1}$ are linearly
separable by the hyperplane $\left\{  tv_{1}^{\ast},\;t\in\mathbb{R}\right\}
$.

\bigskip

\textbf{Remark.} Problem 4.9., p. 58 in [DGL] claims that there is a
distribution of $(X,Y)$, $X\in\mathbb{R}^{2}$ such that Fisher's rule is even
worse: every split rule using projection onto the vector $\hat{m}_{1}-\hat
{m}_{0}$ has error probability $1/2-\varepsilon$ and Fisher's rule has error
probability $1-\varepsilon$, In our example above, these numbers are $1/6$ and
$1/3$ respectively.\pagebreak

\section{Kernelization}

There is a trick called kernelization for improving a computationally simple
classifier $h$. The idea is to map the covariate $X$ which takes values in
$\mathbb{R}^{d}$ into a higher dimensional space $\mathcal{Z}$ and apply the
classifier in the bigger space $\mathcal{Z}$. This can yield a more flexible
classifier while retaining computationally simplicity.

The standard example of this idea is the following. Let $d=2$ and assume a
training set $(X_{i},Y_{i})$, $i=1,\ldots,n$ can be separated into two groups
using an ellipse. Suppose we are using "ellipse classifiers" given by (where
$x=(x_{1},x_{2})$)%
\[
h(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $x^{\top}Ax>c$\\
$0$ otherwise
\end{tabular}
\right.
\]
where $A$ is a symmetric positive definite $2\times2$ matrix and $c>0$. The
decision boundary here is an ellipse:%
\[
\left\{  x:x^{\top}Ax=c\right\}  =\left\{  x:x_{1}^{2}a_{11}+2x_{1}x_{2}%
a_{12}+x_{2}^{2}a_{22}=c\right\}
\]
with shape given by the matrix $A$ and "width" given by $c$. This does not
give all ellipses in the plane; these would be given by
\[
\left\{  x:(x-b)^{\top}A(x-b)=c\right\}  ,
\]
with a vector $b\in\mathbb{R}^{2}$. But for the example we set $b=0$ and
consider only ellipses around the origin. Define a mapping $\phi$ by
\[
z=(z_{1},z_{2},z_{3})=\phi(x)=(x_{1}^{2},\sqrt{2}x_{1}x_{2},x_{2}^{2}).
\]
Thus, $\phi$ maps $\mathbb{R}^{2}$ into $\mathcal{Z}=\mathbb{R}^{3}$. If the
original data are separable by an elliptic decision boundary, then, in the
higher-dimensional space $\mathcal{Z}$, the $(X_{i},Y_{i})$ are separable by a
linear decision boundary. In other words, a linear classifier in a
higher-dimensional space corresponds to a nonlinear classifier in the original space.

The point is that to get a richer set of classifiers we do not need to give up
the convenience of linear classifiers. We simply map the covariates to a
higher-dimensional space. This is akin to making linear regression more
flexible by using polynomials. The space $\mathcal{Z}$ of higher dimension is
often called the \textbf{feature space} and the original $\mathbb{R}^{d}$ is
called input space.

There is a potential drawback. If we significantly expand the dimension of the
problem, we might increase the computational burden.

\bigskip

\textbf{Example. } Suppose $x$ has dimension $d=56$ and we wanted to use all
fourth-order terms, i.e. all terms $x_{i}^{a}x_{j}^{b}x_{k}^{c}x_{l}^{d}$
where the sum of powers $a+b+c+d$ equals $4$. These are all terms of type
$x_{i}^{4}$, $x_{i}^{3}x_{j},$ $x_{i}^{2}x_{j}^{2}$ etc. For the number of
these, consider only terms of type $x_{i}x_{j}x_{k}x_{l}$ where all 4 indices
$i,j,k,l$ are different. There are
\[
\left(
%TCIMACRO{\QATOP{56}{4}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{56}{4}%
%EndExpansion
\right)  =\frac{56\cdot55\cdot54\cdot53}{1\cdot2\cdot3\cdot4}=367,290
\]
of these, so the dimension of $\mathcal{Z}$ exceeds this number.

\bigskip

We are spared this computational nightmare by the following two facts. First,
many classifiers do not require that we know the values of the individual
points but, rather, just the inner product between pairs of points. Second,
notice in our example that the inner product in $\mathcal{Z}$ can be written%
\begin{align*}
\left\langle z,\tilde{z}\right\rangle  & =\left\langle \phi(x),\phi(\tilde
{x})\right\rangle =x_{1}^{2}\tilde{x}_{1}^{2}+2x_{1}\tilde{x}_{1}x_{2}%
\tilde{x}_{2}+x_{2}^{2}\tilde{x}_{2}^{2}\\
& =\left(  \left\langle x,\tilde{x}\right\rangle \right)  ^{2}=:K\left(
x,\tilde{x}\right)  .
\end{align*}
Thus we can compute $\left\langle z,\tilde{z}\right\rangle $ without ever
computing $Z_{i}=\phi\left(  X_{i}\right)  $.

To summarize, kernelizalion involves finding a mapping $\phi:\mathbb{R}%
^{d}\mapsto\mathcal{Z}$ and a classifier such that:

\begin{enumerate}
\item $\mathcal{Z}$ has higher dimension than $X$ and so leads a richer set of classifiers.

\item The classifier only requires computing inner products.

\item There is a function $K$, called a kernel such that $\left\langle
\phi(x),\phi(\tilde{x})\right\rangle =K\left(  x,\tilde{x}\right)  .$

\item Everywhere the term $\left\langle x,\tilde{x}\right\rangle $ appears in
the algorithm, replace it with $K\left(  x,\tilde{x}\right)  .$
\end{enumerate}

In fact, we never need to construct the mapping $\phi$ at all. We only need to
specify a kernel $K\left(  x,\tilde{x}\right)  $ that corresponds to
$\left\langle \phi(x),\phi(\tilde{x})\right\rangle $ for some $\phi$. This
raises an interesting question: given a function of two variables $K(x,y)$,
does there exist a function $\phi(x)$ such that $K(x,y)=\left\langle
\phi(x),\phi(y)\right\rangle $ \ ? The answer is provided by

\bigskip%
%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
give a proper reference here, e.g. Dieudonne ? or the standard book on
functional analysis used here
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\textbf{Mercer's Theorem. } \textit{Let }$C$\textit{\ be a compact subset of
}$\mathbb{R}^{d}$\textit{\ and let }$L_{2}(C)$\textit{\ be the corresponding
}$L_{2}$\textit{-space, i..e. the Hilbert space of square integrable functions
on }$C$\textit{\ endowed with inner product }%
\[
\left\langle f,g\right\rangle =\int f(u)g(u)du\text{, }f,g\in L_{2}(C)
\]
\textit{and associated norm }$\left\Vert f\right\Vert =\left(  \left\langle
f,f\right\rangle \right)  ^{1/2}$\textit{. Suppose }$K(u,v)$\textit{\ is a
continuous symmetric (}$K(u,v)=K(v,u)$\textit{) and real valued function
(kernel) on }$C\times C$\textit{\ which is positive definite, i.e. for all
}$f\in L_{2}(C)$\textit{\ we have}%
\begin{equation}
\int_{C\times C}K(u,v)f(u)f(v)dudv\geq0\text{. }\label{posdef}%
\end{equation}
\textit{Then we can expand }$K(u,v)$\textit{\ in a uniformly convergent series
(on }$C\times C$\textit{) in terms of normalized eigenfunctions }$\phi_{j}\in
L_{2}(C)$\textit{\ with }$\left\Vert \phi_{j}\right\Vert =1$\textit{\ and
nonnegative associated eigenvalues }$\lambda_{j}\geq0$%
\begin{equation}
K(u,v)=\sum_{j=1}^{\infty}\lambda_{j}\phi_{j}(u)\phi_{j}%
(v).\label{kern-expand}%
\end{equation}

\bigskip

\bigskip

\textbf{\ Remark. }The theorem is analogous to the spectral decomposition of
real symmetric nonnegative definite matrices. Replace the functions $f,g$ by
finite dimensional vectors $f,g\in\mathbb{R}^{k}$ and assume we have a
symmetric matrix $k\times k$ matrix $K=\left(  K(i,j)\right)  $ where in
analogy to (\ref{posdef}) for all $f\in\mathbb{R}^{k}$ with i-th component
$f(i)$%
\[
f^{\top}Kf=\sum_{i,j=1}^{k}f(i)f(j)K(i,j)\geq0.
\]
Then we can expand $K$ into a finite sum in terms of normalized eigenvectors
$\phi_{j}\in\mathbb{R}^{k}$ with $\left\Vert \phi_{j}\right\Vert =1$
(euclidean norm) and nonnegative associated eigenvalues $\lambda_{j}\geq0$
\[
K=\sum_{j=1}^{k}\lambda_{j}\phi_{j}\phi_{j}^{\top}%
\]
which can also be written, if $\phi_{j}(i)$ denotes the i-th component of
vector $\phi_{j}$
\[
K(i,s)=\sum_{j=1}^{k}\lambda_{j}\phi_{j}(i)\phi_{j}(s)
\]
which is analogous to (\ref{kern-expand}).

\bigskip

\textbf{Use of Mercer's Theorem with kernelization.} \ Suppose we use a map
$\phi$: $\mathbb{R}^{d}\longmapsto\mathbb{R}^{s}$ from input space
$\mathbb{R}^{d}$ into a higher dimensional feature space $\mathbb{R}^{s}$, and
that the transformed training set $\left(  \phi(X_{1}),Y_{1}\right)
,\ldots,\left(  \phi(X_{n}),Y_{n}\right)  $ is linearly separable (just for
illustration; the kernelization method does not require this assumption). A
separating hyperplane for the transformed training set can be expressed for
$z\in\mathbb{R}^{s}$
\[
\left(  \left\langle a,z\right\rangle +a_{0}\right)  =0
\]
thus in terms of the input space argument $x\in\mathbb{R}^{d}$%
\[
\left(  \left\langle a,\phi(x)\right\rangle +a_{0}\right)  =0.
\]
In all classifiers described so far, computation of the optimal vector vector
$a$ (for classification in feature space) requires only evaluation of scalar
products $\left\langle Z_{i},Z_{j}\right\rangle $ where $Z_{j}=\phi(X_{j})$.
Thus we have to compute all scalar products $\left\langle \phi(X_{i}%
),\phi(X_{j})\right\rangle .$ Define
\[
K(x,y)=\left\langle \phi(x),\phi(y)\right\rangle
\]
and assume that $x,y$ are from a compact set $C\subset\mathbb{R}^{d}$ (the
training set is finite and thus contained in a ball of finite radius in
$\mathbb{R}^{d}$). We will show that the function $K(x,y)$ defines a positive
definite kernel. Suppose \textit{\ } $f\in L_{2}(C)$, then if $\phi_{j}(x)$,
$j=1,\ldots,s$ are the components of $\phi(x)\in\mathbb{R}^{s}$
\[
\int_{C\times C}K(x,y)f(x)f(y)dxdy=\int_{C\times C}\left\langle \phi
(x),\phi(y)\right\rangle f(x)f(y)dxdy
\]%
\begin{align*}
& =\int_{C\times C}\left(  \sum_{j=1}^{s}\phi_{j}(x)\phi_{j}(y)\right)
f(x)f(y)dxdy=\sum_{j=1}^{s}\int_{C\times C}\phi_{j}(x)f(x)\phi_{j}%
(y)f(y)dxdy\\
& =\sum_{j=1}^{s}\left(  \int_{C}\phi_{j}(x)f(x)dx\right)  ^{2}\geq0.
\end{align*}
Clearly $K(x,y)$ is also symmetric and continuous in $x,y$ if the mapping
$\phi(x)$ is continuous. Thus $K(x,y)$ is a kernel satisfying the conditions
of Mercer's theorem.

\textit{The idea of kernelized classification now is to replace the special
kernel }$K(x,y)=\left\langle \phi(x),\phi(y)\right\rangle $\textit{\ where
}$\phi(x)$\textit{\ is a map into feature space, with a general kernel
}$K(x,y)$\textit{\ satisfying the conditions of Mercer's theorem. The question
arises: can the resulting classification procedure still be understood as
linear classification in some feature space ? }

The answer is given by Mercer's theorem. According to (\ref{kern-expand}) we
have
\[
K(x,y)=\sum_{j=1}^{\infty}\lambda_{j}\phi_{j}(x)\phi_{j}(y).
\]
We have to consider an infinite dimensional feature space $l_{2}$ (the space
of sequences $\left(  m_{i}\right)  _{i=1}^{\infty}$ such that $\sum
_{i=1}^{\infty}m_{i}^{2}<\infty$) and define a mapping from $\mathbb{R}^{d}$
into this space by
\[
\phi(x)=\left(  \lambda_{j}^{1/2}\phi_{j}(x)\right)  _{j=1}^{\infty}%
\]
Then, if the scalar product in $l_{2}$ is given by the obvious infinite series
then
\[
K(x,y)=\left\langle \phi(x),\phi(x)\right\rangle =\sum_{j=1}^{\infty}%
\lambda_{j}\phi_{j}(x)\phi_{j}(y).
\]

\subsection{Kernelization of Fisher's LDA}

Let us first assume that we use a map $\phi$: $\mathbb{R}^{d}\longmapsto
\mathbb{R}^{s}$ from input space $\mathbb{R}^{d}$ into a higher dimensional
feature space $\mathbb{R}^{s}$ ($s>d$). We perform the LDA\ (linear
discrimination analysis) on the basis of the transformed sample $Z_{i}%
=\phi\left(  X_{i}\right)  $ and we classify a "new" $x$ from input space on
the basis of the transformed value $\phi\left(  x\right)  $. We will write the
method in such a way that it appears as a function of the scalar products
$\left\langle \phi\left(  X_{i}\right)  ,\phi\left(  X_{j}\right)
\right\rangle $ and $\left\langle \phi\left(  x\right)  ,\phi\left(
X_{i}\right)  \right\rangle $ (for the $x$ to be classified), and in a second
step we will replace these expressions by $K(X_{i},X_{j})$ and $K(x,X_{i})$
respectively, for a positive definite kernel $K$.

Thus, to be able to kernelize any classification method, we need two facts.

\begin{enumerate}
\item The method depends on the (transformed) training set $Z_{i}=\phi\left(
X_{i}\right)  $ only via their Gram matrix
\[
G_{Z}:=\left(  \left\langle Z_{i},Z_{j}\right\rangle \right)  _{i,j=1}^{n}%
\]
and the vector $Y=\left(  Y_{1},\ldots,Y_{n}\right)  $ of recorded $Y_{i}$'s.

\item Then classification of $z$ is a given function $\left\langle z,\hat
{a}\right\rangle $ for a certain $\hat{a}$.
\end{enumerate}

Fisher's rule in terms of $Z_{i}=\phi\left(  X_{i}\right)  $ and
$z=\phi\left(  x\right)  $ is as follows. Let
\[
n_{0}=\sum_{i:Y_{i}=-1}1\text{,\ \ }n_{1}=\sum_{i:Y_{i}=1}1
\]
be the size of the respective group ($n_{0}+n_{1}=n$) and let
\[
\hat{m}_{0}=\frac{1}{n_{0}}\sum_{i:Y_{i}=-1}Z_{i}\text{,\ \ }\hat{m}_{1}%
=\frac{1}{n_{1}}\sum_{i:Y_{i}=1}Z_{i}\text{ }%
\]
be the two respective group means. Let the two scatter matrices be
\[
S_{0}=\sum_{i:Y_{i}=-1}\left(  Z_{i}-\hat{m}_{0}\right)  \left(  Z_{i}-\hat
{m}_{0}\right)  ^{\top}\text{,\ \ }S_{1}=\sum_{i:Y_{i}=1}\left(  Z_{i}-\hat
{m}_{1}\right)  \left(  Z_{i}-\hat{m}_{1}\right)  ^{\top}\text{ }%
\]
and define $S=S_{0}+S_{1}$. Then Fishers's rule is: classify $z\in
\mathbb{R}^{s}$ according to
\begin{equation}
g(z)=\mathrm{sgn}\left(  \left\langle \hat{a},z\right\rangle +a_{0}\right)
\label{fish-rule}%
\end{equation}
where
\begin{align*}
\hat{a}  & =S^{-1}\left(  \hat{m}_{1}-\hat{m}_{0}\right)  ,\\
\hat{a}_{0}  & =\frac{1}{2}\left(  \hat{m}_{0}^{\top}S^{-1}\hat{m}_{0}-\hat
{m}_{1}^{\top}S^{-1}\hat{m}_{1}\right)  .
\end{align*}
(Indeed, we derived Fisher's rule from the normal Bayes rule
(\ref{normal-bayes-rule}) by substituting estimates $\hat{m}_{i},$
$\hat{\Sigma}$ for $m_{i}$ and $\Sigma$. But $\hat{\Sigma}=S/(n-1),$ and
multiplying $\hat{a}$ and $a_{0}$ in (\ref{fish-rule}) by a scalar gives the
same rule).

We will have to show that $\hat{a}$, $\hat{a}_{0}$ depend on the training set
only via its Gram matrix $G_{Z}$.

\begin{lemma}
\label{lem-maxim-problem}For any vector $m\in\mathbb{R}^{s}$ and any
nonsingular $s\times s$ matrix $S$
\[
m^{\top}S^{-1}m=\max_{u\in\mathbb{R}^{s}}\frac{\left(  \left\langle
u,m\right\rangle \right)  ^{2}}{u^{\top}Su}.
\]
and a maximizing vector is
\[
u^{\ast}=S^{-1}m
\]
or any scalar multiple of $u^{\ast}$.
\end{lemma}

\begin{proof}
For the maximization problem on the right, write $v=S^{1/2}u$, then the
problem becomes%
\[
\max_{v\in\mathbb{R}^{s}}\frac{\left(  v^{\top}S^{-1/2}m\right)  ^{2}}%
{v^{\top}v}%
\]
Now apply Cauchy-Schwartz:
\[
\left(  v^{\top}S^{-1/2}m\right)  ^{2}\leq\left(  v^{\top}v\right)  \left\Vert
S^{-1/2}m\right\Vert ^{2}%
\]
and equality is attained if and only if $v$ is a scalar multiple of
$S^{-1/2}m$. Hence
\[
\frac{\left(  v^{\top}S^{-1/2}m\right)  ^{2}}{v^{\top}v}\leq\left\Vert
S^{-1/2}m\right\Vert ^{2}=m^{\top}S^{-1}m
\]
and equality is attained if and only if $v=S^{1/2}u=\lambda S^{-1/2}m$ for
some $\lambda$, which means $u=\lambda S^{-1}m$.
\end{proof}

For the kernelized version, note that we made the implicit assumption that $S
$ is nonsingular. That is fulfilled if the $Z_{i}=\phi(X_{i})$ are not
concentrated on a linear subspace in $\mathbb{R}^{s}$, i.e. under suitable
assumptions on $\phi$, such that $\phi$ is not a linear mapping, and if $n$ is
large enough. Then we can claim that $\hat{a}$ is a linear combination of the
$Z_{i}$. Indeed in this case the vectors $Z_{i}$, $i=1,\ldots,n$ span the
whole of $\mathbb{R}^{s}$, and hence $\hat{a}\in\mathbb{R}^{s}$ is a linear
combination
\[
\hat{a}=\sum_{i=1}^{n}\alpha_{i}Z_{i}.
\]
We will now first obtain $\hat{a}$ as the solution of a maximization as in the
above lemma for $m=\hat{m}_{1}-\hat{m}_{0}$, and then reformulate the problem
as a maximization problem in terms of $\alpha=\left(  \alpha_{1},\ldots
,\alpha_{n}\right)  ^{\top}$. Similarly we assume $\hat{a}_{0}$ given by
\begin{equation}
\hat{a}_{0}=\frac{1}{2}\left(  \max_{u\in\mathbb{R}^{s}}\frac{\left(
\left\langle u,\hat{m}_{0}\right\rangle \right)  ^{2}}{u^{\top}Su}-\max
_{u\in\mathbb{R}^{s}}\frac{\left(  \left\langle u,\hat{m}_{1}\right\rangle
\right)  ^{2}}{u^{\top}Su}\right) \label{a0-explicit}%
\end{equation}
and will represent this in terms of maximization problems for $\alpha$. For
the numerators in these problems, write $\hat{m}_{0}=\sum c_{i}Z_{i}$ for
certain $c_{i}$ depending only on $Y$ and $u=\sum\alpha_{i}Z_{i}$, then
\[
\left\langle u,\hat{m}_{0}\right\rangle =\sum_{i.j}\alpha_{i}c_{j}\left\langle
Z_{i},Z_{j}\right\rangle
\]
\ and hence the numerators are expressions of type
\[
\left(  \sum_{i.j}\alpha_{i}c_{j}\left\langle Z_{i},Z_{j}\right\rangle
\right)  ^{2}%
\]
where the sum ranges over certain sets of indices $i,j$ and $c_{j}$ are
certain coefficients depending only on $Y$ . Consider now the denominators,
which are of form $a^{\top}Sa$ where $a=\sum_{i=1}^{n}\alpha_{i}Z_{i}$. For
any $j,k$ we have
\[
Z_{j}^{\top}SZ_{k}=\sum_{i:Y_{i}=-1}Z_{j}^{\top}\left(  Z_{i}-\hat{m}%
_{0}\right)  \left(  Z_{i}-\hat{m}_{0}\right)  ^{\top}Z_{k}+\sum_{i:Y_{i}%
=1}Z_{j}^{\top}\left(  Z_{i}-\hat{m}_{1}\right)  \left(  Z_{i}-\hat{m}%
_{1}\right)  ^{\top}Z_{k}.
\]
We see that this term again is a function of the vector $Y$ and the scalar
products $\left\langle Z_{i},Z_{j}\right\rangle $ and $\left\langle
Z_{i},Z_{k}\right\rangle $. In the same way it is argued that $\hat{a}_{0}$
given by (\ref{a0-explicit}) is a function of $Y$ and the Gram matrix $G_{Z}$.
Thus condition 1.) above for kernelization is satisfied. Condition 2.) is
satisfied in view of the form of Fisher's rule (\ref{fish-rule}).

\bigskip

\textbf{Explicit expressions in terms of the Gram matrix. } Define a $s\times
n_{0}$ matrix $\mathbf{Z}_{0}$ the colums of which are the $Z_{i}$ with
$Y_{i}=-1$ and a $s\times n_{1}$ matrix $\mathbf{Z}_{1}$ the the colums of
which are the $Z_{i}$ with $Y_{i}=1$. Let $\mathbf{1}_{0}$ be the $n_{0}%
$-vector of $1$'s and $\mathbf{1}_{1}$ be the $n_{1}$-vector of $1$'s$.$ Then%
\[
\hat{m}_{0}=n_{0}^{-1}\mathbf{Z}_{0}\mathbf{1}_{0},\;\hat{m}_{1}=n_{1}%
^{-1}\mathbf{Z}_{1}\mathbf{1}_{1},
\]%
\begin{align*}
S_{0}  & =\sum_{i:Y_{i}=-1}\left(  Z_{i}-\hat{m}_{0}\right)  \left(
Z_{i}-\hat{m}_{0}\right)  ^{\top}=\sum_{i:Y_{i}=-1}Z_{i}Z_{i}^{\top}-n_{0}%
\hat{m}_{0}\hat{m}_{0}^{\top}\\
& =\mathbf{Z}_{0}\mathbf{Z}_{0}^{\top}-\mathbf{Z}_{0}n_{0}^{-1}\mathbf{1}%
_{0}\mathbf{1}_{0}^{\top}\mathbf{Z}_{0}^{\top}=\mathbf{Z}_{0}P_{0}%
\mathbf{Z}_{0}^{\top}%
\end{align*}
where
\[
P_{0}=I_{n_{0}}-n_{0}^{-1}\mathbf{1}_{0}\mathbf{1}_{0}^{\top}.
\]
Similarly \
\[
S_{1}=\mathbf{Z}_{1}P_{1}\mathbf{Z}_{1}^{\top}\text{, }P_{1}=I_{n_{1}}%
-n_{1}^{-1}\mathbf{1}_{1}\mathbf{1}_{1}^{\top}.
\]
Consequently
\[
S=\mathbf{Z}_{0}P_{0}\mathbf{Z}_{0}^{\top}+\mathbf{Z}_{1}P_{1}\mathbf{Z}%
_{1}^{\top}.
\]
Let the total $s\times n$ matrix of the $Z_{i}$ be
\[
\mathbf{Z}=\left(  \mathbf{Z}_{0}\;\mathbf{Z}_{1}\right)
\]
Then the matrix of scalar products $\left\langle Z_{i},Z_{j}\right\rangle $
is
\[
\tilde{G}_{Z}=\mathbf{Z}^{\top}\mathbf{Z}=%
\begin{pmatrix}
\mathbf{Z}_{0}^{\top}\mathbf{Z}_{0} & \mathbf{Z}_{0}^{\top}\mathbf{Z}_{1}\\
\mathbf{Z}_{1}^{\top}\mathbf{Z}_{0} & \mathbf{Z}_{1}^{\top}\mathbf{Z}_{1}%
\end{pmatrix}
.
\]
The matrix $\tilde{G}_{Z}$ represents just a permutation of the Gram matrix
$G_{Z}$.

As argued above, the vectors $Z_{i}$, $i=1,\ldots,n$ span the whole of
$\mathbb{R}^{s}$. Hence we can find $s$ of these vectors which already span
$\mathbb{R}^{s}$. Write these vectors as columns in an $s\times s$ matrix and
call this matrix $\mathbf{Z}_{s}$. Then $\mathbf{Z}_{s}$ is nonsingular and we
write $a=\mathbf{Z}_{s}\alpha$ for some $\alpha=\left(  \alpha_{1}%
,\ldots,\alpha_{s}\right)  ^{\top}$ Then the target function in the
maximization problem for $\hat{a}$ (according to Lemma \ref{lem-maxim-problem}%
) can be written
\begin{align*}
\frac{\left(  a^{\top}\left(  \hat{m}_{1}-\hat{m}_{0}\right)  \right)  ^{2}%
}{a^{\top}Sa}  & =\frac{\left(  \alpha^{\top}\mathbf{Z}_{s}^{\top}\left(
n_{1}^{-1}\mathbf{Z}_{1}\mathbf{1}_{1}-n_{0}^{-1}\mathbf{Z}_{0}\mathbf{1}%
_{0}\right)  \right)  ^{2}}{\alpha^{\top}\mathbf{Z}_{s}^{\top}S\mathbf{Z}%
_{s}\alpha}\\
& =\frac{\left(  \alpha^{\top}\left(  n_{1}^{-1}\mathbf{Z}_{s}^{\top
}\mathbf{Z}_{1}\mathbf{1}_{1}-n_{0}^{-1}\mathbf{Z}_{s}^{\top}\mathbf{Z}%
_{0}\mathbf{1}_{0}\right)  \right)  ^{2}}{\alpha^{\top}\left(  \mathbf{Z}%
_{s}^{\top}\mathbf{Z}_{0}P_{0}\mathbf{Z}_{0}^{\top}\mathbf{Z}_{s}%
+\mathbf{Z}_{s}^{\top}\mathbf{Z}_{1}P_{1}\mathbf{Z}_{1}^{\top}\mathbf{Z}%
_{s}\right)  \alpha}%
\end{align*}
Since the matrix $\mathbf{Z}_{s}$ was formed using $s$ selected columns of the
matrix $\mathbf{Z}$, we see that the minimization problem in terms of $\alpha$
now involves only the elements of the matrix $\tilde{G}_{Z}$. Furthermore,
since $S$ was assumed nonsingular, the matrix $\mathbf{Z}_{s}^{\top
}S\mathbf{Z}_{s}$ is then also nonsingular. This allows us to write for the
solution $\hat{\alpha},$ using Lemma \ref{lem-maxim-problem}
\begin{align*}
\hat{\alpha}  & =\mathbf{M}^{-1}\left(  n_{1}^{-1}\mathbf{Z}_{s}^{\top
}\mathbf{Z}_{1}\mathbf{1}_{1}-n_{0}^{-1}\mathbf{Z}_{s}^{\top}\mathbf{Z}%
_{0}\mathbf{1}_{0}\right)  ,\\
\mathbf{M}  & =\mathbf{Z}_{s}^{\top}\mathbf{Z}_{0}P_{0}\mathbf{Z}_{0}^{\top
}\mathbf{Z}_{s}+\mathbf{Z}_{s}^{\top}\mathbf{Z}_{1}P_{1}\mathbf{Z}_{1}^{\top
}\mathbf{Z}_{s}.
\end{align*}
A similar expression can be found for $\hat{a}_{0},$ namely
\begin{align*}
\hat{a}_{0}  & =\frac{1}{2}\left(  \hat{m}_{0}^{\top}S^{-1}\hat{m}_{0}-\hat
{m}_{1}^{\top}S^{-1}\hat{m}_{1}\right) \\
& =\frac{1}{2}\left(  n_{1}^{-2}\mathbf{1}_{1}^{\top}\mathbf{Z}_{1}^{\top
}\mathbf{Z}_{s}\mathbf{M}^{-1}\mathbf{Z}_{s}^{\top}\mathbf{Z}_{1}%
\mathbf{1}_{1}-n_{0}^{-2}\mathbf{1}_{0}^{\top}\mathbf{Z}_{0}^{\top}%
\mathbf{Z}_{s}\mathbf{M}^{-1}\mathbf{Z}_{s}^{\top}\mathbf{Z}_{0}\mathbf{1}%
_{0}\right)  .
\end{align*}
\

\subsection{Kernelization of the perceptron algorithm\ }

Recall that the perceptron is a linear classifier
\[
h(x)=\mathrm{sgn}\left(  \left\langle w,x\right\rangle +b\right)
\]
where $w$ and $b$ are found from the perceptron training algorithm. This
algorithm starts with $w=\mathbf{0}$ and successively adds expressions $\eta
y_{i}x_{i}$ to it, and similarly the algorithm starts with $b=0$ and adds
expressions $\eta y_{i}R^{2}$. So when the algorithm stops, the final $w$ is
of form $w=\sum_{i=1}^{n}\alpha_{i}$ $\eta y_{i}x_{i}$ for certain
coefficients $\alpha_{i}$ and the final $b$ is of form $b=\sum_{i=1}^{n}%
\beta_{i}$ $\eta y_{i}R^{2}$. We see that the number $\eta$ influences only
the scaling of $\left(  w,b\right)  $ in each step, not the hyperplane and the
decisions in each step (i.e. $\mathrm{sgn}\left(  \left\langle
w,x\right\rangle +b\right)  =\mathrm{sgn}\left(  \left\langle \eta
w,x\right\rangle +\eta b\right)  $), so that we may equivalently set $\eta=1$.
Furthermore, let us represent all the successive $w^{\prime}$s in the form
$w=\sum_{i=1}^{n}\alpha_{i}$ $\eta y_{i}x_{i}$ where coefficients $\alpha_{i}$
may change in each step. Then we can write the perceptron training algorithm
in the following "dual" form. Write $\alpha=\left(  \alpha_{1},\ldots
,\alpha_{n}\right)  $ for the vector of coefficients. Recall that $\left(
x_{1},\ldots,x_{n}\right)  $ denotes the training set. \newpage

\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_

\textbf{Algorithm. }

$\alpha\leftarrow\mathbf{0}\in\mathbb{R}^{d}$; $b\leftarrow0,$ $k\leftarrow
0$\newline$R\leftarrow\max_{1\leq i\leq n}\left\Vert x_{i}\right\Vert
$\newline%

\begin{tabular}
[c]{llll}%
\textbf{repeat} &  &  & \\
& \textbf{for} $i=1$ \textbf{to} $n$ &  & \\
&  & \textbf{if} & $y_{i}\left(  \sum_{j=1}^{n}\alpha_{j}\left\langle
x_{j},x_{i}\right\rangle +b\right)  \leq0$ \textbf{then}\\
&  &  & $\alpha_{i}\leftarrow\alpha_{i}+1$\\
&  &  & $b\leftarrow b+y_{i}R^{2}$\\
&  &  & \\
&  & \textbf{endif} & \\
& \textbf{endfor} &  & \\
&  &  & \\
&  &  &
\end{tabular}
$_{{}}$\newline\textbf{until} no mistakes made within the \textbf{for}
loops\newline\textbf{return} $\left(  \alpha,b\right)  $ to define a linear
classifier
\[
h(x)=\mathrm{sgn}\left(  \sum_{j=1}^{n}\alpha_{j}y_{j}\left\langle
x_{j},x\right\rangle +b\right)  .
\]

We see immediately that the perceptron algorithm and the resulting linear
classifier are now written in terms of scalar products $\left\langle
x_{i},x_{j}\right\rangle $ and $\left\langle x_{i},x\right\rangle $, i.e. they
are ready to be kernelized.

\subsection{Examples of kernels}

Commonly used kernels are for $x,y\in\mathbb{R}^{d}$
\begin{align}
\text{polynomial}\text{: }  & K(x,y)=\left(  \left\langle x,y\right\rangle
+b\right)  ^{r}\text{, where }b\geq0\text{, }r=1,2,\ldots\label{kern-poly}\\
\text{Gaussian}\text{: }  & K(x,y)=\exp\left(  -\left\Vert x-y\right\Vert
^{2}/c\right)  \text{, where }c>0\text{.}\label{kern-gauss}\\
\text{sigmoid}\text{: }  & K(x,y)=\tanh\left(  a\left\langle x,y\right\rangle
+a_{0}\right)  \text{, }%
\end{align}
where $\tanh(x)$ means the hyperbolic tangent of $x$,
\[
\tanh(x)=\frac{\exp(x)-\exp(-x)}{\exp(x)+\exp(-x)}%
\]
A kernel satisfying the conditions of Mercer's theorem is called a
\textit{Mercer kernel. }The sigmoid kernel is known not to be a Mercer kernel
(not positive definite), but it has been sucessfully used in practice.

\begin{lemma}
Let $C$ be a ball in $\mathbb{R}^{d}$: for some $M>0$%
\[
C=\left\{  x\in\mathbb{R}^{d}:\left\Vert x\right\Vert \leq M\right\}  .
\]
The polynomial kernel is a Mercer kernel on $C$.
\end{lemma}

\begin{proof}
Assume $b>0$; the proof for $b=0$ is similar. Without loss of generality then
assume $b=1$. For $v=(v_{1},\ldots,v_{d})\in\mathbb{R}^{d}$ note that
\[
\left(  \sum_{i=1}^{d}v_{i}+1\right)  ^{r}=\sum_{s_{1}+\ldots+s_{d}+s_{d+1}%
=r}c_{s_{1},\ldots,s_{d}}%
%TCIMACRO{\dprod \limits_{i=1}^{d}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{d}}
%EndExpansion
v_{i}^{s_{i}}1^{s_{d+1}}=\sum_{0\leq s_{1}+\ldots+s_{d}\leq r}c_{s_{1}%
,\ldots,s_{d}}%
%TCIMACRO{\dprod \limits_{i=1}^{d}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{d}}
%EndExpansion
v_{i}^{s_{i}}.
\]
where all coefficients $c_{s_{1},\ldots,s_{d}}$ are nonnegative. (As an
example consider $d=2$, $r=2$; then
\[
\left(  v_{1}+v_{2}+1\right)  ^{2}=v_{1}^{2}+v_{1}^{2}+2v_{1}v_{2}%
+2v_{1}+2v_{2}+1\text{).}%
\]
We introduce notation for a multi-index $s=(s_{1},\ldots,s_{d})$ with integer
nonnegative components, where $\left\vert s\right\vert =\sum_{i=1}^{d}s_{i}$
and
\[
v^{s}=%
%TCIMACRO{\dprod \limits_{i=1}^{d}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{d}}
%EndExpansion
v_{i}^{s_{i}}.
\]
Then we have
\begin{align}
\left(  \sum_{i=1}^{d}v_{i}+1\right)  ^{r}  & =\sum_{\left\vert s\right\vert
\leq r}c_{s}v^{s},\label{effective}\\
\left(  \left\langle x,y\right\rangle +1\right)  ^{r}  & =\left(  \sum
_{i=1}^{d}x_{i}y_{i}+1\right)  ^{r}=\sum_{\left\vert s\right\vert \leq r}%
c_{s}x^{s}y^{s}.\nonumber
\end{align}
Let $f$ be any function $f\in L_{2}(C)$; then
\begin{align*}
\int_{C\times C}K(x,y)f(x)f(y)dxdy  & =\sum_{\left\vert s\right\vert \leq
r}\int_{C\times C}c_{s}x^{s}y^{s}f(x)f(y)dxdy\\
& =\sum_{\left\vert s\right\vert \leq r}c_{s}\left(  \int_{C}x^{s}%
f(x)dx\right)  ^{2}\geq0
\end{align*}
since all $c_{s}$ are nonnegative.
\end{proof}

Note that the result holds true for more general sets $C$, not only balls. In
(\ref{effective}) we effectively showed that a possible map $\phi$ into
feature space is $\phi(x)=\left(  c_{s}^{1/2}x^{s}\right)  _{\left\vert
s\right\vert \leq r}$ which takes values in a high dimensional space
$\mathbb{R}^{k}$ with $k=\#\left\{  s:\left\vert s\right\vert \leq r\right\}
.$ \newpage

\begin{lemma}
\label{lem-gauss-kernel-mercer}Let $C$ be as in the previous lemma. The
Gaussian kernel is a Mercer kernel on $C$.
\end{lemma}

\begin{proof}
Let $N_{d}(\mu,\Sigma)$ be the $d$-dimensional normal distribution with
expectation $\mu$ and covariance matrix $\Sigma$. It is well known that the
characteristic function of $N_{d}(0,I_{d})$ is
\[
E\exp\left(  i\left\langle t,Z\right\rangle \right)  =\exp\left(  -\left\Vert
t\right\Vert ^{2}/2\right)  .
\]
It follows that for $\sigma>0$ the characteristic function of $N_{d}%
(0,\sigma^{2}I_{d})$ is, if $Z$ is standard $d$-variate normal
\[
E\exp\left(  i\left\langle t,\sigma Z\right\rangle \right)  =\exp\left(
-\sigma^{2}\left\Vert t\right\Vert ^{2}/2\right)  .
\]
Let $\sigma^{2}=2/c$ and let $f$ be any function $f\in L_{2}(C)$; then
\begin{align*}
\int_{C\times C}K(x,y)f(x)f(y)dxdy  & =\int_{C\times C}\exp\left(  -\sigma
^{2}\left\Vert x-y\right\Vert ^{2}/2\right)  f(x)f(y)dxdy\\
& =\int_{C\times C}E\exp\left(  i\left\langle x-y,\sigma Z\right\rangle
\right)  f(x)f(y)dxdy
\end{align*}%
\begin{equation}
=E\left(  \int_{C}\exp\left(  i\left\langle x,\sigma Z\right\rangle \right)
f(x)dx\right)  \left(  \int_{C}\exp\left(  -i\left\langle y,\sigma
Z\right\rangle \right)  f(y)dy\right) \label{above-1}%
\end{equation}
Let the complex valued random variable $\xi$ be defined by
\[
\xi=\int_{C}\exp\left(  i\left\langle x,\sigma Z\right\rangle \right)
f(x)dx\text{ }%
\]
and let $\bar{\xi}$ be its complex conjugate; then in view of $\overline
{\exp\left(  iu\right)  }=\exp\left(  -iu\right)  $ we have as a consequence
of (\ref{above-1})
\[
\int_{C\times C}K(x,y)f(x)f(y)dxdy=E\xi\bar{\xi}=E\left\vert \xi\right\vert
^{2}\geq0.
\]

\end{proof}

In order to understand what is the corresponding map in to feature space
$\phi(x)$ here, we would have to know the series expansion of the kernel.
However the theory of reproducing kernels (see below) provides an alternative
to finding a map $\phi(x)$.

\subsection{Reproducing kernels}

Suppose we have a symmetric positive definite kernel $K(x,y)$ defined on
$C\times C$ (where $C$ is a compact subset of $\mathbb{R}^{d}$ ) which serves
as the basis of kernelization of a linear classifier. Such a kernel is also
called a Mercer kernel. We will now discuss another way of assigning a feature
space to the kernel, i.e. represent the kernel as
\begin{equation}
K(x,y)=\left\langle \Phi(x),\Phi(y)\right\rangle _{\ast}%
\label{feature-map-def}%
\end{equation}
where $\Phi:C\mapsto\mathcal{H}$ where $\mathcal{H}$ is a linear space endowed
with a scalar product $\left\langle \cdot,\cdot\right\rangle _{\ast}$.

Above we saw that in the case of a Mercer kernel, the map given by
\[
\tilde{\Phi}(x)=\left(  \lambda_{j}^{1/2}\phi_{j}(x)\right)  _{j=1}^{\infty}%
\]
which maps $C$ into the space of sequences $l_{2}$ (which are square summable)
Then, if the scalar product in $l_{2}$ is given by the obvious infinite series
and denoted $\left\langle \cdot,\cdot\right\rangle _{\ast} $ then
\[
K(x,y)=\left\langle \tilde{\Phi}(x),\tilde{\Phi}(y)\right\rangle _{\ast}%
=\sum_{j=1}^{\infty}\lambda_{j}\phi_{j}(x)\phi_{j}(y).
\]
anf for $\mathcal{H=}l_{2}$ we have found a corresponding feature space.
However now we want the result of the feature map $\tilde{\Phi}(x)$ to be a
function rather than a sequence.

Define the feature map as
\[
\Phi(x)=K(\cdot,x)
\]
where the dot $\cdot$ in the first argument signifies that the value of a map
is the function $z\rightarrow K(z,x)$. In this connection we need to develop
some theory.

Let $L_{2}(C)$ be the usual Hilbert space of square integrable functions on
$C$, with scalar product
\[
\left\langle f,g\right\rangle =\int_{C}f(x)g(x)dx.
\]
and associated norm $\left\Vert f\right\Vert =\sqrt{\left\langle
f,f\right\rangle .}$ If $\left\{  \phi_{j}\right\}  _{j=1}^{\infty}$ is an
orthonormal basis (ONB) of $L_{2}(C)$, consisting of functions $\phi_{j}$ then
every $f\in L_{2}(C)$ has a series representation $f=\sum_{j=1}^{\infty}%
f_{j}\phi_{j}$ where $f_{j}=\left\langle f,\phi_{j}\right\rangle $. Moreower,
for any two $f,g\in L_{2}(C)$ we have
\[
\left\langle f,g\right\rangle =\sum_{j=1}^{\infty}f_{j}g_{j}%
\]
(Parseval's theorem), in particular
\[
\left\Vert f\right\Vert ^{2}=\left\langle f,f\right\rangle =\sum_{j=1}%
^{\infty}f_{j}^{2}.
\]
Now, given the kernel $K$, define a set $\mathcal{H}$ as follows: if $\left\{
\phi_{j}\right\}  _{j=1}^{\infty}$ is the ONB appearing in the Mercer theorem
for $K$ and $\lambda_{j}$ are the corresponding numbers there ($\lambda_{j}%
>0$) then
\[
\mathcal{H=}\left\{  f\in L_{2}(C):\sum_{j=1}^{\infty}\frac{1}{\lambda_{j}%
}f_{j}^{2}<\infty\right\}  .
\]
Since in general we have $\lambda_{j}\rightarrow0$ as $j\rightarrow\infty$,
not all functions in $L_{2}(C)$ will be in $\mathcal{H}$. In fact if
$\sum_{j=1}^{\infty}f_{j}^{2}$ is finite then there is no guarantee that
$\sum_{j=1}^{\infty}f_{j}^{2}/\lambda_{j}$ converges, precisely because
$\lambda_{j}\rightarrow0$ as $j\rightarrow\infty$.

Thus $\mathcal{H}$ is a subset of $L_{2}(C)$. On $\mathcal{H}$ define a scalar
product
\[
\left\langle f,g\right\rangle _{\ast}=\sum_{j=1}^{\infty}\frac{1}{\lambda_{j}%
}f_{j}g_{j}.
\]
It is easy to see that this is defined for any $f,g\in\mathcal{H}$: by the
Cauchy-Schwartz inequality in the series space $l_{2}$, for any $k>0$ the tail
expression of the above series satisfies
\[
\left\vert \sum_{j=k}^{\infty}\frac{1}{\lambda_{j}}f_{j}g_{j}\right\vert
=\left\vert \sum_{j=k}^{\infty}\left(  \frac{f_{j}}{\lambda_{j}^{1/2}}\right)
\left(  \frac{g_{j}}{\lambda_{j}^{1/2}}\right)  \right\vert \leq\left(
\sum_{j=k}^{\infty}f_{j}^{2}/\lambda_{j}\right)  ^{1/2}\left(  \sum
_{j=k}^{\infty}g_{j}^{2}/\lambda_{j}\right)  ^{1/2}%
\]
and the expressions on the right tend to zero as $k\rightarrow\infty$ by
assumption, hence the series expression $\left\langle f,g\right\rangle _{\ast
}$ exists and is finite. A little more (standard) reasoning shows that
$\mathcal{H}$ becomes a Hilbert space under the scalar product $\left\langle
\cdot,\cdot\right\rangle _{\ast}$. Thus, as a set, $\mathcal{H\subset}%
L_{2}(C)$ but $\mathcal{H}$ is a Hilbert space with a different scalar product
(not the scalar product of $L_{2}(C)$, i.e. $\left\langle \cdot,\cdot
\right\rangle _{\ast}$). The Hilbert space $\mathcal{H}$ constructed is called
the\textbf{\ reproducing kernel Hilbert space (RKHS)} of $K.$

To justify that name, for any given $x\in C$ consider the function
$y\rightarrow K(y,x)$, written as $K(\cdot,x)$. Since $K(\cdot,x)$ is a
continuous function on $C,$ it must be in $L_{2}(C)$. Moreover, we have by the
Mercer expansion
\[
K(\cdot,x)=\sum_{j=1}^{\infty}\lambda_{j}\phi_{j}(\cdot)\phi_{j}(x)
\]
so the function $K(\cdot,x)$ has a series expansion in terms of the
ONB\ $\left\{  \phi_{j}\right\}  _{j=1}^{\infty}$ with coefficients
$\lambda_{j}\phi_{j}(x)$. Let us check whether $K(\cdot,x)\in\mathcal{H}$:%
\[
\sum_{j=1}^{\infty}\frac{1}{\lambda_{j}}\lambda_{j}^{2}\phi_{j}^{2}%
(x)=\sum_{j=1}^{\infty}\lambda_{j}\phi_{j}(x)\phi_{j}(x)=K(x,x)<\infty
\]
since $K$ was a continuous function with finite values everywhere on $C\times
C$. Hence $K(\cdot,x)\in\mathcal{H}$. Now for any $f\in\mathcal{H} $ we have
\begin{equation}
\left\langle f,K(\cdot,x)\right\rangle _{\ast}=\sum_{j=1}^{\infty}\frac
{1}{\lambda_{j}}f_{j}\lambda_{j}\phi_{j}(x)=\sum_{j=1}^{\infty}f_{j}\phi
_{j}(x)=f(x).\label{repro-prop}%
\end{equation}
The last equality defines the\textbf{\ reproducing property of the kernel}
$K,$ using the scalar product $\left\langle \cdot,\cdot\right\rangle _{\ast}$:
if we take the function $K(\cdot,x)$ and scalarproduct it with any
$f\in\mathcal{H}$ then we reproduce the function $f$, more specifically the
value $f(x)$.

Recall that we wanted to use the map $\Phi(x)=K(\cdot,x)$ (which to $x\in C $
assigns a function in $\mathcal{H}$) as a feature map, i.e. we wanted to
achieve (\ref{feature-map-def}). To this end apply the reproducing equality
(\ref{repro-prop}) for $f=$ $K(\cdot,y)$, given some $y\in C$. In this case
$f(x)=$ $K(x,y)$, so (\ref{repro-prop}) becomes
\[
\left\langle K(\cdot,y),K(\cdot,x)\right\rangle _{\ast}=K(x,y)
\]
which we now can write
\begin{equation}
\left\langle \Phi(x),\Phi(y)\right\rangle _{\ast}=\left\langle \Phi
(y),\Phi(x)\right\rangle _{\ast}=K(x,y).\label{key-rkhs-2}%
\end{equation}
Thus we have succeeded in representing $K\left(  x,y\right)  $ as a scalar
product of the values of feature maps.

\bigskip

\begin{remark}
\label{rem-feature-map-def}\emph{For a given Mercer kernel }$K$\emph{\ we have
obtained a map }$\Phi(x)$\emph{\ into some feature space }$H$\emph{\ (the RKHS
of }$K$\emph{) endowed with a scalar product }$\left\langle \cdot
,\cdot\right\rangle _{\ast}$\emph{\ such that (\ref{key-rkhs-2}) holds. Recall
that this has been achieved earlier in another way: suppose the expansion of
the kernel is }%
\[
K(x,y)=\sum_{j=1}^{\infty}\lambda_{j}\phi_{j}(x)\phi_{j}(y)
\]
\emph{and consider the map }$\phi(x)=\left(  \lambda_{j}^{1/2}\phi
_{j}(x)\right)  _{j=1}^{\infty}$\emph{\ with values in }$l_{2}.$\emph{\ Let
}$\left\langle \cdot,\cdot\right\rangle _{l_{2}}$\emph{\ be the scalar product
in }$l_{2}$\emph{, i.e. for sequences }$u,v\in l_{2}$\emph{\ }%
\[
\left\langle u,v\right\rangle _{l_{2}}=\sum_{j=1}^{\infty}u_{j}v_{j}\text{.}%
\]
\emph{Then as we have seen before }%
\[
K(x,y)=\left\langle \phi(x),\phi(y)\right\rangle _{l_{2}}.
\]
\emph{Now the two approaches are basically equivalent: consider the function
}$\Phi(x)=K(\cdot,x)$\emph{\ and its series expansion }$K(\cdot,x)=\sum
_{j=1}^{\infty}\lambda_{j}\phi_{j}(\cdot)\phi_{j}(x).$\emph{\ The functions
}$\tilde{\phi}_{j}=\lambda_{j}^{1/2}\phi_{j}(\cdot)$\emph{, }$j=1,2,\ldots
$\emph{are an orthonormal basis (ONB) wrt the scalar product }$\left\langle
\cdot,\cdot\right\rangle _{\ast}$\emph{, and }%
\[
\Phi(x)=K(\cdot,x)=\sum_{j=1}^{\infty}\lambda_{j}^{1/2}\tilde{\phi}_{j}%
(\cdot)\phi_{j}(x),
\]
\emph{thus }$\lambda_{j}^{1/2}\phi_{j}(x)$\emph{\ are the coefficients of
}$\Phi(x)$\emph{\ in that basis. A function in a Hilbert space can
equivalently represented by is coefficients wrt an ONB, thus we a have a
one-to-one map}%
\[
\Phi(x)\leftrightarrow\left(  \lambda_{j}^{1/2}\phi_{j}(x)\right)
_{j=1}^{\infty}=\phi(x)
\]
\emph{and by Parseval's theorem for Hilbert spaces }%
\[
\left\langle \Phi(x),\Phi(y)\right\rangle _{\ast}=\left\langle \phi
(x),\phi(y)\right\rangle _{l_{2}}%
\]
\emph{thus }$\Phi(x)$\emph{\ and }$\phi(x)$\emph{\ basically represent the
same map, up to an isomorphism.}
\end{remark}

\begin{remark}
\label{rem-other-positive-definiteness-kernel}There is another definition of
positive definiteness which is more common. Let us call a symmetric kernel
$K\left(  x,y\right)  $ defined on $C\subset\mathbb{R}^{d}$ positive
definite$^{\ast}$ if for all $n$ and every $n$-tuple $x_{1},\ldots,x_{n}$, the
matrix $\tilde{K}=\left(  K\left(  x_{i},x_{j}\right)  \right)  _{i,j=1}^{n}$
is positive definite, i.e. for every $y\in\mathbb{R}^{n}$ we have $y^{\top
}\tilde{K}y\geq0$. It can easily be seen that every Mercer kernel is positive
definite$^{\ast}$: if $K(u,v)=\sum_{j=1}^{\infty}\lambda_{j}\phi_{j}%
(u)\phi_{j}(v)$ then
\begin{align*}
y^{\top}\tilde{K}y  & =\sum_{j=1}^{\infty}\lambda_{j}\sum_{i,k=1}^{n}%
y_{i}y_{k}\phi_{j}(x_{i})\phi_{j}(x_{k})\\
& =\sum_{j=1}^{\infty}\lambda_{j}\left(  \sum_{i=1}^{n}y_{i}\phi_{j}%
(x_{i})\right)  ^{2}\geq0.
\end{align*}

\end{remark}

\subsubsection{Example: the kernel $\min(x,y)$}

Suppose $C=[0,1]$ and consider the kernel
\[
K\left(  x,y\right)  =\min(x,y).
\]
In the learning context, this is a "toy example" since the $x_{i}$ are assumed
to be on $[0,1]$. First let us demonstrate that the kernel is positive
definite in the sense used in Mercer's theorem, i.e. in the sense of
(\ref{posdef}). We have
\[
\min(x,y)=\int_{0}^{1}\mathbf{1}_{[0,x]}(t)\mathbf{1}_{[0,y]}(t)dt
\]
and thus
\begin{align*}
\int_{C\times C}K(x,y)f(x)f(y)dxdy  & =\int_{0}^{1}\int_{C\times C}%
\mathbf{1}_{[0,x]}(t)\mathbf{1}_{[0,y]}(t)f(x)f(y)dxdydt\\
& =\int_{0}^{1}\left(  \int_{t}^{1}f(x)dx\right)  ^{2}dt\geq0.
\end{align*}

To find the Mercer expansion of $K\left(  x,y\right)  ,$ we first note:

\begin{lemma}
\label{lem-eigenfunc}(i) The set of functions
\[
\phi_{j}(x)=\sqrt{2}\sin\left(  \left(  j-1/2\right)  \pi x\right)  \text{,
}j=1,2,\ldots
\]
is a complete orthonormal basis of $L_{2}(0,1)$.\newline(ii) Likewise, the set
of functions
\[
\psi_{j}(x)=\sqrt{2}\cos\left(  \left(  j-1/2\right)  \pi x\right)
,j=1,2,\ldots
\]
is a complete orthonormal basis of $L_{2}(0,1)$.
\end{lemma}%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
A proof should be along the lines of the half-trigonometric bases, as in our
674 notes, in the chapters on "nonparametric smoothing". Possibly give as
homework, with the proof of half-trigonometric bases as background material.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

To comment, recall the standard trigonometric orthonormal basis in
$L_{2}(0,1)$, consisting of functions
\begin{align*}
f_{0}  & =1,\\
f_{j}(x)  & =\sqrt{2}\cos\left(  2\pi x\right)  ,f_{-j}(x)=\sqrt{2}\sin\left(
2\pi x\right)  \text{, }j=1,2,\ldots
\end{align*}
The two bases considered in the lemma are derived from this; they use
different arguments in $\sin(\cdot)$ and $\cos(\cdot)$.

\begin{proposition}
\label{prop-mercer-min-kernel}The Mercer expansion of $K\left(  x,y\right)
=\min(x,y)$ on $C=[0,1]$ is
\[
K(x,y)=\sum_{j=1}^{\infty}\lambda_{j}\phi_{j}(x)\phi_{j}(y)
\]
for the functions $\phi_{j}$ given in Lemma \ref{lem-eigenfunc} and
\[
\lambda_{j}=\frac{1}{\left(  j-1/2\right)  ^{2}\pi^{2}}\text{, }j=1,2,\ldots.
\]

\end{proposition}

\begin{proof}
We show that the $\phi_{j}$ are eigenfunctions, i.e. fulfill
\[
\int_{C}K(x,y)\phi_{j}(y)dy=\lambda_{j}\phi_{j}(x).
\]
Write $\gamma_{j}=\left(  j-1/2\right)  \pi$ and note that $\phi_{j}%
(y)=\sqrt{2}\sin\left(  \gamma_{j}y\right)  $, $\lambda_{j}=\gamma_{j}^{-2}$.
The left side above fulfills
\[
\int_{C}\min(x,y)\phi_{j}(y)dy=\int_{0}^{x}y\phi_{j}(y)dy+\int_{x}^{1}%
x\phi_{j}(y)dy
\]
and using partial integration on the first integral on the right,%
\[
=\sqrt{2}\left[  y\frac{1}{\gamma_{j}}\left(  -\cos\left(  \gamma_{j}y\right)
\right)  \right]  _{0}^{x}+\sqrt{2}\int_{0}^{x}\frac{1}{\gamma_{j}}\cos\left(
\gamma_{j}y\right)  dy+\sqrt{2}x\left[  \frac{1}{\gamma_{j}}\left(
-\cos\left(  \gamma_{j}y\right)  \right)  \right]  _{x}^{1}%
\]%
\begin{align*}
& =-\sqrt{2}\frac{x}{\gamma_{j}}\cos\left(  \gamma_{j}x\right)  +\sqrt{2}%
\frac{0}{\gamma_{j}}\cos\left(  0\right)  +\sqrt{2}\int_{0}^{x}\frac{1}%
{\gamma_{j}}\cos\left(  \gamma_{j}y\right)  dy\\
& -\sqrt{2}\frac{x}{\gamma_{j}}\cos\gamma_{j}+\sqrt{2}\frac{x}{\gamma_{j}}%
\cos\left(  \gamma_{j}x\right)
\end{align*}%
\[
=-\sqrt{2}\frac{x}{\gamma_{j}}\cos\gamma_{j}+\sqrt{2}\int_{0}^{x}\frac
{1}{\gamma_{j}}\cos\left(  \gamma_{j}y\right)  dy
\]
and since $\cos\gamma_{j}=\cos\left(  \left(  j-1/2\right)  \pi\right)  =0$,
this equals
\begin{align*}
& =0+\sqrt{2}\left[  \frac{1}{\gamma_{j}^{2}}\sin\left(  \gamma_{j}y\right)
\right]  _{0}^{x}=\frac{1}{\gamma_{j}^{2}}\sqrt{2}\left(  \sin\left(
\gamma_{j}x\right)  -\sin\left(  0\right)  \right) \\
& =\lambda_{j}\phi_{j}(x).
\end{align*}
In the Mercer theorem, the set of eigenfunctions is unique if all $\lambda
_{j}$ are different. Since $\phi_{j}$ used here are a complete ONB, there can
be no further eigenfunctions.
\end{proof}

We can now identify the reproducing kernel Hilbert space\textbf{\ }associated
to kernel $K$ and the associated scalar product $\left\langle \cdot
,\cdot\right\rangle _{\ast}$. Consider functions $f=\sum f_{j}\phi_{j}$ with
the property
\begin{equation}
\left\langle f,f\right\rangle _{\ast}=\sum_{j=1}^{\infty}\frac{1}{\lambda_{j}%
}f_{j}^{2}<\infty.\label{summability-for-H}%
\end{equation}
To understand this, assume that $f$ has a finite expansion in the basis
$\phi_{j}$: $f=\sum_{j=1}^{n}f_{j}\phi_{j}.$ Then
\begin{align*}
f^{\prime}(x)  & =\sqrt{2}\sum_{j=1}^{n}f_{j}\frac{d}{dx}\sin\left(
\gamma_{j}y\right)  =\sqrt{2}\sum_{j=1}^{n}f_{j}\gamma_{j}\cos\left(
\gamma_{j}y\right) \\
& =\sum_{j=1}^{n}f_{j}\gamma_{j}\psi_{j}(x).
\end{align*}
as noted in Lemma \ref{lem-eigenfunc} the set of functions $\psi_{j}%
(x)=\sqrt{2}\cos\left(  \gamma_{j}\cdot\right)  $, $j=1,2,\ldots$ is
orthonormal on $[0,1].$ Therefore
\begin{equation}
\int\left(  f^{\prime}(x)\right)  ^{2}dx=\sum_{j=1}^{n}f_{j}^{2}\gamma_{j}%
^{2}=\sum_{j=1}^{n}\frac{1}{\lambda_{j}}f_{j}^{2}.\label{smoothnessnorm-1}%
\end{equation}
Moreover, all $f$ with a finite expansion $f=\sum_{j=1}^{n}f_{j}\phi_{j}$
satisfy $f(0)=0$. A limiting argument for $n\rightarrow\infty$ (standard in
functional analysis) shows that the set $\mathcal{H}$ of functions $f$
fulfilling (\ref{summability-for-H}) can be identified with the set of all
$f\in L_{2}(0,1)$ which are absolutely continuous, with (generalized )
derivative $f^{\prime}$ satisfying $\int_{0}^{1}\left(  f^{\prime}(x)\right)
^{2}dx<\infty$ and additionally satisfying $f(0)=0$. An argument entirely
analogous to (\ref{smoothnessnorm-1}) shows that the scalar product in
$\mathcal{H}$ must be
\begin{equation}
\left\langle f,g\right\rangle _{\ast}=\int_{0}^{1}f^{\prime}(x)g^{\prime
}(x)dx.\label{sclar-prod-rkhs}%
\end{equation}
Finally let us check the reproducing property of $K$: for $f\in\mathcal{H}$:
since
\[
K(t,x)=\min(t,x)=\int_{0}^{t}\mathbf{1}_{\left[  0.x\right]  }(u)du,
\]
we have
\begin{align*}
\left\langle f,K\left(  \cdot,x\right)  \right\rangle _{\ast}  & =\int_{0}%
^{1}f^{\prime}(t)\left(  \frac{d}{dt}K(t,x)\right)  dt=\int_{0}^{1}f^{\prime
}(t)\mathbf{1}_{\left[  0.x\right]  }(t)dt\\
& =\int_{0}^{x}f^{\prime}(t)dt=\left[  f(t)\right]  _{0}^{x}=f(x).
\end{align*}
\textbf{Summary.} This example exhibits all the basic features of the general
RKHS theory. Starting with a Mercer kernel $K(\cdot,\cdot)$ on $C\times C$,
one finds an associated subspace of functions in the space $L_{2}(C)$. This
space can be endowed with a scalar product $\left\langle \cdot,\cdot
\right\rangle _{\ast}$ under which it becomes a Hilbert space $\mathcal{H}$.
The scalar product is not the scalar product of $L_{2}(C)$ and it is not
defined for all functions in $L_{2}(C)$. In fact the functions in
$\mathcal{H}$ are usually smooth functions and the scalar product
$\left\langle \cdot,\cdot\right\rangle _{\ast}$ is a scalar product for smooth
functions, operating on the derivatives.

The scalar product (\ref{sclar-prod-rkhs}) is of simple form, but the scalar
products for general kernels on $C\subset\mathbb{R}^{d}$ can be involved and
may use various kinds of derivatives. In analysis, one also asks the reverse
question: given a scalar product on smooth functions in $L_{2}(C)$, and
associated Hilbert space $\mathcal{H}$, find a kernel $K$ such that
$\mathcal{H}$ becomes the RKHS\ of $\mathcal{H}$.

\textbf{Learning with kernel }$\min(x,y)$. Recall that with kernelization, we
replace the training set $\left(  x_{i},y_{i}\right)  $ with $\left(  K\left(
\cdot,x_{i}\right)  ,y_{i}\right)  $, i.e. our $x_{i}$ are replaced by
functions $K\left(  \cdot,x_{i}\right)  $. Let us see how a training set which
is not linearly separable on $[0,1]$ becomes so after kernelization. Assume
that $x_{1}<x_{2}<\ldots<x_{n}$ and the $y_{i}$ alternate, i.e. $y_{1}=1$,
$y_{2}=-1$ and so on. Such a set is not not linearly separable on $[0,1],$
since a linear rule is just a cut of the interval: all $x_{i}$ on one side of
an $h_{0}$ are classified as $1$ and all $x_{i}$ on the other side are
classified as $-1$. If $n$ is large, the error of any such rule approaches
$1/2.$

Now what is a linear rule in the RKHS $\mathcal{H}$ ? Let $\tilde{h}%
\in\mathcal{H}$; then a rule would be
\[
h(x_{i})=\mathrm{sgn}\left\langle \tilde{h},K\left(  \cdot,x_{i}\right)
\right\rangle _{\ast}%
\]
(we take a hyperplane through the origin as an example here). By the
reproducing property, we have
\[
\left\langle \tilde{h},K\left(  \cdot,x_{i}\right)  \right\rangle _{\ast
}=\tilde{h}(x_{i}),
\]
so for separation of the training set we need a function $\tilde{h}%
\in\mathcal{H}$ such that $\tilde{h}(x_{i})=y_{i}$ for all $i$. This is
certainly possible for any training set: a function $\tilde{h}$ which is
differentiable with $\tilde{h}(0)=0$ and $\tilde{h}(x_{i})=y_{i}$,
$i=1,\ldots,,$ i.e. an interpolating function. Such a function always exists
in the space $\mathcal{H}$ (it can be taken to be a continuously
differentiable function).

For a special configuration of the $x_{i}$, the basis functions $\phi_{n}$ are
suitable: note that
\begin{align*}
\phi_{n}\left(  \frac{1}{2n-1}\right)   & =\sin\left(  (2n-1)\frac{\pi}%
{2}\frac{1}{2n-1}\right)  =1,\phi_{n}\left(  \frac{2}{2n-1}\right)  =0,\\
\phi_{n}\left(  \frac{3}{2n-1}\right)   & =-1,\phi_{n}\left(  \frac{4}%
{2n-1}\right)  =0,\phi_{n}\left(  \frac{5}{2n-1}\right)  =1,\ldots
\end{align*}
so that if $x_{i}=\frac{2i-1}{2n-1}$, $i=1,\ldots,n$ then $\tilde{h}=\phi_{n}$ separates.

\section{Interpolation and Regularization}

In the last example we have seen that with a high dimensional feature space,
such as the function space (RKHS ) $\mathcal{H}$ figuring there, every
training becomes linearly separable. So the perceptron training algorithm
would separate the training set; Fisher's LDA in feature space would not
necessarily do this, but still it would work on a linearly separable training
set. But we also saw that a completely separating rule is somewhat
nonintuitive: the space where the $x_{i}$ live is split up into many small
regions; essentially every $x_{i}$ has its own classification region around
it. Such rules could be seen as erratic; for instance if one point $x_{i}$
from the $-1$ class appears completely surrounded by many points of the $+1 $
class , we should not give credence to this single point and classify the
whole neighboring region as "$+1$".

A method to deal with this problem is called \textit{regularization}. It
essentially consists of imposing a penalty term on rules such the above
interpolating rule $\tilde{h}$ which penalizes "wiggliness". In the above
example, the interpolating rule $\tilde{h}$ was a differentiable function with
$\tilde{h}(0)=0$, $\tilde{h}(x_{i})=y_{i}$, $i=1,\ldots,n$. This means that
$\tilde{h}$ might be very wiggly, i. e. $\int_{0}^{1}\left(  \tilde{h}%
^{\prime}(t)\right)  ^{2}dt=\left\Vert \tilde{h}^{\prime}\right\Vert _{2}^{2}$
might be large. If we seek a compromise between approximation of the training
set and smallness of the smoothness term $\left\Vert \tilde{h}^{\prime
}\right\Vert _{2}^{2}$, we migh arrive at a reasonable classifier.

Approximation of the training set by a rule $h$ might be measured by the
empical risk, i.e the proportion of misclassified points:
\[
R_{emp}(h)=n^{-1}\sum_{i=1}^{n}\mathbf{1}_{\left\{  h(x_{i})\neq
y_{i}\right\}  }.
\]
However it is common to use a somewhat modified expression, called
\textit{soft margin risk}. For that recall that the classification problem can
be understood in such a way that our object is the regression function (if
$Y\in\left\{  0,1\right\}  $)
\begin{equation}
r(x)=E\left(  Y|X=x\right)  =P\left(  Y=1|X=x\right)
\end{equation}
and the Bayes decision is such that
\begin{equation}
h^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $r(x)>1/2$\\
$0$ otherwise
\end{tabular}
\right. \label{bayes-rule-again}%
\end{equation}
So an classifier can be obtained by first estimating the regression function
$r$ by an estimator $\hat{r}$ and then deriving the classification rule from
it according to (\ref{bayes-rule-again}). Presently we are assuming
$Y\in\left\{  -1,1\right\}  $; then again $r(x)=E\left(  Y|X=x\right)  $ and
the classification rule derived from an $\hat{r}$ is
\[
\hat{h}(x)=\mathrm{sgn\;}\hat{r}(x).
\]
Recall that for a linear rule in the RKHS we had above
\begin{equation}
h(x)=\mathrm{sgn}\left\langle \tilde{h},K\left(  \cdot,x\right)  \right\rangle
_{\ast}.\label{lin-rule-rkhs}%
\end{equation}
or a more simple linear rule in the original space, if $x\in\mathbb{R}^{d}$,
is
\begin{equation}
h(x)=\mathrm{sgn}\left(  \left\langle a,x_{i}\right\rangle +a_{0}\right)
.\label{hyppl-rule}%
\end{equation}
This suggests to regard $\left\langle a,x_{i}\right\rangle +a_{0}$ or
$\left\langle \tilde{h},K\left(  \cdot,x\right)  \right\rangle _{\ast}%
=\tilde{h}(x)$ as estimators of the regression function $r(x)$. We then use
the following modification of the empirical risk: define
\begin{equation}
l_{sm}(y,r(x))=\max\left(  0,1-yr(x)\right)  =\left\{
\begin{tabular}
[c]{l}%
$0$ if $yr(x)>1$\\
$1-yr(x)$ otherwise
\end{tabular}
\right.  .\label{soft-margin-loss}%
\end{equation}
The rationale is the following. If $yr(x_{i})>0$ then $x_{i}$ is correctly
classified, if $yr(x_{i})<0$ then there is an error. We regard $\left\vert
r(x_{i})\right\vert $ as the "strength" of "confidence" with which the sign of
$r(x_{i})$ is predicted. If there is a wrong classification of $x_{i}$, then
the above loss is $1+\left\vert r(x_{i})\right\vert $, i.e. the loss increases
with the expressed "confidence" for that wrong prediction. If there is correct
classification, $yr(x_{i})>0$, then the loss is set $0$ only if $\left\vert
r(x_{i})\right\vert >1,$ i.e. for a high enough confidence. If $0<\left\vert
r(x_{i})\right\vert <1$ then there is still some penalty, even though $x_{i}$
is correctly classified.

For a simple linear (hyperplane) rule (\ref{hyppl-rule}), the expression
$\left\vert \left\langle a,x_{i}\right\rangle +a_{0}\right\vert $ was seen to
be the distance of $x_{i}$ to the hyperplane $\left\langle a,x\right\rangle
+a_{0}=0$. In this case the soft margin loss (\ref{soft-margin-loss})
penalized wrong classification, with increasing distance from the hyperplane,
and it penalizes also correct classification, to a lesser extent, if the
distance to the hyperplane is less than 1.

Consider any linear classifier (\ref{lin-rule-rkhs}) in a feature space which
is an RKHS, and define the empirical soft margin risk
\[
R_{emp}^{sm}(\tilde{h})=n^{-1}\sum_{i=1}^{n}l_{sm}(y_{i},\tilde{h}(x_{i})).
\]
Rather than achieving $0$ by interpolation ($\tilde{h}(x_{i})=y_{i}$) we now
impose a roughness penalty term and try to minimize the \textit{regularized
risk}:
\begin{equation}
R_{reg}(\tilde{h})=R_{emp}^{sm}(\tilde{h})+\lambda\left\Vert \tilde
{h}\right\Vert _{\ast}^{2}\label{regul-risk}%
\end{equation}
where $\left\Vert \cdot\right\Vert _{\ast}^{2}$ is the squared norm in the
RKHS. Recall that in our example $\left\Vert \tilde{h}\right\Vert _{\ast}%
^{2}=\left\Vert \tilde{h}^{\prime}\right\Vert _{2}^{2}$, so the term penalizes
indeed "wiggliness" or complexity of the function $\tilde{h}$. The number
$\lambda$ can be chosen: it expresses the weight of the penalty term relative
to sample approximation. For $\lambda\rightarrow0$, since we equivalently
minimize $\lambda^{-1}R_{emp}^{sm}(\tilde{h})+\left\Vert \tilde{h}\right\Vert
_{\ast}^{2}$, in the limit we will enforce $R_{emp}^{sm}(\tilde{h})=0$, i.e.
interpolation, while still minimizing $\left\Vert \tilde{h}\right\Vert _{\ast
}^{2}.$ This will amount to something like "minimum norm interpolation".

\begin{theorem}
\label{theor-representer}(Representer Theorem). Each minimizer $\tilde{h}$ of
the regularized risk functional (\ref{regul-risk}) admits a representation of
the form
\[
\tilde{h}(x)=\sum_{i=1}^{n}\alpha_{i}K\left(  x,x_{i}\right)  .
\]
where $\alpha_{i}\in\mathbb{R}$.
\end{theorem}

\begin{proof}
We may decompose any $f\in\mathcal{H}$ into a part contained in the span of
the kernel functions $K\left(  \cdot,x_{1}\right)  $,\ldots, $K\left(
\cdot,x_{n}\right)  $ and one in the orthogonal complement:
\[
\tilde{h}(x)=h_{\Vert}(x)+h_{\bot}(x)=\sum_{i=1}^{n}\alpha_{i}K\left(
x,x_{i}\right)  +h_{\bot}(x).
\]
Here $h_{\bot}\in\mathcal{H}$ and $\left\langle h_{\bot},K\left(  \cdot
,x_{i}\right)  \right\rangle _{\ast}=0$ for all $i=1,\ldots,n$. We also have
\begin{align}
\tilde{h}(x_{j})  & =\left\langle \tilde{h},K\left(  \cdot,x_{j}\right)
\right\rangle _{\ast}=\sum_{i=1}^{n}\alpha_{i}K\left(  x_{i},x_{j}\right)
+\left\langle h_{\bot},K\left(  \cdot,x_{i}\right)  \right\rangle _{\ast}\\
& =\sum_{i=1}^{n}\alpha_{i}K\left(  x_{i},x_{j}\right)  .\label{kept-fixed}%
\end{align}
Furthermore, for all $h_{\bot}$
\begin{equation}
\left\Vert \tilde{h}\right\Vert _{\ast}^{2}=\left\Vert \sum_{i=1}^{n}%
\alpha_{i}K\left(  \cdot,x_{i}\right)  \right\Vert _{\ast}^{2}+\left\Vert
h_{\bot}\right\Vert _{\ast}^{2}\geq\left\Vert \sum_{i=1}^{n}\alpha_{i}K\left(
\cdot,x_{i}\right)  \right\Vert _{\ast}^{2}.\label{kept-fixed-2}%
\end{equation}
We may now minimize $R_{reg}(\tilde{h})$ in two steps: first fix $\alpha_{i}
$, $i=1,\ldots,n$ and minimize over $h_{\bot}$, and then minimize over
$\alpha_{i}$. In the first step, in view of (\ref{kept-fixed}), all $\tilde
{h}(x_{j})$ are fixed too, hence the term $R_{emp}^{sm}(\tilde{h})$ is also
fixed. Thus minimization in the first step concerns only the term $\left\Vert
\tilde{h}\right\Vert _{\ast}^{2}$. From (\ref{kept-fixed-2}) we see that the
optimal choice in the first step is $h_{\bot}=0$. Thus the overall solution is
always of the form $\tilde{h}(x)=$ $h_{\Vert}(x)=$ $\sum_{i=1}^{n}\alpha
_{i}K\left(  \cdot,x_{i}\right)  $.
\end{proof}

The importance of this theorem lies in the fact that the infinite-dimensional
minimization problem over $\tilde{h}\in\mathcal{H}$ is reduced to a finite
dimensional one, over the linear span of $K\left(  \cdot,x_{1}\right)
$,\ldots, $K\left(  \cdot,x_{n}\right)  $. Let us examine how these functions
look like in our example. We have
\[
K\left(  t,x_{i}\right)  =\min\left(  t,x_{i}\right)  =\left\{
\begin{tabular}
[c]{l}%
$t$ if $t\leq x_{i}$\\
$x_{i}$ otherwise
\end{tabular}
\right.  .
\]
Such a function is a special \textit{piecewiese linear spline on }$[0,1]$. It
is linear for $t\leq x_{i}$ and constant for $t>x_{i}$ and it is continuous in
the point $x_{i}$.

In general, a spline on $\Omega$ is a function which is polynomial on certain
regions of $\Omega$ and the polynomials are "joined" in the borders of the
regions such that the overall function has some continuity and smoothness properties.

\subsection{Linear estimation of the regression function}

\textbf{Question:} \textit{Above it was claimed that if we use a simple linear
discrimination rule in the original space ( }$x\in\mathbb{R}^{d}$\textit{\ ) }%
\[
h(x)=\mathrm{sgn}\left(  \left\langle a,x\right\rangle +a_{0}\right)
\]
\textit{we can interpret the procedure as follows: the expression
}$\left\langle a,x\right\rangle +a_{0}$\textit{\ is an estimator of the
regression function }$r(x)=E\left(  Y|X=x\right)  $\textit{\ for }%
$Y\in\left\{  -1,1\right\}  $,\textit{\ and we then mimic the Bayes rule
}$h^{\ast}(x)=\mathrm{sgn}\left(  r(x)\right)  $\textit{. However for }%
$Y\in\left\{  -1,1\right\}  $\textit{\ the regression function takes values
}$\left\vert r(x)\right\vert <1,$\textit{\ but }$\left\langle a,x\right\rangle
+a_{0}$\textit{\ is linear in }$x$\textit{\ and hence unbounded in value. How
does this fit together ? }

\bigskip

\textbf{Answer. }The expression $\left\langle a,x\right\rangle +a_{0}$ can be
seen as a linear approximation to $r(x)$ in the following sense. Consider the
case where the class-conditional densities $f_{i}$ are both normal, i.e. the
densities of $N_{d}\left(  m_{i},\Sigma\right)  $, $i=0,1.$ This case was
treated in detail in the section on Fisher's LDA. For simplicity let us assume
$\Sigma=I_{d}$. We now use a class indicator $Y\in\{-1,1\}$ but we keep
notation $f_{0},f_{1}$ for the class-conditional densities. There is a
one-to-one correspondence to the previously used class indicator $0$ or $1$
$(\tilde{Y}\in\left\{  0,1\right\}  $, say) given by $Y=2\tilde{Y}-1$ and
$\tilde{Y}=\left(  Y+1\right)  /2$. We have
\[
f_{i}(x)=\frac{1}{\sqrt{2\pi}}\exp\left(  -\frac{1}{2}\left\Vert
x-m_{i}\right\Vert ^{2}\right)  .
\]
The Bayes rule was shown to be $h^{\ast}(x)=\mathrm{sgn}\left(  f_{1}%
(x)-f_{0}(x)\right)  $ (recall we now formally decide between $-1$ and $1,$
not between $0$ and $1$) or equivalently
\begin{equation}
h^{\ast}(x)=\mathrm{sgn}\left(  \left\langle a,x\right\rangle +a_{0}\right)
\text{ for }a=m_{1}-m_{0},\text{ }a_{0}=\frac{1}{2}\left(  \left\Vert
m_{0}\right\Vert ^{2}-\left\Vert m_{1}\right\Vert ^{2}\right)
\label{normal-bayes-rule-b}%
\end{equation}
(cf. (\ref{normal-bayes-rule}). At the same time the Bayes rule can be stated
in terms of the regression function $r(x)=E\left(  Y|X=x\right)  $ as
$h^{\ast}(x)=\mathrm{sgn}\left(  r(x)\right)  $. Indeed, when we used a class
indicator $\tilde{Y}\in\left\{  0,1\right\}  $ the regression function was
\[
\tilde{r}(x)=E\left(  \tilde{Y}|X=x\right)  =\left(  r(x)+1\right)  /2,
\]
so $r(x)\geq0$ if and only if $\tilde{r}(x)\geq1/2$. Let us compute $r(x)$ for
uniform prior probabilities ($P(Y=1)=P(Y=-1)=1/2$).
\begin{align*}
r(x)  & =E\left(  Y|X=x\right)  =1\cdot P\left(  Y=1|X=x\right)  +\left(
-1\right)  \mathit{\ }P\left(  Y=-1|X=x\right) \\
& =\frac{\frac{1}{2}f_{1}(x)}{\frac{1}{2}f_{0}(x)+\frac{1}{2}f_{1}(x)}%
-\frac{\frac{1}{2}f_{0}(x)}{\frac{1}{2}f_{0}(x)+\frac{1}{2}f_{1}(x)}\\
& =\frac{f_{1}(x)-f_{0}(x)}{f_{1}(x)+f_{0}(x)}.
\end{align*}
By using the fact
\[
f_{i}(x)=\frac{1}{\sqrt{2\pi}}\exp\left(  -\frac{1}{2}\left\Vert x\right\Vert
^{2}\right)  \exp\left(  \left\langle m_{i},x\right\rangle -\frac{1}%
{2}\left\Vert m_{i}\right\Vert ^{2}\right)  ,\;i=0,1
\]
we obtain
\begin{align*}
r(x)  & =\frac{\exp\left(  \left\langle m_{1},x\right\rangle -\frac{1}%
{2}\left\Vert m_{1}\right\Vert ^{2}\right)  -\exp\left(  \left\langle
m_{0},x\right\rangle -\frac{1}{2}\left\Vert m_{0}\right\Vert ^{2}\right)
}{\exp\left(  \left\langle m_{1},x\right\rangle -\frac{1}{2}\left\Vert
m_{1}\right\Vert ^{2}\right)  +\exp\left(  \left\langle m_{0},x\right\rangle
-\frac{1}{2}\left\Vert m_{0}\right\Vert ^{2}\right)  }\\
& =\frac{\exp\left(  z_{1}\right)  -\exp\left(  z_{0}\right)  }{\exp\left(
z_{1}\right)  +\exp\left(  z_{0}\right)  },
\end{align*}
where
\[
z_{i}=\left\langle m_{i},x\right\rangle -\frac{1}{2}\left\Vert m_{i}%
\right\Vert ^{2},\;i=0,1.
\]
The $z_{i}$ are linear functionals of $x$ and $r(x)$ depends on $x$ only via
those linear functionals. Let us make a further linear transformation
\[
u_{1}=z_{1}-z_{0},\;u_{2}=z_{1}+z_{0}.
\]
Then
\[
z_{0}=\frac{u_{2}-u_{1}}{2},\;z_{1}=\frac{u_{2}+u_{1}}{2}%
\]
and
\begin{align*}
r(x)  & =\frac{\exp\left(  \frac{u_{2}+u_{1}}{2}\right)  -\exp\left(
\frac{u_{2}-u_{1}}{2}\right)  }{\exp\left(  \frac{u_{2}+u_{1}}{2}\right)
+\exp\left(  \frac{u_{2}-u_{1}}{2}\right)  }=\frac{\exp\left(  u_{1}/2\right)
-\exp\left(  -u_{1}/2\right)  }{\exp\left(  u_{1}/2\right)  +\exp\left(
-u_{1}/2\right)  }\\
& =\tanh\left(  u_{1}/2\right)  .
\end{align*}
Here $u_{1}$ is the linear functional of $x$
\[
u_{1}=z_{1}-z_{0}=\left\langle a,x\right\rangle +a_{0}%
\]
where $a,a_{0}$ are given by (\ref{normal-bayes-rule-b}). Thus
\begin{equation}
r(x)=\tanh\left(  \frac{1}{2}\left(  \left\langle a,x\right\rangle
+a_{0}\right)  \right)  .\label{reg-func-normal}%
\end{equation}
The function $\tanh(x)$ is antisymmetric ($\tanh(x)=-\tanh(-x)$) and smooth.
Therefore $\tanh(0)=0$ and also $\tanh^{\prime}(0)=1$. Consider a Taylor
expansion of the function $r(x)$, $x\in\mathbb{R}^{d}$ around any point
$x_{0}$ on the decision boundary. i.e. $r(x_{0})=0.$ The last relation is
equivalent to $\left\langle a,x_{0}\right\rangle +a_{0}=0$, i.e. $x_{0}$ is on
the hyperplane given by $a,a_{0}$. Setting $z=x-x_{0}$ and letting
$\Delta_{x_{0}}$ be the gradient of $r$ at $x_{0}$ we obtain
\begin{equation}
r(x)=r(x_{0}+z)=r(x_{0})+\Delta_{x_{0}}^{\top}z+o(1)\text{ as }\left\Vert
z\right\Vert \rightarrow0\label{taylor-expan-reg-func}%
\end{equation}
and we find by the chain rule
\[
\Delta_{x_{0}}=\tanh^{\prime}(\frac{1}{2}\left(  \left\langle a,x_{0}%
\right\rangle +a_{0}\right)  )\frac{1}{2}a=\frac{1}{2}a.
\]
Note that $\left\langle a,x_{0}\right\rangle +a_{0}=0$ implies that
$a_{0}=-\left\langle a,x_{0}\right\rangle $ does not depend on $x_{0}$, so we
can write
\begin{equation}
r(x)=\Delta_{x_{0}}^{\top}\left(  x-x_{0}\right)  +o(1)=\frac{1}%
{2}\left\langle a,x\right\rangle +\frac{1}{2}a_{0}+o(1)\text{ as }\left\Vert
x-x_{0}\right\Vert \rightarrow0.\label{taylor-expan-reg-func-2}%
\end{equation}
We see that we can approximate the regression function in the normal case by
\textit{the same linear function} $\frac{1}{2}\left(  \left\langle
a,x\right\rangle +a_{0}\right)  $ in every point $x_{0}$ on the decision
boundary. In contrast, if $r$ is an arbitrary regression function for the
joint distribution $(X,Y)$, assumed smooth, then we can approximate it near a
point $x_{0}$ on the decision boundary, .according to
(\ref{taylor-expan-reg-func}) by a linear function $\Delta_{x_{0}}^{\top
}x-\Delta_{x_{0}}^{\top}x_{0}$ which depends on $x_{0}$ in general. Then the
decision boundary near $x_{0}$ is approximately a hyperplane $\left\{
x:\Delta_{x_{0}}^{\top}x-\Delta_{x_{0}}^{\top}x_{0}=0\right\}  $ which depends
on $x_{0}$ in general.
\[
\Delta_{x_{0}}^{\top}x-\Delta_{x_{0}}^{\top}x_{0}%
\]
In that sense, under the present normality assumption, the regression is
approximately linear near the decision boundary.

Recall that if $\left\Vert a\right\Vert =1$ then $\left\vert \left\langle
a,x\right\rangle +a_{0}\right\vert $ is the distance of $x$ from the
hyperplane $\left\langle a,x\right\rangle +a_{0}=0$. In general, $\left\vert
\left\langle a,x\right\rangle +a_{0}\right\vert $ is proportional to that
distance (with factor $\left\Vert a\right\Vert $). Then, using only a Taylor
expansion of $\tanh(x)$ at $x=0$ we find from (\ref{reg-func-normal})%
\[
r(x)=\frac{1}{2}\left(  \left\langle a,x\right\rangle +a_{0}\right)
+o(1)\text{ as }\left\vert \left\langle a,x\right\rangle +a_{0}\right\vert
\rightarrow0
\]
and we have obtained (\ref{taylor-expan-reg-func-2}) again, but as an
approximation directly in terms of the distance to the hyperplane.

In that sense, we can say that $\left\langle a,x\right\rangle +a_{0}$
approximates the regression function, up to a constant factor, for small
values of the distance of $x$ from the optimal hyperplane, in the Gaussian
case with equal class-conditional covariance matrices $\Sigma=I_{d}$.

\subsection{More on spline smoothing}

So far we have considered the regularized risk (\ref{regul-risk}) where the
first term is the empirical soft margin risk $R_{emp}^{sm}(\tilde{h})$ and the
second term is the regularization term or smoothness penalty $\lambda
\left\Vert \tilde{h}\right\Vert _{\ast}^{2}$. In the representer theorem
(Theor. \ref{theor-representer}) the minimization was over $\tilde{h}%
\in\mathcal{H}$. In our first example the RKHS $\mathcal{H}$ was
\[
H_{0}^{1}(0,1):=\left\{  f\in L_{2}(0,1):f\text{ abs. continuous,
}f(0)=0,f^{\prime}\in L_{2}(0,1)\right\}
\]
with norm $\left\Vert f\right\Vert _{\ast}^{2}=\left\Vert f^{\prime
}\right\Vert _{2}^{2}$. The decision rule was
\begin{equation}
h(x_{i})=\mathrm{sgn}\left\langle \tilde{h},K\left(  \cdot,x_{i}\right)
\right\rangle _{\ast}\label{lin-rule-in-rkhs}%
\end{equation}
where by the reproducing property, we have $\left\langle \tilde{h},K\left(
\cdot,x_{i}\right)  \right\rangle _{\ast}=\tilde{h}(x_{i})$, so in fact our
rule is $h(x_{i})=\mathrm{sgn}\tilde{h}(x_{i})$. However the general form of a
linear rule is $h(x_{i})=\mathrm{sgn}\left(  \left\langle a,x_{i}\right\rangle
+a_{0}\right)  $ where a constant $a_{0}$ is present. We might wish to include
a constant $a_{0}$ in the linear rule in RKHS (\ref{lin-rule-in-rkhs}), i.e.
to consider
\[
h(x_{i})=\mathrm{sgn}\left(  \left\langle \tilde{h},K\left(  \cdot
,x_{i}\right)  \right\rangle _{\ast}+a_{0}\right)  =\mathrm{sgn}\left(
\tilde{h}(x_{i})+a_{0}\right)
\]
and then consider minimization of the regularized empirical risk functional.
We then define $f=\tilde{h}+a$, $\tilde{h}\in\mathcal{H}$, $a\in\mathbb{R}$
and define
\[
R_{emp}^{sm}(f)=n^{-1}\sum_{i=1}^{n}l_{sm}(y_{i},f(x_{i})).
\]
and the regularized version by adding $\lambda\left\Vert \tilde{h}\right\Vert
_{\ast}^{2}$. Note that in our example the expression $\left\Vert f^{\prime
}\right\Vert _{2}^{2}$ is defined since $f^{\prime}=\tilde{h}^{\prime}$.

A second extension we will consider is regression estimation, separate from
classification but somewhat similar in methodology. We have i.i.d.
observations $\left(  X_{i},Y_{i}\right)  $, $i=1,\ldots,n$ where $X_{i}%
\in\mathbb{R}^{d}$, $Y_{i}\in\mathbb{R}$ and the purpose is to estimate
$r(x)=E\left(  Y|X=x\right)  $. The empiricial risk is then just the squared
risk:
\[
R_{emp}^{squ}(f)=n^{-1}\sum_{i=1}^{n}(y_{i}-f(x_{i}))^{2}.
\]
where $f=\tilde{h}+g$, $\tilde{h}\in\mathcal{H}$ (some RKHS) and
$g\in\mathcal{G},$ where $\mathcal{G}$ is a \textit{unisolvent set of
functions} for the given $\left\{  x_{1},\ldots,x_{n}\right\}  $:
\[
\left\{  g_{1},g_{2}\in\mathcal{G}\text{, }g_{1}(x_{i})=g_{2}(x_{i})\text{,
}i=1,\ldots,n\right\}  \Longrightarrow g_{1}=g_{2}\text{.}%
\]
For instance, if $\mathcal{G}$ is the set of constants then the values on one
$x_{i}$ uniquely determine $g\in\mathcal{G}$, if $\mathcal{G}$ is the set of
linear functions then the values on two distinct $x_{i}$ uniquely determine
$g\in\mathcal{G}$ and so forth. Our primary example below will be linear
functions and an RKHS\ $\mathcal{H}$ where the norm is given by the second derivative.

\begin{theorem}
\label{theor-genlzd-representer}(Generalized Representer Theorem). For
functions $f=\tilde{h}+g$, $\tilde{h}\in\mathcal{H}$ (RKHS) and $g\in
\mathcal{G}$ (unisolvent), consider a regularized risk functional
\[
R_{reg}(f)=R_{emp}(f)+\lambda\left\Vert \tilde{h}\right\Vert _{\ast}^{2}%
\]
where $R_{emp}(f)$ is either $R_{emp}^{sm}(f)$ or $R_{emp}^{squ}(f)$. Each
minimizer $f$ of $R_{reg}(f)$ admits a representation of the form
\[
f(x)=\sum_{i=1}^{n}\alpha_{i}K\left(  x,x_{i}\right)  +g.
\]
where $\alpha_{i}\in\mathbb{R}$ and $g\in\mathcal{G}$.
\end{theorem}

\begin{proof}
We may decompose any $\tilde{h}\in\mathcal{H}$ into a part contained in the
span of the kernel functions $K\left(  \cdot,x_{1}\right)  $,\ldots, $K\left(
\cdot,x_{n}\right)  $ and one in the orthogonal complement:
\[
\tilde{h}(x)=h_{\Vert}(x)+h_{\bot}(x)=\sum_{i=1}^{n}\alpha_{i}K\left(
x,x_{i}\right)  +h_{\bot}(x).
\]
Here $h_{\bot}\in\mathcal{H}$ and $\left\langle h_{\bot},K\left(  \cdot
,x_{i}\right)  \right\rangle _{\ast}=0$ for all $i=1,\ldots,n$. We also have
\begin{align}
\tilde{h}(x_{j})  & =\left\langle \tilde{h},K\left(  \cdot,x_{j}\right)
\right\rangle _{\ast}=\sum_{i=1}^{n}\alpha_{i}K\left(  x_{i},x_{j}\right)
+\left\langle h_{\bot},K\left(  \cdot,x_{i}\right)  \right\rangle _{\ast}\\
& =\sum_{i=1}^{n}\alpha_{i}K\left(  x_{i},x_{j}\right)  .
\end{align}
hence
\begin{equation}
f(x_{j})=\tilde{h}(x_{j})+g(x_{j})=\sum_{i=1}^{n}\alpha_{i}K\left(
x_{i},x_{j}\right)  +g(x_{j})\label{kept-fixed-a}%
\end{equation}
Furthermore, for all $h_{\bot}$
\begin{equation}
\left\Vert \tilde{h}\right\Vert _{\ast}^{2}=\left\Vert \sum_{i=1}^{n}%
\alpha_{i}K\left(  \cdot,x_{i}\right)  \right\Vert _{\ast}^{2}+\left\Vert
h_{\bot}\right\Vert _{\ast}^{2}\geq\left\Vert \sum_{i=1}^{n}\alpha_{i}K\left(
\cdot,x_{i}\right)  \right\Vert _{\ast}^{2}.
\end{equation}
We now may minimize $R_{reg}(f)$ in two steps: first fix $\alpha_{i}$,
$i=1,\ldots,n$ and $g\in\mathcal{G}$, and minimize over $h_{\bot}$, and then
minimize over $\alpha_{i}$ and $g$. In the first step, in view of
(\ref{kept-fixed-a}), all $\tilde{h}(x_{j})$ are fixed too, hence the term
$R_{emp}^{sm}(\tilde{h})$ is also fixed. Thus minimization in the first step
concerns only the term $\left\Vert \tilde{h}\right\Vert _{\ast}^{2}$. From
(\ref{kept-fixed-2}) we see that the optimal choice in the first step is
$h_{\bot}=0$. Thus the overall solution is always of the form claimed.
\end{proof}

\textbf{\ } \bigskip\bigskip

\textbf{Remark.} The theorem is true without the unisolvency condition on
$\mathcal{G}$, but this one is useful for other purposes such as uniqueness of
the solution. \bigskip\bigskip

\subsubsection{Example 1: linear smoothing spline, special}

The RKHS $\mathcal{H}$ is\textbf{\ }$H_{0}^{1}(0,1)$ with norm $\left\Vert
f\right\Vert _{\ast}^{2}=\left\Vert f^{\prime}\right\Vert _{2}^{2}$,
$\mathcal{G}$ is empty, $R_{emp}(f)$ is $R_{emp}^{squ}(f$ $)$. We are in the
general regression situation and minimizing
\begin{equation}
n^{-1}\sum_{i=1}^{n}\left(  y_{i}-f(x_{i})\right)  ^{2}+\lambda\left\Vert
f^{\prime}\right\Vert _{2}^{2}\label{linear-spline-objective-func}%
\end{equation}
over $f\in$ $H_{0}^{1}(0,1)$. We assume the points $x_{i}$ ordered and all
distinct: $0<x_{1}<\ldots<x_{n}<1$. By the representer theorem, the result is
a piecewise linear spline $f$ with $f(0)=0$ and $f(1)=f(x_{n})$ (since any
linear combination $\sum_{i=1}^{n}\alpha_{i}K\left(  x_{i},\cdot\right)  $ is
of this form). Write
\[
y=\left(  y_{1},\ldots,y_{n}\right)  ^{\top}\text{, }a=\left(  \alpha
_{1},\ldots,\alpha_{n}\right)  ^{\top}\text{, }K=\left(  K\left(  x_{i}%
,x_{j}\right)  \right)  _{i,j=1}^{n}%
\]
According to Remark \ref{rem-other-positive-definiteness-kernel}, the kernel
$K(t,u)$ on $[0,1]\times\lbrack0,1]$ is positive definite$^{\ast},$ which
implies that the symmetric matrix $K$ is nonnegative definite (meaning
$c^{\top}Kc\geq0$ for every $c\in\mathbb{R}^{n})$. Furthermore we have

\begin{lemma}
The matrix $K$ is nonsingular.
\end{lemma}

The proof is left as an exercise.

A symmetric matrix which is nonnegative definite and singular is
\textit{positive definite} (meaning meaning $c^{\top}Kc>0$ for every
$c\in\mathbb{R}^{n},$ $c\neq\mathbf{0}_{n}$)\footnote{For matrices and kernels
we use a slightly different language here: a kernel was called positive
definite (or p.d.*) if the relevant inequality for a function $f$ involves
$"\geq0"$ whereas symmetric matrices with an inequality $"\geq0"$are called
nonnegative definite. For symmetric matrices "positive definite" means the
strict inequality $">0"$ holds for any nonzero vectors.}.

Now our target function in the minimization problem
(\ref{linear-spline-objective-func}) is (with $\left\Vert \cdot\right\Vert $
the euclidean norm)
\begin{align*}
& n^{-1}\left\Vert y-Ka\right\Vert ^{2}+\lambda\left\Vert \sum_{i=1}^{n}%
\alpha_{i}K\left(  x_{i},\cdot\right)  \right\Vert _{\ast}^{2}\\
& =n^{-1}\left\Vert y-Ka\right\Vert ^{2}+\lambda\sum_{i,j=1}^{n}\alpha
_{i}\alpha_{j}\left\langle K\left(  x_{i},\cdot\right)  ,K\left(  x_{j}%
,\cdot\right)  \right\rangle _{\ast}\\
& =n^{-1}\left\Vert y-Ka\right\Vert ^{2}+\lambda\sum_{i,j=1}^{n}\alpha
_{i}\alpha_{j}K\left(  x_{i},x_{j}\right) \\
& =n^{-1}\left\Vert y-Ka\right\Vert ^{2}+\lambda a^{\top}Ka.
\end{align*}
We set $\hat{f}=Ka$ where $\hat{f}=\left(  f\left(  x_{1}\right)
,\ldots,f\left(  x_{n}\right)  \right)  ^{\top}$; then our target function is
\begin{equation}
n^{-1}\left\Vert y-\hat{f}\right\Vert ^{2}+\lambda\hat{f}^{\top}K^{-1}\hat
{f}\label{targ-func-smoo-spli}%
\end{equation}
To minimize this quadratic form, we differentiate and get a normal equation
(using $K^{\top}=K$)
\[
-2n^{-1}\left(  y-\hat{f}\right)  +2\lambda K^{-1}\hat{f}=0
\]
with solution
\begin{align}
\left(  n^{-1}I_{n}+\lambda K^{-1}\right)  \hat{f}  & =n^{-1}y,\nonumber\\
\hat{f}  & =\hat{D}y\text{ for }\hat{D}=\left(  I_{n}+\lambda nK^{-1}\right)
^{-1}\label{solution-lin-smooth-spline}%
\end{align}

\textbf{Shrinkage property of the solution.} From the last equation we see
that the solution, i.e. the linear smooting spline, is the result of a linear
operator applied to the data $y$ (if $\lambda$ is chosen a priori, i.e. not
depending on $y$). This operator is given by the matrix $\hat{D}=\left(
I_{n}+\lambda nK^{-1}\right)  ^{-1},$ also called a \textit{smoother matrix}
(or \textit{hat matrix}). For $\lambda=0$ we obtain $\hat{f}=y$, i.e. the
linear spline which interpolates the data. For $\lambda>0$ the matrix $\lambda
nK^{-1}$ is positive definite. Thus
\begin{equation}
I_{n}+\lambda nK^{-1}\geq I_{n}\label{inequ-pos-def}%
\end{equation}
(in the sense of the order of symmetric matrices, induced by positive
definiteness: $A\geq B$ if $A-B$ is nonnegative definite, i.e. if $a^{\top
}\left(  A-B\right)  a\geq0$ for all $a\in\mathbb{R}^{n}$). From
(\ref{inequ-pos-def}) we obtain by inverting
\[
\hat{D}=\left(  I_{n}+\lambda nK^{-1}\right)  ^{-1}\leq I_{n}%
\]
which means that the "hat matrix" $\hat{D}$ is smaller than the unit matrix,
i.e. the data vector $y$ is "shrunk" towards the origin by $\hat{f}=\hat{D}%
y$.\bigskip\bigskip

\textbf{Smoothing property of the solution.} Consider spectral decomposition
of $K$
\[
K=\sum_{j=1}^{n}\gamma_{j,n}k_{j}k_{j}^{\top}%
\]
where $k_{j},$ $j=1,\ldots,n$ is an orthonormal set of eigenvectors in
$\mathbb{R}^{n}$ ($\left\Vert k_{j}\right\Vert =1$ for the euclidean norm )
and $\gamma_{j,n}>0,$ $j=1,\ldots,n$ are the eigenvalues. (Here of course the
$k_{j}$ also depend on $n,$ but we emphasise this dependence on $n$ in the
eigenvalues). Then it is a well known fact that
\[
K^{-1}=\sum_{j=1}^{n}\gamma_{j,n}^{-1}k_{j}k_{j}^{\top}%
\]
and the hat matrix is
\[
\hat{D}=\left(  I_{n}+\lambda nK^{-1}\right)  ^{-1}=\sum_{j=1}^{n}\frac
{1}{1+\lambda n\gamma_{j,n}^{-1}}k_{j}k_{j}^{\top}%
\]
Thus the operation $\hat{f}=\hat{D}y$ may be interpreted as follows: get new
coordinates of the data $y$ by taking $k_{j}^{\top}y$, multiply each
coordinate by $\left(  1+\lambda n\gamma_{j,n}^{-1}\right)  ^{-1}$, change
back to old coodinates. Since $\left(  1+\lambda n\gamma_{j,n}^{-1}\right)
^{-1}<1$ (if $\lambda>0$), we again see the shrinkage character.

Now consider the Mercer decomposition of the kernel $K\left(  x,y\right)
=\min(x,y)$ given by Proposition \ref{prop-mercer-min-kernel}: it is
\[
K(x,y)=\sum_{j=1}^{\infty}\gamma_{j}\phi_{j}(x)\phi_{j}(y)
\]
for the functions $\phi_{j}(x)=\sqrt{2}\sin\left(  \left(  j-1/2\right)  \pi
x\right)  $, $j=1,2,\ldots$ given in Lemma \ref{lem-eigenfunc} and
\[
\gamma_{j}=\frac{1}{\left(  j-1/2\right)  ^{2}\pi^{2}}\text{, }j=1,2,\ldots.
\]

\begin{claim}
For the matrix $K=\left(  K\left(  x_{i},x_{j}\right)  \right)  _{i,j=1}^{n}$
we can expect that for large $n$, the eigenvectors $k_{j}$ are close to
$\hat{\phi}_{j}$ where
\[
\hat{\phi}_{j}:=n^{-1/2}\left(  \phi_{j}\left(  x_{1}\right)  ,\ldots,\phi
_{j}\left(  x_{n}\right)  \right)  ^{\top}%
\]
and the eigenvalues $\gamma_{j,n}$ are approximately $n\gamma_{j}$.
\end{claim}

\begin{proof}
[Reasoning]Indeed approximating first the kernel $K$ by a truncated series
\[
K(x,y)\approx\sum_{j=1}^{n}\gamma_{j}\phi_{j}(x)\phi_{j}(y)
\]
and then noting for the matrix $K$
\[
K=\left(  K\left(  x_{i},x_{j}\right)  \right)  _{i,j=1}^{n}\approx\left(
\sum_{s=1}^{n}\gamma_{s}\phi_{s}(x_{i})\phi_{s}(x_{j})\right)  _{i,j=1}%
^{n}=\sum_{s=1}^{n}n\gamma_{s}\hat{\phi}_{s}\hat{\phi}_{s}^{\top}%
\]
where the vectors $\hat{\phi}_{s}=\left(  \phi_{s}\left(  x_{1}\right)
,\ldots,\phi_{s}\left(  x_{n}\right)  \right)  ^{\top}$ are approximately
orthogonal if the values $x_{1},\ldots,x_{n}$ are nearly uniformly spaced
within $[0,1]$:
\begin{align*}
\left\langle \hat{\phi}_{r},\hat{\phi}_{s}\right\rangle  & =\hat{\phi}%
_{r}^{\top}\hat{\phi}_{s}=n^{-1}\sum_{i=1}^{n}\phi_{r}(x_{i})\phi_{s}(x_{i})\\
& \approx\int_{0}^{1}\phi_{r}(x)\phi_{s}(x)dx=\left\{
\begin{tabular}
[c]{l}%
$1$ if $r=s$\\
$0$ otherwise
\end{tabular}
\right.  .
\end{align*}
Thus $K=\sum_{s=1}^{n}n\gamma_{s}\hat{\phi}_{s}\hat{\phi}_{s}^{\top}$ is an
approximate spectral decomposition of the matrix $K.$
\end{proof}

\bigskip\bigskip

We obtain an approximation for $\hat{D}$: since
\[
n\gamma_{j,n}^{-1}\approx\gamma_{j}^{-1}=\left(  j-1/2\right)  ^{2}\pi
^{2}\approx\pi^{2}j^{2}%
\]
(for large $n)$, we get
\[
\hat{D}=\sum_{j=1}^{n}\frac{1}{1+\lambda n\gamma_{j,n}^{-1}}k_{j}k_{j}^{\top
}\approx\sum_{j=1}^{n}\frac{1}{1+\lambda\pi^{2}j^{2}}\hat{\phi}_{j}\hat{\phi
}_{j}^{\top}%
\]
Here we may see the action of the smoothing operator on the data. The $j$-th
Fourier coefficient, corresponding to basis function $\sqrt{2}\sin\left(
\left(  j-1/2\right)  \pi x\right)  $, of the smoothed function $\hat{f}$ is
approximately
\[
\left(  \int_{0}^{1}\varphi_{j}(x)\hat{f}(x)dx\right)  \approx n^{-1/2}%
\hat{\phi}_{j}^{\top}\hat{f}=\frac{1}{1+\lambda\pi^{2}j^{2}}n^{-1/2}\hat{\phi
}_{j}^{\top}y
\]
where $n^{-1/2}\hat{\phi}_{j}^{\top}y$ is the approximate Fourier coefficient
of the data, corresponding to the same frequency. We see that smoothing means
multiplication of the coefficient by the multiplier $\left(  1+\lambda\pi
^{2}j^{2}\right)  ^{-1}$. The higher the frequency $j$, the smaller the
multiplier; \textit{this means "dampening" or "filtering" of high
frequencies.} Thus the smoothing spline is close to a tapered orthogonal
series estimator.

The map $j\rightarrow\left(  1+\lambda\pi^{2}j^{2}\right)  ^{-1}$ may be
called the filter characteristic of the smoothing spline. Thus the smoothing
spline is close to a "tapered" orthogonal series estimator. A
\textit{truncated }orthogonal series estimator would have filter
characteristic $\mathbf{1}_{[0,s]}(j)$where $s$ is the truncation index.
\bigskip\bigskip

\subsubsection{Example 2: linear smoothing spline, general}

The setup is as in example 1, but now $\mathcal{G}$ is the set of constant
functions on $[0,1]$. Thus $f=\tilde{h}+g$ where $\tilde{h}\in H_{0}^{1}(0,1)
$ and $g$ is constant. Consider a version of the space $H_{0}^{1}(0,1)$
without boundary conditions:
\[
H^{1}(0,1):=\left\{  f\in L_{2}(0,1):f\text{ abs. continuous, },f^{\prime}\in
L_{2}(0,1)\right\}  .
\]
Obviously any function $f\in H^{1}(0,1)$ has a unique decomposition
$f=\tilde{h}+g$, $\tilde{h}\in H_{0}^{1}(0,1)\mathcal{\ }$and $g$ a constant:
determine $g$ by $g(0)=f(0)$, then it is clear that $\tilde{h}:=f-g\in
H_{0}^{1}(0,1)$. Since $\tilde{h}^{\prime}=f^{\prime}$ we may equivalently
consider minimization of (\ref{linear-spline-objective-func}) over functions
$f\in H^{1}(0,1)$. According to the generalized representer theorem (Theorem
\ref{theor-genlzd-representer}) any solution can be written
\[
f=\sum_{i=1}^{n}\alpha_{i}K\left(  \cdot,x_{i}\right)  +g
\]
where $K\left(  t,x_{i}\right)  =\min\left(  t,x_{i}\right)  $. This is a
piecewise linear spline $f$ with $f(1)=f(x_{n}).$ Note that the condition
$f(0)=0$ is no longer present since we are adding a constant $g,$ but it can
be shown that now $f(0)=f(x_{1})$. Indeed, any other linear spline $f$ with
the same values of $f(x_{i})$ would have a higher value of $\left\Vert
f^{\prime}\right\Vert _{2}^{2}$.

\begin{corollary}
\label{coroll-lin-spline-general}Any solution of the problem of minimizing
\[
n^{-1}\sum_{i=1}^{n}(y_{i}-f(x_{i}))^{2}+\lambda\int_{0}^{1}\left(  f^{\prime
}(t)\right)  ^{2}dt
\]
over functions $f\in H^{1}(0,1)$, for given $x_{i}\in\left[  0,1\right]  $ and
$y_{i}$, is a continuous, piecewiese linear spline function with knots in
$x_{i}$, $i=1,\ldots,n$.
\end{corollary}

This solution is called the linear smoothing spline, pertaining to data
$x_{i},y_{i}$, $i=1,\ldots,n$.

\subsubsection{Example 3: Cubic smoothing spline}

The RKHS $\mathcal{H}$ is
\[
H_{0}^{2}(0,1):=\left\{  f\in L_{2}(0,1):f^{\prime}\text{ exists, abs.
continuous, },f^{\prime\prime}\in L_{2}(0,1)\text{, }f(0)=f^{\prime
}(0)=0\right\}
\]
with scalar product
\[
\left\langle f_{1},f_{2}\right\rangle _{\ast}=\int_{[0,1]}f_{1}^{\prime\prime
}(x)f_{2}^{\prime\prime}(x)dx
\]
and norm $\left\Vert f\right\Vert _{\ast}^{2}=\left\Vert f^{\prime\prime
}\right\Vert _{2}^{2}$. The reproducing kernel \cite{BTA}, p. 287
\footnote{Berlinet, A., Thomas-Agnan, C., Reproducing kernel Hilbert spaces in
Probability and Statistics, Kluwer, 2004, p. 287} is%
\begin{equation}
K\left(  s,t\right)  =\int_{0}^{1}\left(  t-w\right)  _{+}\left(  s-w\right)
_{+}dw.\label{integral-repres}%
\end{equation}
Let us evaluate the integral: for $s\leq t$ we have
\begin{align*}
K\left(  s,t\right)   & =\int_{0}^{s}\left(  t-w\right)  \left(  s-w\right)
dw=\int_{0}^{s}\left(  w^{2}-(s+t)w+st\right)  dw\\
& =s^{3}/3-(s+t)s^{2}/2+s^{2}t=-s^{3}/6+s^{2}t/2.
\end{align*}
For $s\geq t$ the roles of $s$ and $t$ switch, thus we have
\begin{align*}
K\left(  s,t\right)   & =\left\{
\begin{tabular}
[c]{l}%
$k\left(  s,t\right)  $ if $s\leq t$\\
$k\left(  t,s\right)  $ if $s\geq t$%
\end{tabular}
\right. \\
\text{for }k\left(  s,t\right)   & =ts^{2}/2-s^{3}/6.
\end{align*}
The symmetry $K\left(  s,t\right)  =K\left(  t,s\right)  $ follows from this
(or also directly from the integral representation (\ref{integral-repres}).
The function $K\left(  \cdot,t\right)  $ is a third degree polynomial on
$\left[  0,t\right]  $ and a linear function on $\left[  t,1\right]  $, with
common value $k\left(  t,t\right)  =t^{3}/3$ at $s=t$. Hence $K\left(
\cdot,t\right)  $ is continuous. The derivatives of the pieces are
\[
\frac{\partial}{\partial s}K\left(  s,t\right)  =\left\{
\begin{tabular}
[c]{l}%
$ts-s^{2}/2$ if $s\leq t$\\
$t^{2}/2$ if $s\geq t$%
\end{tabular}
\right.
\]
with common value $t^{2}/2$ at $s=t$, hence $K\left(  \cdot,t\right)  $ is
continuously differentiable. The second derivatives of the pieces are%
\[
\frac{\partial^{2}}{\partial s^{2}}K\left(  s,t\right)  =\left\{
\begin{tabular}
[c]{l}%
$t-s$ if $s\leq t$\\
$0$ if $s\geq t$%
\end{tabular}
\right.
\]
hence is $K\left(  \cdot,t\right)  $ twice continuously differentiable (i.e.
is in $C^{2}$) and the second derivative is a continuous piecewise linear
spline. The third derivative is the step function $-\mathbf{1}_{[0,t]}$.
Functions like these (piecewise polynomial of 3rd degree) with smoothness
properties are called \textit{cubic splines}; thus $K\left(  \cdot,t\right)  $
is a cubic spline function which is in $C^{2}$. The points where the
polynomials change are called \textit{knots}; thus $K\left(  \cdot,t\right)  $
has one knot in $t.$

To check the reproducing property, note
\begin{align*}
\left\langle f,K\left(  \cdot,x\right)  \right\rangle _{\ast}  & =\int
_{[0,1]}f^{\prime\prime}(s)\frac{\partial^{2}}{\partial s^{2}}K\prime\left(
s,x\right)  ds=\int_{0}^{x}f^{\prime\prime}(s)\left(  x-s\right)  ds\\
& =\left[  f^{\prime}(s)\left(  x-s\right)  \right]  _{0}^{x}+\int_{0}%
^{x}f^{\prime}(s)ds=\\
& =f^{\prime}(x)\left(  x-x\right)  -f^{\prime}(0)x+f(x)-f(0)=f(x).
\end{align*}
Positive definiteness follows from the integral representation
(\ref{integral-repres}).

We may now consider the minimization problem
\[
n^{-1}\sum_{i=1}^{n}(y_{i}-f(x_{i}))^{2}+\lambda\int_{0}^{1}\left(
f^{\prime\prime}(t)\right)  ^{2}dt
\]
over $f=\tilde{h}+g$, $\tilde{h}\in H_{0}^{2}(0,1)\mathcal{\ }$and $g$ a
linear function $\left(  g(x)=g_{1}x+g_{0}\right)  .$ Here $f^{\prime\prime
}=\tilde{h}^{\prime\prime}$ so that the original penalty term $\int_{0}%
^{1}\left(  \tilde{h}^{\prime\prime}(t)\right)  ^{2}dt$ can also be written in
terms of $f.$ According to the generalized representer theorem (Theorem
\ref{theor-genlzd-representer}) any solution can be written
\[
f=\sum_{i=1}^{n}\alpha_{i}K\left(  \cdot,x_{i}\right)  +g.
\]
Moreover, consider a version of the space $H_{0}^{2}(0,1)$ without boundary
conditions:
\[
H^{2}(0,1):=\left\{  f\in L_{2}(0,1):f^{\prime}\text{ exists, abs. continuous,
},f^{\prime\prime}\in L_{2}(0,1)\right\}  .
\]
Obviously any function $f\in H^{2}(0,1)$ has a unique decomposition
$f=\tilde{h}+g$, $\tilde{h}\in H_{0}^{2}(0,1)\mathcal{\ }$and $g$ a linear
function: determine $g$ by $g(0)=f(0)$, $g^{\prime}(0)=f^{\prime}(0)$, then it
is clear that $f-g\in H_{0}^{2}(0,1)$.

\begin{corollary}
Any solution of the problem of minimizing
\[
n^{-1}\sum_{i=1}^{n}(y_{i}-f(x_{i}))^{2}+\lambda\int_{0}^{1}\left(
f^{\prime\prime}(t)\right)  ^{2}dt
\]
over functions $f\in H^{2}(0,1)$, for given $x_{i}\in\left[  0,1\right]  $ and
$y_{i}$, is a cubic spline function in $C^{2}$ with knots in $x_{i}$,
$i=1,\ldots,n$.
\end{corollary}

This solution is called the cubic smoothing spline, pertaining to data
$x_{i},y_{i}$, $i=1,\ldots,n$. For more insight on spline smoothing and RKHS
see Wahba \cite{Wah} and Berlinet, Thomas-Agnan \cite{BTA}

\subsection{Smoothing in classification}

Above we discussed examples of smoothing splines for regression. Our main
interest is classification however, and we originally introduced the
regularization idea in RKHS in that context. In (\ref{soft-margin-loss}) we
defined the soft margin loss for misclassification, we considered the
corresponding soft margin risk and its penalized version (\ref{regul-risk}).
Our concrete examples of RKHS were for $x\in(0,1)$ however, and for
classification we should certainly discuss some examples where $x\in
\mathbb{R}^{d}$. Before that however, let us discuss another approach to make
the smoothing idea work in classification. This approach is based on logistic regression.

\subsubsection{ Logistic regression}

In the classification framework, assume that $Y\in\left\{  0,1\right\}  $ and
$X\in\Omega\subset\mathbb{R}^{d}$, and $\left(  X,Y\right)  $ have a joint
distribution. We observe an i.i.d. training set $\left(  X_{i},Y_{i}\right)
$, $i=1,\ldots,n$ all distributed like $\left(  X,Y\right)  $ and we are
required, for a "new" incoming pair $\left(  X,Y\right)  $ of which only $X$
is observed, to predict $Y$. Denote
\[
p_{x}:=P\left(  Y=1|X=x\right)
\]
and note that the joint law of $\left(  X,Y\right)  $ is determined by $p_{x}$
and the marginal law of $X$, say $\mathcal{L}(X)$. Indeed, given $X=x$, the
conditional law of $Y$ is a Bernoulli distribution with parameter $p_{x}$ and
$P\left(  Y=0|X=x\right)  =1-p_{x}$. In logistic regression it is assumed
that
\begin{equation}
\log\frac{p_{x}}{1-p_{x}}=\beta_{0}+\left\langle \beta,x\right\rangle
\label{solving}%
\end{equation}
for some $\beta\in\mathbb{R}^{d}$ and $\beta_{0}\in\mathbb{R}$. The parameters
$\beta,\beta_{0}$ are estimated from the training set as $\hat{\beta}%
,\hat{\beta}_{0}$, giving for each $x$ an estimated $\hat{p}_{x}$ by solving
(\ref{solving}). This $\hat{p}_{x}$ provides an obvious classifier $h $:
$h(x)=1$ if $\hat{p}_{x}>1/2$. Recall that the Bayes classifier $h^{\ast} $ is
a function of the posterior probability $p_{x}$ and classifies $1$ if
$p_{x}>1/2$.

Since $\hat{p}_{x}>1/2$ is equivalent to $\log\left(  \hat{p}_{x}/\left(
1-\hat{p}_{x}\right)  \right)  >0$, the classifier $\phi$ is linear:%
\[
\phi(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $\beta_{0}+\left\langle \beta,x\right\rangle >0$\\
$0$ otherwise.
\end{tabular}
\right.
\]
Note that solving (\ref{solving}) for $p_{x}$ gives
\[
p_{x}=\frac{\exp\left(  \beta_{0}+\left\langle \beta,x\right\rangle \right)
}{1+\exp\left(  \beta_{0}+\left\langle \beta,x\right\rangle \right)  }.
\]
The function
\[
l(t)=\frac{\exp\left(  t\right)  }{1+\exp\left(  t\right)  }%
\]
is known as the \textit{logistic function }and its inverse
\[
\text{\textrm{logit}}\left(  u\right)  =\log\frac{u}{1-u}%
\]
as the \textit{logit function} (check \textrm{logit}$\left(  l(t)\right)
=t$). Since $p_{x}$ is also the expectation $E\left(  Y|X=x\right)  $, we have
for the regression function in the logistic regression model
\[
r(x)=E\left(  Y|X=x\right)  =p_{x}=l(\beta_{0}+\left\langle \beta
,x\right\rangle )
\]

\bigskip\bigskip\textbf{Relation to the normal model. }When Fisher's LDA was
discussed, a normal model for the class-conditional distributions
$\mathcal{L}\left(  X|Y=i\right)  $ was assumed:
\begin{equation}
\mathcal{L}\left(  X|Y=i\right)  =N_{d}\left(  m_{i},\Sigma\right)  \text{,
}i=0,1\label{class-condit-normal-1}%
\end{equation}
with equal prior probabilities
\begin{equation}
\pi_{i}=P\left(  Y=i\right)  =1/2,i=0,1.\label{class-condit-normal-2}%
\end{equation}
Together these two assumptions determine the joint distribution of $\left(
X,Y\right)  $. In the logistic regression model, one proceeds the other way
round: one specifies%
\begin{equation}
\mathcal{L}\left(  Y|X=x\right)  =\mathrm{Bernoulli}\left(  p_{x}\right)
\text{, }p_{x}=l(\beta_{0}+\left\langle \beta,x\right\rangle
)\label{lgist-reg-mod}%
\end{equation}
and one leaves $\mathcal{L}\left(  X\right)  $, the marginal law of $X$,
unspecified. Thus, to build a joint distribution of $\left(  X,Y\right)  $
here, one may assume \textit{any } distribution for $X$.

\begin{proposition}
\label{prop-logistic-is true-if-normal}The class-conditional normal model
(\ref{class-condit-normal-1}), (\ref{class-condit-normal-2}) is a logistic
regression model (\ref{lgist-reg-mod}) with
\begin{align*}
\beta & =\Sigma^{-1}\left(  m_{1}-m_{0}\right)  ,\\
\beta_{0}  & =\frac{1}{2}\left(  m_{0}{}^{\top}\Sigma^{-1}m_{0}-m_{1}{}^{\top
}\Sigma^{-1}m_{1}\right)
\end{align*}
where $\mathcal{L}\left(  X\right)  $ is a mixture of normals: if $\phi_{i}$
is the density of $N_{d}\left(  m_{i},\Sigma\right)  $, $i=0,1$ then the
density $\phi$ of $X$ is
\[
\phi(x)=\frac{1}{2}\phi_{1}(x)+\frac{1}{2}\phi_{2}(x)\text{. }%
\]

\end{proposition}

\begin{proof}
We assume $\Sigma=I_{d}$; the proof for general $\Sigma$ is analogous with a
transformation of $X$ by $\Sigma^{-1/2}$ first. In the subsection "Linear
estimation of the regression function" (in relation (\ref{reg-func-normal}))
we proved that in the normal model (\ref{class-condit-normal-1}),
(\ref{class-condit-normal-2}) when we use a class index $\tilde{Y}=2Y-1$
(taking values in $\left\{  -1,1\right\}  $) then
\[
\tilde{r}(x):=E\left(  \tilde{Y}|X=x\right)  =\tanh\left(  \frac{1}{2}\left(
\left\langle a,x\right\rangle +a_{0}\right)  \right)
\]
for $a=m_{1}-m_{0},$ $a_{0}=\frac{1}{2}\left(  \left\Vert m_{0}\right\Vert
^{2}-\left\Vert m_{1}\right\Vert ^{2}\right)  $, where
\[
\tanh\left(  u\right)  =\frac{\exp\left(  u\right)  -\exp\left(  -u\right)
}{\exp\left(  u\right)  +\exp\left(  -u\right)  }.
\]
Since $Y=\left(  \tilde{Y}+1\right)  /2$, we obtain for $u=\left\langle
a,x\right\rangle +a_{0}$
\begin{align*}
r(x)  & =E\left(  Y|X=x\right)  =\frac{1}{2}\left(  \tanh\left(  u/2\right)
+1\right) \\
& =\frac{1}{2}\frac{2\exp\left(  u/2\right)  }{\exp\left(  u/2\right)
+\exp\left(  -u/2\right)  }=\frac{\exp\left(  u\right)  }{\exp\left(
u\right)  +1}=l(u)
\end{align*}
i.e.
\[
r(x)=l(\left\langle a,x\right\rangle +a_{0}).
\]
For $\Sigma=I_{d}$ we have $a=\beta$ and $a_{0}=\beta_{0}$ as claimed. It
remains to find the marginal distribution of $X$. This is immediate: for any
measurable $A\subset\mathbb{R}^{d}$
\begin{align*}
P\left(  X\in A\right)   & =\frac{1}{2}P\left(  X\in A|Y=0\right)  +\frac
{1}{2}P\left(  X\in A|Y=1\right) \\
& =\frac{1}{2}\int_{A}\phi_{0}(x)dx+\frac{1}{2}\int_{A}\phi_{1}(x)dx=\int
_{A}\phi(x)dx
\end{align*}
so that the density $\phi$ of $X$ is as claimed.
\end{proof}

\bigskip\bigskip Thus the claim (\ref{normal-bayes-rule}) about the Bayes rule
in the normal model
\[
h_{N}^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $x^{\top}a+a_{0}>0$\\
$0$ otherwise
\end{tabular}
\right.
\]
is just a special case of the fact that in the logistic regression model, for
any marginal distribution of $X$, the Bayes rule is
\[
h^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $r(x)>1/2$\\
$0$ otherwise
\end{tabular}
\right.  =\left\{
\begin{tabular}
[c]{l}%
$1$ if $\beta_{0}+\left\langle \beta,x\right\rangle >0$\\
$0$ otherwise
\end{tabular}
\right.  .
\]

\bigskip\bigskip\textbf{Estimation of parameters.} The parameters $\beta
,\beta_{0}$ are estimated from the realized training set $\left(  x_{i}%
,y_{i}\right)  $, $i=1,\ldots,n$ by maximum likelihood, conditionally on
$X_{1},\ldots,X_{n}$. Taking this conditional point of view is logical, since
we left $\mathcal{L}(X)$ unspecified and the parameters of interest concern
only the conditional distribution $\mathcal{L}(Y_{i}|X_{i})$. To obtain the
likelihood function, note that if $Y\sim\mathrm{Bernoulli}\left(  p\right)  $
then the probability function can be expressed (for $y\in\left\{  0,1\right\}
$) as
\[
p^{y}(1-p)^{1-y}%
\]
which takes value $p$ if $y=1$ and value $1-p$ if $y=0$. The log-likelihood
function for $Y_{i}$, $i=1,\ldots,n$ thus is
\begin{align*}
L(\beta_{0},\beta)  & =\log%
%TCIMACRO{\dprod \limits_{i=1}^{n}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{n}}
%EndExpansion
p_{x_{i}}^{y_{i}}\left(  1-p_{x_{i}}\right)  ^{1-y_{i}}\\
& =\sum_{i=1}^{n}y_{i}\log r(x_{i})+\left(  1-y_{i}\right)  \log\left(
1-r(x_{i})\right) \\
& =\sum_{i=1}^{n}y_{i}\log l(\beta_{0}+\left\langle \beta,x\right\rangle
)+\left(  1-y_{i}\right)  \log\left(  1-l(\beta_{0}+\left\langle
\beta,x\right\rangle )\right)
\end{align*}
where $l$ is the logistic function. This can be slightly simplified, in view
of the form of $l.$ The maximization is performed numerically.

\subsubsection{Nonparametric version}

The above approach might be called \textit{linear logistic regression} since
in $r(x)=l(\beta_{0}+\left\langle \beta,x\right\rangle )$, the term $\beta
_{0}+\left\langle \beta,x\right\rangle $ is linear (even though the regression
function $r(x)$ is nonlinear). We may now substitute $\beta_{0}+\left\langle
\beta,x\right\rangle $ by a function $f(x)$ and build a regularized risk
functional, analogously to (\ref{regul-risk})
\[
R_{reg}(f)=R_{emp}^{lik}(f)+\lambda\left\Vert f\right\Vert _{\ast}^{2}%
\]
where
\[
R_{emp}^{lik}(f)=-L(f)=-\left(  \sum_{i=1}^{n}y_{i}\log l(f(x_{i}))+\left(
1-y_{i}\right)  \log\left(  1-l(f(x_{i}))\right)  \right)  .
\]
Indeed taking the \textit{negative} log-likelihood leads to a minimization
problem, and the expression $R_{emp}^{lik}(f)$ is then analogous to the other
risk criteria such as soft margin risk, misclassification risk and also
quadratic risk for regression. The term $\lambda\left\Vert f\right\Vert
_{\ast}^{2}$ is a penalty where $\left\Vert f\right\Vert _{\ast}^{2}$ is a
smoothness measure and $\lambda$ is chosen. In that framework, smoothing
spline theory can now be applied to classification.

It is possible to obtain this framework formally from the feature space idea
and RKHS theory. Recall that in this setting, $\left(  X_{i},Y_{i}\right)  $
is substituted by $\left(  K\left(  \cdot,X_{i}\right)  ,Y_{i}\right)  $ where
$K$ is a kernel. For linear logistic regression, the regression function then
is
\[
r(x)=l\left(  \beta_{0}+\left\langle g,K\left(  \cdot,x\right)  \right\rangle
_{\ast}\right)  =l\left(  \beta_{0}+g(x)\right)
\]
so the classification rule obtained from estimating $f=\beta_{0}+g$ is no
longer linear.

\subsection{Bayesian smoothing}

Let us return to the regression (non-classification) setup discussed in
example 1 above. We are in the general regression situation and minimizing
\[
n^{-1}\sum_{i=1}^{n}\left(  y_{i}-f(x_{i})\right)  ^{2}+\lambda\left\Vert
f^{\prime}\right\Vert _{2}^{2}%
\]
over $f\in$ $H_{0}^{1}(0,1)$. The result was a piecewise linear spline $f$
with $f(0)=0$ and $f(1)=f(x_{n})$. We will show that this estimator of the
regression function $f$ can also be obtained in a \textit{Bayesian setting}
with a certain (Gaussian) prior on the regression function $f.$ We will then
develop the analog in the classification problem, via the logistic regression
setup. This gives rise to the \textit{Gaussian process (GP) classifiers},
closely related to the penalized kernel and RKHS\ methods discussed
previously. GP classifiers are treated in the book [RW]$\footnote{Rasmussen,
C. E. and Wiliams, C.K. I., \textsl{Gaussian Processes for Machine Learning},
MIT Press, 2006.}$

\subsubsection{Regression with Gaussian prior}

Let us specify the regression model as normal: given $X_{i}=x_{i}$, the
$Y_{i}$ are normal $N\left(  f(x_{i}),\sigma^{2}\right)  $ where $\sigma^{2}$
is known. With regard to the $x_{i},$ we will take the conditional point of
view and assume they are nonrandom. Our parameter of interest is the function
$f.$ The model can thus equivalently be written
\begin{equation}
Y_{i}=f(x_{i})+\varepsilon_{i}\label{reg-model-bay-1}%
\end{equation}
where $\varepsilon_{i}$ are i.i.d. $N(0,\sigma^{2})$. It is well known that
when we assume a parameter space $f\in\mathcal{F}$ and consider maximum
likelihood estimation, the maximum likelihood criterion is equivalent to the
least squares criterion. Indeed the likelihood function for $f$ is
\[
\frac{1}{\left(  2\pi\right)  ^{n/2}\sigma^{n}}\exp\left(  -\frac{1}%
{2\sigma^{2}}\sum_{i=1}^{n}\left(  y_{i}-f(x_{i})\right)  ^{2}\right)
\]
and taking the negative log-likelihood, recalling that $\sigma$ is known, we
see that maximizing the likelihood over $f\in\mathcal{F}$ is equivalent to
minimizing
\[
\sum_{i=1}^{n}\left(  y_{i}-f(x_{i})\right)  ^{2}%
\]
over $f\in\mathcal{F}$, i.e. the least squares criterion. Let us now consider
a Bayesian approach to estimation where we place a prior distribution on $f$,
in the following form. Consider the kernel $K(x,y)=\min(x,y)$ and write its
Mercer expansion on $[0,1]$ as
\begin{equation}
K(x,y)=\sum_{j=1}^{\infty}\lambda_{j}\phi_{j}(x)\phi_{j}%
(y)\label{mercer-expan-bayes}%
\end{equation}
for the functions $\phi_{j}(x)=\sqrt{2}\sin\left(  \left(  j-1/2\right)  \pi
x\right)  $, $j=1,2,\ldots$ and $\lambda_{j}=\left(  j-1/2\right)  ^{-2}%
\pi^{-2}$ (see Proposition \ref{prop-mercer-min-kernel}). Assume $f$ is square
integrable on $[0,1]$ and write the expansion of $f$ in terms of the basis
$\left\{  \phi_{j},j=1,2,\ldots\right\}  $ as
\[
f=\sum_{j=1}^{\infty}f_{j}\phi_{j}\text{ where }f_{j}=\left\langle f,\phi
_{j}\right\rangle
\]
(where $\left\langle \cdot,\cdot\right\rangle $ is the scalar product in
$L_{2}[0,1]$). Recall that we considered the function space as
\[
H_{0}^{1}(0,1):=\left\{  f\in L_{2}(0,1):f\text{ abs. continuous,
}f(0)=0,f^{\prime}\in L_{2}(0,1)\right\}
\]
as RKHS\ for $K$, with norm $\left\Vert f\right\Vert _{\ast}^{2}=\left\Vert
f^{\prime}\right\Vert _{2}^{2}$. For any $f\in H_{0}^{1}(0,1)$ this norm is
for $\gamma_{j}=\lambda_{j}^{-1/2}=\left(  j-1/2\right)  \pi$
\[
\left\Vert f^{\prime}\right\Vert _{2}^{2}=\sum_{j=1}^{\infty}\frac{1}%
{\lambda_{j}}f_{j}^{2}=\sum_{j=1}^{\infty}f_{j}^{2}\gamma_{j}^{2}.
\]
\bigskip

\textbf{Bayesian prior distribution on} $f$:%
\begin{equation}
\text{\textit{assume}}\mathit{\ }f\mathit{\ }\text{\textit{is random such
that}}\mathit{\ }f_{j}\mathit{\ }\text{\textit{are independent}}%
\mathit{\ }N\left(  0,\lambda_{j}\right)  .\label{gaussian-prior}%
\end{equation}
$\bigskip$

With this prior, we are assuming that the Fourier coefficients of $f$ are
random, and since $\lambda_{j}\rightarrow0$ as $j\rightarrow\infty$ and small
variance means that the $f_{j}$ is more likely to be close to $0$, this prior
can be interpreted as a \textit{smoothness prior}. However it does not mean
that $f$ is in $H_{0}^{1}(0,1)$ almost surely: $\lambda^{-1/2}f_{j}$ are
i.i.d. standard normal variables, $\xi_{j}$ say, and
\[
\left\Vert f\right\Vert _{\ast}^{2}=\sum_{j=1}^{\infty}\frac{1}{\lambda_{j}%
}f_{j}^{2}=\sum_{j=1}^{\infty}\xi_{j}^{2}=\infty\text{ almost surely.}%
\]

\begin{lemma}
For a sequence $\xi_{j}$ of i.i.d. standard normal random variables,
$\sum_{j=1}^{\infty}\xi_{j}^{2}=\infty$ almost surely.
\end{lemma}

\begin{proof}
Let $S=\sum_{j=1}^{\infty}\xi_{j}^{2}$. Then, for $K>0$ and $m>K$, since
$E\xi_{j}^{2}=1$, and $\mathrm{Var}\left(  \xi_{j}^{2}\right)  =2$,
\begin{align*}
P\left(  S\leq K\right)   & \leq P\left(  \sum_{j=1}^{m}\xi_{j}^{2}\leq
K\right)  =P\left(  \sum_{j=1}^{m}\xi_{j}^{2}-m\leq K-m\right) \\
& \leq P\left(  \left\vert \sum_{j=1}^{m}\left(  \xi_{j}^{2}-E\xi_{j}%
^{2}\right)  \right\vert \geq m-K\right)  \leq\frac{m\mathrm{Var}\left(
\xi_{1}^{2}\right)  }{\left(  m-K\right)  ^{2}}=\frac{2m}{\left(  m-K\right)
^{2}}%
\end{align*}
according to Chebyshev's inequality, where the upper bound can be made
arbitrarily small for fixed $K$, by letting $m\rightarrow\infty$. Thus
$P\left(  S\leq K\right)  =0$ for all $K>0$, hence $P\left(  S=\infty\right)
=1$.
\end{proof}

Now $f$ itself can be represented
\[
f(x)=\sum_{j=1}^{\infty}f_{j}\phi_{j}(x)=\sum_{j=1}^{\infty}\lambda_{j}%
^{1/2}\xi_{j}\phi_{j}(x)
\]
where as above, $\xi_{j}$ are i.i.d. standard normal. If this is a
well-defined random variable for every $x$, i.e. the series converges almost
surely, then $f(x)$ represents a stochastic process. Consider the partial sums
$f_{m}(x)=\sum_{j=1}^{m}\lambda_{j}^{1/2}\xi_{j}\phi_{j}(x)$; this is a
Gaussian stochastic process with
\begin{align*}
Ef_{m}(x)  & =0\text{; }\\
Ef_{m}(x)f_{m}(y)  & =E\left(  \sum_{i=1}^{m}\lambda_{i}^{1/2}\xi_{i}\phi
_{i}(x)\right)  \left(  \sum_{j=1}^{m}\lambda_{j}^{1/2}\xi_{j}\phi
_{j}(y)\right)  =\sum_{j=1}^{m}\lambda_{j}E\xi_{j}^{2}\phi_{j}(x)\phi_{j}(y)\\
& =\sum_{j=1}^{m}\lambda_{j}\phi_{j}(x)\phi_{j}(y)\rightarrow_{m\rightarrow
\infty}K(x,y)=\min\left(  x,y\right)
\end{align*}
in view of the Mercer expansion (\ref{mercer-expan-bayes}). Thus we have
heuristic evidence for:

\begin{proposition}
The Bayesian prior on $f$ given by (\ref{gaussian-prior}) makes $f$ a Gaussian
stochastic process with zero expectation and covariance function
\[
Ef(t)f(u)=\min\left(  t,u\right)
\]
i. e. the Wiener process $W(t)$ (or Brownian motion) on $[0,1]$.
\end{proposition}

(A rigorous proof would argue that the r.v.'s $f_{m}(x)$ converge in quadratic
mean to a r.v. which is finite a.s., then show that this is a Gaussian r.v.
via the characteristic function, then show it jointly for every finite
collection $f_{m}(x_{1}),\ldots,f_{m}(x_{k})$ etc. This would be analogous to
the construction of Brownian motion in [D]\footnote{Durret, R.,
\textsl{Probability: Theory and Examples}, 2nd ed., chap. 7.1})

Recall that the Brownian motion can also be characterized as zero mean
Gaussian with independent increments and $\mathrm{Var}(W(t))=t$. Let us
compute the covariance function of $W(t)$ from there. For $t\leq u$, the r.v.
$W(t)$ and the increment $W(u)-W(t)$ are independent, hence the covariance
function is
\begin{align*}
EW(u)W(t)  & =E\left(  W(t)+W(u)-W(t)\right)  W(t)=EW^{2}(t)+E\left(
W(u)-W(t)\right)  EW(t)\\
& =EW^{2}(t)=\mathrm{Var}(W(t))=t=\min(t,u).
\end{align*}
It is well known that the Brownian motion is a continuous random function with
$W(0)=0$ which is nowhere differentiable. Thus we can say that the prior
distribution we put on $f$ in (\ref{gaussian-prior}) makes $f$ "somewhat"
smooth (at least continuous), but certainly we do not have $f\in H_{0}%
^{1}(0,1)$, as also shown before. It is possible to achieve "genuine" or
higher smoothness with Gaussian priors, e.g. by setting $f(t)\simeq\int
_{0}^{t}W(u)du$ (integrated Brownian motion) or by taking other values
$\lambda_{j}\rightarrow0$ with faster rate.%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
At this point we gave a heuristic construction of BM\ as a limit of scaled
random walk, with nornal increments, on a uniform grid in $[0,1]$ when grid
size $n\rightarrow\infty$. Write this down next time.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\textbf{Bayesian inference with Gaussian process prior.} Our model is the
regression model (\ref{reg-model-bay-1})%
\[
Y_{i}=f(x_{i})+\varepsilon_{i}%
\]
where $x_{i}$ are nonrandom and given $x_{i}$ and $f$, the $Y_{i}$ are
independent $N(f(x_{i}),\sigma^{2})$. In a Bayesian setting, $f$ now has
"become" random. But the distribution of the $Y_{i}$ does not depend on the
function $f$ as a whole, but only on the values $f(x_{1}),\ldots,f(x_{n})$.
Thus effectively we have put a prior on the vector $\bar{f}=$ $\left(
f(x_{1}),\ldots,f(x_{n})\right)  ^{\top}$: it is Gaussian with $0$ expectation
and covariance matrix
\[
Ef(x_{i})f(x_{j})=K\left(  x_{i},x_{j}\right)  =\min\left(  x_{i}%
,x_{j}\right)  \text{, }i,j=1,\ldots,n.
\]
Write%
\[
\bar{K}=\left(  K\left(  x_{i},x_{j}\right)  \right)  _{i,j=1}^{n}\text{,
}y=\left(  y_{1},\ldots,y_{n}\right)  ^{\top}.
\]
We showed before that the matrix $\bar{K}$ is nonsingular if all $x_{i}$ are
different (in connection with (\ref{linear-spline-objective-func})).

Consider the Bayesian formalism: we have a parametric family of densities for
observed $y$: $p(y|\vartheta)$, $y\in\mathbb{R}^{n}$, $\vartheta\in\Theta$
where $\Theta=$ $\mathbb{R}^{n}$. Here $\vartheta=$ $\bar{f}$ and
$p(y|\vartheta)\doteq N_{n}(\hat{f},\sigma^{2}I_{n})$ (here is "$\doteq$"
means "is the density of"). In a Bayesian statistical approach, assume that
$\vartheta$ \textquotedblright becomes random\textquotedblright\ as well.
Suppose we have a prior density $\pi(\vartheta)$; presently $\pi
(\vartheta)\simeq N_{n}(\mathbf{0},\bar{K})$ where $\mathbf{0}$ is the
$0$-vector. The joint density of $y$ and $\vartheta$ is
\[
p(y|\vartheta)\pi(\vartheta)
\]
and the posterior density of $\vartheta$ is
\[
p(\vartheta|y)=\frac{p(y|\vartheta)\pi(\vartheta)}{\int p_{\vartheta}%
(y)\pi(\vartheta)d\vartheta}.
\]
It is well known that if both $p(y|\vartheta)$ and $\pi(\vartheta)$ are
Gaussian densities, then $p(\vartheta|y)$ is also Gaussian. Common Bayesian
estimators are the mode of the posterior density (MAP estimator, maximum a
posteriori) and the posterior expectation, i.e. the expected value of
$p(\vartheta|y)$. Since $p(\vartheta|y)$ is Gaussian, both coincide, and the
posterior expectation is
\begin{align*}
\arg\max_{\vartheta}p(\vartheta|y)  & =\arg\max_{\vartheta}p(y|\vartheta
)\pi(\vartheta)=\arg\max_{\vartheta}\left(  \log p(y|\vartheta)+\log
\pi(\vartheta)\right) \\
& =\arg\min_{\vartheta}\left(  -\log p(y|\vartheta)-\log\pi(\vartheta)\right)
\\
& =\arg\min_{\hat{f}}\left(  \frac{1}{2\sigma^{2}}\left\Vert y-\bar
{f}\right\Vert ^{2}+\frac{1}{2}\bar{f}^{\top}\bar{K}^{-1}\bar{f}.\right)
\end{align*}
This is closely related to the minimization problem for the special linear
smoothing spline obtained in (\ref{targ-func-smoo-spli}), i.e where the target
turned out to be
\[
n^{-1}\left\Vert y-\bar{f}\right\Vert ^{2}+\lambda\bar{f}^{\top}K^{-1}\bar{f}.
\]
For more flexibility, we may also consider a prior on $f$ as a $f_{j}\simeq
N\left(  0,\delta\lambda_{j}\right)  $ in (\ref{gaussian-prior}) where
$\delta>0$.

\begin{theorem}
Suppose that in the regression model
\[
Y_{i}=f(x_{i})+\varepsilon_{i}%
\]
where $\varepsilon_{i}$ are i.i.d. $N(0,\sigma^{2})$ and $x_{i}$ are
nonrandom, we assume a prior on $f$ as $f\simeq\delta^{1/2}W,$ where
$\delta>0$ and $W$ is the standard Brownian motion on $[0,1]$. Then the Bayes
estimator of $\bar{f}=\left(  f(x_{1}),\ldots,f(x_{n})\right)  ^{\top}$
coincides with the values at $x_{i}$ of the minimizer of
\[
n^{-1}\sum_{i=1}^{n}\left(  y_{i}-f(x_{i})\right)  ^{2}+\lambda\left\Vert
f^{\prime}\right\Vert _{2}^{2}%
\]
over $f\in H_{0}^{1}(0,1)$, i.e. the special linear smoothing spline, for
$\lambda=\sigma^{2}/n\delta$.
\end{theorem}

With more reasoning, it can also be shown that for $f$ itself, the Bayes
estimator is the smoothing spline; one then has to take the expectation of the
random pieces of $f$ between the $x_{i}$ conditionally on $f(x_{i})$ (Brownian bridges).

\textbf{Example b: Cubic smoothing spline }

Consider a prior distribution on $f$ as
\[
f\simeq\delta^{1/2}V
\]
where
\[
V(t)=\int_{0}^{t}W(u)du
\]
is the integrated Brownian motion on $[0,1]$. Since $W$ is continuous a.s.,
the integral is well defined and produces a Gaussian process. Let us compute
the covariance function:
\[
EV(s)V(t)=\int_{0}^{s}\int_{0}^{t}EW(u)W(v)dvdu=\int_{0}^{s}\int_{0}^{t}%
\min\left(  u,v\right)  dvdu.
\]
When discussing the kernel $\min\left(  u,v\right)  $ we used the
representation
\[
\min(u,v)=\int_{0}^{1}\mathbf{1}_{[0,u]}(w)\mathbf{1}_{[0,v]}(w)dw.
\]
Plugging in we obtain
\begin{align*}
EV(s)V(t)  & =\int_{0}^{s}\int_{0}^{t}\left(  \int_{0}^{1}\mathbf{1}%
_{[0,u]}(w)\mathbf{1}_{[0,v]}(w)dw\right)  dvdu\\
& =\int_{0}^{1}\left(  \int_{0}^{s}\mathbf{1}_{[0,u]}(w)du\right)  \left(
\int_{0}^{t}\mathbf{1}_{[0,v]}(w)dv\right)  dw\\
& =\int_{0}^{1}\left(  \int_{0}^{s}\mathbf{1}_{\left\{  u\geq w\right\}
}(u)du\right)  \left(  \int_{0}^{t}\mathbf{1}_{\left\{  v\geq w\right\}
}(v)dv\right)  dw\\
& =\int_{0}^{1}\left(  s-w\right)  _{+}\left(  t-w\right)  _{+}dw.
\end{align*}
This is exactly the kernel $K(s,t)$ we found as reproducing for the space
$H_{0}^{2}(0,1)$ when discussing the cubic smoothing spline, cf.
(\ref{integral-repres}):%
\begin{align*}
K\left(  s,t\right)   & =\left\{
\begin{tabular}
[c]{l}%
$k\left(  s,t\right)  $ if $s\leq t$\\
$k\left(  t,s\right)  $ if $s\geq t$%
\end{tabular}
\right. \\
\text{for }k\left(  s,t\right)   & =ts^{2}/2-s^{3}/6.
\end{align*}
Thus we have shown:

\begin{theorem}
Suppose that in the regression model
\[
Y_{i}=f(x_{i})+\varepsilon_{i}%
\]
where $\varepsilon_{i}$ are i.i.d. $N(0,\sigma^{2})$ and $x_{i}$ are
nonrandom, we assume a prior on $f$ as $f\simeq\delta^{1/2}V,$ where
$\delta>0$ and $V(t)=\int_{0}^{t}W(u)du$ is the integrated Wiener process on
$[0,1]$. Then the Bayes estimator of $\bar{f}=\left(  f(x_{1}),\ldots
,f(x_{n})\right)  ^{\top}$ coincides with the values at $x_{i}$ of the
minimizer of
\[
n^{-1}\sum_{i=1}^{n}\left(  y_{i}-f(x_{i})\right)  ^{2}+\lambda\left\Vert
f^{\prime\prime}\right\Vert _{2}^{2}%
\]
over $f\in H_{0}^{2}(0,1)$, i.e. the (special) cubic linear smoothing spline,
for $\lambda=\sigma^{2}/n\delta$.
\end{theorem}

\newpage

\subsubsection{Gaussian process classifiers}

\bigskip\bigskip%
%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
In an introduction, we need to stress the difference between using a Bayesian
estimate of $w$ (and $c$), which will be a hyperplane, for prediction, and
using the predictive distribution for prediction. The latter will have a
nonlinear decision boundary in general (see also HW 3.1, where this point
should also be mentioned). Also, for Bayesian estimate of $w$, distinguish
between MAP estimate and posterior expectation (coincide if the posterior is normal).

\textit{This comment also occurs a fe w lines below, almost identically. }
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

In the classification problem we assume a logistic link function:
\[
P\left(  Y=1|X=x\right)  =l\left(  f(x)\right)
\]
and we put a Gaussian process prior on the function $f$ (also called latent
function). The negative log-likelihood is, for $Y\in\left\{  0,1\right\}  $
\begin{align*}
R_{emp}^{lik}(f)  & =-L(f)=-\left(  \sum_{i=1}^{n}y_{i}\log l(f(x_{i}%
))+\left(  1-y_{i}\right)  \log\left(  1-l(f(x_{i}))\right)  \right)  =\\
& =-\sum_{i=1}^{n}L_{i}(f)\text{ where }L_{i}(f)=\left\{
\begin{tabular}
[c]{l}%
$-\log l(f(x_{i}))$ if $y_{i}=1$\\
$-\log\left(  1-l(f(x_{i}))\right)  $ if $y_{i}=0$%
\end{tabular}
.\right.  .
\end{align*}
Let us give an alternative expression in case we use $\tilde{Y}=2Y-1$ as a
class index. The logistic function $l(t)=\frac{\exp(t)}{1+\exp(t)}$ has the
property that $l(-t)=1-l(t)$. Hence $\log\left(  1-l(f(x_{i}))\right)  =\log
l(-f(x_{i}))$ which is $\log l(\tilde{y}_{i}f(x_{i}))$ if $y_{i}=0$. Thus we
may write
\[
-L(f)=-\sum_{i=1}^{n}L_{i}(f)\text{ where }L_{i}(f)=\left\{
\begin{tabular}
[c]{l}%
$\log l(f(x_{i}))$ if $\tilde{y}_{i}=1$\\
$\log l(-f(x_{i}))$ if $\tilde{y}_{i}=-1$%
\end{tabular}
\right.  =\log l(\tilde{y}_{i}f(x_{i}))
\]
Then the Bayesian negative log-posterior density is (up to a fixed additive
term)
\begin{equation}
-\sum_{i=1}^{n}\log l(\tilde{y}_{i}f(x_{i}))+\frac{1}{2}\bar{f}^{\top}\bar
{K}^{-1}\bar{f}.\label{neg-log-posterior}%
\end{equation}
This is minimized numerically, to obtain Bayesian a estimate of $\bar{f}$. The
classifier then produces $1$ if $l\left(  \bar{f}(x_{i})\right)  >1/2.$

The most commonly used covariance function is the squared exponential: for
$x_{1},x_{2}\in\mathbb{R}^{d}$
\[
\mathrm{cov}\left(  f(x_{1}),f(x_{2})\right)  =K(x_{1},x_{2})=\sigma_{f}%
^{2}\exp\left(  -\frac{1}{2\ell}\left\Vert x_{1}-x_{2}\right\Vert ^{2}\right)
\]
where $\sigma_{f}^{2}$, $\ell$ are hyperparameters of the prior distribution-
they add flexibility to the Bayesian analysis. In Lemma
\ref{lem-gauss-kernel-mercer} we have shown that this kernel is a Mercer
kernel.\bigskip\bigskip

\textbf{Predictions. } Minimization of the expression (\ref{neg-log-posterior}%
) does not give the whole story for Bayesian classification, since it does not
describe how to make a prediction for a test case $x_{\ast}$. From
(\ref{neg-log-posterior}) we only obtain a Bayes (vector) estimate $\hat
{f}=\left(  \hat{f}(x_{1}),\ldots,\hat{f}(x_{n})\right)  ^{\top}$ which
produces classifications of the points $x_{i}$, $i=1,\ldots,n$ of the training
set which can then be compared with the observed $y_{i}$. However the purpose
of classification is to predict $y_{\ast}$ for a "new" test points $x_{\ast}$.
To obtain the full Bayesian treatment of this problem, note that when we put a
Gaussian process prior on $f$, all the values $f(x_{1}),\ldots,f(x_{n}%
),f(x_{\ast})$ have a joint Gaussian distribution, with $0$ expectation and
covariance matrix induced by the kernel $K$. Let again $\bar{f}=\left(
f(x_{1}),\ldots,f(x_{n})\right)  ^{\top}$ and $f_{\ast}=f(x_{\ast})$, recall
that all $x_{i},x_{\ast}$ are nonrandom and let the posterior density (given
observed $y=\left(  y_{1},\ldots,y_{n}\right)  ^{\top}$) of $\bar{f}$ be
\begin{equation}
p\left(  \bar{f}|y\right)  =\frac{p\left(  y|\bar{f}\right)  \pi(\bar{f}%
)}{\int p\left(  y|\bar{f}\right)  \pi(\bar{f})d\bar{f}}\label{formal}%
\end{equation}
where $\pi(\bar{f})$ is the prior density on $\bar{f}$. By taking $-\log
p\left(  \bar{f}|y\right)  $ we obtain (\ref{neg-log-posterior}). Consider now
the conditional density $\pi(f_{\ast}|\bar{f})$ derived from the joint prior
distribution of $\bar{f},f_{\ast}$. Then
\[
p\left(  \bar{f},f_{\ast}|y\right)  =\pi(f_{\ast}|\bar{f})p\left(  \bar
{f}|y\right)
\]
gives the joint posterior density of $\bar{f},f_{\ast}$. \bigskip\bigskip

\begin{proof}
We may carry out the formalism (\ref{formal}) jointly for $\bar{f},f_{\ast}%
,y$: first build a joint density of $\bar{f},f_{\ast},y$ by noting that
$p\left(  y|\bar{f},f_{\ast}\right)  =p\left(  y|\bar{f}\right)  $ and hence
the joint density of $v,f_{\ast},y$ is
\[
p\left(  y|\bar{f},f_{\ast}\right)  \pi\left(  \bar{f},f_{\ast}\right)
=p\left(  y|\bar{f}\right)  \pi\left(  \bar{f},f_{\ast}\right)  =p\left(
y|\bar{f}\right)  \pi(f_{\ast}|\bar{f})\pi(\bar{f}).
\]
Then the joint posterior of $f,f_{\ast}$ is
\begin{align*}
p\left(  \bar{f},f_{\ast}|y\right)   & =\frac{p\left(  y|\bar{f},f_{\ast
}\right)  \pi\left(  \bar{f},f_{\ast}\right)  }{\int\int p\left(  y|\bar
{f},f_{\ast}\right)  \pi\left(  \bar{f},f_{\ast}\right)  df_{\ast}d\bar{f}%
}=\frac{p\left(  y|\bar{f}\right)  \pi(f_{\ast}|\bar{f})\pi(\bar{f})}{\int
p\left(  y|\bar{f}\right)  \left(  \int\pi(f_{\ast}|\bar{f})df_{\ast}\right)
\pi(\bar{f})d\bar{f}}\\
& =\frac{p\left(  y|\bar{f}\right)  \pi(f_{\ast}|\bar{f})\pi(\bar{f})}{\int
p\left(  y|\bar{f}\right)  \pi(\bar{f})d\bar{f}}=p\left(  \bar{f}|y\right)
\pi(f_{\ast}|\bar{f}).
\end{align*}

\end{proof}

\bigskip\bigskip By integrating out $\bar{f}$ we obtain the (marginal)
posterior density of $f_{\ast}$:%
\[
p\left(  f_{\ast}|y\right)  =\int p\left(  \bar{f}|y\right)  \pi(f_{\ast}%
|\bar{f})d\bar{f}.
\]
Now $P\left(  Y_{\ast}=1|f(x_{\ast}),y\right)  =l\left(  f(x_{\ast})\right)
=l\left(  f_{\ast}\right)  $ is a deterministic function of $f_{\ast}$ where
$l$ is the logistic function, so when $f_{\ast}$ is random with its posterior
density $p\left(  f_{\ast}|y\right)  $ then $Y_{\ast}$ is Bernoulli with
expectation
\begin{equation}
P\left(  Y_{\ast}=1|y\right)  =\int l\left(  f_{\ast}\right)  p\left(
f_{\ast}|y\right)  df_{\ast}.\label{predictive-probability}%
\end{equation}
The natural prediction (classification of $x_{\ast}$) then is $h(x_{\ast})=1$
if $P\left(  Y_{\ast}=1|y\right)  \geq1/2.$

The expression (\ref{predictive-probability}) depends on $x_{\ast}$ which was
assumed fixed; for clarity we may write it $P\left(  Y_{\ast}=1|y,x_{\ast
}\right)  $. The book \cite{RW} in sections 3.4, 3.6 describes two different
computational approaches (Laplace approximation, EP algorithm) to obtaining
$P\left(  Y_{\ast}=1|y,x_{\ast}\right)  $ for an arbitrary $x_{\ast}$. It also
describes experiments with the resulting Gaussian classifiers for handwritten
digit recognition, on the US Postal service database of handwritten digits (p
63 ff. in \cite{RW}).

\bigskip\bigskip%
%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
In the following, example, we need to stress the difference between using a
Bayesian estimate of $w$ (and $c$), which will be a hyperplane, for
prediction, and using the predictive distribution for prediction. The latter
wil have a nonlinear decision boundary in general (see also HW 3.1, where this
point should also be mentioned). Also, for Bayesian estimate of $w$,
distinguish between MAP estimate and posterior expectation (coincide if the
posterior is normal).
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\textbf{Simple special case: linear logistic regression, Bayesian.} A Gaussian
process is a random function $f(x)$ such that all values $f(x_{1}%
),\ldots,f(x_{m})$ have a joint normal distribution for arbitrary
$x_{1},\ldots,x_{m}$ (generally in $\mathbb{R}^{d}$). It is possible to give
such a process as a linear function of $x$: set
\[
f(x)=w^{\top}x\text{, }w\sim N_{d}\left(  0,\Sigma\right)
\]
where the $d\times d$ matrix $\Sigma$ is positive definite. In this case our
model is linear logistic regression
\[
P\left(  Y=1|X=x\right)  =l\left(  w^{\top}x\right)
\]
where a normal prior $N_{d}\left(  0,\Sigma\right)  $ is put on the weight
vector $w\in\mathbb{R}^{d}$.

The book \cite{RW} describes an instructive toy example for $d=2$ and a given
training set of 6 points in the plane (3 in each class). The prior on $w$ is
chosen as standard normal $N_{2}\left(  0,I\right)  $. The posterior of $w$ is
shown in the lower left figure (c). The maximum appears around the vector
$(1/2,1/2)^{\top}$ which agrees with the fact that the best classifying
hyperplane through the origin appears to be perpendicular to this vector.
(Note that in this model, the Bayesian classifier hyperplane is forced to go
through the origin since we did not include a constant term in $f$, i.e did
not set $P\left(  Y=1|X=x\right)  =l\left(  w^{\top}x+c\right)  $ ). In this
simple model, we automatically obtain the posterior of $f_{\ast}=f(x_{\ast
})=w^{\top}x_{\ast}$ from the posterior on $w$, i.e. we shortcut the reasoning
leading to (\ref{predictive-probability}), but it nevertheless is a special
case. Then (\ref{predictive-probability}) simplifies to
\begin{equation}
P\left(  Y_{\ast}=1|y,x_{\ast}\right)  =\int l\left(  w^{\top}x_{\ast}\right)
p\left(  w|y\right)  dw.\label{predictive-probability-simplified}%
\end{equation}
As a function of $x_{\ast}$, this is called the predictive distribution; it is
plotted in the lower right figure (d). The contour $P\left(  Y_{\ast
}=1|y,x_{\ast}\right)  =1/2$ is linear and gives the Bayes classifier
hyperplane through the origin.

\bigskip

\textbf{Bayesian logistic regression } (summary of example from book
Rasmussen, Williams, \cite{RW}, p. 39))

$Y\in\left\{  1,-1\right\}  $, $X\in\mathbb{R}^{2}$, linear logistic
regression
\[
P\left(  Y=1|X=x\right)  =l\left(  w^{\top}x\right)
\]

Gaussian process prior on function $f(x)=w^{\top}x$: vector $w$ is normal
$N_{2}\left(  0,\Sigma\right)  $ for $\Sigma=I$

implies bilinear covariance function for Gaussian process $f(x)$: for
$x_{1},x_{2}\in\mathbb{R}^{2}$
\begin{align*}
\mathrm{cov}\left(  f(x_{1}),f(x_{2})\right)   & =K(x_{1},x_{2})=E\left(
w^{\top}x_{1}\right)  \left(  w^{\top}x_{2}\right) \\
& =x_{1}^{\top}\Sigma x_{2}=x_{1}^{\top}x_{2}.
\end{align*}

Panel (a): plot of prior distribution $w\sim N_{2}\left(  0,I\right)  $

Panel (b): training set, 3 points in each class

Panel (c): plot of posterior distribution $w\sim p(w|y)$

Panel (d): predictive distribution $P\left(  Y_{\ast}=1|y,x_{\ast}\right)  $
plotted as a function of $x_{\ast}$

\bigskip

\textbf{Gaussian process classifier} (summary of example from book Rasmussen,
Williams \cite{RW}, p. 61))

$Y\in\left\{  1,-1\right\}  $, $X\in\mathbb{R}^{2}$, logistic link function:
\[
P\left(  Y=1|X=x\right)  =l\left(  f(x)\right)
\]

Gaussian process prior on function $f:$ $\mathbb{R}^{2}\rightarrow\mathbb{R} $

covariance function: squared exponential kernel: for $x_{1},x_{2}\in
\mathbb{R}^{2}$
\[
\mathrm{cov}\left(  f(x_{1}),f(x_{2})\right)  =K(x_{1},x_{2})=\sigma_{f}%
^{2}\exp\left(  -\frac{1}{2\ell}\left\Vert x_{1}-x_{2}\right\Vert ^{2}\right)
\]

$\sigma_{f}^{2}$, $\ell$ hyperparameters

choices $\sigma_{f}^{2}=9$, $\ell$ varying (see below)

Panel (a): training set on $[0,1]^{2},$ 10 points in each class

Panel (b): predictive distribution $P\left(  Y_{\ast}=1|y,x_{\ast}\right)  $
plotted as a function of $x_{\ast},$ for $\ell=0,1$

Panel (c): same for $\ell=0,2$

Panel (d): same for $\ell=0,3$

Interpretation: higher values of $\ell$ mean more correlation (for fixed
$x_{1},x_{2},$ the value $K(x_{1},x_{2})$ increases in $\ell$), hence less
oscillation in $f$ is assumed a priori. Thus less oscillation of $f$ is also
expressed in the posterior distribution for $f,$ and the decision boundary is
less wiggly.

\section{Support vector machines}%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
For a \textit{good account of Lagrange, Kuhn-Tucker etc see also \cite{CST},
chap. 5, "optimization theory", p. 79.}
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

Suppose again that $S=\left\{  \left(  x_{i},y_{i}\right)  ,i=1,\ldots
,n\right\}  $ is a training set where $X_{i}$ are $\mathbb{R}^{d}$-valued and
$Y_{i}\in\{-1,1\}$. Assume the training set is linearly separable, i.e.
separable by a hyperplane. Let a hyperplane in $\mathbb{R}^{d}$ be given by
$\left(  b,b_{0}\right)  $ where $b\in\mathbb{R}^{d}$ and $b_{0}\in\mathbb{R}%
$, by the equation
\begin{equation}
\left\langle b,x\right\rangle +b_{0}=0\text{, }x\in\mathbb{R}^{d}%
\end{equation}
where $\left\langle b,x\right\rangle =b^{\top}x$ is the scalar product. Recall
that for any scalar $\lambda$, the pairs $\left(  b,b_{0}\right)  $ and
$\left(  \lambda b,\lambda b_{0}\right)  $ define the same hyperplane. Under
the assumption $\left\Vert b\right\Vert =1$ the correspondence $\left(
b,b_{0}\right)  $ to hyperplanes is one-to-one.

Recall that a training set $\left(  x_{1},y_{1}\right)  ,\ldots,(x_{n},y_{n})$
is called \textit{linearly separable} if there is a hyperplane given by
$\left(  b,b_{0}\right)  $ such that all $x_{i}$ with $y_{i}=1$ are on one
side of the hyperplane, while all $x_{i}$ with $y_{i}=-1$ are on the other
side (in the strict sense, not \textit{on} the hyperplane). Obviously we may
assume $\left\Vert b\right\Vert =1.$ Thus a training set is linarly separable
if there exists a pair $\left(  b,b_{0}\right)  $ with $\left\Vert
b\right\Vert =1$ defining a hyperplane such that all
\[
\gamma_{i}=y_{i}\left(  \left\langle b,x_{i}\right\rangle +b_{0}\right)
\]
are nonzero and have the same sign. Note that if we find a hyperplane such
that all $\gamma_{i}<0$, we can simply take $\left(  -b,-b_{0}\right)  $, and
for this hyperplane we have all $\gamma_{i}>0$. Then we defined the
\textit{geometric margin} of $\left(  x_{i},y_{i}\right)  $ with respect to
$\left(  b,b_{0}\right)  $ ($\left\Vert b\right\Vert =1$) by $\gamma_{i}$,
assuming all $\gamma_{i}>0$. This is just the distance of a correctly
classified $x_{i}$ to the hyperplane. Then we call $\gamma=\min_{1\leq i\leq
n}\gamma_{i}$ the \textit{geometric margin of the hyperplane} wrt the training
set $S $. \bigskip\bigskip

\textbf{Idea of the support vector classifier:}\textit{\ find a hyperplane
which has maximal geometric margin for a given (linearly separable) training
set }$S$\textit{. For any hyperplane, the points }$x_{i}$\textit{\ which
attain }$\gamma_{i}=\gamma$ , \textit{i.e. which have minimal distance to the
hyperplane } \textit{are called the \textbf{support vectors}. The method to
find a maximal margin hyperplane, to be described, centrally utilizes the
concept of support vectors. \bigskip\bigskip}

Consider any separating hyperplane given by a pair $\left(  b,b_{0}\right)  $
($\left\Vert b\right\Vert =1$). Consider $a=b/\gamma$ and $a_{0}=b_{0}/\gamma
$; the hyperplane given by the pair $\left(  a,a_{0}\right)  $ is the same,
but now $y_{i}\left(  \left\langle a,x_{i}\right\rangle +a_{0}\right)  \geq1$
for all $i.$ Thus we have shown

\begin{lemma}
The data $\left(  x_{1},y_{1}\right)  ,\ldots,(x_{n},y_{n})$ are linearly
separable if and only if there exists a hyperplane $\left\{  x:H(x)=0\right\}
$ where
\[
H(x)=\left\langle a,x_{i}\right\rangle +a_{0}%
\]
such that
\[
y_{i}H(x_{i})\geq1,\text{ }i=1,\ldots,n
\]

\end{lemma}

\begin{proposition}
\label{prop-max-margin-hyppl}The hyperplane
\[
\hat{H}(x)=\left\langle \hat{a},x_{i}\right\rangle +\hat{a}_{0}%
\]
that separates the data and maximizes the margin $\gamma$ is given by solving
the problem
\[
\min\left\Vert a\right\Vert ^{2}\text{ subject to }y_{i}H(x_{i})\geq1\text{,
}i=1,\ldots,n.
\]

\end{proposition}

\begin{proof}
For every separating hyperplane $H(x)=\left\langle b,x_{i}\right\rangle
+b_{0}$ with $\left\Vert b\right\Vert =1$ let $\gamma_{b,b_{0}}$ be its
margin. Let $a=b/\gamma_{b,b_{0}}$ and $a_{0}=b_{0}/\gamma_{b,b_{0}}$; then
the hyperplane given by $H(x)=\left\langle a,x_{i}\right\rangle +a_{0}$ is the
same and fulfills $y_{i}H(x_{i})\geq1$, and $\left\Vert a\right\Vert
=1/\gamma_{b,b_{0}}$. Thus maximizing $\gamma_{b,b_{0}}$ under $\left\Vert
b\right\Vert =1$ is equivalent to minimizing $\left\Vert a\right\Vert $ under
$y_{i}H(x_{i})\geq1$, $i=1,\ldots,n.$
\end{proof}

\bigskip\bigskip The problem of finding a maximal margin hyperplane now turns
out to be a \textbf{convex optimization problem.} In that connection, let us
first discuss the \textit{Kuhn-Tucker theorem} from convex optimization.
Suppose we want to minimize a function $f_{0}$:%
\begin{align*}
f_{0}(x)  & \rightarrow\text{minimum on }x\in A\text{, }A\in\mathbb{R}^{d}\\
\text{subject to constraints }f_{k}(x)  & \leq0\text{, }k=1,\ldots,m.\text{ }%
\end{align*}
where $A$ is a convex set and all $f_{k}$ are convex functions also. Consider
a Lagrange function
\[
L=L(x,\lambda_{0},\lambda)=\sum_{k=0}^{m}\lambda_{k}f_{k}(x)
\]
and denote $\lambda=\left(  \lambda_{1},\ldots,\lambda_{m}\right)  .$

\begin{theorem}
\label{theorK-T}(Kuhn-Tucker) If $x^{\ast}$ minimizes the function $f_{0}(x) $
under constaints $x\in A$ and $f_{k}(x)\leq0$, $k=1,\ldots,m$ then there exist
Lagrange multipliers $\lambda_{0}^{\ast}$ and $\lambda^{\ast}=\left(
\lambda_{1}^{\ast},\ldots,\lambda_{m}^{\ast}\right)  $ (not all $0$) such that
\newline(a) minimum principle:%
\[
\min_{x\in A}L(x,\lambda_{0}^{\ast},\lambda^{\ast})=L(x^{\ast},\lambda
_{0}^{\ast},\lambda^{\ast})
\]
$\newline$(b) Nonnegativity conditions
\[
\lambda_{k}^{\ast}\geq0\text{, }k=0,\ldots,m
\]
\newline(c) Kuhn-Tucker-conditions%
\[
\lambda_{k}^{\ast}f_{k}(x^{\ast})=0\text{, }k=1,\ldots,m.
\]
If $\lambda_{0}^{\ast}\neq0$ then (a), (b), (c) are also sufficient for
$x^{\ast}$ to be the solution of the optimization problem. For $\lambda
_{0}^{\ast}\neq0$ to hold it is sufficient that the \emph{Slater conditions}
are satisfied: there exist $\bar{x}$ such that
\[
f_{k}(\bar{x})<0\text{, }k=1,\ldots,m.
\]

\end{theorem}

\bigskip\bigskip

\begin{corollary}
\label{cor-to-KT}\textbf{a)} If the Slater condition is satisfied, then one
can choose $\lambda_{0}^{\ast}=1$ and rewrite the Langrangian
\[
L(x,1,\lambda)=f_{0}(x)+\sum_{k=1}^{m}\lambda_{k}f_{k}(x)
\]
\textbf{b)} If $\lambda_{0}^{\ast}=1$ then the Kuhn-Tucker theorem implies
that $\left(  x^{\ast},\lambda^{\ast}\right)  $ defines a saddlepoint of the
Lagrangian:
\begin{equation}
\min_{x\in A}L(x,1,\lambda^{\ast})=L(x^{\ast},1,\lambda^{\ast})=\max
_{\lambda\geq0}L(x^{\ast},1,\lambda)\label{two-equalities}%
\end{equation}
(where $\lambda\geq0$ means $\lambda_{1}\geq0,\ldots,\lambda_{n}\geq0$).
\end{corollary}

\begin{proof}
a) If the $\lambda_{0}^{\ast},\lambda_{1}^{\ast},\ldots,\lambda_{m}^{\ast}$
fulfill (a), (b), (c) of the Kuhn-Tucker theorem and $\lambda_{0}^{\ast}\neq
0$, then $1,\lambda_{1}^{\ast}/\lambda_{0}^{\ast},\ldots,\lambda_{m}^{\ast
}/\lambda_{0}^{\ast}$ also fulfill (a), (b), (c), in particular (a) which
concerns minimization in $x$.

b) The first equality in (\ref{two-equalities}) is claim (a) of the
Kuhn-Tucker theorem. For the second equality, note
\begin{align*}
L(x^{\ast},1,\lambda^{\ast})  & =f_{0}(x^{\ast})+\sum_{k=1}^{m}\lambda
_{k}^{\ast}f_{k}(x^{\ast})=f_{0}(x^{\ast})\text{ (by Kuhn-Tucker conditions
(c)}\\
& \geq f_{0}(x^{\ast})+\sum_{k=1}^{m}\lambda_{k}f_{k}(x^{\ast})\text{ }%
\end{align*}
since $\lambda_{k}\geq0$ and $f_{k}(x^{\ast})\leq0$, $k=1,\ldots,m$ ($x^{\ast
}$ maximizes $f_{0}(x)$ under the constraints, hence fulfills the constraints).
\end{proof}

\bigskip\bigskip

\begin{proof}
[A trivial example]\textbf{\ }Consider minimization of the function
$f_{0}(x)=x^{2}$ over $A=\mathbb{R}$ with constraint $f_{1}(x)=\left(
x-a\right)  ^{2}-1\leq0$ where $a\geq0$. By making a picture, we see
immediately: if $a>1$ then the solution is $x^{\ast}=a-1$ and if $0\leq
a\leq1$ then $x^{\ast}=0$. Let us derive that solution from the Kuhn-Tucker theorem.

First, by a basic reasoning we find that a minimizer $x^{\ast}$ under the
constraint $f_{1}(x)\leq0$ must exist. The Slater condition is satisfied: take
$\bar{x}=a$; then $f_{1}(\bar{x})=-1<0$. Hence there exists $\lambda^{\ast}$
such that $\left(  x^{\ast},\lambda^{\ast}\right)  $ satisfy (a), (b), (c)
with $\lambda_{0}^{\ast}=1$. Now (a) means
\[
\min_{x\in\mathbb{R}}\left(  x^{2}+\lambda^{\ast}\left(  \left(  x-a\right)
^{2}-1\right)  \right)  =\left(  x^{\ast2}+\lambda^{\ast}\left(  \left(
x^{\ast}-a\right)  ^{2}-1\right)  \right)  ,
\]
hence by considering the behaviour of the target function at $x\rightarrow
\pm\infty$ and taking a derivative, we see that $x^{\ast}$ fulfills
\begin{equation}
2x^{\ast}+2\lambda^{\ast}\left(  x^{\ast}-a\right)
=0\label{cond-1-fromKT-preced}%
\end{equation}
which implies
\begin{equation}
\lambda^{\ast}=\frac{x^{\ast}}{a-x^{\ast}}.\label{cond-1-fromKT}%
\end{equation}

\textbf{Case 1.} $\lambda^{\ast}>0$. Then by (c) we have $f_{1}(x^{\ast
})=0=\left(  x^{\ast}-a\right)  ^{2}-1$, hence $x^{\ast}=a\pm1$. If $x^{\ast
}=a+1>0$ then by (\ref{cond-1-fromKT}) $\lambda^{\ast}=-1-a<0$ (since $a>0$)
which contradicts $\lambda^{\ast}\geq0$ from (b). Hence we must have $x^{\ast
}=a-1$, and from (\ref{cond-1-fromKT}) we obtain $\lambda^{\ast}=a-1$. By
assumption $\lambda^{\ast}>0$ this implies $a>1$.

\textbf{Case 2.} $\lambda^{\ast}=0$. From (\ref{cond-1-fromKT-preced}) we
obtain $x^{\ast}=0$. Since $x^{\ast}$ fulfills the constraint, we obtain
\[
f_{1}(x^{\ast})=a^{2}-1\leq0
\]
hence $a\leq1$.

We have thus reproduced the solution: if $a>1$ then $x^{\ast}=a-1$ (Case 1)
and if $0\leq a\leq1$ then $x^{\ast}=0$ (Case 2).
\end{proof}

\bigskip\bigskip

Let us return to the problem of finding a maximal margin hyperplane, i.e. to
the problem formulated in Proposition \ref{prop-max-margin-hyppl}. The problem
is to minimize, in $a\in\mathbb{R}^{d}$ and real valued $a_{0}$ the function
\[
f_{0}(a,a_{0})=\frac{1}{2}\left\Vert a\right\Vert ^{2}%
\]
subject to restrictions $y_{i}\left(  \left\langle a,x_{i}\right\rangle
+a_{0}\right)  \geq1$, $i=1,\ldots,n$ . Here $(x_{i},y_{i})$, $i=1,\ldots,n $
is the training set where $x_{i}\in\mathbb{R}^{d}$ and $y_{i}\in\{-1,1\}$. The
restrictions can be written
\[
f_{k}(a,a_{0})=1-y_{k}\left(  \left\langle a,x_{k}\right\rangle +a_{0}\right)
\leq0,k=1,\ldots,n.
\]

\begin{theorem}
\label{theor-SVM-dual-optim-problem-}Let
\[
\hat{H}(x)=\left\langle \hat{a},x\right\rangle +\hat{a}_{0}%
\]
be the optimal (largest margin) hyperplane. Then
\[
\hat{a}=\sum_{i=1}^{n}\hat{\alpha}_{i}y_{i}x_{i}%
\]
and $\hat{\alpha}=\left(  \hat{\alpha}_{1},\ldots,\hat{\alpha}_{n}\right)  $
is the vector that maximizes
\[
\sum_{i=1}^{n}\alpha_{i}-\frac{1}{2}\sum_{i,j=1}^{n}\alpha_{i}\alpha_{j}%
y_{i}y_{j}\left\langle x_{i},x_{j}\right\rangle
\]
subject to
\[
\alpha_{i}\geq0\text{, }i=1,\ldots,n\text{ and }\sum_{i=1}^{n}\alpha_{i}%
y_{i}=0.
\]
The \textbf{support vectors} (i.e. the $x_{i}$ attaining $y_{i}\hat{H}%
(x_{i})=1$) are the points $x_{i}$ where $\hat{\alpha}_{i}\neq0$. The value
$\hat{a}_{0}$ can be found by solving
\[
y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle +\hat{a}_{0}\right)  =1
\]
for any support vector $x_{i}$.
\end{theorem}

\textbf{Comments. }

\begin{enumerate}
\item The $\alpha_{i}$ are Lagrange multipliers, the problem is a dual convex
optimization problem where maximization is in $\alpha$, not the original
$a,a_{0}$ (the parameters of the hyperplane).

\item Moreover, vectors from the training set $x_{i}$ which are known not to
be support vectors can be excluded from the problem (since their $\alpha_{i} $
must be $0$). Thus the dimensionality of the problem can be reduced
considerably, and iterative algorithms can be devised.

\item The vectors $x_{i}$ enter only through their scalar products
$\left\langle x_{i},x_{j}\right\rangle $. This allows computational
simplification and also kernelization.
\end{enumerate}

\textbf{Exercise. }Consider a linearly separable training set in the case
$d=1$. Then the support vector classifier should find the maximal margin
split, i.e. the point where all $x_{i}$ with $y_{i}=1$ are one one side and
all $x_{i}$ with $x_{i}=-1$ are on the other, and which is exactly in the
middle between the two adjacent points (these must be the support vectors).
Specifically, assume $x_{i}\in\mathbb{R}$, $i=1,\ldots,n$ and $y_{i}=-1$,
$i=1,\ldots,m$ while $y_{i}=1$, $i=m+1,\ldots,n$. Then the support vector
classifier should split at the point $\left(  x_{m+1}+x_{m}\right)  /2$, the
maximal geometric margin is $\gamma=\left(  x_{m+1}-x_{m}\right)  /2$, and the
support vectors are $x_{m+1}$, $x_{m}$ (see below for a detailed treatment).
We will treat this in detail in Example 1 below. \bigskip\bigskip

\begin{proof}
[Proof of the theorem]The basis is the Kuhn-Tucker theorem, Theorem
\ref{theorK-T}. The function $f_{0}(a,a_{0})$ is convex in $(a,a_{0}%
)\in\mathbb{R}^{d+1}$ and all functions $f_{k}(a,a_{0}),$ $k=1,\ldots,n$ are
linear in $(a,a_{0})$, hence also convex. The convex set $A$ is taken to be
$\mathbb{R}^{d+1}$. The Slater conditions are satisfied: there exist $(\bar
{a},\bar{a}_{0})$ such that
\[
y_{i}\left(  \left\langle \bar{a},x_{i}\right\rangle +\bar{a}_{0}\right)
>1,\;i=1,\ldots,n
\]
since for any separating hyperplane $(b,b_{0})$ with margin $\gamma$ we can
take $(\bar{a},\bar{a}_{0})=\frac{2}{\gamma}(b,b_{0})$ so that $Y_{i}\left(
\left\langle \bar{a},X_{i}\right\rangle +\bar{a}_{0}\right)  \geq2$. For the
Lagange multipliers in Theorem \ref{theorK-T} we now write $\lambda_{i}%
=\alpha_{i}$, $i=1,\ldots,n$. The Lagrangian then is
\begin{align*}
L\left(  a,a_{0},\alpha\right)   & =f_{0}(a,a_{0})+\sum_{k=1}^{n}\lambda
_{k}f_{k}(a,a_{0})=\\
& =\frac{1}{2}\left\langle a,a\right\rangle -\sum_{i=1}^{n}\alpha_{i}\left(
y_{i}\left(  \left\langle a,x_{i}\right\rangle +a_{0}\right)  -1\right)  .
\end{align*}
Theorem \ref{theorK-T} now asserts that a necessary and sufficient condition
for $\hat{a},\hat{a}_{0}$ to be minimizers of $f_{0}(a,a_{0})$ subject to the
restrictions $f_{k}(a,a_{0})\leq0$ is the existence of $\hat{\alpha}=\left(
\hat{\alpha}_{1},\ldots,\hat{\alpha}_{n}\right)  $ such that conditions (a),
(b) and (c) are fulfilled. It is easy to see that a minimizer $\hat{a},\hat
{a}_{0}$ exists (i.e a maximum margin hyperplane; use standard results about
minimizing a squared norm over a closed convex domain). Hence a corresponding
$\hat{\alpha}=\left(  \hat{\alpha}_{1},\ldots,\hat{\alpha}_{n}\right)  $ does
exist; let us specify what conditions (a), (b) and (c) mean in this case.

Condition (a) of the Kuhn-Tucker theorem (minimum principle) implies that
partial derivatives wrt to $a$ and $a_{0}$ are $0,$ hence%
\[
\frac{\partial}{\partial a}L\left(  a,a_{0},\alpha\right)  |_{\hat{a},\hat
{a}_{0},\hat{\alpha}}=0=\hat{a}-\sum_{i=1}^{n}\hat{\alpha}_{i}y_{i}x_{i}%
\]
hence
\begin{equation}
\hat{a}=\sum_{i=1}^{n}\hat{\alpha}_{i}y_{i}x_{i}.\label{star-1-firstclaim}%
\end{equation}
Furthermore
\begin{equation}
\frac{\partial}{\partial a_{0}}L\left(  a,a_{0},\alpha\right)  |_{\hat{a}%
,\hat{a}_{0},\hat{\alpha}}=0=\sum_{i=1}^{n}\hat{\alpha}_{i}y_{i}\label{star-2}%
\end{equation}
Condition (b) (nonnegativity)\ means
\[
\hat{\alpha}_{i}\geq0\text{, }i=1,\ldots,n
\]
and condition (c) (Kuhn-Tucker-conditions)
\[
\hat{\alpha}_{i}\left(  y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle
+\hat{a}_{0}\right)  -1\right)  =0,\text{ }i=1,\ldots,n
\]
where $\hat{\alpha}_{i}>0$ only for the support vectors $x_{i}$ of the optimal
hyperplane $\hat{a},\hat{a}_{0}$ (i.e. for $x_{i}$ fulfilling $y_{i}\left(
\left\langle \hat{a},x_{i}\right\rangle +\hat{a}_{0}\right)  =1$). Now write
the Lagrangian as
\begin{align*}
L\left(  \hat{a},\hat{a}_{0},\hat{\alpha}\right)   & =\frac{1}{2}\left\langle
\hat{a},\hat{a}\right\rangle -\sum_{i=1}^{n}\hat{\alpha}_{i}\left(
y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle +\hat{a}_{0}\right)
-1\right) \\
& =\frac{1}{2}\left\langle \hat{a},\hat{a}\right\rangle -\sum_{i=1}^{n}%
\hat{\alpha}_{i}y_{i}\left\langle \hat{a},x_{i}\right\rangle -\hat{a}_{0}%
\sum_{i=1}^{n}\hat{\alpha}_{i}y_{i}+\sum_{i=1}^{n}\hat{\alpha}_{i}.
\end{align*}
The third term on the right vanishes in view of (\ref{star-2}). Using
(\ref{star-1-firstclaim}) to expand $\hat{a}$ in the first and second terms
yields
\[
L\left(  \hat{a},\hat{a}_{0},\hat{\alpha}\right)  =
\]%
\begin{align*}
& =\frac{1}{2}\sum_{i,j=1}^{n}\hat{\alpha}_{i}\hat{\alpha}_{j}y_{i}%
y_{j}\left\langle x_{i},x_{j}\right\rangle -\sum_{i,j=1}^{n}\hat{\alpha}%
_{i}\hat{\alpha}_{j}y_{i}y_{j}\left\langle x_{i},x_{j}\right\rangle
+\sum_{i=1}^{n}\hat{\alpha}_{i}\\
& =\sum_{i=1}^{n}\hat{\alpha}_{i}-\frac{1}{2}\sum_{i,j=1}^{n}\hat{\alpha}%
_{i}\hat{\alpha}_{j}y_{i}y_{j}\left\langle x_{i},x_{j}\right\rangle
=:W(\hat{\alpha})
\end{align*}
i. e. we denote $W(\alpha)$ the resulting functional of $\alpha$. We claim
\begin{equation}
W(\hat{\alpha})\geq W(\alpha)\text{ for all }\alpha\text{ satisfying }%
\sum_{i=1}^{n}\alpha_{i}y_{i}=0\text{ and }\alpha_{i}\geq0\text{, }%
i=1,\ldots,n.\text{ }\label{maxim-alpha}%
\end{equation}
Indeed, according to Corollary \ref{cor-to-KT} to the Kuhn-Tucker theorem
about the saddlepoint of the Lagrange function we have
\[
W(\hat{\alpha})=L\left(  \hat{a},\hat{a}_{0},\hat{\alpha}\right)  \geq
L\left(  \hat{a},\hat{a}_{0},\alpha\right)
\]
for all $\alpha$ with nonnegative components. Define a function of
$a\in\mathbb{R}^{n}$ and $\alpha\in\mathbb{R}^{n}$
\[
L_{0}\left(  a,\alpha\right)  :=\frac{1}{2}\left\langle a,a\right\rangle
-\sum_{i=1}^{n}\alpha_{i}y_{i}\left\langle a,x_{i}\right\rangle +\sum
_{i=1}^{n}\alpha_{i}.
\]
For all $\alpha$ satisfying $\sum_{i=1}^{n}\alpha_{i}y_{i}=0$ we have
\[
L\left(  \hat{a},\hat{a}_{0},\alpha\right)  =\frac{1}{2}\left\langle \hat
{a},\hat{a}\right\rangle -\sum_{i=1}^{n}\alpha_{i}y_{i}\left\langle \hat
{a},x_{i}\right\rangle +\sum_{i=1}^{n}\alpha_{i}=L_{0}\left(  \hat{a}%
,\alpha\right)
\]
Let $\tilde{a}_{\alpha}$ be the minimizer of $L_{0}\left(  a,\alpha\right)  $
in $a$ for given $\alpha$; since $L_{0}\left(  a,\alpha\right)  $ is quadratic
in $a,$ we obtain by taking a derivative, analogously to
(\ref{star-1-firstclaim}),
\[
\tilde{a}_{\alpha}=\sum_{i=1}^{n}\alpha_{i}y_{i}x_{i}.
\]
Hence
\[
L_{0}\left(  \hat{a},\alpha\right)  \geq L_{0}\left(  \tilde{a}_{\alpha
},\alpha\right)  =W(\alpha)
\]
so that (\ref{maxim-alpha}) is proved.
\end{proof}

\bigskip\bigskip

\textbf{Example 1.} \textbf{\ }Consider a linearly separable training set in
the case $d=1$. Then the support vector classifier should find the maximal
margin split, i.e. the point where all $x_{i}$ with $y_{i}=1$ are one one side
and all $x_{i}$ with $x_{i}=-1$ are on the other, and which is exactly in the
middle between the two adjacent points (these must be the support vectors).
Specifically, assume that the real numbers $x_{i}$ are ordered: $x_{1}<$
$\ldots<x_{n}$ and $y_{i}=-1$, $i=1,\ldots,m$ while $y_{i}=1$, $i=m+1,\ldots
,n$. Then the support vector classifier should split at the point $\left(
x_{m+1}+x_{m}\right)  /2$, the maximal geometric margin is $\gamma=\left(
x_{m+1}-x_{m}\right)  /2$, and the support vectors are $x_{m+1} $, $x_{m}$.
Formally we may write that the best "separating hyperplane" with a "unit
vector" $b=1$ fulfills%
\begin{equation}
y_{i}\left(  bx_{i}+b_{0}\right)  \geq\gamma\label{hyppl-simpl-case-1}%
\end{equation}
Indeed for $b_{0}=-\left(  x_{m+1}+x_{m}\right)  /2$ we have
\begin{align*}
y_{m}\left(  bx_{m}+b_{0}\right)   & =(-1)\left(  x_{m}-\left(  x_{m+1}%
+x_{m}\right)  /2\right) \\
& =(-1)\left(  \left(  x_{m}-x_{m+1}\right)  /2\right)  =\gamma
\end{align*}
and similarly
\begin{align*}
y_{m+1}\left(  bx_{m+1}+b_{0}\right)   & =(+1)\left(  x_{m+1}-\left(
x_{m+1}+x_{m}\right)  /2\right) \\
& =\left(  x_{m+1}-x_{m}\right)  /2=\gamma
\end{align*}
and for all other $x_{i}$ we clearly have $y_{i}\left(  bx_{i}+b_{0}\right)
>\gamma$. Thus indeed $x_{m+1}$, $x_{m}$ are the support vectors.
Reformulating (\ref{hyppl-simpl-case-1}) using $a=b/\gamma$, $a_{0}%
=b_{0}/\gamma$ such that
\[
y_{i}\left(  ax_{i}+a_{0}\right)  \geq1
\]
we obtain
\begin{equation}
a=\frac{1}{\gamma}=\frac{2}{x_{m+1}-x_{m}},\;a_{0}=\frac{b_{0}}{\gamma}%
=-\frac{x_{m+1}+x_{m}}{x_{m+1}-x_{m}}.\label{obvious-sol}%
\end{equation}
Let us check whether these values are also obtained as solutions of the dual
optimization problem for SVM\ of Theorem \ref{theor-SVM-dual-optim-problem-}.
In the notation, we omit the hat from the solutions $\hat{a},\hat{\alpha}_{i}%
$. First, the optimal $a$ is $a=\sum_{i=1}^{n}\alpha_{i}y_{i}x_{i}$ for the
solutions $\alpha_{i}$. Since only $x_{m},x_{m+1}$ are support vectors, only
$\alpha_{m},\alpha_{m+1}$ are nonzero. Moreover, the optimal $\alpha_{i}$
fulfill $\sum_{i=1}^{n}\alpha_{i}y_{i}=0$ hence $-\alpha_{m}+\alpha_{m+1}=0$,
which implies $\alpha_{m+1}=\alpha_{m}$. Let us denote $\alpha$ the common
value of $\alpha_{m+1},\alpha_{m}$. Then
\begin{equation}
a=\alpha\left(  x_{m+1}-x_{m}\right)  .\label{a-in-termr-of}%
\end{equation}
Let us find $\alpha$ from maximizing%
\begin{equation}
\sum_{i=1}^{n}\alpha_{i}-\frac{1}{2}\sum_{i,j=1}^{n}\alpha_{i}\alpha_{j}%
y_{i}y_{j}\left\langle x_{i},x_{j}\right\rangle \text{, subject to }\alpha
_{i}\geq0\text{, }i=1,\ldots,n\text{ and }\sum_{i=1}^{n}\alpha_{i}%
y_{i}=0.\label{maxim-2}%
\end{equation}
This now becomes
\begin{align*}
2\alpha-\frac{1}{2}\alpha^{2}\sum_{i,j=m}^{m+1}y_{i}y_{j}x_{i}x_{j}  &
=2\alpha-\frac{1}{2}\alpha^{2}\left(  x_{m+1}-x_{m}\right)  ^{2}%
=2\alpha-2\alpha^{2}\gamma^{2}\\
\text{subject to }\alpha & \geq0.
\end{align*}
The maximizing $\alpha$ without restrictions is given by taking a derivative:
\[
2-4\alpha\gamma^{2}=0\text{ hence }\alpha=\frac{1}{2\gamma^{2}}.
\]
Since this solution fulfills $\alpha\geq0$, it is the solution also for the
restricted problem. Now plugging this $\alpha$ into (\ref{a-in-termr-of})
yields
\[
a=\alpha\left(  x_{m+1}-x_{m}\right)  =\frac{\left(  x_{m+1}-x_{m}\right)
}{2\gamma^{2}}=\frac{\gamma}{\gamma^{2}}=\frac{1}{\gamma}%
\]
i.e. (\ref{obvious-sol}) for $a$ is obtained. For $a_{0}$ we may solve
$y_{i}\left(  \left\langle a,x_{i}\right\rangle +a_{0}\right)  =1$ for any
support vector $x_{i}$. Taking $i=m+1$ we obtain
\begin{align*}
x_{m+1}/\gamma+a_{0}  & =1,\\
a_{0}  & =1-\frac{x_{m+1}}{\gamma}=\frac{\gamma-x_{m+1}}{\gamma}%
=\frac{-x_{m+1}/2-x_{m}/2}{\gamma}\\
& =-\frac{x_{m+1}+x_{m}}{x_{m+1}-x_{m}}%
\end{align*}
i.e. we have reproduced (\ref{obvious-sol}) for $a_{0}.\bigskip\bigskip$

\textbf{Example 2.} Suppose in the general $\mathbb{R}^{d}$ space we have only
two points: $x_{1}$ with $y_{1}=-1$ and $x_{2}$ with $y_{2}=1$. Then the
optimal $a$ is given by $a=\sum_{i=1}^{n}\alpha_{i}y_{i}x_{i}=\alpha_{2}%
x_{2}-\alpha_{1}x_{1}$. Since again $\sum_{i=1}^{n}\alpha_{i}y_{i}=0,$ we have
$\alpha_{1}=\alpha_{2}=:\alpha$ and $a=\alpha\left(  x_{2}-x_{1}\right)  $,
which means that the maximal margin hyperplane is perpendicular to the vector
$z:=x_{2}-x_{1}$. Similarly to the above reasoning we now have to maximize
(\ref{maxim-2}) which means for $\gamma=\left\Vert z\right\Vert /2$
\[
2\alpha-\frac{1}{2}\alpha^{2}\sum_{i,j=1}^{2}y_{i}y_{j}\left\langle
x_{i},x_{j}\right\rangle =2\alpha-\frac{1}{2}\alpha^{2}\left\Vert z\right\Vert
^{2}=2\alpha-2\alpha^{2}\gamma^{2}.
\]
We obtain $\alpha=1/2\gamma^{2}$ as before, hence $a=\alpha z=z/2\gamma
^{2}=z/\left\Vert z\right\Vert \gamma=2z/\left\Vert z\right\Vert ^{2}$ and
obtain $a_{0}$ from
\begin{align*}
y_{2}\left(  \left\langle a,x_{2}\right\rangle +a_{0}\right)   & =1,\\
a_{0}  & =1-2\left\langle z,x_{2}\right\rangle /\left\Vert z\right\Vert
^{2}=\frac{\left\langle z,z\right\rangle -2\left\langle z,x_{2}\right\rangle
}{\left\Vert z\right\Vert ^{2}}\\
& =-\frac{\left\langle z,x_{2}+x_{1}\right\rangle }{\left\Vert z\right\Vert
^{2}}=-\frac{\left\langle x_{2}-x_{1},x_{2}+x_{1}\right\rangle }{\left\Vert
x_{2}-x_{1}\right\Vert ^{2}}.
\end{align*}

\subsection{The nonseparable case}

Recall that in the separable case, the margin of a separating hyperplane
$\left\langle b,x\right\rangle +b_{0}=0$ with unit vector $b$ ($\left\Vert
b\right\Vert =1$) is
\[
\gamma=\min_{1\leq i\leq n}y_{i}\left(  \left\langle b,x_{i}\right\rangle
+b_{0}\right)
\]
and the optimization problem for the maximum margin hyperplane can be written
\begin{align}
& \max_{b,b_{0},\left\Vert b\right\Vert =1}\gamma\text{ }\nonumber\\
\text{subject to }y_{i}\left(  \left\langle b,x_{i}\right\rangle
+b_{0}\right)   & \geq\gamma\text{, }i=1,\ldots,n.\label{relax}%
\end{align}
We saw (dividing the conditions by $\gamma$ and replacing $b$ by $a=b/\gamma$,
$b_{0}$ by $a_{0}=b_{0}/\gamma$) that this problem is equivalent to
\begin{align*}
& \min_{a,a_{0}}\left\Vert a\right\Vert \text{ }\\
\text{subject to }y_{i}\left(  \left\langle a,x_{i}\right\rangle
+a_{0}\right)   & \geq1\text{, }i=1,\ldots,n.
\end{align*}
Assume that we relax the condition (\ref{relax}) in such a way that for
certain $i$, we allow $_{{}}$%
\[
y_{i}\left(  \left\langle b,x_{i}\right\rangle +b_{0}\right)  \geq\gamma
(1-\xi_{i})
\]
for certain $\xi_{i}\geq0$. In the separable case, and if $\xi_{i}<1$, if that
means that these $x_{i}$ are allowed to be closer to the margin than $\gamma$,
but still on the "right" side of the hyperplane. (In this case $\gamma$ is no
longer the margin, but can be seen as something like a "soft margin". Recall
that $y_{i}\left(  \left\langle b,x_{i}\right\rangle +b_{0}\right)  $ for a
unit vector $b$ is the distance of $x_{i}$ to the hyperplane). We may also
allow $\xi_{i}\geq1,$ thus allowing misclassification. In this way we may be
able to treat the linearly nonseparable case where some misclassification is inevitable.

In this approach it is natural to impose a restriction, in addition to
$\xi_{i}\geq0$
\begin{equation}
\sum_{i=1}^{n}\xi_{i}\leq B\label{budget}%
\end{equation}
which represents a certain "budget" which one may spend in allowing $x_{i}$ to
be closer to the hyperplane or to be misclassified. The new variables $\xi
_{i}$ are called \textit{slack variables. }It will be part of the optimization
problem to determine which $\xi_{i}$\textit{\ }are $0$ and which are $>0$.

In the same way as before we see that the new optimization problem can
equivalently be formulated
\begin{align}
& \min_{a,a_{0}}\text{\ }\left\Vert a\right\Vert ^{2}\text{ }\label{opti-1}\\
\text{subject to }y_{i}\left(  \left\langle a,x_{i}\right\rangle
+a_{0}\right)   & \geq(1-\xi_{i}),\text{\ }i=1,\ldots,n\nonumber\\
\text{ }\xi_{i}  & \geq0,\text{ \ }i=1,\ldots,n\nonumber\\
\sum_{i=1}^{n}\xi_{i}  & \leq B\text{.}\nonumber
\end{align}
Let us consider a related problem: for a $\delta>0$
\begin{align}
& \min_{a,a_{0},\xi}\text{\ }\left(  \frac{1}{2}\left\Vert a\right\Vert
^{2}+\delta\sum_{i=1}^{n}\xi_{i}\right)  \text{ }\label{opti-2}\\
\text{subject to }y_{i}\left(  \left\langle a,x_{i}\right\rangle
+a_{0}\right)   & \geq(1-\xi_{i}),\text{\ }i=1,\ldots,n\nonumber\\
\text{ }\xi_{i}  & \geq0,\text{ \ }i=1,\ldots,n\text{.}\nonumber
\end{align}
Here we omitted the constraint $\sum_{i=1}^{n}\xi_{i}\leq B$, but we added a
"penalty term" to the objective function, which is weighted with a "weight"
$\delta$. On p. 420 (top) of the book Hastie et al. [HTF] it is claimed that
the two previous problems are equivalent in some sense. This can be justified
from the Kuhn-Tucker Theorem. For our optimal hyperplane problem, we note that
the optimization problem (\ref{opti-1}) may not have a solution (i.e. there
are no $a,a_{0},\xi$ satisfying the constraints), if the training set is
"highly nonseparable" in the sense that the given bound $B$ does not suffice
to accomodate all the inevitable misclassifications. On the other hand, the
problem (\ref{opti-2}) always has a solution for any $\delta>0$; because for
"highly nonseparable" data sets, the $\xi_{i}$ are allowed to take large
values as necessary, but large $\xi_{i}$ are penalized by the term in the
objective function $\delta\sum_{i=1}^{n}\xi_{i}$. Thus the optimization
problem (\ref{opti-2}) is more flexible; then $\delta$ is a tuning parameter
which specifies how much we penalize "slack" with regard to the margin or
misclassification. The case $\delta=\infty$ corresponds to the hard margin
case for a separable data set, where we try to prohibit any slack with regard
to the margin and thus find the maximal margin hyperplane. In the initial
problem (\ref{opti-1}) $B$ would be such a tuning parameter which we have to
choose first, but in the nonseparable case $B$ would first have to be large enough.

Let us formulate the analog of the previous result on SVM for the soft margin
case.\newpage

\begin{theorem}
\label{theor-SVM-dual-optim-problem-2}Let $\hat{H}(x)=\hat{a}_{0}+\left\langle
\hat{a},x\right\rangle $ denote the optimal hyperplane in the sense of solving
optimization problem (\ref{opti-2}) for a given tuning parameter $\delta>0$.
Then
\[
\hat{a}=\sum_{i=1}^{n}\hat{\alpha}_{i}y_{i}x_{i}%
\]
(sum of vectors), where $\hat{\alpha}=\left(  \hat{\alpha}_{1},\ldots
,\hat{\alpha}_{n}\right)  $ is the vector that maximizes
\[
\sum_{i=1}^{n}\alpha_{i}-\frac{1}{2}\sum_{i,j=1}^{n}\alpha_{i}\alpha_{j}%
y_{i}y_{j}\left\langle x_{i},x_{j}\right\rangle
\]
subject to
\begin{align*}
0  & \leq\alpha_{i}\leq\delta\text{, }i=1,\ldots,n,\\
\sum_{i=1}^{n}\alpha_{i}y_{i}  & =0.
\end{align*}
The points $x_{i}$ for which $\hat{\alpha}_{i}\neq0$ are the \textbf{support
vectors}. Then the value $\hat{a}_{0}$ is determined from
\[
y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle +\hat{a}_{0}\right)  =1
\]
for any $i$ such that $0<\hat{\alpha}_{i}<\delta$, and the $\hat{\xi}_{i}$ are
determined from
\[
y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle +\hat{a}_{0}\right)
=1-\hat{\xi}_{i}%
\]
for all support vectors, and $\hat{\xi}_{i}=0$ for other values of $i.$
\end{theorem}

\begin{proof}
The basis is again the Kuhn-Tucker theorem, Theorem \ref{theorK-T}. Let
$\xi=(\xi_{1},\ldots,\xi_{n})$; the function $f_{0}(a,a_{0},\xi)$
\[
f_{0}(a,a_{0},\xi)=\frac{1}{2}\left\Vert a\right\Vert ^{2}+\delta\sum
_{i=1}^{n}\xi_{i}%
\]
is convex in $(a,a_{0},\xi)\in\mathbb{R}^{d+1+n}$. Define functions
\begin{align*}
f_{k}(a,a_{0},\xi)  & =\left(  1-\xi_{k}\right)  -y_{k}\left(  \left\langle
a,x_{k}\right\rangle +a_{0}\right)  \text{, }k=1,\ldots,n,\\
f_{n+k}(a,a_{0},\xi)  & =-\xi_{k}\text{, }k=1,\ldots,n\text{;}%
\end{align*}
then all functions $f_{k}(a,a_{0},\xi),$ $k=1,\ldots,2n$ are linear in
$(a,a_{0},\xi)$, hence convex. The convex set $A$ is taken to be
$\mathbb{R}^{d+1+n}$.

For the Slater conditions, consider any $(\bar{a},\bar{a}_{0})$ and
\begin{equation}
\mu=\min\left\{  0,y_{i}\left(  \left\langle \bar{a},x_{i}\right\rangle
+\bar{a}_{0}\right)  ,i=1,\ldots,n\right\}
\end{equation}
then $\mu\leq0$ with possibly large $\left\vert \mu\right\vert $ since the
data may be highly nonseparable. Then take $\bar{\xi}_{i}=1-\mu+\varepsilon$,
$\varepsilon>0$, $i=1,\ldots,n$; then $\mu>1-\bar{\xi}_{i} $ and $\bar{\xi
}_{i}>0$. This implies
\begin{align*}
f_{k}(\bar{a},\bar{a}_{0},\bar{\xi})  & =\left(  1-\bar{\xi}_{k}\right)
-y_{k}\left(  \left\langle a,x_{k}\right\rangle +a_{0}\right)  \leq\left(
1-\bar{\xi}_{k}\right)  -\mu<0\text{, }\\
f_{k+n}(\bar{a},\bar{a}_{0},\bar{\xi})  & =-\bar{\xi}_{k}<0\text{, }%
\end{align*}
for $k=1,\ldots,n$, thus the Slater conditions are satisfied.

Introduce Lagrange multipliers $\alpha_{i},\beta_{i}$, $i=1,\ldots,n$ and
write the Lagrangian%
\[
L\left(  a,a_{0},\xi,\alpha,\beta\right)  =
\]%
\begin{equation}
=\frac{1}{2}\left\langle a,a\right\rangle +\delta\sum_{i=1}^{n}\xi_{i}%
-\sum_{i=1}^{n}\alpha_{i}\left(  y_{i}\left(  \left\langle a,x_{i}%
\right\rangle +a_{0}\right)  -\left(  1-\xi_{i}\right)  \right)  -\sum
_{i=1}^{n}\beta_{i}\xi_{i}.\label{Lagrange-1-slack}%
\end{equation}
Let us specify what conditions (a), (b) and (c) of Theorem \ref{theorK-T} mean
in this case.

Condition (a) of the Kuhn-Tucker theorem (minimum principle) implies that
partial derivatives wrt to $a,a_{0}$ and $\xi$ are $0,$ hence%
\[
\frac{\partial}{\partial a}L\left(  a,a_{0},\xi,\alpha,\beta\right)
|_{\hat{a},\hat{a}_{0},\hat{\xi},\hat{\alpha},\hat{\beta}}=0=\hat{a}%
-\sum_{i=1}^{n}\hat{\alpha}_{i}y_{i}x_{i}%
\]
hence
\begin{equation}
\hat{a}=\sum_{i=1}^{n}\hat{\alpha}_{i}y_{i}x_{i}.\label{star-1-firstclaim-a}%
\end{equation}
Furthermore
\begin{equation}
\frac{\partial}{\partial a_{0}}L\left(  a,a_{0},\xi,\alpha,\beta\right)
|_{\hat{a},\hat{a}_{0},\hat{\xi},\hat{\alpha},\hat{\beta}}=0=\sum_{i=1}%
^{n}\hat{\alpha}_{i}y_{i}\label{star-2-a}%
\end{equation}
and
\begin{equation}
\frac{\partial}{\partial\xi_{i}}L\left(  a,a_{0},\xi,\alpha,\beta\right)
|_{\hat{a},\hat{a}_{0},\hat{\xi},\hat{\alpha},\hat{\beta}}=0=\delta-\alpha
_{i}-\beta_{i}=0\text{, }i=1,\ldots,n\label{star-3-a}%
\end{equation}
Condition (b) (nonnegativity)\ means
\[
\hat{\alpha}_{i}\geq0\text{, }\hat{\beta}_{i}\geq0\text{, }i=1,\ldots,n
\]
and condition (c) (Kuhn-Tucker-conditions)
\begin{align}
\hat{\alpha}_{i}\left(  y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle
+\hat{a}_{0}\right)  -\left(  1-\xi_{i}\right)  \right)   &
=0,\label{KT-slack-1}\\
\hat{\beta}_{i}\hat{\xi}_{i}  & =0,\text{ }i=1,\ldots,n\label{KT-slack-2}%
\end{align}
where $\hat{\alpha}_{i}>0$ only for the support vectors. We can eliminate
$\hat{\beta}_{i}$ using (\ref{star-3-a}) and $\hat{\beta}_{i}\geq0$: this
together yields
\begin{equation}
\hat{\alpha}_{i}+\hat{\beta}_{i}=\delta\text{ , }\beta_{i}\geq
0\label{add-alpha-beta}%
\end{equation}
which implies a condition for $\alpha_{i}$
\[
0\leq\hat{\alpha}_{i}\leq\delta\text{.}%
\]
The remainder of the reasoning is as before in the proof of Theorem
\ref{theor-SVM-dual-optim-problem-}. For the Lagrangian at the saddlepoint
$\left(  \hat{a},\hat{a}_{0},\hat{\xi},\hat{\alpha},\hat{\beta}\right)  $ we
find (first rearranging only the terms in (\ref{Lagrange-1-slack}))
\begin{equation}
L\left(  \hat{a},\hat{a}_{0},\hat{\xi},\hat{\alpha},\hat{\beta}\right)
=\frac{1}{2}\left\langle \hat{a},\hat{a}\right\rangle -\sum_{i=1}^{n}%
\hat{\alpha}_{i}\left(  y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle
+\hat{a}_{0}\right)  -1\right)  +\delta\sum_{i=1}^{n}\hat{\xi}_{i}-\sum
_{i=1}^{n}\hat{\alpha}_{i}\hat{\xi}_{i}-\sum_{i=1}^{n}\hat{\beta}_{i}\hat{\xi
}_{i}.\label{elim-1}%
\end{equation}
To simplify that, note relation (\ref{add-alpha-beta}) which eliminates all
terms containing $\hat{\xi}_{i}$. Condition (\ref{star-2-a}) eliminates
$\hat{a}_{0}$, so that
\begin{equation}
L\left(  \hat{a},\hat{a}_{0},\hat{\xi},\hat{\alpha},\hat{\beta}\right)
=\frac{1}{2}\left\langle \hat{a},\hat{a}\right\rangle -\sum_{i=1}^{n}%
\hat{\alpha}_{i}y_{i}\left\langle \hat{a},x_{i}\right\rangle +\sum_{i=1}%
^{n}\hat{\alpha}_{i}.\label{elim-2}%
\end{equation}
Substituting $\hat{a}$ according to (\ref{star-1-firstclaim-a}) yields
\[
L\left(  \hat{a},\hat{a}_{0},\hat{\xi},\hat{\alpha},\hat{\beta}\right)
=\sum_{i=1}^{n}\hat{\alpha}_{i}-\frac{1}{2}\sum_{i,j=1}^{n}\hat{\alpha}%
_{i}\hat{\alpha}_{j}y_{i}y_{j}\left\langle x_{i},x_{j}\right\rangle
=:W(\hat{\alpha})
\]
i. e. we denote $W(\alpha)$ the resulting functional of $\alpha$. We claim
\begin{equation}
W(\hat{\alpha})\geq W(\alpha)\text{ for all }\alpha\text{ satisfying }%
\sum_{i=1}^{n}\alpha_{i}y_{i}=0\text{ and }0\leq\alpha_{i}\leq\delta\text{,
}i=1,\ldots,n.\text{ }\label{maxim-alpha-2}%
\end{equation}
Indeed, according to Corollary \ref{cor-to-KT} to the Kuhn-Tucker theorem
about the saddlepoint of the Lagrange function we have
\[
W(\hat{\alpha})=L\left(  \hat{a},\hat{a}_{0},\hat{\xi},\hat{\alpha},\hat
{\beta}\right)  \geq L\left(  \hat{a},\hat{a}_{0},\hat{\xi},\alpha
,\beta\right)
\]
for all $\alpha,\beta$ with nonnegative components; in particular for those
satisfying additionally $\alpha_{i}+\beta_{i}=\delta,$ $\sum_{i=1}^{n}%
\alpha_{i}y_{i}=0$. For these we have (eliminating all terms containing
$\hat{a}_{0}$ and $\hat{\xi}_{i}$, as we did in (\ref{elim-1} obtaining
(\ref{elim-2}))
\[
L\left(  \hat{a},\hat{a}_{0},\hat{\xi},\alpha,\beta\right)  =\frac{1}%
{2}\left\langle \hat{a},\hat{a}\right\rangle -\sum_{i=1}^{n}\alpha_{i}%
y_{i}\left\langle \hat{a},x_{i}\right\rangle +\sum_{i=1}^{n}\alpha_{i}.
\]
Define a function of $a\in\mathbb{R}^{n}$ and $\alpha\in\mathbb{R}^{n}$
\[
L_{0}\left(  a,\alpha\right)  :=\frac{1}{2}\left\langle a,a\right\rangle
-\sum_{i=1}^{n}\alpha_{i}y_{i}\left\langle a,x_{i}\right\rangle +\sum
_{i=1}^{n}\alpha_{i}.
\]
For all $\alpha,\beta$ satisfying $\sum_{i=1}^{n}\alpha_{i}y_{i}=0$ and
$\alpha_{i}+\beta_{i}=\delta$ we have
\[
L\left(  \hat{a},\hat{a}_{0},\hat{\xi},\alpha,\beta\right)  =L_{0}\left(
\hat{a},\alpha\right)  .
\]
Let $\tilde{a}_{\alpha}$ be the minimizer of $L_{0}\left(  a,\alpha\right)  $
in $a$ for given $\alpha$; since $L_{0}\left(  a,\alpha\right)  $ is quadratic
in $a,$ we obtain by taking a derivative, analogously to
(\ref{star-1-firstclaim-a}),
\[
\tilde{a}_{\alpha}=\sum_{i=1}^{n}\alpha_{i}y_{i}x_{i}.
\]
Hence
\[
L_{0}\left(  \hat{a},\alpha\right)  \geq L_{0}\left(  \tilde{a}_{\alpha
},\alpha\right)  =W(\alpha)
\]
so that (\ref{maxim-alpha-2}) is proved.

If $0<\hat{\alpha}_{i}<1$ then by (\ref{add-alpha-beta}) we have $\hat{\beta
}_{i}>0$, hence by the Kuhn-Tucker condition (\ref{KT-slack-2}) $\hat{\xi}%
_{i}=0$. Also in this case, by (\ref{KT-slack-1})
\[
\left(  y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle +\hat{a}%
_{0}\right)  -\left(  1-\xi_{i}\right)  \right)  =0
\]
hence
\[
y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle +\hat{a}_{0}\right)  -1=0
\]
which determines $\hat{a}_{0}$. To determine $\hat{\xi}_{i}$, for the
non-support vectors ($\hat{\alpha}_{i}=0$) we have $\hat{\beta}_{i}=1$ and
hence by (\ref{KT-slack-2}) $\hat{\xi}_{i}=0$. Also for $0<\hat{\alpha}_{i}<1
$ we have $\hat{\xi}_{i}=0$, as argued above. For $\hat{\alpha}_{i}=1$ we have
by (\ref{KT-slack-1})
\[
y_{i}\left(  \left\langle \hat{a},x_{i}\right\rangle +\hat{a}_{0}\right)
-\left(  1-\hat{\xi}_{i}\right)  =0
\]
which determines $\hat{\xi}_{i}$.
\end{proof}

\bigskip\bigskip On p. 422 of [HTF] a simulation example for various values of
$\gamma$ is discussed.

Note that support vectors in this case are all the points on the wrong side of
their margin or on their margin (as opposed to the hard margin case, where
they are exactly \textit{on }their margin). More specifically, if
$0<\hat{\alpha}_{i}<\delta$ then $\hat{\xi}_{i}=0,$ and $x_{i}$ is exactly
\textit{on the }(soft) margin. If $\hat{\alpha}_{i}=\delta$ then $\hat{\xi
}_{i}>0$ and $x_{i}$ is \textit{inside the }(soft) margin. If $\hat{\xi}_{i}>1
$ then $x_{i}$ is misclassified; all these $x_{i}$ are support vectors.

\subsection{Kernelization}

Kernelization is now straightforward: for a positive definite kernel
$K(\cdot,\cdot)$ and RKHS of functions with scalar product we may now replace
$\left\langle x_{i},x_{j}\right\rangle $ in the dual optimization problem of
Theorem \ref{theor-SVM-dual-optim-problem-2} by $\left\langle K(\cdot
,x_{i}),K(\cdot,x_{j})\right\rangle _{\ast}=K(x_{i},x_{j})$. Before we discuss
that in detail let us give another perspective on the minimization problem
(\ref{opti-2}): write it as
\begin{align*}
& \min_{a,a_{0},\xi}\text{\ }\left(  \frac{1}{2}\left\Vert a\right\Vert
^{2}+\delta\sum_{i=1}^{n}\xi_{i}\right)  \text{ }\\
\text{subject to }\xi_{i}  & \geq1-y_{i}\left(  \left\langle a,x_{i}%
\right\rangle +a_{0}\right)  ,\text{\ }i=1,\ldots,n\\
\text{ }\xi_{i}  & \geq0,\text{ \ }i=1,\ldots,n\text{.}%
\end{align*}
Obviously, for fixed $a,a_{0},$ we try to make $\xi_{i}$ as small as possible
given the constraints. The two constraints may be written as one:
\[
\xi_{i}\geq\max\left\{  0,1-y_{i}\left(  \left\langle a,x_{i}\right\rangle
+a_{0}\right)  \right\}  =\left(  1-y_{i}\left(  \left\langle a,x_{i}%
\right\rangle +a_{0}\right)  \right)  _{+}%
\]
where $a_{+}=\max\left\{  0,a\right\}  $. Clearly for fixed $a,a_{0}$ the
minimizing $\xi_{i}$ attains the bound:
\[
\xi_{i}=\left(  1-y_{i}\left(  \left\langle a,x_{i}\right\rangle
+a_{0}\right)  \right)  _{+}%
\]
so that, by plugging this into the target function, we obtain a problem
\[
\min_{a,a_{0}}\text{\ }\left(  \frac{1}{2}\left\Vert a\right\Vert ^{2}%
+\delta\sum_{i=1}^{n}\left(  1-y_{i}\left(  \left\langle a,x_{i}\right\rangle
+a_{0}\right)  \right)  _{+}\right)  .\text{ }%
\]
Set $f(x)=\left(  \left\langle a,x_{i}\right\rangle +a_{0}\right)  $ and
recall the definition of the \textit{soft margin loss} $l_{sm}:$ (also called
\textit{hinge loss})
\begin{equation}
l_{sm}(y,f(x))=\left(  1-yf(x)\right)  _{+}=\left\{
\begin{tabular}
[c]{l}%
$0$ if $yf(x)>1$\\
$1-yf(x)$ otherwise
\end{tabular}
\right.  .
\end{equation}
If $yf(x_{i})>0$ then $x_{i}$ is correctly classified, if $yf(x_{i})<0$ then
there is an error. We regard $\left\vert f(x_{i})\right\vert $ as the
"strength" of "confidence" with which the sign of $f(x_{i})$ is predicted. If
there is a wrong classification of $x_{i}$, then the above loss is
$1+\left\vert f(x_{i})\right\vert $, i.e. the loss increases with the
expressed "confidence" for that wrong prediction. If there is correct
classification, $yf(x_{i})>0$, then the loss is set $0$ only if $\left\vert
f(x_{i})\right\vert >1,$ i.e. for a high enough confidence. If $0<\left\vert
f(x_{i})\right\vert <1$ then there is still some penalty, even though $x_{i}$
is correctly classified.

Now our minimization problem may be formulated
\[
\min_{a,a_{0}}\text{\ }\left(  \frac{1}{2}\left\Vert a\right\Vert ^{2}%
+\delta\sum_{i=1}^{n}l_{sm}(y_{i},\left\langle a,x_{i}\right\rangle
+a_{0})\right)  .\text{ }%
\]
or equivalently, setting $\lambda=1/2n\delta$%
\[
\min_{a,a_{0}}\text{\ }\left(  n^{-1}\sum_{i=1}^{n}l_{sm}(y_{i},\left\langle
a,x_{i}\right\rangle +a_{0})+\lambda\left\langle a,a\right\rangle \right)
.\text{ }%
\]
In kernelization, we replace the scalar product $\left\langle \cdot
,\cdot\right\rangle $ by $\left\langle \cdot,\cdot\right\rangle _{\ast}$, the
vector $x_{j}$ is replaced $K(\cdot,x_{j})$ and the linear coefficient vector
$a$ is replaced by a function $f$ in the RKHS. Then the target function
becomes
\[
n^{-1}\sum_{i=1}^{n}l_{sm}(y_{i},\left\langle f,K(\cdot,x_{i})\right\rangle
_{\ast}+a_{0})+\lambda\left\langle f,f\right\rangle _{\ast}=
\]%
\begin{equation}
=n^{-1}\sum_{i=1}^{n}l_{sm}(y_{i},f(x_{i})+a_{0})+\lambda\left\langle
f,f\right\rangle _{\ast}\label{minimizing}%
\end{equation}
and minimization is over $f,a_{0}$. By the representer theorem Theorem
(\ref{theor-genlzd-representer}) each minimizer $f$ of the regularized risk
functional (\ref{regul-risk}) admits a representation of the form
\[
f(x)=\sum_{i=1}^{n}\alpha_{i}K\left(  x,x_{i}\right)  .
\]
where $\alpha_{i}\in\mathbb{R}$. We see that the problem coincides with the
\textit{regularization and spline smoothing problems} discussed before, where
$\lambda\left\langle f,f\right\rangle _{\ast}$ may be seen as a penalty term.

Recall the least squares analog for regression; there is an explicit solution:
the linear smoothing spline or regularizer. For soft margin loss, we did not
describe a solution in the section about regularization; we have now obtained
one in the form of the SVM, i.e. by the dual optimization program.

\subsection{Support vector regression}

The standard regression analog, using squared error loss, of the problem of
minimizing (\ref{minimizing}) over $f,a_{0}$ would be to minimize%
\[
n^{-1}\sum_{i=1}^{n}(y_{i}-f(x_{i}))^{2}+\lambda\left\langle f,f\right\rangle
_{\ast}%
\]
over $f$. Consider an alternative to the squared errror loss, the so-called
$\varepsilon$-insensitive loss function:
\[
l_{\varepsilon}\left(  y,f(x)\right)  :=\left(  \left\vert y-f(x)\right\vert
-\varepsilon\right)  _{+}.
\]
This loss is $0$ when ever $\left\vert y-f(x)\right\vert $ is below the
threshold $\varepsilon$, and otherwise grows linearly with the distance
$\left\vert y-f(x)\right\vert .$ Thus it shares some properties with the soft
margin loss $l_{sm}(y,f(x))$ for classification where $y\in\left\{
-1,1\right\}  $. In support vector regression, one looks for a regression
function estimator by minimizing%
\[
n^{-1}\sum_{i=1}^{n}l_{\varepsilon}\left(  y_{i},f(x_{i})\right)
+\lambda\left\langle f,f\right\rangle _{\ast}.
\]
The solution can be described in the spirit of the SVM for classification.
Support vectors would be those $x_{i}$ for which $\left\vert y_{i}%
-f(x_{i})\right\vert >\varepsilon$. See chap. 9 of Schoellkopf, Smola.

\subsection{Summary and further developments}

The soft margin loss function
\[
l_{sm}(y,f)=\left(  1-yf\right)  _{+}=\left\{
\begin{tabular}
[c]{l}%
$0$ if $yf>1$\\
$1-yf$ otherwise
\end{tabular}
\right.  .
\]
is also often called a \textit{hinge loss}, due to the shape of its
graph\footnote{hinge: a jointed or flexible device on which a door, lid, or
other swinging part turns} as a function of $yf$. For any function $f(x)$
define the empirical hinge loss
\[
L_{n}(f)=n^{-1}\sum_{i=1}^{n}l_{sm}(y_{i},f(x_{i}))
\]
where in comparison to (\ref{minimizing}) the constant $a_{0}$ has been
absorbed into $f$. Here the function $f:\mathbb{R}^{d}\rightarrow\mathbb{R}$
is considered a classifier; actually $h(x)=\mathrm{sign}\left(  f(x)\right)  $
is the derived classifier with values in $\{-1,1\}$. The SVM is the minimizer
in $f$ over a class of functions $\mathcal{F}$ of
\begin{equation}
L_{n}(f)+\lambda J(f)\label{penalized-sm-loss}%
\end{equation}
where $J(f)$ is some penalty functional. A motivation for using the hinge loss
rather than the misclassification loss
\[
l_{0}(y,f)=\mathrm{sign}\left(  yf\right)  =\left\{
\begin{tabular}
[c]{l}%
$0$ if $yf(x)\geq0$\\
$1$ otherwise
\end{tabular}
\right.
\]
is that $l_{0}(y,f)$ poses computational problems for high dimensional data
sets. In contrast, minimizing the penalized empirical soft margin loss
(\ref{penalized-sm-loss}) leads to a computationally tractable problem
(Theorem \ref{theor-SVM-dual-optim-problem-2}). We have the following cases.

\begin{description}
\item[a] Linear SVM\textbf{: }$\mathcal{F}$ is the class of linear functions
$f(x)=\left\langle a,x\right\rangle +a_{0}$ where $a\in\mathbb{R}^{d}$,
$a_{0}\in\mathbb{R}$ and $J(f)=\left\Vert a\right\Vert ^{2}$.

\item[b] kernelized SVM: $\mathcal{F}$ coincides with a RKHS $\mathcal{H}$,
pertaining to kernel $K$, equipped with inner product $\left\langle
f,g\right\rangle _{\ast}$ and $J(f)=\left\langle f,f\right\rangle _{\ast}$. By
the representer theorem, the solution is of form
\[
f_{\mathbf{\alpha}}(x)=\sum_{i=1}^{n}\alpha_{i}K\left(  x,x_{i}\right)
\]
where $\mathbf{\alpha}=(\alpha_{1},\ldots,\alpha_{n})$ and the problem becomes
to minimize
\[
L_{n}(f_{\mathbf{\alpha}})+\lambda\alpha^{\top}\bar{K}\alpha
\]
over $\mathbf{\alpha}$.
\end{description}

For more insight on SVM\ and the hinge loss cf. Burges \cite{Burg}, Blanchard
et al. \cite{blanchard-massart-svm}., Tarigan and van de Geer
\cite{Tarig-van-deGeer}%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
The following is an aborted attempt of discussing the polynomial kernel, see
Schoellkopf, Smola, or French papers if this ever is to be written out.

**************

Consider the special case of b) where $K$ is the polynomial kernel
\[
K(x,y)=\left(  \left\langle x,y\right\rangle +b\right)  ^{r},\text{where
}b\geq0,r=1,2,\ldots
\]
(cp. \ref{kern-poly}). The Mercer expansion for this kernel on a compact
$C\subset\mathbb{R}^{d}$ is finite: for some integer $m$ we have
\[
K(x,y)=\sum_{j=1}^{m}\lambda_{j}\phi_{j}(x)\phi_{j}(y)
\]
where $\phi_{j}(x)$ are the orthonormal (in $L_{2}(C)$) eigenfunctions and
$\lambda_{j}$ are the eigenvalues. Then
\[
\left\langle f,f\right\rangle _{\ast}=\sum_{j=1}^{m}f_{j}^{2}/\lambda
_{j}\text{ where }f_{j}=\left\langle f,\phi_{j}\right\rangle
\]
and $\left\langle \cdot,\cdot\right\rangle $ is the scalar product in
$L_{2}(C)$.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\newpage

\section{Neural Networks}

Suppose again that $X$ is an $\mathbb{R}^{d}$-valued random vector and $Y$ is
a r.v. with two possible values, and $\left(  X,Y\right)  $ have a joint
probability distribution. The values of $Y$ are in $\{0,1\}$. Recall that for
a hyperplane classifier, the decision was according to the sign of $c^{\top
}x+\tilde{c}_{0}$ where $c\in\mathbb{R}^{d}$, $\tilde{c}_{0}\in\mathbb{R}$. We
may write this in the equivalent form
\begin{equation}
\phi(x)=\left\{
\begin{tabular}
[c]{l}%
$0$ if $\psi(x)\leq1/2$\\
$1$ otherwise
\end{tabular}
\right. \label{nn-basic}%
\end{equation}
where $\psi(x)=c^{\top}x+c_{0}$ and $c=\left(  c_{1},\ldots,c_{n}\right)
\in\mathbb{R}^{d}$ and $c_{0}=\tilde{c}_{0}+1/2$ is a scalar. Here $x=\left(
x^{(1)},\ldots,x^{(d)}\right)  ^{\top}\in\mathbb{R}^{d}$ are the inputs. This
decision rule, historically called a perceptron, now defines a neural network
without hidden layers.

In a feed-forward neural network with one hidden layer, one takes
\begin{equation}
\psi(x)=c_{0}+\sum_{i=1}^{k}c_{i}\sigma\left(  \psi_{i}\left(  x\right)
\right)  ,\label{nn-one-hidden-layer}%
\end{equation}
where the $c_{i}$ are as before and each $\psi_{i}$ is of the form
\[
\psi_{i}\left(  x\right)  =a_{i0}+\sum_{j=1}^{d}a_{ij}x^{(j)}%
\]
where $c_{0},c_{i}$, $a_{i0}$, $a_{ij}$, $i=1,\ldots,k$, $j=1,\ldots,d$ are
coefficients, also called \textit{weights}, $\sigma$ is a sigmoid function
(nondecreasing with $\sigma(x)\rightarrow-1$ as $x\rightarrow-\infty$ and
$\sigma(x)\rightarrow1$ as $x\rightarrow\infty$).

\textit{The principle underlying the neural network} thus is: in a linear
classifier (or perceptron) $\psi(x)=c^{\top}x+c_{0}$, replace the components
$x^{(j)}$ by $\sigma\left(  a_{j0}+\left\langle a_{j},x\right\rangle \right)
$. Thus we have introduced a "hidden layer" of additional transformation of
the input $x$. This principle may be iterated, i.e we may again replace $x$ by
another another hidden layer etc. This gives rise to multilayer neural
networks. The basic questions of designing a network then are: how many layers
should be used, and which linear coefficients should be zero.

\textbf{Sigmoid functions.} The most commonly used sigmoid function (also
called \textit{activation function}) is the logistic
\[
\sigma(x)=\frac{1-\exp(-x)}{1+\exp(-x)}.
\]
also called the\textit{\ standard sigmoid.} Training the neural network means
choosing the weights as functions of a sample (the training set) $\left(
x_{i},y_{i}\right)  $, $i=1,\ldots,n$ where $y_{i}\in\left\{  0,1\right\}  $.
Other examples are the \textit{threshold sigmoid}
\[
\sigma(x)=\left\{
\begin{tabular}
[c]{l}%
$-1$ if $x\leq0$\\
$1$ otherwise
\end{tabular}
\right.  ,
\]
the arctan sigmoid
\[
\sigma(x)=\frac{2}{\pi}\arctan(x)
\]
and the gaussian sigmoid
\[
\sigma(x)=2\Phi(x)-1
\]
where $\Phi$ is the standard normal distribution function.

\begin{remark}
(Classification and regression).\emph{\ Training a neural network is very
similar to estimating coefficients in a nonlinear regression model. Suppose we
have an i.i.d. sample }$\left(  X_{i},Y_{i}\right)  $\emph{, }$i=1,\ldots,n
$\emph{\ where }$X_{i}\in$\emph{\ }$\mathbb{R}^{d}$\emph{\ and }$Y_{i}%
$\emph{\ are real valued instead of 0-1 valued. Suppose we are interested in
estimating the regression function }%
\begin{equation}
r(x)=E\left(  Y|X=x\right) \label{reg-func}%
\end{equation}
\emph{and we try to fit a family of functions }$f_{\vartheta}(x)$\emph{,
}$\vartheta\in\Theta$\emph{\ to the observed values }$\left(  X_{i}%
,Y_{i}\right)  $\emph{, }$i=1,\ldots,n$\emph{. Here }$\vartheta$\emph{\ is an
}$s$\emph{-dimensional parameter, thus }$\Theta\subset\mathbb{R}^{s}%
$\emph{\ and }$f_{\vartheta}(x)$\emph{\ may be nonlinear both in }$\vartheta
$\emph{\ and }$x$\emph{. The least squares principle may be used to obtain an
estimate of }$\vartheta$\emph{\ }%
\[
\min_{\vartheta}\sum_{i=1}^{n}\left(  Y_{i}-f_{\vartheta}(X_{i})\right)  ^{2}.
\]
\emph{In the neural network described above in (\ref{nn-one-hidden-layer}), we
would have to set }%
\[
\vartheta=\left(  c_{0},c_{i},a_{i0},a_{ij},i=1,\ldots,k,j=1,\ldots,d\right)
\]
\emph{thus }$s=k(d+2)+1$\emph{\ and }%
\[
f_{\vartheta}(x)=c_{0}+\sum_{i=1}^{k}c_{i}\sigma\left(  a_{i0}+\left\langle
a_{i},x\right\rangle \right)
\]
\emph{for }$a_{i}=\left(  a_{i1},\ldots,a_{id}\right)  ^{T}$\emph{. In
\ regression the main purpose may be estimation of }$r(x)$\emph{, but it may
also be prediction of }$Y$\emph{\ (for a "new" }$X$\emph{\ to be observed). In
classification the goal is prediction of }$Y\in\left\{  0,1\right\}  $\emph{.
We already observed several times that classification and prediction are
closely related. In fact in classification we may also consider the regression
function (\ref{reg-func}) ; then }$r(x)$\emph{\ is the Bernoulli parameter of
the posterior distribution of }$Y$\emph{. Recall that for classification, the
function }%
\[
r(x)=P\left(  Y=1|X=x\right)
\]
\emph{is the basis for the optimal (Bayes) decision: the Bayes decision is}%
\[
h^{\ast}(x)=\left\{
\begin{tabular}
[c]{l}%
$0$\emph{\ if }$r(x)\leq1/2$\\
$1$\emph{\ otherwise}%
\end{tabular}
\ \right.
\]
\emph{\ Thus in a sense, classification also aims to estimate the regression
function (since it always aims to get as close as possible to the Bayes
decision }$h^{\ast}(x)$\emph{). Therefore it may be justified to use the least
squares principle also for classification, and indeed we shall see that it
will play a role in training neural network classifiers.}
\end{remark}

\begin{proof}
[Sigmoids between 0 and 1]If $\sigma$ is a sigmoid as above, it is equivalent
to use
\[
\sigma^{\ast}(x)=\frac{1}{2}\left(  \sigma(x)+1\right)
\]
which is nondecreasing with $\sigma(x)\rightarrow0$ as $x\rightarrow-\infty$
and $\sigma(x)\rightarrow1$ as $x\rightarrow\infty$ (i.e. a probability
distribution function). For the standard sigmoid we obtain
\[
\sigma^{\ast}(x)=\frac{1}{2}\left(  \frac{1-\exp(-x)}{1+\exp(-x)}+1\right)
=\frac{1}{1+\exp(-x)}=\frac{\exp(x)}{1+\exp(x)}%
\]
i.e. the logistic distribution function with density
\[
\frac{1+\exp(-x)}{\left(  1+\exp(-x)\right)  ^{2}}.
\]
Using $\sigma^{\ast}$ and $\sigma$ is equivalent since the linear
transformation involved can be incorporated into the coefficients $c_{0}$,
$c_{i}$ in (\ref{nn-one-hidden-layer}).
\end{proof}

\bigskip

\begin{proof}
[Logistic regression]Note that in the section on logistic regression, we used
notation $l(x)=\exp(x)/\left(  1+\exp(x)\right)  $ for the logistic function;
thus $l(x)=\sigma^{\ast}(x).$ One models the regression function $r(x)$ (for
classification) as
\[
r(x)=l(\beta_{0}+\left\langle \beta,x\right\rangle )
\]
and estimates the parameters $\beta_{0},\beta$ by maximum likelihood. Thus,
for estimated $\hat{\beta}_{0},\hat{\beta}$, the decision for $x$ is
\[
\phi(x)=\left\{
\begin{tabular}
[c]{l}%
$1$ if $\hat{r}(x)\leq1/2$\\
$0$ otherwise
\end{tabular}
\right.
\]
where $\hat{r}(x)=l(\hat{\beta}_{0}+\left\langle \hat{\beta},x\right\rangle
).$ Thus linear logistic regression is formally a special of a neural network
with one hidden layer (\ref{nn-one-hidden-layer}) and only one neuron, that is
with $k=1$. However the power of logistic regression derives from the
nonparametric version where we set $r(x)=l(g(x))$ where $g$ is some function,
which is then estimated using penalization. With neural networks, one adds
complexity in a different way (multilayer networks).
\end{proof}

\bigskip

\textbf{Multilayer networks. }In the neural network with one hidden layer, we
say there are $k$ hidden neurons- the output of the $j$-th hidden neuron is
$u^{(j)}=\sigma(\psi_{j}\left(  x\right)  )$. Thus (\ref{nn-one-hidden-layer})
may be rewritten
\[
\psi(x)=c_{0}+\sum_{j=1}^{k}c_{j}u^{(j)}%
\]
which is similar in form to the simple linear classifier (perceptron). To
obtain e.g. a two-hidden layer network we set
\begin{align}
\psi(x) &  =c_{0}+\left\langle c,z\right\rangle ,\label{nn-two-hid-lay-1}\\
z^{(i)} &  =\sigma\left(  b_{i0}+\left\langle b_{i},u\right\rangle \right)
\text{, }i=1,\ldots,l,\nonumber\\
u^{(j)} &  =\sigma\left(  a_{j0}+\left\langle a_{j},x\right\rangle \right)
\text{, }j=1,\ldots,k\nonumber\\
u &  =(u^{(1)},\ldots,u^{(k)})^{\top},z=(z^{(1)},\ldots,z^{(l)})^{\top
},x=(x^{(1)},\ldots,x^{(d)})^{\top}%
\end{align}
where $c_{0}$, $a_{j0}$, $b_{i0}$ are constants and $a_{j}$ are $d$-vectors,
$b_{i}$ are $k$-vectors and $c$ is a $l$-vector. The first hidden layer has
$l$ hidden neurons while the second hidden layer has $k$ hidden neurons. Here
a \textit{neuron} is an expression of form $\sigma\left(  b+\left\langle
a,u\right\rangle \right)  $ for an input vector $u$ and coefficients $b,a$.

\bigskip

\begin{proof}
[K-class classification]In this case the first layer which consists of one
linear function in (\ref{nn-one-hidden-layer}) or (\ref{nn-two-hid-lay-1}) is
replaced by $K$ linear terms
\[
\psi^{(s)}(x)=c_{0s}+\left\langle c_{s},z\right\rangle \text{, }s=1,\ldots,K
\]
and the classification into one of $K$ classes is obtained in the following
way. Consider the \textit{softmax functions} for argument $t=(t^{(1)}%
,\ldots,t^{(K)})^{\top}$%
\begin{equation}
g_{s}(t)=\frac{\exp(t^{(s)})}{\sum_{l=1}^{K}\exp(t^{(l)})}\text{, }%
s=1,\ldots,K\label{softmax-func}%
\end{equation}
then all values are positive with $\sum_{s=1}^{K}g_{s}(t)=1$; and if we define
a vector valued map $\tilde{\psi}(x)=(\psi^{(1)}(x),\ldots,\psi^{(K)}%
(x))^{\top}$ then the transformation
\[
g_{s}(\tilde{\psi}(x))\text{, }s=1,\ldots,K
\]
is used as\ a basis for classification. The vector $x$ will be classified into
the class $s$ for which $g_{s}(\tilde{\psi}(x))$ is largest.

In this approach, the values $g_{s}(\tilde{\psi}(x))$ are obviously considered
estimates of the posterior probability of class $s$ given $X=x$, in the same
way as $\psi(x)$ in (\ref{nn-basic}) is treated as an estimate of the
posterior probability $r(x)$ $=$ $P\left(  Y=1|X=x\right)  $ (though we have
not required for the neural network that $\psi(x)$ takes values between $0$
and $1$). Consider the case of two classes $K=2$ and rename the classes $0,1$;
then we decide%
\[
\phi(x)=\left\{
\begin{tabular}
[c]{l}%
$0$ if $g_{0}(\tilde{\psi}(x))\leq1/2$\\
$1$ otherwise
\end{tabular}
\ \right.
\]
or equivalently we decide $0$ if
\[
\exp(\psi^{(0)}(x))\leq\frac{1}{2}\left(  \exp(\psi^{(0)}(x))+\exp(\psi
^{(1)}(x))\right)
\]
which means $\exp(\psi^{(0)}(x))\leq\exp(\psi^{(1)}(x))$ and is equivalent to
\[
\psi^{(0)}(x)\leq\psi^{(1)}(x).
\]
Now the difference $\psi^{(0)}(x)-\psi^{(1)}(x)$ is again of form $\psi(x) $
as in (\ref{nn-basic})
\[
\psi(x)=\psi^{(0)}(x)-\psi^{(1)}(x)=c_{00}-c_{01}+\left\langle c_{0}%
-c_{1},z\right\rangle
\]
so the decision is made according to $\psi(x)\leq0$ or $\psi(x)>0$, or by
adding $1/2$ to both sides, is the same as in (\ref{nn-basic}). Thus binary
classification is a special case.
\end{proof}

\subsection{Fitting neural networks}

Consider a single hidden layer neural network which is a classifier for the
K-class problem. We write the neural network to be trained
\begin{align}
z^{(i)} &  =\sigma\left(  a_{i0}+\left\langle a_{i},x\right\rangle \right)
\text{, }i=1,\ldots,m,\label{nn-HTF-notat}\\
t^{(k)} &  =c_{0k}+\left\langle c_{k},z\right\rangle \text{, }k=1,\ldots
,K,\nonumber\\
\psi_{k}(x) &  =g_{k}(t),k=1,\ldots,K\nonumber
\end{align}
where $x=(x^{(1)},\ldots,x^{(d)})^{\top}$, $z=(z^{(1)},\ldots,z^{(m)})^{\top}$
and $t=(t^{(1)},\ldots,t^{(K)})^{\top}$. Here $g_{k}(t)$ is the $k$-th softmax
function (\ref{softmax-func}). Thus we consider a the network which is a
classifier for the K-class problem. Also we consider the simplest case of only
one hidden layer, represented by the $z_{i}$.

The neural network model has unknown parameters, often called weights, and we
seek values for them that make the model fit the training data well. We denote
the complete set of weights by $\theta$, thus
\[
\theta=\left(  a_{10},a_{1},\ldots,a_{m0},a_{m},c_{01},c_{1},\ldots
,c_{0K},c_{K}\right)
\]
which is $m(d+1)+K(m+1)$ dimensional. The components of $\theta$ are often
called "weights"; thus we have $m(d+1)+K(m+1)$ weights in this neural network.
However, implicitly additional constraints on $\theta$ are present,
effectively reducing the dimensionality (i.e. the number of weights, see
Remark \ref{rem-constraints-NN}) below).

For classification, we use sum-of-squared errors as our measure of fit (error
function): for a training set $(x_{i},y_{i})$ $i=1,\ldots,n$ where $x_{i}%
\in\mathbb{R}^{d}$ and $y_{i}=(y_{i}^{(1)},\ldots,y_{i}^{(K)})$ is an
indicator vector (where $y_{i}^{(k)}=1$ if $x_{i}$ is in class $k$, $0$
otherwise)
\begin{equation}
R(\theta)=\sum_{i=1}^{n}\sum_{k=1}^{K}\left(  y_{i}^{(k)}-\psi_{k}%
(x_{i})\right)  ^{2}\label{least-squares-NN}%
\end{equation}
and the corresponding classifier is $G(x)=\arg\max_{k\in\{1,\ldots,K\}}%
\psi_{k}(x)$.

.

The generic approach to minimizing $R(\theta)$ is by gradient descent, called
\textit{back-propagation} in this setting. Because of the compositional form
of the model, the gradient can be easily derived using the chain rule for
differentiation. This can be computed by a forward and backward sweep over the
network, keeping track only of quantities local to each unit.

An alternative measure of fit is \textit{cross-entropy} (deviance):%
\begin{equation}
R(\theta)=-\sum_{i=1}^{n}\sum_{k=1}^{K}y_{i}^{(k)}\log\psi_{k}(x_{i}%
)\label{cross-entropy}%
\end{equation}
With the softmax activation function and the cross-entropy error function, the
neural network model is exactly a linear logistic regression model in the
hidden units, and all the parameters are estimated by maximum likelihood.

\bigskip

\textbf{Multinomial logistic regression. }In linear logistic regression, cp
(\ref{solving}), we set $p_{x}:=P\left(  Y=1|X=x\right)  $ and note that the
joint law of $\left(  X,Y\right)  $ is determined by $p_{x}$ and the marginal
law of $X$, say $\mathcal{L}(X)$. Indeed, given $X=x$, the conditional law of
$Y$ is a Bernoulli distribution with parameter $p_{x}$ and $P\left(
Y=0|X=x\right)  =1-p_{x}$. In logistic regression it is assumed that%
\begin{equation}
p_{x}=l(a_{0}+\left\langle a,x\right\rangle ),\label{ass-logistic}%
\end{equation}
for some $a\in\mathbb{R}^{d}$ and $a_{0}\in\mathbb{R}$, where $l(t)=\exp
(t)/\left(  1+\exp(t)\right)  $ is the logistic function. The marginal
distribution of $X$ is not specified; thus we assume that our training set
$(x_{i},y_{i})$ has been generated from a model%
\begin{equation}
Y_{i}\sim\mathrm{Bernoulli}\left(  l(a_{0}+\left\langle a,x_{i}\right\rangle
)\right) \label{distr-ass-logistic-regr}%
\end{equation}
The parameters $a,a_{0}$ are estimated from the training set, by maximum
likelihood based (\ref{distr-ass-logistic-regr}) with fixed on $x_{1}%
,\ldots,x_{n}$, as $\hat{a},\hat{a}_{0}$, giving for each $x$ an estimated
$\hat{p}_{x}=l(\hat{a}_{0}+\left\langle \hat{a},x\right\rangle )$. This
$\hat{p}_{x}$ provides an obvious classifier $h$: $h(x)=1$ if $\hat{p}%
_{x}>1/2$. Recall that the (unknown) Bayes classifier $h^{\ast}$ for a joint
law of $(X,Y)$ is a function of the (unkown) posterior probability $p_{x}=$
$P\left(  Y=1|X=x\right)  $ and classifies $1$ if $p_{x}>1/2$. The assumption
(\ref{ass-logistic}) is true if the two class-conditional laws of $X$ are
normal:\ the law of $(X,Y)$ is such that $\mathcal{L}(X|Y=i)=N_{d}(\mu
_{i},\Sigma)$, $i=0,1$ and $P(Y=1)=1/2$ (Proposition
\ref{prop-logistic-is true-if-normal}).

Let us consider a generalization of this setup to the $K$-classes case. Note
that $Y\sim$ $\mathrm{Bernoulli}(p)$ holds if an only if the random vector
$\mathbf{Y}=(1-Y,Y)$ has a multinomial distribution
\[
\mathfrak{M}_{2}(1,\mathbf{p})\text{ where }\mathbf{p}=\left(  1-p,p\right)  .
\]
Write the constant $a_{0}$ in (\ref{distr-ass-logistic-regr}) as a difference
$a_{0}=a_{20}-a_{10}$ and similarly for the vector $a$: $a=a_{2}-a_{1}$. Then
the assumption (\ref{ass-logistic}) means for the probability vector
$\mathbf{p}$ and $t^{(1)}=a_{10}+\left\langle a_{1},x\right\rangle ,$
$t^{(2)}=a_{20}+\left\langle a_{2},x\right\rangle $
\begin{align*}
\mathbf{p}  & \mathbf{=}\left(  1-l(t^{(2)}-t^{(1)}),l(t^{(2)}-t^{(1)})\right)
\\
& =\left(  \frac{1}{1+\exp(t^{(2)}-t^{(1)})},\frac{\exp(t^{(2)}-t^{(1)}%
)}{1+\exp(t^{(2)}-t^{(1)})}\right)  =\\
& =\left(  \frac{\exp(t^{(1)})}{1+\exp(t^{(2)}-t^{(1)})},\frac{\exp(t^{(2)}%
)}{1+\exp(t^{(2)}-t^{(1)})}\right) \\
& =\left(  g_{1}(\mathbf{t}),g_{2}(\mathbf{t})\right)
\end{align*}
where $g_{1},g_{2}$ are the softmax functions (\ref{softmax-func}) for $2$
dimensional input $\mathbf{t}=(t^{(1)},t^{(2)})$.

\begin{remark}
\label{rem-constraints-NN}The original logistic regression contained $d+1$
parameters $a_{0}$,$a,$ whereas we are now using a $2(d+1)$-dimensional
parameter $\theta=\left(  a_{10},a_{1},a_{20},a_{2}\right)  $ (related to
$a_{0}$,$a$ by $a_{0}=a_{20}-a_{10},$ $a=a_{2}-a_{1}$). Clearly the model is
overparametrized now, that is $\theta$ is not identifiable (different values
of $\theta$ give rise to the same law of $\mathbf{Y}$, for all possible values
of $x$). To eliminate this nonidentifiability, we have to introduce $d+1$
constraints on $\theta$; an example would be $a_{20}=0,a_{2}=0.$ In general we
write $H(\theta)=0$ for such constraints, where $H$ is a $\mathbb{R}^{d+1}%
$-valued map (linear or nonlinear). Note that such constraints should also be
present on the $c_{0k},c_{k}$, $k=1,\ldots,K$ in the second line of
(\ref{nn-HTF-notat}), cf also below. .
\end{remark}

Thus the natural generalization of (\ref{distr-ass-logistic-regr}) to the
$K$-class case is, for observed indicator vectors $\mathbf{Y}_{i}$
($K$-vectors having a $1$ in one position and a $0$ in all others)
\begin{equation}
\mathbf{Y}_{i}\sim\mathfrak{M}_{K}(1,\mathbf{p}_{x_{i}})\text{ }%
\label{mult-logistic-reg-1}%
\end{equation}
where $\mathbf{p}_{x}$ is a vector valued function of $x\in\mathbb{R}^{d}$,
described as follows. Define the vector valued map $\mathbf{g}(\mathbf{t})$
for $K$- dimensional argument $\mathbf{t}=(t^{(1)},\ldots,t^{(K)})$ as
\[
\mathbf{g}(\mathbf{t})=\left(  g_{1}(\mathbf{t}),\ldots,g_{K}(\mathbf{t}%
)\right)
\]
where $g_{s}(\mathbf{t})$ is the $s$-th softmax function (\ref{softmax-func}).
Consider $a_{k0}$ and $d$-vectors $a_{k}$, $k=1,\ldots,K$ $;$ then the model
assumption on $\mathbf{p}_{x}$ is
\begin{equation}
\mathbf{p}_{x}=\mathbf{g}\left(  (a_{10}+\left\langle a_{1},x\right\rangle
,\ldots,a_{K0}+\left\langle a_{K},x\right\rangle )\right)
.\label{mult-logistic-reg-2}%
\end{equation}
with $d+1$ additional constraints $H(\theta)=0$ (where $\theta=\left(
a_{10},a_{1},\ldots,a_{K0},a_{K}\right)  $), possibly given by $a_{10}%
=0,a_{1}=0$. Together assumptions (\ref{mult-logistic-reg-1}),
(\ref{mult-logistic-reg-2}) constitute the multinomial logistic regression
model. Note that here $\mathbf{Y}_{i}$ are independent multinomial, but not
identically distributed.

As in the univariate model, given a training set $\left(  x_{i},\mathbf{y}%
_{i}\right)  $, $i=1,\ldots,n$, one would estimate the parameters $a_{k0}$,
$a_{k}$, $k=1,\ldots,K$ by maximum likelihood. For shortness write for an
observed $\mathbf{y}=\left(  y^{(1)},\ldots,y^{(K)}\right)  $ and a
probability vector $\mathbf{p=}\left(  p_{1},\ldots,p_{K}\right)  $%
\[
\mathbf{p}^{\mathbf{y}}=%
%TCIMACRO{\dprod \limits_{k=1}^{K}}%
%BeginExpansion
{\displaystyle\prod\limits_{k=1}^{K}}
%EndExpansion
p_{k}^{y^{(k)}},
\]
then the likelihood function (for fixed $x_{i}$) is
\[%
%TCIMACRO{\dprod \limits_{i=1}^{n}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{n}}
%EndExpansion
\mathbf{p}_{x_{i}}^{\mathbf{y}_{i}}.
\]
Thus the negative log-likelihood is
\[
-\sum_{i=1}^{n}\log\mathbf{p}_{x_{i}}^{\mathbf{y}_{i}}=-\sum_{i=1}^{n}%
\sum_{k=1}^{K}y_{i}^{(k)}\log g_{k}\left(  (a_{10}+\left\langle a_{1}%
,x_{i}\right\rangle ,\ldots,a_{K0}+\left\langle a_{K},x_{i}\right\rangle
)\right)  .
\]
Comparing this with (\ref{cross-entropy}) we see that the expressions coincide
if we identify $\psi_{k}(x_{i})$ and \linebreak\ $g_{k}\left(  (a_{10}%
+\left\langle a_{1},x_{i}\right\rangle ,\ldots,a_{K0}+\left\langle a_{K}%
,x_{i}\right\rangle )\right)  $. However in (\ref{cross-entropy}) we actually
have, according to the network structure (\ref{nn-HTF-notat}),
\begin{equation}
\psi_{k}(x)=g_{k}(t)=g_{k}(c_{01}+\left\langle c_{1},z\right\rangle
,\ldots,c_{0K}+\left\langle c_{K},z\right\rangle )\label{mult-logistic-reg-3}%
\end{equation}
where $z=(z^{(1)},\ldots,z^{(m)})$ are the hidden units, so
(\ref{cross-entropy}) is indeed the multinomial logistic neg-loglikelihood in
the hidden units. In the latter, we need $m+1$ additional constraints to make
$c_{01},c_{1},\ldots,c_{0m},c_{m}$ identifiable, such as $c_{01}=0,c_{1}=0$.

\subsection{An example: handwritten ZIP codes}

We describe an experiment with neural networks by Le Cun (1989)\footnote{Le
Cun, Y. (1989). Generalization and Network Design Strategies. Technical Report
CRG-TR-89-4, Department of Computer Science, University of Toronto.}, also
discussed in [HTF], p. 404. A training set of $n=320$ handwritten ZIP-code
digits was considered; each handwritten digit came classified into one of
$K=10$ classes. Thus the training set was $(x_{i},y_{i})$ where $y_{i}%
\in\left\{  0,1,\ldots,9\right\}  $ and $x_{i}\in\mathbb{R}^{d}$.
(Equivalently, $y_{i}$ could be a 10-dimensional indicator vector). Here
$d=256,$ as each handwritten digit $x_{i}$ came as a $16\times16=256$ pixel
image, each pixel having an $8$-bit certain grayscale value (that is a
grayscale value from 0 to 255).

Along with the training set, there was a test set of 160 handwritten data
$(\tilde{x}_{i},\tilde{y}_{i})$, $i=1,\ldots,160$. Five different neural
networks were fit to the data $(x_{i},y_{i})$ (trained on the training set)
and then their preformance on the test set was evaluated. Table 1 (picture) in
column 4 gives the performance (\% of correct recognition) of each of the five
networks on the test set. All 5 networks were fitted to the training set using
the least squares error criterion (\ref{least-squares-NN}). In each case,
complete separation of the classes was achieved on the training set (no
misclassifications). Note that already in Net-1 there are more parameters
(weights) than observations, namely $n=320$ observations and $2570 $
parameters. This explains the error-free performance of all networks on the
training set.

We proceed to describe the 5 models used (Net-1 to Net-5).

\bigskip

\textbf{Net-1.} This is a single layer (no hidden layer) network, that is a
network with no hidden units. Thus for $K=10$
\[
\psi_{k}(x)=g_{k}((a_{10}+\left\langle a_{1},x\right\rangle ,\ldots
,a_{K0}+\left\langle a_{K},x\right\rangle ),k=1,\ldots,K
\]
Here the parameter is
\[
\theta=\left(  a_{10},a_{1},\ldots,a_{K0},a_{K}\right)
\]
which is of dimension $K(d+1)=10\cdot257=2570$ (except for the $d+1=11$
additional constraints on $\theta$). According to (\ref{mult-logistic-reg-1}%
)-(\ref{mult-logistic-reg-3}), this is equivalent ot fitting a multinomial
logistic regression, except for the fact that maximum likelihood is not used
here for training, but rather the least squares criterion. Line 1 of table 1
gives then number of links is 2570 ( each of the $257$ components of the
augmented input $(x,1)$ influences each of the 10 components of $\psi_{k}(x)
$). However the effective number of weights is less, due to the $d+1=257$
implicit constraints $H(\theta)=0$. Net-1 correctly classifies 80\% of the
test set.

\bigskip

\textbf{Net-2.} This is a single hidden layer (two layer) network, fully
connected. A network which has all possible links and no additional
constraints on the parameter (except the necessary $H(\theta)=0$ for the
softmax function) may be called a "plain vanilla network". Here there are 12
hidden units; thus the model can be written as (\ref{nn-HTF-notat}) with
$m=12,$ $K=10$, $d=256$. Thus the dimension of $\theta$ (number of weights)
is
\[
m(d+1)+K(m+1)=12\cdot257+10\cdot13=3214
\]
(the $3240$ in the "number of weights" column for this network has been
corrected to $3214$ in [HTF]). Since this is a plain vanilla network, the
number of links equals the number of weights. Net-2 correctly classifies 87\%
of the test set.

\bigskip

\textbf{Net-3.} Here we have two hidden layers (3 layers), that is the
architecture given in (\ref{nn-two-hid-lay-1}) but not as a plain vanilla
network but with "local connectivity". That means that not all possible links
are present; rather each unit in the two hidden layers is influenced only by
\textit{some} (not all) units in the layer below. Suppose we write a two
hidden layer network for $K$ classes, analogously to (\ref{nn-two-hid-lay-1})
which was for $K=2$ classes, as
\begin{align}
u^{(i)} &  =\sigma\left(  a_{i0}+\left\langle a_{i},x\right\rangle \right)
\text{, }i=1,\ldots,m,\label{two-hiddenl-layer-again}\\
z^{(j)} &  =\sigma\left(  b_{j0}+\left\langle b_{j},u\right\rangle \right)
\text{, }j=1,\ldots,l\\
t^{(k)} &  =c_{0k}+\left\langle c_{k},z\right\rangle \text{, }k=1,\ldots
,K,\nonumber\\
\psi_{k}(x) &  =g_{k}(t),k=1,\ldots,K\nonumber
\end{align}
where $u=(u^{(1)},\ldots,u^{(m)})^{\top},$ $z=(z^{(1)},\ldots,z^{(l)})^{\top
},$ $x=(x^{(1)},\ldots,x^{(d)})^{\top}$. In Net-3 the first hidden layer is
represented as a $8\times8$ pixel picture, hence there are $m=64$ units
$u^{(i)}$ in the first hidden layer and $l=4\times4=16$ units $z^{(j)}$ in the
second hidden layer. Local connectivity here means that each 256-vector
$a_{i}$ has only 9 nonzero components, and each $64$-vector $b_{j}$ has only
25 nonzero components. Thus Net-3, even though it has two hidden layers, has
fewer weights and links than the plain vanilla Net-2, and achieves comparable
performance 88.5 \%.

\bigskip

\textbf{Net-4. }This adds "shared weights" to the two hidden layer
architecture of Net-3. The first (lower) hidden layer is divided into two
parts, represented as two $8\times8$ "pictures", and with in each of the two
parts the weights are equal. That means that in (\ref{two-hiddenl-layer-again}%
), $m=8\times8\times2=128$ and we introduce equality constraints%
\begin{align*}
(a_{10},a_{1})  & =\ldots=(a_{64,0},a_{64})\text{, }\\
(a_{65,0},a_{65})  & =\ldots=(a_{128,0},a_{128}).
\end{align*}
In addition, local connectivity is present, i.e. most of the components of
$a_{i}$ are zero. The second hiden layer is not split (no weight sharing
there). This network has fewer weights than Net-3 but better performance (94 \%).

\bigskip

\textbf{Net -5}. Here the second hidden layer is also split, into 4 parts now,
and within each part there is weight sharing (the first hidden layer has the
same structure of the first hidden layer as Net-4). This net has the best
performance: 98.4 \%.

\newpage

\section{Histogram and Tree Methods}

\subsection{Partitioning rules}

We mostly follow [DGL], sec. 6.3 here. Many important classification rules
partition $\mathbb{R}^{d}$ into disjoint cells $A_{1},A_{2},\ldots$and and
classify in each cell according to the majority vote among the labels of the
$X_{i}$`s falling in the same cell. More precisely, if $A(x)$ denotes the cell
into which $x\in\mathbb{R}^{d}$ is falling,%
\begin{equation}
h_{n}(x)=\left\{
\begin{tabular}
[c]{l}%
$0$ if \textrm{card}$\left\{  (X_{i},Y_{i}):X_{i}\in A(x),Y_{i}=1\right\}
\leq$\textrm{card}$\left\{  (X_{i},Y_{i}):X_{i}\in A(x),Y_{i}=0\right\}  $\\
$1$ otherwise
\end{tabular}
\right. \label{partition-rule-def}%
\end{equation}

The decision is zero if the number of ones does not exceed the number of zeros
in the cell where $X$ falls, and vice versa. The partitions we consider in
this section may change with $n$, and they may also depend on the points
$X_{1},\ldots,X_{n},$, but we assume that they do not depend on the $Y_{i}$.
We develop a general consistency result for such partitioning rules. It
requires two properties of the partition: first, cells should be small enough
so that local changes of the distribution can be detected. On the other hand,
cells should be large enough to contain a large number of points so that
averaging among the labels is effective. $\mathrm{diam}(A)$ denotes the
diameter of a set $A$, that is,%
\[
\mathrm{diam}(A)=\sup_{x,y\in A}\left\Vert x-y\right\Vert
\]
As usual, we assume that $(X,Y),$ $(X_{i},Y_{i}),$ $i=1,\ldots,n$ are i.i.d.
distributed ($Y\in\left\{  0,1\right\}  $), and let $\mu=\mathcal{L}(X)$
denote the marginal distribution of $X$. Let also $\mu_{n}$ denote the
empirical measure of the $X_{i}$:
\[
\mu_{n}\left(  A\right)  =n^{-1}\sum_{i=1}^{n}\mathbf{1}_{A}(X_{i})
\]
and denote
\[
N(x)=n\mu_{n}\left(  A(x)\right)  =\sum_{i=1}^{n}\mathbf{1}_{A(x)}(X_{i})
\]
the number of $X_{i}$`s falling in the same cell as $x$. \bigskip

\textbf{Error probabilities.} Recall that a classification rule is a mapping
$h:\mathbb{R}^{d}\rightarrow\left\{  0,1\right\}  $, which also depends on the
training set $T_{n}:=\left(  (X_{1},Y_{1}),\ldots,(X_{n},Y_{n})\right)  $. So
actually $h$ is a (measurable) function%
\[
h:\mathbb{R}^{d}\times\left\{  \mathbb{R}^{d}\times\left\{  0,1\right\}
\right\}  ^{\times n}\rightarrow\left\{  0,1\right\}
\]
and we occasionally (but not always) write $h(X,T_{n})$. For a given decision
$\kappa\in\left\{  0,1\right\}  $ and a realized value $y$ of the r.v. $Y$ we
write the loss (or prediction error)
\[
L\left(  \kappa,y\right)  =\mathbf{1}\left\{  \kappa\neq y\right\}
\]
that is, the loss is $0$ if $\kappa$ and $y$ coincide, and $1$ otherwise. For
a given (fixed) training set $t_{n}$, the performance of a rule $h$ is
measured by the probability of error when $(X,Y)$ follow their joint
distribution and $h(X,t_{n})$ is used to predict $Y$
\begin{equation}
P\left(  h(X,t_{n})\neq Y\right)  =EL\left(  h(X,t_{n}),Y\right)
.\label{err-probab-class}%
\end{equation}
Following the notation in [DGL], we shall always write $L_{n}=L\left(
h(X,T_{n}),Y\right)  $ if it is clear from the context which rule $h$ is being
considered. Furthermore, the above error probability (\ref{err-probab-class})
for given training set $t_{n}$ is written as a conditional probability:%
\[
E\left(  L_{n}|T_{n}=t_{n}\right)  :=EL\left(  h(X,t_{n}),Y\right)
\]%
\[
=P\left(  h_{n}\neq Y|T_{n}=t_{n}\right)  =P\left(  h(X,t_{n})\neq Y\right)
\]
where on the right side, the expectation refers to the random $(X,Y)$ only.
(Here we wrote $h_{n}=h(X,t_{n}).$) The final measure of performance for the
rule $h_{n}$ is obtained by averaging over the training set $T_{n}$:%
\begin{equation}
E\left(  L_{n}\right)  =E^{T}\left(  E\left(  L_{n}|T_{n}\right)  \right)
,\label{err-probab-class-total}%
\end{equation}%
\[
=P\left(  h_{n}\neq Y\right)  =E^{T}\left(  P\left(  h(X,T_{n})\neq Y\right)
\right)
\]
where on the right side, the expectation $E^{T}$ refers to the random training
set $T_{n}$.\bigskip

\textbf{Bayes rule. }According to Theorem \ref{theor-bayes-rule}, the Bayes
rule $h^{\ast}$ is optimal among all rules $h:\mathbb{R}^{d}\rightarrow
\left\{  0,1\right\}  $, including those which depend on a training set
$T_{n}$. The Bayes rule requires knowledge of $\mathcal{L}(X,Y)$ and is
therefore not available in practice. Denoting
\[
L^{\ast}:=EL\left(  h^{\ast}(X),Y\right)  =P\left(  h^{\ast}\neq Y\right)
\]
where $E$ and $P$ refer only to $\mathcal{L}(X,Y)$, Theorem
\ref{theor-bayes-rule} implies for any rule $h_{n}=h(X,T_{n})$
\begin{equation}
P\left(  h_{n}\neq Y|T_{n}=t_{n}\right)  =E\left(  L_{n}|T_{n}=t_{n}\right)
\geq L^{\ast}\label{lower-bound-loss-Bayes}%
\end{equation}
and therefore also
\[
P\left(  h_{n}\neq Y\right)  =E\left(  L_{n}\right)  \geq L^{\ast}.
\]

.

\bigskip

\textbf{Consistency.} Recall that in (ordinary) statistical parameter
estimation, an estimator $\hat{\theta}_{n}$ of a real valued parameter
$\theta$ is called \textit{consistent} if $\hat{\theta}_{n}\rightarrow
_{P}\theta$ (convergence in probability). If $\theta$ is an element of a
metric space and $d(\cdot,\cdot)$ the pertaining metric, then consistency at
$\theta$ is defined as
\begin{equation}
d\left(  \hat{\theta}_{n},\theta\right)  \rightarrow_{P}0\text{ as
}n\rightarrow\infty.\label{consist-classical-estim}%
\end{equation}
In classification, the analog of the metric $d(\cdot,\cdot)$, is
\[
P\left(  h_{n}\neq Y|T_{n}\right)  -L^{\ast}=E\left(  L_{n}|T_{n}\right)
-L^{\ast}%
\]
which in some sense measures the distance of $h_{n}$ to the Bayes rule. Note
that here the role of $\hat{\theta}_{n}$ is played by $h_{n}$ and the role of
$\theta$ (true parameter) is played by $\mathcal{L}(X,Y)$. Thus,
classification fits very much into the framework of statistical decision
theory: we try to capture a certain characteristic of the unknown law
$\mathcal{L}(X,Y)$ by a decision function $h_{n}$, based on a sample
$T_{n}=\left(  (X_{1},Y_{1}),\ldots,(X_{n},Y_{n})\right)  $.

Thus, in analogy to (\ref{consist-classical-estim}), a classification rule
$h_{n}$ may be called \textit{consistent} at $\mathcal{L}(X,Y)$ if
\[
E\left(  L_{n}|T_{n}\right)  -L^{\ast}\rightarrow_{P}0.
\]
Note that $\zeta_{n}:=E\left(  L_{n}|T_{n}\right)  -$ $L^{\ast}$ is a bounded
random variable (as a function of $T_{n}$): in view of
(\ref{lower-bound-loss-Bayes}) we have $0\leq\zeta_{n}\leq1-L^{\ast}\leq1$. If
$0\leq\zeta_{n}\leq1$ then $\zeta_{n}\rightarrow_{P}0$ is equivalent to
$E\zeta_{n}\rightarrow0$. (If $E\zeta_{n}\rightarrow0$ then $\zeta
_{n}\rightarrow_{P}0$ by Chebyshev's inequality. For the other direction, cf.
the topic "Integration to the Limit", Theorem 25.12 in Billingsley
(1995)$^{\text{\footnote{Billingsley, P. (1995). Probability and Measure, 3red
Ed. Wiley, NY. }}}$)

According to (\ref{err-probab-class-total}) we have $E\zeta_{n}=E\left(
L_{n}\right)  -L^{\ast}$. Thus we may equivalently define consistency of a
classification rule $h_{n}$ at $\mathcal{L}(X,Y)$ by
\[
E\left(  L_{n}\right)  \rightarrow L^{\ast}%
\]
In (ordinary) statistical parameter estimation, consistency of an estimator
$\hat{\theta}_{n}$ may not hold at all unknown values of $\theta$ or only
under certain conditions. For instance a certain estimator for an expectation
may be consistent only if the expectation exists, or if it exists and
additionally a certain variance is finite etc. In classification, the lower
bound $L^{\ast}$ for the error probability (\ref{lower-bound-loss-Bayes}) is
always defined. Thus a rule $h_{n}$ may be called \textit{universally
consistent } if
\[
E\left(  L_{n}\right)  \rightarrow_{P}L^{\ast}\text{ for all unknown
distributions }\mathcal{L}(X,Y)\text{. }%
\]
Below we will establish universal consistency of the histogram rule.

\bigskip

Return first to the general partition rules (\ref{partition-rule-def}). The
conditions of the theorem below require that a random cell---selected
according to the distribution of $X$- has a small diameter, and contains many
points with large probability.%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
The following proof needs to be checked and possibly elaborated.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\begin{theorem}
\label{theor-consist-partitionrules}Consider a partitioning classification
rule as defined above in (\ref{partition-rule-def}). Then $E\left(
L_{n}\right)  \rightarrow L^{\ast}$ if \newline(i) $\mathrm{diam}%
(A(X))\rightarrow0$ in probability\newline(ii) $N(X)\rightarrow\infty$ in probability.
\end{theorem}

\begin{proof}
Define $r(x)=P\left(  Y=1|X=x\right)  $ (the regression function in
$\mathcal{L}(X,Y)$). Define
\[
\hat{r}_{n}(x)=\frac{\mathrm{card}\left\{  (X_{i},Y_{i}):X_{i}\in
A(x),Y_{i}=1\right\}  }{N(x)}=\frac{1}{N(x)}%
%TCIMACRO{\dsum \limits_{i=1}^{n}}%
%BeginExpansion
{\displaystyle\sum\limits_{i=1}^{n}}
%EndExpansion
Y_{i}\mathbf{1}_{A(x)}(X_{i}).
\]
Then the rule $h_{n}$ in (\ref{partition-rule-def}) may equivalently be
written
\begin{align}
h_{n}(x)  & =\left\{
\begin{tabular}
[c]{l}%
$0$ if $\hat{r}_{n}(x)\leq1-\hat{r}_{n}(x)$\\
$1$ otherwise
\end{tabular}
\right. \nonumber\\
& =\left\{
\begin{tabular}
[c]{l}%
$0$ if $\hat{r}_{n}(x)\leq1/2$\\
$1$ otherwise
\end{tabular}
\right.  .\label{using-form-2}%
\end{align}
Thus we may consider $\hat{r}_{n}(x)$ an estimator of $r(x)$ and $h_{n}(x)$
decides in analogy to the Bayes rule, with a threshold $1/2$. In this
situation we claim
\begin{equation}
E\left(  L_{n}|T_{n}\right)  -L^{\ast}\leq2\int_{\mathbb{R}^{d}}\left\vert
r(x)-\hat{r}_{n}(x)\right\vert d\mu(x)=2E\left(  \left\vert r(X)-\hat{r}%
_{n}(X)\right\vert |T_{n}\right) \label{claim-risk-plugin}%
\end{equation}
To prove this, note that given $X=x$, (and also $T_{n}=t_{n}$, but we
initially do not write possible dependence on $t_{n}$) the conditional error
probability of any rule $h$ may be expressed as
\[
P\left(  h(X)\neq Y|X=x\right)  =1-P\left(  h(X)=Y|X=x\right)
\]%
\begin{align*}
& =1-P\left(  Y=1,h(X)=1|X=x\right)  -P\left(  Y=0,h(X)=0|X=x\right) \\
& =1-\mathbf{1}_{\left\{  h(x)=1\right\}  }P\left(  Y=1|X=x\right)
-\mathbf{1}_{\left\{  h(x)=0\right\}  }P\left(  Y=0|X=x\right) \\
& =1-\mathbf{1}_{\left\{  h(x)=1\right\}  }r(x)-\mathbf{1}_{\left\{
h(x)=0\right\}  }\left(  1-r(x)\right)  .
\end{align*}
Thus for every $x\in\mathbb{R}^{d}$, if $h^{\ast}$ is the Bayes rule%
\begin{align*}
& P\left(  h(X)\neq Y|X=x\right)  -P\left(  h^{\ast}(X)\neq Y|X=x\right) \\
& =r(x)\left(  \mathbf{1}_{\left\{  h^{\ast}(x)=1\right\}  }-\mathbf{1}%
_{\left\{  h(x)=1\right\}  }\right)  +\left(  1-r(x)\right)  \left(
\mathbf{1}_{\left\{  h^{\ast}(x)=0\right\}  }-\mathbf{1}_{\left\{
h(x)=0\right\}  }\right) \\
& =r(x)\left(  \mathbf{1}_{\left\{  h^{\ast}(x)=1\right\}  }-\mathbf{1}%
_{\left\{  h(x)=1\right\}  }\right)  +\left(  1-r(x)\right)  \left(
-\mathbf{1}_{\left\{  h^{\ast}(x)=1\right\}  }+\mathbf{1}_{\left\{
h(x)=1\right\}  }\right) \\
& =\left(  2r(x)-1\right)  \left(  \mathbf{1}_{\left\{  h^{\ast}(x)=1\right\}
}-\mathbf{1}_{\left\{  h(x)=1\right\}  }\right)  .
\end{align*}
Note that $\mathbf{1}_{\left\{  h^{\ast}(x)=1\right\}  }-\mathbf{1}_{\left\{
h(x)=1\right\}  }$ is nonzero if and only if $h^{\ast}(x)\neq h(x)$; if
$\mathbf{1}_{\left\{  h^{\ast}(x)=1\right\}  }-\mathbf{1}_{\left\{
h(x)=1\right\}  }=-1$ then $h^{\ast}(x)=0$ and hence ($h^{\ast}$ being the
Bayes rule) $r(x)\leq1/2$, hence $2r(x)-1\leq0$; if $\mathbf{1}_{\left\{
h^{\ast}(x)=1\right\}  }-\mathbf{1}_{\left\{  h(x)=1\right\}  }=1$ then
$h^{\ast}(x)=1$ and hence $r(x)>1/2$, consequently $2r(x)-1>0$. This reasoning
entails
\begin{equation}
P\left(  h(X)\neq Y|X=x\right)  -P\left(  h^{\ast}(X)\neq Y|X=x\right)
=2\left\vert r(x)-\frac{1}{2}\right\vert \mathbf{1}_{\left\{  h^{\ast}(x)\neq
h(x)\right\}  }\label{entails}%
\end{equation}
Now setting $h_{n}=h$ and using the description (\ref{using-form-2}) in terms
of $\hat{r}_{n}(x)$ we note: if $h^{\ast}(x)\neq h_{n}(x)$ then $r(x)$ and
$\hat{r}_{n}(x)$ are at opposite sides of $1/2,$ hence
\begin{equation}
\left\vert r(x)-\frac{1}{2}\right\vert \leq\left\vert r(x)-\hat{r}%
_{n}(x)\right\vert .\label{inequ-r-hat-r}%
\end{equation}
Taking an expectation in (\ref{entails}) over $X$ (with $T_{n}=t_{n}$ still
fixed ) and using (\ref{inequ-r-hat-r}) we obtain (\ref{claim-risk-plugin}).

Thus we need only show
\[
E\left(  \left\vert r(X)-\hat{r}_{n}(X)\right\vert |T_{n}\right)
\rightarrow_{P}0
\]
or equivalently
\[
E\left(  \left\vert r(X)-\hat{r}_{n}(X)\right\vert \right)  \rightarrow0.
\]
Introduce
\begin{equation}
\bar{r}(x)=E\left(  r(X)|X\in A(x)\right)  =P\left(  Y=1|X\in A(x)\right)
.\label{r-bar-def}%
\end{equation}
By the triangle inequality
\[
E\left(  \left\vert r(X)-\hat{r}_{n}(X)\right\vert \right)  \leq E\left(
\left\vert \hat{r}_{n}(X)-\bar{r}(X)\right\vert \right)  +E\left(  \left\vert
\bar{r}(X)-r(X)\right\vert \right)  .
\]
By conditioning on the random variable $N(x)$, it is easy to see that
$N(x)\hat{r}_{n}(x)=%
%TCIMACRO{\dsum \limits_{i=1}^{n}}%
%BeginExpansion
{\displaystyle\sum\limits_{i=1}^{n}}
%EndExpansion
Y_{i}\mathbf{1}_{A(x)}(X_{i})$ is a binomial random variable $B\left(
N(x),\bar{r}(x)\right)  $ with parameters $N(x)$ and $\bar{r}(x)$. (That
requires a short reasoning. Note that unconditionally, the r.v. $%
%TCIMACRO{\dsum \limits_{i=1}^{n}}%
%BeginExpansion
{\displaystyle\sum\limits_{i=1}^{n}}
%EndExpansion
Y_{i}\mathbf{1}_{A(x)}(X_{i})$ is binomial $B\left(  n,P\left(  Y=1,X\in
A(x)\right)  \right)  $). Thus
\begin{align*}
& E\left(  \left\vert \hat{r}_{n}(x)-\bar{r}(x)\right\vert |\mathbf{1}%
_{\left\{  X_{1}\in A(x)\right\}  },\ldots,\mathbf{1}_{\left\{  X_{n}\in
A(x)\right\}  }\right) \\
& \leq E\left(  \left\vert \frac{B\left(  N(x),\bar{r}(x)\right)  }{N(x)}%
-\bar{r}(x)\right\vert \mathbf{1}_{\left\{  N(x)>0\right\}  }|\mathbf{1}%
_{\left\{  X_{1}\in A(x)\right\}  },\ldots,\mathbf{1}_{\left\{  X_{n}\in
A(x)\right\}  }\right)  +\\
& +E\left(  \mathbf{1}_{\left\{  N(x)=0\right\}  }|\mathbf{1}_{\left\{
X_{1}\in A(x)\right\}  },\ldots,\mathbf{1}_{\left\{  X_{n}\in A(x)\right\}
}\right)
\end{align*}
For a binomial $B(m,q),$ we have by Cauchy-Schwarz
\[
E\left\vert \frac{B\left(  m,q\right)  }{m}-q)\right\vert \leq\left(
\frac{\mathrm{Var}\left(  B\left(  m,q\right)  \right)  }{m^{2}}\right)
^{1/2}=\left(  \frac{q(1-q)}{m}\right)  ^{1/2}%
\]
and applying this to the previous display we obtain%
\begin{align*}
& E\left(  \left\vert \hat{r}_{n}(x)-\bar{r}(x)\right\vert |\mathbf{1}%
_{\left\{  X_{1}\in A(x)\right\}  },\ldots,\mathbf{1}_{\left\{  X_{n}\in
A(x)\right\}  }\right) \\
& \leq E\left(  \left(  \frac{\bar{r}(x)\left(  1-\bar{r}(x)\right)  }%
{N(x)}\right)  ^{1/2}\mathbf{1}_{\left\{  N(x)>0\right\}  }|\mathbf{1}%
_{\left\{  X_{1}\in A(x)\right\}  },\ldots,\mathbf{1}_{\left\{  X_{n}\in
A(x)\right\}  }\right) \\
& +E\left(  \mathbf{1}_{\left\{  N(x)=0\right\}  }|\mathbf{1}_{\left\{
X_{1}\in A(x)\right\}  },\ldots,\mathbf{1}_{\left\{  X_{n}\in A(x)\right\}
}\right)  .
\end{align*}
Taking an expectation over $X=x$ and $X_{1},\ldots,X_{1}$ we see that (noting
$\bar{r}(x)\left(  1-\bar{r}(x)\right)  \leq1/4$)
\begin{align*}
E\left(  \left\vert \hat{r}_{n}(x)-\bar{r}(x)\right\vert \right)   & \leq
E\left(  \frac{1}{2\sqrt{N(X)}}\mathbf{1}_{\left\{  N(X)>0\right\}  }\right)
+P\left(  N(X)=0\right) \\
& \leq\frac{1}{2}P\left(  N(X)\leq k\right)  +\frac{1}{2\sqrt{k}}+P\left(
N(X)=0\right)
\end{align*}
for any $k$, and this can be made small, first by choosing $k$ large enough
and then by using condition (ii).

For $\varepsilon>0$, find a uniformly continuous $[0,1]$-valued function
$r_{\varepsilon}$ on a bounded set $C$ and vanishing off $C$ so that
$E\left\vert r_{\varepsilon}(X)-r(X)\right\vert <\varepsilon$. (Indeed, $r(x)$
is measurable and bounded with values in $[0,1]$. Such an approximation is
similar to an approximation of $r$ by step functions, see basic texts about
integration). Next, setting analogously to (\ref{r-bar-def})
\[
\bar{r}_{\varepsilon}(x)=E\left(  r_{\varepsilon}(X)|X\in A(x)\right)
\]
we employ the triangle inequality:
\begin{align*}
E\left\vert \bar{r}(X)-r(X)\right\vert  & \leq E\left\vert \bar{r}(X)-\bar
{r}_{\varepsilon}(X)\right\vert \\
& +E\left\vert \bar{r}_{\varepsilon}(X)-r_{\varepsilon}(X)\right\vert \\
& +E\left\vert r_{\varepsilon}(X)-r(X)\right\vert \\
& =I+II+III.
\end{align*}
Clearly, $III<\varepsilon$ by choice of $r_{\varepsilon}$. Since
$r_{\varepsilon}$ is uniformly continuous, and $\bar{r}_{\varepsilon}(x)$
represents an average of the function $r$ over the set $A(x)$, we can find a
$\theta=\theta\left(  \varepsilon\right)  >0$ such that
\[
II\leq\varepsilon+P\left(  \mathrm{diam}(A(X))>\theta\right)
\]
There $II<2\varepsilon$ for $n$ large enough, by condition (i). Finally,
$I\leq III<\varepsilon$ since $\bar{r}$, $\bar{r}_{\varepsilon}$ represent
averages of $r,r_{\varepsilon}$ over sets $A(x)$.
\end{proof}

\subsection{Universal consistency}

\subsubsection{Histogram\label{subsec-histogr}}

The cubic histogram rule partitions $\mathbb{R}^{d}$ into cubes of the same
size, and makes a decision according to the majority vote among the $Y_{i}$'s
such that the corresponding $X_{i}$ falls in the same cube as $X$. Formally,
let $\mathcal{P}_{n}=\left\{  A_{n1,}A_{n2},\ldots\right\}  $ be a partition
of $\mathbb{R}^{d}$ into cubes of size $h_{n}>0.$ More precisely, if
$U=[0,1)^{\times d}$ is the (half-open) unit cube in $\mathbb{R}^{d}$\ then
the sets $A_{n2}$ are of form $\mathbf{k}+h_{n}U$ where $\mathbf{k}%
=(k_{1},\ldots,k_{d})$ is a vector of integers. For every $x\in\mathbb{R}^{d}$
let $A_{n}(x)=A_{ni}$ if $x\in A_{ni}$. The histogram rule is defined as the
pertaining special case of (\ref{partition-rule-def}), i.e. by
\[
h_{n}(x)=\left\{
\begin{tabular}
[c]{l}%
$0$ if $\sum_{i=1}^{n}\mathbf{1}_{\left\{  Y_{i}=1\right\}  }\mathbf{1}%
_{A_{n}(x)}(X_{i})\leq\sum_{i=1}^{n}\mathbf{1}_{\left\{  Y_{i}=10\right\}
}\mathbf{1}_{A_{n}(x)}(X_{i})$\\
$1$ otherwise
\end{tabular}
\right.  .
\]
The next theorem establishes universal consistency of certain cubic histogram
rules (cf\ [DGL], p. 96).%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
The proof needs to be checked and possibly elaborated.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\begin{theorem}
\label{theor-consist-histogr}If $h_{n}\rightarrow0$ and $nh_{n}^{d}%
\rightarrow\infty$ as $n\rightarrow\infty$ then the cubic histogram rule is
universally consistent.
\end{theorem}

\begin{proof}
We check the two simple conditions of of Theorem
\ref{theor-consist-partitionrules}. Clearly, the diameter of each cell is
$\sqrt{d}h^{d}$. Therefore condition (i) follows trivially:
\[
\mathrm{diam}(A(X))\leq\sqrt{d}h^{d}\rightarrow0.
\]
To show condition (ii), we need to prove that for any $M<\infty$, $P\left(
N(X)\leq M\right)  \rightarrow0$. Let $S$ be an arbitrary ball centered at the
origin. Then the number of cells intersecting $S$ is not more than
$c_{1}+c_{2}/h^{d}$ for some positive constants $c_{1},c_{2}$. Then
\[
P\left(  N(X)\leq M\right)  \leq\sum_{j:A_{nj}\cap S\neq\emptyset}P\left(
X\in A_{nj},N(X)\leq M\right)  +P\left(  X\in S^{c}\right)
\]%
\[
\leq\sum_{j:A_{nj}\cap S\neq\emptyset,\text{ }\mu(A_{nj})\leq2M/n}\mu\left(
A_{nj}\right)  +\sum_{j:A_{nj}\cap S\neq\emptyset,\text{ }\mu(A_{nj})>2M/n}%
\mu\left(  A_{nj}\right)  P\left(  n\mu_{n}(A_{nj})\leq M\right)  +\mu(S^{c})
\]
(for the second term above, recall that $X$ and $X_{1},\ldots,X_{n}$ are
independent),%
\[
\leq\frac{2M}{n}\left(  c_{1}+\frac{c_{2}}{h^{d}}\right)  +\sum_{j:A_{nj}\cap
S\neq\emptyset,\text{ }\mu(A_{nj})>2M/n}\mu\left(  A_{nj}\right)  P\left(
\mu_{n}(A_{nj})-E\mu_{n}(A_{nj})\leq M/n-\mu(A_{nj})\right)  +\mu(S^{c})
\]%
\[
\leq\frac{2M}{n}\left(  c_{1}+\frac{c_{2}}{h^{d}}\right)  +\sum_{j:A_{nj}\cap
S\neq\emptyset,\text{ }\mu(A_{nj})>2M/n}\mu\left(  A_{nj}\right)  P\left(
\mu_{n}(A_{nj})-E\mu_{n}(A_{nj})\leq\frac{-\mu(A_{nj})}{2}\right)  +\mu(S^{c})
\]
and applying Chebyshev's inequality%
\[
\leq\frac{2M}{n}\left(  c_{1}+\frac{c_{2}}{h^{d}}\right)  +\sum_{j:A_{nj}\cap
S\neq\emptyset,\text{ }\mu(A_{nj})>2M/n}4\mu\left(  A_{nj}\right)
\frac{\mathrm{Var}\left(  \mu_{n}(A_{nj})\right)  }{\left(  \mu(A_{nj}%
)\right)  ^{2}}+\mu(S^{c})
\]%
\[
\frac{2M}{n}\left(  c_{1}+\frac{c_{2}}{h^{d}}\right)  +\sum_{j:A_{nj}\cap
S\neq\emptyset,\text{ }\mu(A_{nj})>2M/n}4\mu\left(  A_{nj}\right)  \frac
{1}{n\mu(A_{nj})}+\mu(S^{c})
\]%
\[
\leq\frac{2M+4}{n}\left(  c_{1}+\frac{c_{2}}{h^{d}}\right)  +\mu
(S^{c})\rightarrow\mu(S^{c})
\]
by the condition $nh^{d}\rightarrow\infty$. Since $S$ is arbitrary, the proof
is complete.
\end{proof}

\subsubsection{$k$-nearest neighbor rule}

This simple classifier is conceptually related to the histogram. Suppose a
prediction of $Y$ is to be made given $X=x$, on the basis of a training set
$T_{n}=\left(  (X_{1},Y_{1}),\ldots,(X_{n},Y_{n})\right)  $. first the data
are ordered according to increasing Euclidean distances of the $X_{j}$'s to
$x$:
\[
\left(  (X_{\left(  1\right)  }(x),Y_{(1)}(x)),\ldots,(X_{(n)}(x),Y_{(n)}%
(x))\right)  ,
\]
that is, $X_{\left(  i\right)  }(x)$ is the $i$-th nearest neighbor of $x$
among the points $X_{1},\ldots,X_{n}$. Distance ties are broken by comparing
indices, that is, in case of $\left\Vert X_{i}-x\right\Vert =\left\Vert
X_{j}-x\right\Vert $, $X_{i}$ is considered to be "closer" to $x$ if $i<j$.

The $k$-nearest neighbor (k-NN) classification rule is defined as
\[
h_{n}(x)=\left\{
\begin{tabular}
[c]{l}%
$0$ if $\sum_{i=1}^{k}\mathbf{1}_{\left\{  Y_{(i)}(x)=1\right\}  }\leq
\sum_{i=1}^{k}\mathbf{1}_{\left\{  Y_{(i)}(x)=0\right\}  }$\\
$1$ otherwise
\end{tabular}
\right.  .
\]
In other words, $h_{n}(x)$ is a majority vote among the labels of the $k$
nearest neighbors of $x$.

\begin{theorem}
If $k\rightarrow\infty$ and $k/n\rightarrow0$ then the $k-$NN rule is
universally consistent.
\end{theorem}

For a proof see Theorem 6.4 in [DGL], p. 101.%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
DGL also has universal consistency of NN, as no. of nodes$\rightarrow\infty$,
see the chapter on NN there
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\subsection{Classification and Regression Trees (CART)}

Let us return to the similarity between classification and regression. Suppose
we have an i.i.d. sample $\left(  X_{i},Y_{i}\right)  $, $i=1,\ldots,n $ where
$X_{i}\in$ $\mathbb{R}^{d}$ and $Y_{i}$ are real valued. In classification,
the $Y_{i}$ would be $0$-$1$ valued for 2 classes. Suppose we are interested
in estimating the regression function
\begin{equation}
r(x)=E\left(  Y|X=x\right)
\end{equation}
and we try to fit a family of functions $f_{\vartheta}(x)$, $\vartheta
\in\Theta$ to the observed values $\left(  X_{i},Y_{i}\right)  $,
$i=1,\ldots,n$. As explained, in the classification case, for a ("new") test
case $x$ we would predict $Y=1$ if the estimated $\hat{\vartheta}$ results in
$f_{\hat{\vartheta}}(x)>1/2$. Let us assume henceforth that $X_{i}$ take
values in a set $S\subset\mathbb{R}^{d}$, assumed compact.

\textit{In the histogram method for regression, } we select a partition of the
set $S$ into subsets $A_{j}$, $j=1,\ldots,m$ and the family of functions is
piecewise constant on the sets $A_{j}$. Then $\vartheta$ may be taken as the
vector of values $\left(  \vartheta_{j}\text{, }j=1,\ldots,m\right)  $ of the
function such that $f_{\vartheta}(x)=\vartheta_{j}$ for $x\in A_{j}$. The
least squares principle may be used to obtain an estimate of $\vartheta$
\[
\min_{\vartheta}\sum_{i=1}^{n}\left(  Y_{i}-f_{\vartheta}(X_{i})\right)  ^{2}%
\]
It is clear that this yields the average of the $Y_{i}\in A_{j}$ as an
estimate of $\vartheta_{j}$:
\[
\hat{\vartheta}_{j}:=\bar{Y}_{j}:=\frac{1}{n_{j}}\sum_{i=1}^{n}Y_{i}%
\mathbf{1}_{A_{j}}(X_{i})\text{ where }n_{j}=\sum_{i=1}^{n}\mathbf{1}_{A_{j}%
}(X_{i}).
\]
Thus the histogram method for regression also results from the least squares
principle, once the partition $A_{j}$, $j=1,\ldots,m$ \ is chosen. The main
problem with histogram estimation is the choice of this partition.

The histogram rule for classification has been treated in section
\ref{subsec-histogr}\textit{. }Here, once the partition is chosen, each
$A_{j}$ is classified acording to whether $\bar{Y}_{j}\geq1/2$, i.e. by a
"majority vote" of the $X_{j}$ in the given set $A_{j}$. \ Other nonparametric
regression estimators also have their classification analogs, such as nearest
neighbor methods.

Let us consider some algorithms in which the partition is chosen recursively,
depending on the data. \bigskip

\textbf{Partition choice for regression.} Assume first $S=[0,1]$ and we
consider a histogram with $m=2$, where $A_{1}=[0,s]$ and $A_{2}=(s,1]$, but
where we try to select also the points $s$ from the data. Now $\vartheta$ may
be taken as the vector of values $(\vartheta_{1}$,$\vartheta_{2},s)$ and the
least squares principle gives (by minimizing first over $\vartheta_{1}%
$,$\vartheta_{2}$ for given $s$)
\[
\min_{\vartheta}\sum_{i=1}^{n}\left(  Y_{i}-f_{\vartheta}(X_{i})\right)
^{2}=\min_{s}\left(  \min_{c_{1}}\sum_{X_{i}\leq s}(Y_{i}-c_{1})^{2}%
+\min_{c_{2}}\sum_{X_{i}>s}(Y_{i}-c_{2})^{2}\right)  ,
\]
where the solutions of the minimum problem over $c_{j}$ are $\bar{Y}_{j,s}$,
$j=1,2$ i.e. the averages of $Y_{i}$ over the sets $X_{i}\leq s$ and $X_{i}>s
$. The function
\begin{equation}
Q(s)=\sum_{X_{i}\leq s}(Y_{i}-\bar{Y}_{1,s})^{2}+\min_{c}\sum_{X_{i}>s}%
(Y_{i}-\bar{Y}_{2,s})^{2}\label{impurity-analog-regress}%
\end{equation}
is the criterion to be minimized over $s$, and can be seen as an
\textit{impurity measure} for a given split point $s.$ For instance, it is $0$
if all values $Y_{i}$ to the right of $s$ are equal and also all values
$Y_{i}$ to the left of $s$ are equal.

\bigskip

\textbf{Impurity for classification}\textit{.} Again, $S=[0,1]$ but now the
data $Y_{i}$ are $0$-$1$ valued. We have to choose a split point $s$ for a
histogram on a partition where $A_{1}=[0,s]$ and $A_{2}=(s,1]$, such that each
side is classified according to a majority vote. The essence of the CART
method (classification and regression trees, Breiman et al. \cite{BFOS}%
)\footnote{Breiman, L., Friedman, J., Olshen, R., Stone, C. (1984).
\textsl{Classification and Regression Trees}. Wadsworth, Belmont, CA.} to be
discussed is the choice of a split point $s$ using an impurity measure.
Intuitively, an impurity function measures the "homogeneity" of data on each
side of the split point $s$. Let $\hat{p}_{1,s}$ be the proportion of
$1^{^{\prime}s}$ in $\left\{  Y_{i}:X_{i}\leq s\right\}  $ and $\hat{p}_{2,s}$
be the proportion of $1^{^{\prime}s}$ in $\left\{  Y_{i}:X_{i}>s\right\}  $.
An impurity measure for the data left of the split point $s$ (that is for
$\left\{  Y_{i}:X_{i}\leq s\right\}  $ ) is a function $\psi$ of $p=\hat
{p}_{1,s}$ with the following properties:

\begin{enumerate}
\item $\psi\left(  \frac{1}{2},\frac{1}{2}\right)  \geq\psi\left(
p,1-p\right)  $ for any $p\in\lbrack0,1]$

\item $\psi\left(  0,1\right)  =\psi\left(  1,0\right)  =0,$

\item $\psi\left(  p,1-p\right)  $ increases in $p$ on on $[0,1/2]$ and
decreases in $p$ on $[1/2,1].$
\end{enumerate}

For a given function $\psi$ to be used, the impurity function to be minimized
for a possible split $s$ is
\begin{equation}
I(s)=n_{1,s}\psi\left(  \hat{p}_{1,s},1-\hat{p}_{1,s}\right)  +n_{2,s}%
\psi\left(  \hat{p}_{2,s},1-\hat{p}_{2,s}\right) \label{impurity-criterion}%
\end{equation}

where $n_{1,s}$, $n_{2,s}$ are the sizes of the two classes:
\[
n_{1,s}=\sum_{i=1}^{n}\mathbf{1}_{[0,s]}(X_{i}),\;n_{2,s}=\sum_{i=1}%
^{n}\mathbf{1}_{(s,1]}(X_{i}).
\]
For a "perfect" split, where $\hat{p}_{1,s}=1$ and $\hat{p}_{2,s}=0$ we have
$I(s)=0$ and also for the case $\hat{p}_{1,s}=0$ and $\hat{p}_{2,s}=1$. If
$\hat{p}_{1,s}=\hat{p}_{2,s}=1/2$ (the worst case split) we have $I(s)=n/2$,
the maximal value. Note that the weighting of $\psi\left(  \hat{p}%
_{i,s},1-\hat{p}_{i,s}\right)  $ by $n_{i}$ $,$ $i=1,2$ makes sense, as seen
by the following example. Suppose there is an $s_{0}\in(0,1)$ such that
$\hat{p}_{2,s_{0}}=1/2$ and $\hat{p}_{1,s_{0}}=0$, and $n_{1}$ is large while
$n_{2} $ is small. Intuitively, a choice of split at $s=s_{0}$ does make
sense, as the class to the left is completey "pure" and contains most of the
data. If were to use the unweighted sum
\[
I_{0}(s)=\psi\left(  \hat{p}_{1,s},1-\hat{p}_{1,s}\right)  +\psi\left(
\hat{p}_{2,s},1-\hat{p}_{2,s}\right)
\]
for a choice of split point $s$ then then $I_{0}(s_{0})=\psi\left(
1/2,1/2\right)  $ and we could decrease $I_{0}(s)$ by choosing $s<s_{0}$ (then
$\hat{p}_{2,s}<1/2$ while $\hat{p}_{1,s}$ remains $0$). We would arrive at a
non-sensible split where $s$ is very small. With the weighted criterion
(\ref{impurity-criterion}) however , a high impurity $\psi\left(  \hat
{p}_{2,s},1-\hat{p}_{2,s}\right)  $ at $s=s_{0}$ is counterbalanced by a small
$n_{2,s}$ so that a choice $s<s_{0}$ is not obviously favored.

Examples of functions $\psi$ include:

\begin{enumerate}
\item The\textit{\ entropy function} $\psi(p,1-p)=-p\log p-(1-p)\log(1-p)$

\item The \textit{Gini function }$\psi(p,1-p)=2p(1-p)$.

\item The\textit{\ probability of misclassification} $\psi(p,1-p)=\min
(p,1-p)$. Indeed, if on the left side we have $\hat{p}_{1,s}>1/2$ then the
left side is classified as "1". The proportion of $0$'s here is $1-p_{1,s}$,
and selecting a random point from the left side, the probability of a
$0$-point (that is a misclassified point) is $1-p_{1,s}$. Analogously if
$\hat{p}_{1,s}\leq1/2$ then the left side is classified as "0" etc., so this
measure is $\min(p,1-p)$ (the proportion of points which are in the minority).
\end{enumerate}

\bigskip

\textbf{CART\ algorithm: forward stage (tree growing). }To build a tree by
successive, data-driven splitting of rectangles, for multivariate
$X_{i}=(X_{i}^{(1)},\ldots,X_{i}^{(d)})\in\mathbb{R}^{d}$ one takes a split
point $s$ for the $k$-th component and a criterion analogous to
(\ref{impurity-criterion}), \textit{applied to the }$k$\textit{-th components
}$X_{i}^{(k)}$, $i=1,\ldots,n$. Successive splits on different components
produce rectangles, which are split further. To give the a recursive
formulation of the algorithm, suppose a rectangle $R\subset\mathbb{R}^{d}$ is
considered for splitting along one of the coordinates, that is, only points
$X_{i}\in R$ are taken into consideration. Define
\begin{align}
n_{1,R,k,s}  & =\sum_{i=1}^{n}\mathbf{1}_{(-\infty,s]}(X_{i}^{(k)}%
)\mathbf{1}_{R}(X_{i}),\;n_{2,R,k,s}=\sum_{i=1}^{n}\mathbf{1}_{(s,\infty
]}(X_{i}^{(k)})\mathbf{1}_{R}(X_{i}),\label{unweighted-multi-1}\\
\hat{p}_{1,R,k,s}  & =\frac{1}{n_{1,R,k,s}}\sum_{i=1}^{n}Y_{i}\mathbf{1}%
_{(-\infty,s]}(X_{i}^{(k)})\mathbf{1}_{R}(X_{i}),\;\hat{p}_{2,R,k,s}=\frac
{1}{n_{2,R,k,s}}\sum_{i=1}^{n}Y_{i}\mathbf{1}_{(s,\infty]}(X_{i}%
^{(k)})\mathbf{1}_{R}(X_{i})\label{unweighted-multi-2}%
\end{align}
and the impurity measure
\begin{equation}
I(R,k,s)=n_{1,R,k,s}\psi\left(  \hat{p}_{1,R,k,s},1-\hat{p}_{1,R,k,s}\right)
+n_{2,R,k,s}\psi\left(  \hat{p}_{2,R,k,s},1-\hat{p}_{2,R,k,s}\right)
\label{impurity-rectangle}%
\end{equation}
The algorithm proceeeds in a greedy manner, where, if at a given stage one has
a collection of rectangles $R\in\mathcal{R}$, one seeks the minimum
\[
\min_{R\in\mathcal{R}\text{,}k=1,\ldots,d,s}I(R,k,s)
\]
where the split point $s$ conforms to the rectangle $R$ (that is, $r_{l}%
^{(k)}<s<r_{u}^{(k)}$ if $R=%
%TCIMACRO{\dprod \limits_{j=1}^{d}}%
%BeginExpansion
{\displaystyle\prod\limits_{j=1}^{d}}
%EndExpansion
\left[  r_{l}^{(j)},r_{u}^{(j)}\right)  $ ). If $\left(  \hat{R},\hat{k}%
,\hat{s}\right)  $ is a minimizer then the rectangle $\hat{R}$ is split along
coordinate $\hat{k}$ at point $\hat{s}$, and $\hat{R}$ gives rise to two new rectangles.

In this manner, a \textit{binary classification tree} is obtained. The splits
are continued until all sets contain less than $C$ elements, eg. $C=5$. One
could imagine another stopping criterion, for instance one where the tree
growing stops if no sensible gain in purity is obtained. However the CART
algorithm does not use such a criterion; it continues regardless of purity
gain until all rectangles contain few elements. Thus the resulting tree can be
expected to have far too many leaves (overfit); it has to be cut back (pruned)
in a second stage. The reason for this will be explained below.

\bigskip

\textbf{CART\ algorithm: backward stage (pruning). }We introduce some
terminology for trees: the top of a binary tree is called the \textit{root}.
Each \textit{node} has either no child (in that case it is called a
\textit{terminal node} or \textit{leaf}), a left child, a right child or a
left child and a right child. Each node is the root of a tree itself.

Let $\tau_{0}$ be the binary tree obtained above in the first stage. We define
a subtree $\tau\subset\tau_{0}$ to be any tree that can be obtained by pruning
$\tau_{0}$, that is collapsing of its internal (non-terminal) nodes (so that
this node becomes terminal). We index terminal nodes by $m$, with node $m$
region $R_{m}$. Let $\left\vert \tau\right\vert $ be the number of terminal
nodes in $\tau$. Letting%

\begin{align*}
n_{m}  & =\sum_{i=1}^{n}\mathbf{1}_{R_{m}}(X_{i}),\\
\hat{p}_{m}  & =\frac{1}{n_{m}}\sum_{i=1}^{n}Y_{i}\mathbf{1}_{R_{m}}(X_{i})
\end{align*}
and for $\psi(p)=\min(p,1-p)$ (misclassification rate)%
\[
\tilde{I}(m,\tau)=n_{m}\psi\left(  \hat{p}_{m},1-\hat{p}_{m}\right)
\]
we define the cost complexity criterion
\[
C_{\alpha}(\tau)=\sum_{m=1}^{\left\vert \tau\right\vert }\tilde{I}%
(m,\tau)+\alpha\left\vert \tau\right\vert
\]

The idea is to find, for each $\alpha$, the subtree $\tau_{\alpha}\subset
\tau_{0}$ to minimize $C_{\alpha}(\tau)$. The tuning parameter $\alpha\geq0$
governs the tradeoff between tree size and its goodness of fit to the data.
Large values of $\alpha$ result in smaller trees $\tau_{\alpha}$ and
conversely for smaller values of $\alpha$. With $\alpha=0$ the solution is the
full tree $\tau_{0}$ (or among all possible trees on the data, it would be a
tree where each pair $(X_{i},Y_{i})$ has its own region $R_{m}$, with zero
classification error $\hat{p}_{m}$. The tree $\tau_{0}$ stopped at $5$ points
per region intuitively comes close to that).

The degree of pruning (tuning parameter $\alpha$ for the penalty) is chosen by
cross validation; the choice is an $\hat{\alpha}$. The final tree is the one
pruned according to the cross validated $\hat{\alpha}$. For further discussion
of cost complexity pruning and "greedy histograms", see Klemel\"{a}
\cite{Kl}\footnote{Klemel\"{a}, J. (2009). \textsl{Smoothing of Multivariate
Data.}Wiley, New York.}. A justification of the pruning algorithm as a model
choice procedure (with oracle type inequalities) in the regression context is
given by Sauv\'{e} \cite{sauv}.

\bigskip

\bigskip\textbf{Reason for the two stage procedure. }There are configurations
of data (that is laws $\mathcal{L}(X,Y)$) where initially, no split
perpendicular to the axes seems reasonable, but there are splits below the
first stage which are very good. Consider two class-conditional laws
$P_{0}=\mathcal{L}(X|Y=0)$, $P_{1}=\mathcal{L}(X|Y=1)$ defined as follows.
Divide the unit cube $U=[0,1]\times\lbrack0,1]$ into two regions:
$U_{0}=\left(  [0,1/2]\times\lbrack0,1/2]\right)  \cup\left(  \lbrack
1/2,1]\times\lbrack1/2,1]\right)  $ and $U_{1}=\left(  [1/2,1]\times
\lbrack0,1/2]\right)  \cup\left(  \lbrack0,1/2]\times\lbrack1/2,1]\right)  $
and let $P_{i}$ be the uniform law on $U_{i}$, $i=1,2$. Set $P(Y=0)=1/2$.
Then, for large sample size, both $\hat{p}_{1,U,k,s}$ and $\hat{p}_{2,U,k,s}$
are close to $1/2$ for $k=1,2$ and all possible split points $s\in(0,1)$. Thus
the selection of the first split seems to be arbitrary; none of these appears
really good. However after the first split has been made, the two resulting
rectangles can be split in a meaningful way (picture).

For consistency results see Olshen \cite{Olsh}\footnote{Olshen, R. (2007).
Tree-structured regression and the differentiation of integrals. Ann. Statist.
\textbf{35} 1-12} and references therein, as well as discussions in
\cite{DGL}, sec. 20.9.

An example: spam data (HTF, p. 313).%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
At this point I showed the picture of the tree with the spam example for HTF,
scanned picture "pic-CART-HTF-copyright".
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\newpage

\section{Boosting}%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
A section on bagging might be inserted before. We have an 05 handwritten ms,
see also Klemela p. 460, also HTF ,amd paper by Yu et al.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion
\bigskip

Boosting is based on the observation that finding many rough prediction rules
can be much easier than finding a single highly accurate predictor. In the
boosting approach, we start with a \textquotedblleft\ base\textquotedblright%
\ learning algorithm, capable of producing moderately accurate prediction
rules (perhaps rules that do slightly better than random guessing). We call
this algorithm on our training set, producing a rough prediction rule. We then
tune the importance, or weight, of our training data, assigning higher weights
to those data misclassified by our first rough predictor, and assigning lower
weight to the data classified correctly. We repeat this procedure, producing
many rough classifiers. Heuristically, each new classifier should perform well
on the data misclassified by the previous predictors. At the end of this
iteration, the algorithm must combine the set of weak classifiers into a
single, and hopefully accurate, prediction rule for the entire data set.

There are two fundamental questions in this approach: first, how should the
distribution on the training data be chosen on each round, and second, how
should the weak rules be combined into a single rule? There is also the
question of what to use for the base learning algorithm, which we will not
address in this discussion. The goal here is to develop a general method for
combining weak learners into a single good predictor.%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
This is generally poorly understood by me. What exactly is the error estimate
given in "training error analysis" ? Student Josef Broder who wrote this
section used references:

[2] Yoav, F. \& Schapire, R. A decision-theoretic generalization of
on-linelearning and an application to boosting. Journal of Computer and
SystemSciences, 55(1):119---139, 1997.[3] Schapire, R. (2001). The Boosting
Approach to Machine Learning: An Overview, Nonlinear Estimation and
Classification, Springer, /2008.

The section "boosting as additive modelling", basically from HTF, needs to be
better understood and digested. Also, add advanced results such as consistency
results by Lugosi, Vayatis (AS) and other recent results by Koltchinski (AS).

In general, references in this section have not been properly written out in
the reference list at the end, needs to be done.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\subsection{Weighted training data}

The AdaBoost algorithm is based on the notion of a training set equipped with
weights $w_{i}$, $i=1,\ldots,n.$ The algorithm uses a "base learner"
(classifier) which is repeatedly applied to the same training set
$(x_{i},y_{i})$, $i=1,\ldots,n$ but with different weights each time. To
understand that, we first have to clarify what does it mean to apply a
classifier to a training set equipped with weights. We focus first on the tree
method, and review the unweighted case first.

\textit{Unweighted impurity measure. }Suppose for ease of notation all $x_{i}
$ are on the interval $[0,1]$ and let $\psi(p,1-p)=2p(1-p)$ be the Gini index.
The impurity function to be minimized for a possible split $s$ is
\begin{equation}
I(s)=n_{1,s}\psi\left(  \hat{p}_{1,s},1-\hat{p}_{1,s}\right)  +n_{2,s}%
\psi\left(  \hat{p}_{2,s},1-\hat{p}_{2,s}\right) \label{again-impurity}%
\end{equation}

where $\hat{p}_{1,s}$ is the proportion of $1^{^{\prime}s}$ in $\left\{
Y_{i}:X_{i}\leq s\right\}  $ and $\hat{p}_{2,s}$ is the proportion of
$1^{^{\prime}s}$ in $\left\{  Y_{i}:X_{i}>s\right\}  ,$ and $n_{1,s}$,
$n_{2,s} $ are the sizes of the two classes:
\begin{align*}
n_{1,s}  & =\sum_{i=1}^{n}\mathbf{1}_{[0,s]}(x_{i}),\;\hat{p}_{1,s}=\frac
{1}{n_{1,s}}\sum_{i=1}^{n}y_{i}\mathbf{1}_{[0,s]}(x_{i}),\\
n_{2,s}  & =\sum_{i=1}^{n}\mathbf{1}_{(s,1]}(x_{i}),\;\hat{p}_{2,s}=\frac
{1}{n_{2,s}}\sum_{i=1}^{n}y_{i}\mathbf{1}_{[0,s]}(x_{i}).
\end{align*}

\textit{Weighted impurity measures.}\textbf{\ }Let $w_{i}$, $i=1,\ldots,n$ be
nonnegative with $\sum_{i=1}^{n}w_{i}=1$. Define the weighted analogs of
$\hat{p}_{1,s}$ and $n_{1,s}$ as
\[
n_{1,s}=\sum_{i=1}^{n}w_{i}\mathbf{1}_{[0,s]}(x_{i}),\;\hat{p}_{1,s}=\frac
{1}{n_{1,s}}\sum_{i=1}^{n}w_{i}y_{i}\mathbf{1}_{[0,s]}(x_{i})
\]
and analogously for $\hat{p}_{2,s}$ and $n_{2,s}$. The impurity function to be
minimized for a possible split $s$ is again (\ref{again-impurity}) but with
the modified $n_{i,s},\hat{p}_{i,s}$, $i=1,2$. The same principle applies when
for multivariate $x_{i}=(x_{i}^{(1)},\ldots,x_{i}^{(d)})\in\mathbb{R}^{d}$ one
takes a split point $s$ for the $k$-th component and a criterion analogous to
(\ref{impurity-criterion}), applied to the $k$-th components $x_{i}^{(k)}$,
$i=1,\ldots,n$. This means that when considering a split of a rectangle $R$
along the $k$-th component, one defines the weighted quantities as
$n_{1,R,k,s}=\sum_{i=1}^{n}w_{i}\mathbf{1}_{(-\infty,s]}(x_{i}^{(k)}%
)\mathbf{1}_{R}(x_{i})$ etc. in analogy to (\ref{unweighted-multi-1}),
(\ref{unweighted-multi-2}).

\textit{Neural network training.} Suppose we have $K$ classes and we use
sum-of-squared errors as our measure of fit (error function): for a training
set $(x_{i},y_{i})$ $i=1,\ldots,n$ where $x_{i}\in\mathbb{R}^{d}$ and
$y_{i}=(y_{i}^{(1)},\ldots,y_{i}^{(K)})$ is an indicator vector (where
$y_{i}^{(k)}=1$ if $x_{i}$ is in class $k$, $0$ otherwise)
\[
R(\theta)=\sum_{i=1}^{n}\sum_{k=1}^{K}\left(  y_{i}^{(k)}-\psi_{k}%
(x_{i})\right)  ^{2}%
\]
(cf. (\ref{least-squares-NN})) and the corresponding classifier is
$G(x)=\arg\max_{k\in\{1,\ldots,K\}}\psi_{k}(x)$. Here $\theta$ represents the
parameters of the neural network (that is, of the the vector valued function
$\psi_{k}(x)$, $k=1,\ldots,K.$ ). The weighted variant of $R(\theta)$ is
\[
R(\theta)=\sum_{i=1}^{n}\sum_{k=1}^{K}w_{i}\left(  y_{i}^{(k)}-\psi_{k}%
(x_{i})\right)  ^{2}%
\]
and the weighted variant of the cross-entropy (\ref{cross-entropy}) is
\[
R(\theta)=-\sum_{i=1}^{n}\sum_{k=1}^{K}w_{i}y_{i}^{(k)}\log\psi_{k}(x_{i}).
\]

\subsection{The AdaBoost algorithm}

Suppose we are given a training set $T_{n}=\left(  \left(  x_{i},y_{i}\right)
\text{,}i=1,\ldots,n\right)  $ where the labels $y_{i}$ are in $\left\{
-1,1\right\}  $. The $x_{i}$ can be in a general set $\mathcal{X}$. The
following algorithm trains $M$ weak learners, and combines them into a final
prediction rule. We assume all classifiers $G(x)$ taskes values in $\left\{
-1,1\right\}  $.

\bigskip

\textbf{Algorithm AdaBoost.M1. }(Freund \& Schapire 1997)\textbf{. }

\begin{description}
\item[(1)] Initialize the observation weights $w_{i}^{(1)}=1/n$,
$i=1,2,\ldots,n$.

\item[(2)] For $m=1$ to $M$:

\begin{description}
\item[(a)] Fit a classifier $G_{m}(x)$ to the training data using weights
$w_{i}^{(m)}.$

\item[(b)] Compute.%
\[
\mathrm{err}_{m}=\frac{\sum_{i=1}^{n}w_{i}^{(m)}\mathbf{1}\left(  y_{i}\neq
G_{m}(x_{i})\right)  }{\sum_{i=1}^{n}w_{i}^{(m)}}%
\]

\item[(c)] Compute%
\[
\alpha_{m}=\log\left(  \frac{1-\mathrm{err}_{m}}{\mathrm{err}_{m}}\right)
\]

\item[(d)] Set $w_{i}^{(m+1)}:=w_{i}^{(m)}\exp\left[  \alpha_{m}%
\mathbf{1}\left(  y_{i}\neq G_{m}(x_{i})\right)  \right]  ,$ $i=1,2,\ldots,n$
\end{description}

\item[(3)] Output%
\begin{equation}
G(x)=\mathrm{sign}\left(  \sum_{m=1}^{M}\alpha_{m}G_{m}(x)\right) \label{10.1}%
\end{equation}
.
\end{description}

We assume that the initial classifier $G_{1}(x)$ has error $\mathrm{err}_{1}$
$<1/2$. We can interpret this as meaning that $G_{1}(x)$ does slightly better
than random guessing. From this assumption, we know that$(1-\mathrm{err}%
_{1})/\mathrm{err}_{1}>1$, so

\begin{enumerate}
\item $\alpha_{1}=\log\left(  \frac{1-\mathrm{err}_{m}}{\mathrm{err}_{m}%
}\right)  >0$, ie $G_{1}(x)$ has positive weight in the final classifier. In
general, any weak learning that does slightly better than guessing will have
positive weight in the final classifier.

\item $w_{i}^{(2)}\longleftarrow w_{i}^{(1)}\left(  1-\mathrm{err}_{1}\right)
/\mathrm{err}_{1}$ for misclassified examples $\left(  x_{i},y_{i}\right)  $,
and $w_{i}^{(2)}\longleftarrow w_{i}^{(1)}$ for correctly classified examples.
This means that the weight of misclassified examples increases, and the weight
of correctly classified examples remains the same.
\end{enumerate}

\subsubsection{ Training error analysis}%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
I don't understand this, what is actually being estimated here. This has been
presented without proper understanding by me, scandalous, almost never
happends. Based on a little paper by Josef Broder; possibly look up his references.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

Let $\mathrm{err}_{m}=1/2-\gamma_{m}$, and suppose that each base learner is
slightly better than random guessing, so that $\gamma_{m}>\gamma>0$ for
$m=1,\ldots,M$. In this case, we will show that the training error of the
final classifier is exponentially small in the number of iterations of the
AdaBoost algorithm. We will prove this result in a series of three steps.

First, define $f(x)=\sum_{m=1}^{M}\alpha_{m}G_{m}(x)$, so that
$G(x)=\mathrm{sign}\left(  f(x)\right)  $, and define $W_{m}=\sum_{i=1}%
^{n}w_{i}^{(m)}$.

\begin{lemma}
$W_{m}=%
%TCIMACRO{\dprod \limits_{i=1}^{m-1}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{m-1}}
%EndExpansion
2\left(  1-\mathrm{err}_{i}\right)  .$
\end{lemma}

\begin{proof}
This follows from the recursion%
\begin{align*}
W_{k+1}  & =\sum_{i=1}^{n}w_{i}^{(k)}\exp\left(  \alpha_{k}\mathbf{1}%
_{\left\{  y_{i}\neq G_{k}(x_{i})\right\}  }\right) \\
& =\left(  \frac{1-\mathrm{err}_{k}}{\mathrm{err}_{k}}\right)  \sum
_{i:\;y_{i}\neq G_{k}(x_{i})}w_{i}^{(k)}+\sum_{i:\;y_{i}=G_{k}(x_{i})}%
w_{i}^{(k)}\\
& =\left(  \frac{1-\mathrm{err}_{k}}{\mathrm{err}_{k}}\right)  \left(
W_{k}\cdot\mathrm{err}_{k}\right)  +\left(  W_{k}-W_{k}\cdot\mathrm{err}%
_{k}\right) \\
& =2\left(  1-\mathrm{err}_{k}\right)  \cdot W_{k}%
\end{align*}
and the base $W_{1}=1$.
\end{proof}

\begin{lemma}%
\[
w_{i}^{(M)}=\frac{1}{n}\exp\left(  \frac{-y_{i}f(x_{i})}{2}\right)
\exp\left(  \frac{1}{2}\sum_{m=1}^{M}\alpha_{m}\right)
\]

\end{lemma}

\begin{proof}
Recall that $f(x)=\sum_{m=1}^{M}\alpha_{m}G_{m}(x)$ and the final classifier
is $G(x)=\mathrm{sign}\left(  f(x)\right)  $. Also note that for any
classifier $G(x)$,%
\[
-yG(x)=2\mathbf{1}_{\left\{  y\neq G(x)\right\}  }-1.
\]
This gives the relation%
\[
\mathbf{1}_{\left\{  y\neq G(x)\right\}  }=\frac{-yG(x)+1}{2}.
\]
Now by \textquotedblleft unwrapping\textquotedblright\ the iterative update of
$w_{i}^{(m)}$, we can use the above observation to calculate%
\begin{align*}
w_{i}^{(M)}  & =\frac{1}{n}\exp\left(  \sum_{m=1}^{M}\alpha_{m}\mathbf{1}%
_{\left\{  y_{i}\neq G_{m}(x_{i})\right\}  }\right) \\
& =\frac{1}{n}\exp\left(  \sum_{m=1}^{M}\alpha_{m}\left(  \frac{-y_{i}%
G_{m}(x_{i})+1}{2}\right)  \right) \\
& =\frac{1}{n}\exp\left(  \frac{-y_{i}}{2}\sum_{m=1}^{M}\alpha_{m}G_{m}%
(x_{i})\right)  \exp\left(  \frac{1}{2}\sum_{m=1}^{M}\alpha_{m}\right)  .
\end{align*}

\end{proof}

We can now combine these results to estimate the training error of the final
classifier $G(x)$, defined as%
\[
\tau(G)=\frac{1}{n}\sum_{i=1}^{n}\mathbf{1}_{\left\{  y_{i}\neq G(x_{i}%
)\right\}  .}%
\]

\begin{theorem}%
\[
\tau(G)\leq\exp\left(  -2(M-1)\gamma^{2}\right)  .
\]

\end{theorem}

\begin{proof}
Note that if $G(x)\neq y$, then $yf(x)\leq0$, so $-yf(x)/2\geq0$ and we have
$\exp\left(  -yf(x)/2\right)  \geq1$. We can use this to estimate the training
error of the final classifier $G(x)$:%
\[
\tau(G)=\frac{1}{n}\sum_{i=1}^{n}\mathbf{1}_{\left\{  y_{i}\neq G(x_{i}%
)\right\}  .}\leq\frac{1}{n}\sum_{i=1}^{n}\exp\left(  \frac{-y_{i}f(x_{i})}%
{2}\right)
\]%
\[
=\left(  \sum_{i=1}^{n}w_{i}^{(M)}\right)  \exp\left(  -\frac{1}{2}\sum
_{m=1}^{M}\alpha_{m}\right)
\]%
\[
\leq\left(
%TCIMACRO{\dprod \limits_{i=1}^{M-1}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{M-1}}
%EndExpansion
2\left(  1-\mathrm{err}_{m}\right)  \right)  \left(
%TCIMACRO{\dprod \limits_{i=1}^{M}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{M}}
%EndExpansion
\sqrt{\frac{\mathrm{err}_{m}}{1-\mathrm{err}_{m}}}\right)
\]%
\[
\leq%
%TCIMACRO{\dprod \limits_{i=1}^{M-1}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{M-1}}
%EndExpansion
2\sqrt{\mathrm{err}_{m}\left(  1-\mathrm{err}_{m}\right)  }%
\]
where the last inequality holds due to
\[
\sqrt{\frac{\mathrm{err}_{M}}{1-\mathrm{err}_{M}}}=\sqrt{\frac{1/2-\gamma_{M}%
}{1/2+\gamma_{M}}}\leq1.
\]
Now using the assumption that $\mathrm{err}_{m}=1/2-\gamma_{m}$ and the fact
that for all real numbers $x$ one has $\exp\left(  -2x^{2}\right)
\geq1-2x^{2}$, we obtain
\begin{align*}
\tau(G)  & \leq%
%TCIMACRO{\dprod \limits_{i=1}^{M-1}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{M-1}}
%EndExpansion
2\sqrt{\mathrm{err}_{m}\left(  1-\mathrm{err}_{m}\right)  }\\
& =%
%TCIMACRO{\dprod \limits_{i=1}^{M-1}}%
%BeginExpansion
{\displaystyle\prod\limits_{i=1}^{M-1}}
%EndExpansion
\sqrt{1-4\gamma_{m}^{2}}\\
& \leq\exp\left(  -2\sum_{m=1}^{M-1}\gamma_{m}^{2}\right)  \leq\exp\left(
-2(M-1)\gamma^{2}\right)
\end{align*}
since $\gamma_{m}>\gamma>0$ for $m=1,\ldots,M$ by assumption.
\end{proof}

\subsubsection{A simulation example}

Let the distribution of $X=\left(  X^{(1)},\ldots,X^{(10)}\right)  $ be
$N_{10}(0,I)$ and let the conditional law $\mathcal{L}(Y|X)$ be given by
\[
\mathcal{L}(Y|X=x)=\left\{
\begin{tabular}
[c]{l}%
$P(Y=1|X=x)=1$ if $\left\Vert X\right\Vert ^{2}>\chi_{10}^{2}(0.5)$\\
$P(Y=-1|X=x)=1$ otherwise
\end{tabular}
\right.
\]
where $\chi_{10}^{2}(0.5)=9.34$ is the median ($0.5$-quantile) of the
$\chi_{10}^{2}$-distribution. Thus $P(Y=1)=P(Y=-1)=1/2$ and the
class-conditional laws $\mathcal{L}(X|Y=1)$, $\mathcal{L}(X|Y=-1)$ have
disjoint support (on $\left\{  X:\left\Vert X\right\Vert ^{2}>\chi_{10}%
^{2}(0.5)\right\}  $ and $\left\{  X:\left\Vert X\right\Vert ^{2}\leq\chi
_{10}^{2}(0.5)\right\}  $ respectively). There were $n=2000$ training cases
generated and also $10,000$ test observations. The base classifier $G_{1}(x)$
is a "stump": a rudimentary classification tree with two terminal nodes, such
that for some $j\in\left\{  1,\ldots,k\right\}  $ and $s\in\mathbb{R}$ and
$z\in\left\{  -1,1\right\}  $
\[
G_{1}(x)=\left\{
\begin{tabular}
[c]{l}%
$z$ if $x^{(j)}\leq s$\\
$-z$ if $x^{(j)}>s$%
\end{tabular}
\right.  .
\]
The test error rate of $G_{1}$ was $46\%$. We can easily compute the
appriximate error rate of a stump with $s=9.34$, $z=-1$ and $k=1$ (or
arbirtrary)$.$ Indeed, all $X_{i}$ with $\left\Vert X_{i}\right\Vert \leq s$
are correctly classified as $-1,$ and also all $X_{i}$ with $X_{i}^{(1)}>s$
are correctly classified as $1$. Thus the error rate is (for large $n$)
\begin{align*}
P\left(  \left\Vert X\right\Vert ^{2}>s,X^{(1)}<s\right)   & =1-P\left(
\left\Vert X\right\Vert ^{2}<s\right)  -P\left(  X^{(1)}>9.34\right) \\
& =1/2-\varepsilon
\end{align*}
for $\varepsilon=P\left(  Z>9.34\right)  $ where $Z$ is standard normal.

After $M=400$ booosting iterations, the test error was 12.2\%. For a
comparison, a CART tree with backpruning achieved a 26\% error rate. The
boosted classifier
\[
G(x)=\mathrm{sign}\left(  \sum_{m=1}^{M}\alpha_{m}G_{m}(x)\right)
\]
should be constant on rectangles, and obviously the sum of the rectangles
where $-1$ is classified approximately cover the ball $\left\{  X:\left\Vert
X\right\Vert ^{2}\leq\chi_{10}^{2}(0.5)\right\}  $.

\subsection{Boosting as additive modeling}

The goal of boosting is to express an accurate classifier $G(x)$ as a weighted
sum of weak learners $G_{m}(x)$. We can view this as a special case of
expressing an arbitrary function $f(x)$ as a weighted sum of elementary
\textquotedblleft basis\textquotedblright\ functions
\[
f(x)=\sum_{m=1}^{M}\beta_{m}b(x,\gamma_{m})
\]
where the $b(x,\gamma_{m})$ are the basis functions depending on some
parameters $\gamma_{m}$ and the $\beta_{m}$ are the expansion coefficients.
This is a common idea in many learning and nonparametric regression
techniques, for example

\begin{itemize}
\item In single hidden layer neural networks ,$b(x;\gamma)=\sigma\left(
\gamma_{0}+\gamma_{1}^{\top}x\right)  $ where $\sigma(\cdot)$ is a sigmoidal
function and $\gamma$ parameterizes a linear combination of the input variables.

\item For wavelets, $\gamma$ parametrizes the location and scale shifts of a
\textquotedblleft mother\textquotedblright\ wavelet. However in the standard
orthogonal series estimator for regression, the parameters $\gamma_{m}$ are
fixed (not estimated), only the coefficients $\beta_{m}$ are estimated in a
linear or nonlinear way. Another example is classical Fourier series where
$b(x;\gamma_{m})=\phi_{m}(x)$ is the $m$-th element of an orthogonal basis in
function space.

\item MARS (not discussed in this course ) uses truncated power spline basis
functions where \ $\gamma$ parameterizes the variables and values for the knots.

\item For trees, $\gamma$ parameterizes the split variables ($k$-th component)
and split points at the internal nodes, and the predictions at the terminal nodes.
\end{itemize}

A typical approach to this problem is to minimize a loss function averaged
over the training data%
\begin{equation}
\min_{\beta_{m},\gamma_{m}}\sum_{i=1}^{n}L\left(  y_{i},\sum_{m=1}^{M}%
\beta_{m}b(x_{i},\gamma_{m})\right) \label{loss-func-total}%
\end{equation}
$.$ where the loss function L could be squared-error or a likelihood-based
loss function. However, for many loss functions, this approach requires
computationally intensive optimization techniques, so we consider the simpler
subproblem of fitting just a single basis function%
\[
\min_{\beta,\gamma}\sum_{i=1}^{n}L\left(  y_{i},\beta b(x_{i},\gamma)\right)
.
\]
This is \textit{Forward Stagewise Additive Modeling (FSAM)}, in which we
approximate the solution to (\ref{loss-func-total}) by sequentially adding new
basis functions to the expansion without adjusting the previously added basis
functions or coefficients. In the algorithm below, at iteration $m$, we solve
for the optimal basis function$b(x,\gamma_{m})$ and coefficient $\beta_{m}$
given the current expansion $f_{m-1}(x)$. We then update the current expansion
and repeat the process.

\bigskip

\textbf{Algorithm FSAM. }

\begin{description}
\item[(1)] Initialize $f_{0}(x)=0$.

\item[(2)] For $m=1$ to $M$:

\begin{description}
\item[(a)] Compute.%
\[
\left(  \beta_{m},\gamma_{m}\right)  =\arg\min_{\beta,\gamma}\sum_{i=1}%
^{N}L\left(  y_{i},f_{m-1}(x_{i})+\beta b(x_{i};\gamma)\right)
\]

\item[(b)] Set $f_{m}(x)=f_{m-1}(x)+\beta_{m}b(x;\gamma_{m})$
\end{description}
\end{description}

We can now see that the AdaBoost algorithm is a special case of FSAM, using
the loss function
\[
L(y,f(x))=\exp\left(  -yf(x)\right)
\]
Note that if $f(x)=G(x)$ is a classifier taking values in $\{-1,1\}$, then
$L(y,f(x))=e$ for misclassification and $L(y,1(x))=e^{-1}$ for correct classification.

So in the FSAM framework, we wish to solve
\begin{equation}
\left(  \beta_{m},G_{m}\right)  =\arg\min_{\beta,G}\sum_{i=1}^{n}\exp\left[
-y_{i}\left(  f_{m-1}(x_{i})+\beta G(x_{i})\right)  \right]
\label{solve-argmin}%
\end{equation}
for the classifier $G_{m}$ and corresponding coefficient $\beta_{m}$. Note
that (\ref{solve-argmin}) is equivalent to%
\begin{equation}
\left(  \beta_{m},G_{m}\right)  =\arg\min_{\beta,G}\sum_{i=1}^{n}w_{i}%
^{(m)}\exp\left(  -\beta y_{i}G(x_{i})\right) \label{solve-argmin-2}%
\end{equation}
where $w_{i}^{(m)}=\exp\left(  -y_{i}f_{m-1}(x_{i})\right)  $. Since each
$w_{i}^{(m)}$ depends neither on $\beta$ nor $G(x)$, it can be regarded as a
weight applied to each observation. This weight depends on $f_{m-1}(x)$, so
the weight values change with each iteration.

\begin{lemma}
For any value of $\beta>0$, the solution to (\ref{solve-argmin-2}) for
$G_{m}(x)$ is%
\[
G_{m}=\arg\min_{G}\sum_{i=1}^{N}w_{i}^{(m)}\mathbf{1}_{\left\{  y_{i}\neq
G(x_{i})\right\}  }%
\]
\qquad which is the classifier that minimizes the weighted error rate in
predicting y.
\end{lemma}

\begin{proof}
Since $y_{i}G(x_{i})=-1$ for correct classification and $y_{i}G(x_{i})=1$ for
misclassification, we can write the criterion in(\ref{solve-argmin-2}) as
\begin{align}
& e^{-\beta}\sum_{y_{i}=G(x_{i})}w_{i}^{(m)}+e^{\beta}\sum_{y_{i}\neq
G(x_{i})}w_{i}^{(m)}\label{solve-argmin-3}\\
& =\left(  e^{\beta}-e^{-\beta}\right)  \sum_{i=1}^{N}w_{i}^{(m)}%
\mathbf{1}_{\left\{  y_{i}\neq G(x_{i})\right\}  }+e^{-\beta}\sum_{i=1}%
^{N}w_{i}^{(m)}.\nonumber
\end{align}
and see that $G_{m}$ satisfies (\ref{solve-argmin-2}) if and only if it
satisfies (\ref{solve-argmin}).
\end{proof}

Now fixing this $G_{m}$ and minimizing the criterion in (\ref{solve-argmin})
with respect to $\beta$, we obtain the solution
\[
\beta_{m}=\frac{1}{2}\log\frac{1-\mathrm{err}_{m}}{\mathrm{err}_{m}}%
\]
(by differentiating (\ref{solve-argmin-3}) and setting to zero) where
$\mathrm{err}_{m}$ is the minimized error rate%
\[
\mathrm{err}_{m}=\frac{\sum_{i=1}^{n}w_{i}^{(m)}\mathbf{1}_{\left\{  y_{i}\neq
G(x_{i})\right\}  }}{\sum_{i=1}^{n}w_{i}^{(m)}}%
\]
The approximation is then updated
\[
f_{m}(x)=.f_{m-1}(x)+\beta_{m}G_{m}(x).
\]
This causes the weight for the next iteration to be
\begin{equation}
w_{i}^{(m+1)}=w_{i}^{(m)}\exp\left(  -\beta_{m}y_{i}G_{m}(x_{i})\right)
.\label{update-step}%
\end{equation}
Since we have $-y_{i}G_{m}(x_{i})=2\cdot\mathbf{1}_{\left\{  y_{i}\neq
G(x_{i})\right\}  }-1$ , the update step (\ref{update-step}) becomes
\[
w_{i}^{(m+1)}=w_{i}^{(m)}\exp\left(  -\alpha_{m}\mathbf{1}_{\left\{  y_{i}\neq
G(x_{i})\right\}  }\right)  e^{-\beta_{m}}%
\]
where $\alpha_{m}=2\beta_{m}$ as in step 2 (c) of the algorithm AdaBoost. The
above calculation is equivalent to the update step in the AdaBoost algorithm,
up to a constant factor $e^{-\beta_{m}}$, so we can interpret AdaBoost as
minimizing the exponential loss function via FSAM.

The use of exponential loss in the context of additive modeling is justified
by the simple reweighting AdaBoost algorithm. However, it is of interest to
inquire about its statistical properties. What does it estimate and how well
is it being estimated? The first question is answered by seeking its
population minimizer.

It is easy to show (Friedman et al., 2000)

\begin{lemma}
We have
\[
f^{\ast}(x)=\arg\min_{f(x)}E_{Y|x}\exp\left(  -Yf(x)\right)  =\frac{1}{2}%
\log\frac{\Pr\left(  Y=1|x\right)  }{\Pr\left(  Y=-1|x\right)  }%
\]
\qquad or equivalently%
\[
\Pr\left(  Y=1|x\right)  =\frac{1}{1+\exp\left(  -2f^{\ast}(x)\right)  }.
\]

\end{lemma}

Thus, the additive expansion produced by AdaBoost is estimating one-half the
log-odds of $\Pr\left(  Y=1|x\right)  $. This justifies using its sign as the
classification rule in (\ref{10.1}).

\newpage

\section{Statistical performance of classifiers}

Consider again a training sample $T_{n}=\left\{  (X_{i},Y_{i}),i=1,\ldots
,n\right\}  $ where $X_{i}\in\mathbb{R}^{d},$ $Y_{i}\in\left\{  0,1\right\}  $
and $Z_{i}=(X_{i},Y_{i})$ are i.i.d. random vectors with common distribution
$Q$ on $\mathbb{R}^{d}\times\left\{  0,1\right\}  $. A classifier is a
(measurable) function%
\[
h:\mathbb{R}^{d}\times\left\{  \mathbb{R}^{d}\times\left\{  0,1\right\}
\right\}  ^{\times n}\rightarrow\left\{  0,1\right\}
\]
Classifiers $f$ should be optimal with respect to the error probability for a
"new", incoming test case $\left(  X,Y\right)  $ of which only $X$ is
observed. For a given (fixed) training set $t_{n}$, the performance of a rule
$h$ is measured by the probability of error for a test case $(X,Y)$ having
joint distribution $Q$ and $h(X,t_{n})$ is used to predict $Y:$
\begin{equation}
P_{Q}\left(  h(X,t_{n})\neq Y\right)  =P_{Q}\left(  h\neq Y|T_{n}%
=t_{n}\right)  .\label{err-probab-class(2)}%
\end{equation}
(Note that now we index expectations and probabilities by the pertaining
unknown law $Q=\mathcal{L}(X,Y)$). We denote the above conditional probability
as a random variable
\[
L_{Q}(h):=P_{Q}\left(  h\neq Y|T_{n}\right)  .
\]
and sometimes we omit the lower index, writing $L(h)=L_{Q}(h)$. The final
measure of performance for the rule $h_{n}$ is obtained by averaging over the
training set $T_{n}$:%
\begin{equation}
E_{Q}L(h)=E_{Q}P_{Q}\left(  h\neq Y|T_{n}\right)  =P_{Q}\left(  h\neq
Y\right)  .
\end{equation}
The classifier minimizing (\ref{err-probab-class(2)}) for given $t_{n}$
\textit{among all }$h:\mathbb{R}^{d}\mapsto\left\{  0,1\right\}  $, does not
depend on $t_{n}$ but depends on $Q$ and is called the Bayes classifier
$h_{Q}^{\ast}$:%
\[
L_{Q}^{\ast}:=P_{Q}\left(  h_{Q}^{\ast}(X)\neq Y\right)  =\inf_{h:\mathbb{R}%
^{d}\mapsto\left\{  0,1\right\}  }P_{Q}\left(  h(X)\neq Y\right)  ;
\]
Having a training set, $T_{n}=\left\{  Z_{i},i=1,\ldots,n\right\}  $, , one
might consider it desirable to approximate the Bayes classifier as much as
possible. In principle statistics gives the methods to do that: estimate $Q$
or estimate the regression function
\[
r(x)=E_{Q}\left(  Y|X=x\right)
\]
consistently from the data, and build a classifier from there. Heuristically,
such a sequence of classifiers $h_{n}$ should be \textit{consistent}:
\begin{equation}
L(h_{n})\rightarrow_{P}L_{Q}^{\ast}\text{ as }\mathcal{L}\left(  T_{n}\right)
=Q^{n}\text{ and }n\rightarrow\infty\text{ }\label{univ-consist}%
\end{equation}
where $L(h)=P_{Q}\left(  h_{n}\neq Y|T_{n}\right)  $ is random (as a function
of $T_{n}$) and convergence is in probability. Consistency of a sequence
$h_{n}$ which holds for any $Q$ is called \textit{universal consistency}.
There are simple classifiers which are universally consistent (histogram rule,
nearest neighbor rule). Since $0\leq L_{Q}(h)\leq1$, the convergence in
probability (\ref{univ-consist}) is equivalent to
\[
E_{Q}L(h_{n})\rightarrow L_{Q}^{\ast}\text{ as }n\rightarrow\infty\text{ }.
\]
The consistency (\ref{univ-consist}) may be written: for every $\alpha>0$
there is a sequence $\varepsilon_{n}=\varepsilon(n,\alpha,Q)\rightarrow0$ such
that that for all $n$%
\[
P_{Q}\left(  L(h_{n})\leq L_{Q}^{\ast}+\varepsilon_{n}\right)  \geq
1-\alpha.\text{ }%
\]
(make a picture; recall that $L(h_{n})\geq L_{Q}^{\ast}$). Generally
$\varepsilon_{n}=\varepsilon(n,\alpha,Q)$ depends on $Q$ which is unknown, and
also $L_{Q}^{\ast}$ is unknown. One may be interested in obtaining statements
where $\varepsilon_{n}$ is replaced by a known (or observable) quantity
$\tilde{b}(n,\alpha)$, which does not depend on $Q$. But it will turn out that
then the Bayes risk $L_{Q}^{\ast}$ is too strong a benchmark; it will have to
be replaced by the best risk over a restricted class of rules $\mathcal{C}$.

\subsection{Vapnik-Chervonenkis Theory: Overview}

Consider the following situation: Let $\mathcal{C}$ be a class of mappings of
the form $h:\mathbb{R}^{d}\rightarrow\{0,1\}$. Recall that a classifier was a
map $h:\mathbb{R}^{d}\times\left\{  \mathbb{R}^{d}\times\left\{  0,1\right\}
\right\}  ^{\times n}\rightarrow\left\{  0,1\right\}  $, so $\mathcal{C}$
describes a class of $h$ not depending on $T_{n}$. Assume however that $T_{n}$
is present and that the error count or the empirical risk
\[
L_{emp}(h)=\frac{1}{n}\sum_{j=1}^{n}\mathbf{1}_{\{h(X_{j})\neq Y_{j}\}}%
\]
is used to estimate the error probability $L_{Q}(h)=P_{Q}\left(  h(X)\neq
Y\right)  $ of each $h\in\mathcal{C}$. Denote by $\hat{h}$ the element of
$\mathcal{C}$ that minimizes the estimated error probability over the class:%
\[
L_{emp}(\hat{h})\leq L_{emp}(h)\text{ for all }h\in\mathcal{C}\text{. }%
\]
This $\hat{h}$ now depends on $T_{n}$ and is thus a classifier. With such an
approach, we do not directly aim at the Bayes rule since we are selecting only
among a restricted class $\mathcal{C}$ of rules. Of course, as $n\rightarrow
\infty$, our class $\mathcal{C}$ may expand and in this way we may be able to
"capture" the Bayes rule. But for now, suppose $n$ fixed; then what we can
hope for is to be as best as possible within the class of rules $\mathcal{C}$.
Thus our target quantity is
\[
\inf_{h\in\mathcal{C}}\;L_{Q}(h)
\]
and since $L_{Q}(h)$ depends on the unknown $Q$, the best possible rule within
$\mathcal{C}$ is also unknown. However for the (random) error probability%
\[
L_{Q}(\hat{h})=P\left(  \hat{h}(X)\neq Y|T_{n}\right)
\]
of $\hat{h}$ we have:

\begin{lemma}
\label{lem-VC-basic}
\begin{align}
L_{Q}(\hat{h})-\inf_{h\in\mathcal{C}}\;L_{Q}(h)  & \leq2\sup_{h\in\mathcal{C}%
}\left\vert L_{emp}(h)-L_{Q}(h)\right\vert ,\label{basic-vc-1}\\
\left\vert L_{emp}(\hat{h})-L_{Q}(\hat{h})\right\vert  & \leq\sup
_{h\in\mathcal{C}}\left\vert L_{emp}(h)-L_{Q}(h)\right\vert
.\label{basic-vc-2}%
\end{align}

\end{lemma}

\begin{proof}
In the proof we omit the subscript $Q.$ We have
\begin{equation}
L(\hat{h})-\inf_{h\in\mathcal{C}}\;L(h)\leq L(\hat{h})-L_{emp}(\hat
{h})+L_{emp}(\hat{h})-\inf_{h\in\mathcal{C}}\;L(h).\label{start-inequ}%
\end{equation}
Let $\varepsilon>0$ and let $h_{\varepsilon}\in\mathcal{C}$ be such that
\[
\inf_{h\in\mathcal{C}}\;L(h)\geq L(h_{\varepsilon})-\varepsilon
\]
i.e. the classifier $h_{\varepsilon}$ is within $\varepsilon$ of attaining
$\inf_{h\in\mathcal{C}}\;L(h)$ ($h_{\varepsilon}$ always exists). Then
\begin{align*}
L_{emp}(\hat{h})-\inf_{h\in\mathcal{C}}\;L(h)  & \leq L_{emp}(\hat
{h})-L(h_{\varepsilon})+\varepsilon\\
& \leq L_{emp}(h_{\varepsilon})-L(h_{\varepsilon})+\varepsilon
\end{align*}
(since $\hat{h}$ is the minimizer of $R_{emp}$), and hence from
(\ref{start-inequ})%
\begin{align*}
L(\hat{h})-\inf_{h\in\mathcal{C}}\;L(h)  & \leq L(\hat{h})-L_{emp}(\hat
{h})+L_{emp}(h_{\varepsilon})-L(h_{\varepsilon})+\varepsilon\\
& \leq\left\vert L(\hat{h})-L_{emp}(\hat{h})\right\vert +\left\vert
L_{emp}(h_{\varepsilon})-L(h_{\varepsilon})\right\vert +\varepsilon\\
& \leq2\sup_{h\in\mathcal{C}}\left\vert L_{emp}(h)-L(h)\right\vert
+\varepsilon.
\end{align*}
Since $\varepsilon>0$ was arbitrary, the first claim follows. The second
inequality is trivially true.
\end{proof}

\bigskip\bigskip The aim of Vapnik-Chervonenkis is to obtain probability
inequalities of the type
\[
P_{Q}\left(  L_{Q}(\hat{h})-\inf_{h\in\mathcal{C}}\;L_{Q}(h)>\varepsilon
\right)  \leq b_{n}(\varepsilon,\mathcal{C})
\]
for some bounds $b(\varepsilon,\mathcal{C})$ independent of $Q$, but depending
on the set of classifiers $\mathcal{C}$. By inequality (\ref{basic-vc-1}) this
reduces to
\[
P_{Q}\left(  \sup_{h\in\mathcal{C}}\left\vert L_{emp}(h)-L_{Q}(h)\right\vert
>\varepsilon/2\right)  \leq b_{n}(\varepsilon,\mathcal{C}).
\]
If $b_{n}(\varepsilon,\mathcal{C})\rightarrow0$ holds for fixed $\mathcal{C}$
then we obtain a type of consistency of the rule $\hat{h}$:
\[
L_{Q}(\hat{h})\rightarrow_{P}\inf_{h\in\mathcal{C}}\;L_{Q}(h)\text{ as
}n\rightarrow\infty
\]
analogous to (\ref{univ-consist}). However the right hand side is a minimum
within the class $\mathcal{C}$, thus $\hat{h}$ does not asymptotically attain
the risk of the Bayes rule. However if $\mathcal{C}$ depends on $n$ and
$b_{n}(\varepsilon,\mathcal{C}_{n})\rightarrow0$ then we potentially obtain
stronger statements on $\hat{h}$. Thus the specific form of the bound
$b_{n}(\varepsilon,\mathcal{C})$ is of central interest.

\subsubsection{Law of Large Numbers in function space}

For any fixed $h\in\mathcal{C}$ we obtain trivially
\[
L_{emp}(h)-L_{Q}(h)\rightarrow_{P}0\text{ as }n\rightarrow\infty.
\]
Indeed
\[
L_{emp}(h)=\frac{1}{n}\sum_{j=1}^{n}\mathbf{1}_{\{h(X_{j})\neq Y_{j}\}}%
\]
is an average of $n$ i.i.d. Bernoulli $\mathrm{Bern}(p)$ random variables with
$p=$ $E_{Q}\mathbf{1}_{\{h(X)\neq Y\}}=$ $P_{Q}\left(  h(X)\neq Y\right)  $
$=L_{Q}(h)$, thus by the LLN
\begin{equation}
P_{Q}\left(  \left\vert L_{emp}(h)-L_{Q}(h)\right\vert >\varepsilon\right)
\rightarrow0.\label{LLN-onerule}%
\end{equation}
However the uniform version
\begin{equation}
P_{Q}\left(  \sup_{h\in\mathcal{C}}\left\vert L_{emp}(h)-L_{Q}(h)\right\vert
>\varepsilon\right)  \rightarrow0\label{uniform-LLN}%
\end{equation}
is not an automatic consequence; its validity depends on how large the class
$\mathcal{C}$ is. The last relation may be interpreted as a uniform LLN over
the set of classifiers $\mathcal{C}$. Thus a notion of \textit{capacity} of
the class $\mathcal{C}$ is likely to play a role in an upper bound
$b_{n}(\varepsilon,\mathcal{C})$.

In fact we may write $L_{Q}(h)$ as the $Q$-probability of a set associated to
$h$: if
\[
A(h)=\left\{  \left(  x,y\right)  :h(x)=1,y=0\right\}  \cup\left\{  \left(
x,y\right)  :h(x)=0,y=1\right\}
\]
then
\[
L_{Q}(h)=P_{Q}\left(  h(X)\neq Y\right)  =Q\left(  A(h)\right)  .
\]
If we define $Q_{n}$ as the empirical measure associated to the training set
$T_{n}$: for measurable $A\subset\mathbb{R}^{d}\times\left\{  0,1\right\}  $
\[
Q_{n}(A)=\frac{1}{n}\sum_{j=1}^{n}\mathbf{1}_{A}(X_{j},Y_{j})
\]
then we have $L_{emp}(h)=Q_{n}(A(h))$ and defining a class of sets in
$\mathbb{R}^{d}\times\left\{  0,1\right\}  $%
\begin{equation}
\mathcal{A}:=\left\{  A(h),\;h\in\mathcal{C}\right\}
\label{corresp-sets-classif}%
\end{equation}
(\ref{uniform-LLN}) writes
\[
P_{Q}\left(  \sup_{A\in\mathcal{A}}\left\vert Q_{n}(A)-Q\left(  A\right)
\right\vert >\varepsilon\right)  \rightarrow0.
\]
Thus the problem is reduced to the convergence of the emprical measure $Q_{n}
$ to $Q$, uniformly over the class of sets $\mathcal{A}$ in $\mathbb{R}%
^{d}\times\left\{  0,1\right\}  $. Clearly, the size of the class $\mathcal{A}
$ should be crucial for the validity of such a result.

\subsubsection{Exponential inequalities}%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
somewhere here we might mention Glivenko-Cantelli; look up van der Vaart for
"G-K classes of sets or functions"
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion
Recall Hoeffding's inequality:

\begin{theorem}
Let $Y_{1},\ldots,Y_{n}$ be independent observations such that $EY_{i}=0$ and
$a\leq Y_{i}\leq b$. Let $\varepsilon>0$. Then
\begin{equation}
P\left(  \left\vert n^{-1}\sum_{i=1}^{n}Y_{i}\right\vert \geq\varepsilon
\right)  \leq2\exp\left(  -2n\varepsilon^{2}/(b-a)^{2}\right)  .\label{Hoeffd}%
\end{equation}

\end{theorem}

Let us apply this to estimating $L_{emp}(h)-L_{Q}(h)$ for one individual
classification rule $h$. Set
\[
V_{i}=\mathbf{1}_{\{h(X_{i})\neq Y_{i}\}}-E_{Q}\mathbf{1}_{\{h(X)\neq
Y\}}=\mathbf{1}_{\{h(X_{i})\neq Y_{i}\}}-L_{Q}(h)
\]
then $V_{i}$ are i.i.d. with $-p=a\leq V_{i}\leq b=1-p$ where $p=L_{Q}(h)$,
and $n^{-1}\sum_{i=1}^{n}V_{i}=L_{emp}(h)-L_{Q}(h)$, hence $b-a=1$ and
(\ref{Hoeffd}) gives
\[
P\left(  \left\vert L_{emp}(h)-L_{Q}(h)\right\vert \geq\varepsilon\right)
\leq2\exp\left(  -2n\varepsilon^{2}\right)  .
\]
We have thus "quantified" the simple Law of Large Numbers result
(\ref{LLN-onerule}). A uniform uniform version over $h\in\mathcal{C}$ (the VC
Theorem for classifiers, Theorem 12.6 in [DGL] p. 199) takes the form%
\begin{equation}
P\left(  \sup_{h\in\mathcal{C}}\left\vert L_{emp}(h)-L_{Q}(h)\right\vert
>\varepsilon\right)  \leq b_{n}(\varepsilon,\mathcal{C}):=8s\left(
\mathcal{C},n\right)  \exp\left(  -n\varepsilon^{2}/32\right)
\label{inequality-VC-theorem}%
\end{equation}
where $s\left(  \mathcal{C},n\right)  $ is a capacity measure for the class
$\mathcal{C}$ of classifiers, the so-called \textbf{shatter coefficient}. The
shatter coefficient is originally a charactistic of a set of subsets of
$\mathbb{R}^{d}$ and is then applied to classifiers via the correspondence
(\ref{corresp-sets-classif}).

\begin{definition}
Let $\mathcal{A}$ be a collection of measurable sets in $\mathbb{R}^{d}$. For
$\left(  z_{1},\ldots,z_{n}\right)  \in\left(  \mathbb{R}^{d}\right)  ^{n} $,
let $N_{\mathcal{A}}\left(  z_{1},\ldots,z_{n}\right)  $ be the number of
different sets in
\[
\left\{  \left\{  z_{1},\ldots,z_{n}\right\}  \cap A,\;A\in\mathcal{A}%
\right\}  .
\]
The $n$-th shatter coefficient of $\mathcal{A}$ is
\[
s\left(  \mathcal{A},n\right)  :=\max_{\left(  z_{1},\ldots,z_{n}\right)
\in\left(  \mathbb{R}^{d}\right)  ^{n}}N_{\mathcal{A}}\left(  z_{1}%
,\ldots,z_{n}\right)  .
\]

\end{definition}

That is, the shatter coefficient is the maximal number of different subsets of
$n$ points that can be picked out by the class of sets $\mathcal{A}$. The
shatter coefficient measures the richness or capacity of the class
$\mathcal{A}$. We also state the VC theorem for general classes $\mathcal{A}$
([DGL], Theorem 12.5, p. 197) of which (\ref{inequality-VC-theorem}) is a consequence.

\begin{theorem}
(Vapnik, Chervonenkis, 1971). Let $Q$ be a probability measure on
$\mathbb{R}^{d}$ and $Q_{n}$ be the corresponding empirical measure based in
an i.i.d. sample. For any collection $\mathcal{A}$ of measurable sets%
\begin{equation}
P_{Q}\left(  \sup_{A\in\mathcal{A}}\left\vert Q_{n}(A)-Q\left(  A\right)
\right\vert >\varepsilon\right)  \leq b_{n}(\varepsilon,\mathcal{A}%
):=8s\left(  \mathcal{A},n\right)  \exp\left(  -n\varepsilon^{2}/32\right)
.\label{inequality-VC-theorem-sets}%
\end{equation}

\end{theorem}

It turn out that, when the shatter coefficient $s\left(  \mathcal{C},n\right)
$ of a class $\mathcal{C}$ of classifiers is computed via the correspondence
(\ref{corresp-sets-classif}) to sets, a simplification is possible. Namely, if
we adopt the more obvious description of $h\in\mathcal{C}$ by the set
$A_{0}(h)=\left\{  x:h(x)=1\right\}  $ and $\mathcal{A}_{0}=\left\{
A_{0}(h)\text{,}h\in\mathcal{C}\right\}  $ then it can be shown that $s\left(
\mathcal{C},n\right)  =s\left(  \mathcal{A}_{0},n\right)  $ ([DGL] Theorem
13.1, p. 216). Therefore, in the examples below, when we discuss $s\left(
\mathcal{A},n\right)  $ for some classes $\mathcal{A}$ of sets then this
directly applies to the respective classifiers given by the indicator function
of these sets.

\bigskip

\textbf{Example 1.} If $\mathcal{A}$ is the class of all subsets of
$\mathbb{R}^{d}$ then from every $\left\{  z_{1},\ldots,z_{n}\right\}  $,
every subset can be picked out, and there are $2^{n}$ different subsets if all
$x_{i}$ are different. In this case $s\left(  \mathcal{A},n\right)  =2^{n}$
and inequality (\ref{inequality-VC-theorem-sets}) loses its meaning:
\[
b_{n}(\varepsilon,\mathcal{A})=8\cdot\left(  2^{n}\right)  \exp\left(
-n\varepsilon^{2}/32\right)  =8\exp\left(  n\left(  \log2-\varepsilon
^{2}/32\right)  \right)
\]
and for $\varepsilon<$ $\sqrt{32\log2}=4.71$ the upper bound $b_{n}%
(\varepsilon,\mathcal{C})$ does not converge to zero, i.e.
(\ref{inequality-VC-theorem-sets}) does not imply the uniform LLN over
$\mathcal{C}$. $\square$\bigskip\bigskip

\textbf{Example 2. }Consider the case $d=1$ and all half spaces in
$\mathbb{R}$: $A=[a,\infty)$ or $A=(-\infty,a]$ for some $a\in\mathbb{R}$.
Consider ordered points $x_{1}<\ldots<x_{n}$; then sets $A=(-\infty,a]$ pick
out all subsets $\left\{  x_{1},\ldots,x_{k}\right\}  $, $k=0,\ldots,n$ (
$k=0$ meaning the empty set); there are $n+1$ of these. Also sets
$A=[a,\infty)$ pick out all subsets $\left\{  x_{k},\ldots,x_{n}\right\}  $,
$k=1,\ldots,n+1 $ ($k=n+1$ meaning the empty set). Here we counted the empty
set and the full set $\left\{  x_{1},\ldots,x_{n}\right\}  $ twice, so
$s\left(  \mathcal{A},n\right)  =2\left(  n+1\right)  -2=2n.$ The upper bound
$b_{n}(\varepsilon,\mathcal{A})$ becomes
\[
b_{n}(\varepsilon,\mathcal{A})=8n\exp\left(  -n\varepsilon^{2}/32\right)
=8\exp\left(  \log n-n\varepsilon^{2}/32\right)
\]
This is still an exponential bound; since $\log n/n\rightarrow0$, e.g. for
sufficiently large $n$ and a small $\delta>0$ an upper bound is $8\exp\left(
-n\left(  \varepsilon^{2}/32-\delta\right)  \right)  .\square$\bigskip\bigskip

In these two examples the growth of the shatter coefficient $s\left(
\mathcal{A},n\right)  $ was either exponential ($\exp(n\log2)$ for the class
of all sets ) or linear ($2n$ for the half lines). The concept of\textbf{\ VC
dimension} of $\mathcal{A}$, to be introduced, serves to describe classes
$\mathcal{A}$ for which $s\left(  \mathcal{A},n\right)  $ grows algebraically
(that is like $n^{s}$), and for which (\ref{inequality-VC-theorem-sets}) hence
is a meaningful uniform LLN. Analogously, the VC dimension of $\mathcal{C}$
will provide a bound on learning capacity by (\ref{inequality-VC-theorem}) and
(\ref{corresp-sets-classif}).

If $s\left(  \mathcal{A},n\right)  =2^{n}$ then the class of sets
$\mathcal{A}$ is said to \textit{shatter} a set of $n$ points (more precisely,
\textit{there is }a set of $n$ distinct points which can be shattered). It is
easy to see that if $s\left(  \mathcal{A},k\right)  <2^{k}$. that is .
$\mathcal{A} $ does not shatter $k$ points (there is no set of $k$ points such
that $\mathcal{A}$ picks out all its subsets) then $\mathcal{A}$ does not
shatter $n$ points for all $n>k.$ The first time when this happens is important:

\begin{definition}
Let $\mathcal{A}$ be a collection of sets with $\left\vert \mathcal{A}%
\right\vert >2$. The largest integer $k\geq1$ for which $s\left(
\mathcal{A},k\right)  =2^{k}$ is denoted by $V_{\mathcal{A}}$ and is called
the Vapnik-Chervonenkis dimension (or VC dimension) of the class $\mathcal{A}%
$. If $s\left(  \mathcal{A},n\right)  =2^{n}$ for all $n$ then by definition
$V_{\mathcal{A}}=\infty$.
\end{definition}

Again, we carry this over to classes $\mathcal{C}$ by
(\ref{corresp-sets-classif}). Thus, the VC dimension of $\mathcal{A}$ is the
maximal number of points which can be arranged so that $\mathcal{A}$ can
shatter them.

It turns out that the VC dimension plays a crucial role in the growth
behaviour of $s\left(  \mathcal{A},n\right)  $ as $n\rightarrow\infty$. Namely
it can be shown [DGL, Theorem 13.3] that if $V_{\mathcal{A}}<\infty$ then
\begin{equation}
s\left(  \mathcal{A},n\right)  \leq\left(  n+1\right)  ^{V_{\mathcal{A}}%
}.\label{bound-on shatter-coeff-VC}%
\end{equation}

\textbf{Example 2 (continued). }Consider again the class $\mathcal{A}$ of half
real lines $[a,\infty)$ and $(-\infty,a]$. It shatters any two points
$x_{1}<x_{2}$ and not does shatter any triple of points $x_{1}<x_{2}<x_{3}$
(the subset $\left\{  x_{2}\right\}  $ cannot be picked out). Thus
$V_{\mathcal{C}}=2$.

\textit{Remark. }Then (\ref{bound-on shatter-coeff-VC}) yields a bound
$s\left(  \mathcal{A},n\right)  \leq\left(  n+1\right)  ^{2},$ but above we
already showed $s\left(  \mathcal{A},n\right)  =2n.$ This shows that
(\ref{bound-on shatter-coeff-VC}) is not sharp. $\square$\bigskip\bigskip

\textbf{Example 3. }Consider the class $\mathcal{A}$ of closed intervals on
the real line, i. e. $[a_{1},a_{2}]$. It shatters any two points $x_{1}<x_{2}
$ and not does shatter any triple of points $x_{1}<x_{2}<x_{3}$ (the subset
$\left\{  x_{1},x_{3}\right\}  $ cannot be picked out). Thus $V_{\mathcal{C}%
}=2$. On the other hand we can directly estimate $s\left(  \mathcal{A}%
,n\right)  $: suppose $x_{1}<\ldots<x_{n}$; the nonempty subsets which can be
picked out are $\left\{  x_{k},\ldots,x_{k+s}\right\}  $ where $1\leq k\leq n$
and $0\leq s\leq n-k$. There are
\[
\sum_{k=1}^{n}(n-k+1)=\sum_{k=1}^{n}k=n\left(  n+1\right)  /2
\]
possibilities, thus, adding the empty set, we obtain $s\left(  \mathcal{A}%
,n\right)  =n\left(  n+1\right)  /2+1$ which is of order $n^{2}$, the same
order as the upper bound in (\ref{bound-on shatter-coeff-VC}). $\square
$\bigskip\bigskip

\textbf{Example 4.} Let $\mathcal{A}$ be all linear half-spaces on the plane.
Any 3-point set (not all on a line) can be shattered. No 4 point set can be
shattered. Consider, for example, 4 points forming a diamond. Let $T$ be the
left and rightmost points. This can't be picked out; other configurations can
also be seen to be unshatterable. So $V_{\mathcal{A}}$ $=3 $. In general,
halfspaces in $\mathbb{R}^{d}$ have VC dimension $d+1$ [DGL, Corollary 13.1,
p. 223 ]. In example 2 we showed this for $d=1.$ $\square$\bigskip\bigskip

\textbf{Example 5.}. Let $\mathcal{A}$ be all rectangles on the plane with
sides parallel to the axes. There is a $4$ point set that can be shattered;
e.g. a diamond. Let $S$ be a $5$ point set. There is one point that is not
leftmost, rightmost, uppermost, or lowermost. Let $T$ be all points in $S$
except this point.Then $T$ can't be picked out. So $V_{\mathcal{A}}$ $=4$. In
general, rectangles in $\mathbb{R}^{d}$ have VC dimension $2d$ [DGL, Theorem
13.8.] $\square$\bigskip\bigskip

\subsection{Confidence intervals}

The combination of inequalities (\ref{bound-on shatter-coeff-VC}) and
(\ref{inequality-VC-theorem}) yields
\begin{align*}
P_{Q}\left(  \sup_{h\in\mathcal{C}}\left\vert L_{emp}(h)-L_{Q}(h)\right\vert
>\varepsilon\right)   & \leq b_{n}(\varepsilon,\mathcal{C}):=8\left(
n+1\right)  ^{V_{\mathcal{C}}}\exp\left(  -n\varepsilon^{2}/32\right) \\
& =8\exp\left(  V_{\mathcal{C}}\log(n+1)-n\varepsilon^{2}/32\right)
\end{align*}
which is a valid exponential bound (since $\log(n+1)/n\rightarrow0$). We may
set the right side $b_{n}(\varepsilon,\mathcal{C})=\alpha$ and solve for
$\varepsilon$; this yields $\varepsilon=\tilde{b}_{n}(\alpha,\mathcal{C})$
for
\[
\tilde{b}_{n}(\alpha,\mathcal{C})=\sqrt{\frac{32}{n}\left(  V_{\mathcal{C}%
}\log(n+1)+\log\frac{8}{\alpha}\right)  }.
\]
and an inequality
\begin{equation}
P_{Q}\left(  \sup_{h\in\mathcal{C}}\left\vert L_{Q}(h)-L_{emp}(h)\right\vert
\leq\tilde{b}_{n}(\alpha,\mathcal{C})\right)  \geq1-\alpha\text{ for all
}Q=\mathcal{L}\left(  X,Y\right)  \text{.}\label{inequ-basic-VC-uniform}%
\end{equation}
From this, two types of confidence statements can be derived, for the
empirical risk minimizing classifier $\hat{h}$ treated in Lemma
\ref{lem-VC-basic}.

\bigskip

\subsubsection{Oracle type inequalities}

By inequality (\ref{basic-vc-1}) we obtain
\[
L_{Q}(\hat{h})-\inf_{h\in\mathcal{C}}\;L_{Q}(h)\leq\tilde{b}_{n}%
(\alpha,\mathcal{C})
\]
valid with probability at least $1-\alpha$, thus a statement
\begin{equation}
P_{Q}\left(  L(\hat{h}_{n})\leq\inf_{h\in\mathcal{C}}\;L_{Q}(h)+\tilde{b}%
_{n}(\alpha,\mathcal{C})\right)  \geq1-\alpha\text{ for all }Q=\mathcal{L}%
\left(  X,Y\right)  .\label{pac-inequa}%
\end{equation}
The name "oracle type" refers to the fact that with the classifier $\hat
{h}_{n}$ we are almost as good (up to a term $\tilde{b}_{n}(\alpha
,\mathcal{C}) $) as if an oracle would have told us which $h\in\mathcal{C}$ is
the best for the given unknown $Q$.%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
The claim that this is indeed an oracle type inequality \textbf{is a bold
one}, verify this using the Massart/Blanchard paper on SVM, and possibly
Tarigan, van de Geer.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\bigskip

\subsubsection{PAC\ bounds}

In the inequality (\ref{pac-inequa}) have an upper bound for unknown
$L(\hat{h}_{n})$ which is itself unknown, indeed $\inf_{h\in\mathcal{C}%
}\;L_{Q}(h)$ depends on $Q$. We may want to replace it by a known or
observable quantity, in order to asses the performance of $\hat{h}_{n}$ for a
given class $\mathcal{C}$ and compare it with another class $\mathcal{C}%
^{\prime}$. Inequality (\ref{basic-vc-2}) of Lemma \ref{lem-VC-basic} can be
used for that: in conjunction with (\ref{inequ-basic-VC-uniform}) it yields
\begin{equation}
P_{Q}\left(  L(\hat{h}_{n})\leq L_{emp}(\hat{h})+\tilde{b}_{n}(\alpha
,\mathcal{C})\right)  \geq1-\alpha\text{ for all }Q=\mathcal{L}\left(
X,Y\right)  .\label{confid-interval-VC}%
\end{equation}
Indeed here the upper bound $L_{emp}(\hat{h})+\tilde{b}_{n}(\alpha
,\mathcal{C})$ is known, and serves as an upper confidence bound of level
$1-\alpha$ for $L(\hat{h}_{n})$. This is sometimes referred to as a PAC
statement ("\textbf{p}robably \textbf{a}pproximately \textbf{c}orrect"),
meaning that with high probability $1-\alpha$ the empirical risk minimizer
within the class $\mathcal{C}$ has $L(\hat{h}_{n})$ bounded above by a known
quantity, regardless of $Q$ which is unknown. Some authors call this the
"fundamental theorem of learning" (cf. \cite{CST}, Theorem 4.1, p. 56, for the
case $L_{emp}(\hat{h})=0$). It should be noted that (\ref{confid-interval-VC})
is valid for any classifierv $\hat{h}$, not just the empirical risk minimizer,
since (\ref{basic-vc-2}) of Lemma \ref{lem-VC-basic} is holds for any $\hat
{h}.$

The inequality (\ref{confid-interval-VC}) also gives rise to the idea of
\textit{structural risk minimization}, a model choice method, described as follows.

\bigskip

\subsubsection{Structural risk minimization}

\textbf{\ }Suppose there is a nested structure on sets of classifiers:
$\mathcal{C}_{r}$, $r=1,2,\ldots$ such that $\mathcal{C}_{r_{1}}%
\subset\mathcal{C}_{r_{2}}$ if $r_{1}<r_{2}$. One is then confronted with the
problem of deciding what $\mathcal{C}$ to choose a priori. Problems of this
kind are generally called \textit{model selection problems}. Suppose that for
each of these classes , there is an inequality
\begin{equation}
L(h)\leq L_{emp}(h)+\tilde{b}_{n}(\alpha,\mathcal{C}_{r})\text{ for all }%
h\in\mathcal{C}_{r}\label{last-inequ-(1)}%
\end{equation}
valid with probability at least $1-\alpha$, and a capacity bound $\tilde
{b}_{n}(\alpha,\mathcal{C}_{r})$ for the particular class $\mathcal{C}_{r}$.
If $V_{\mathcal{C}_{r}}$ is the VC dimension of $\mathcal{C}_{r}$ then as
shown above
\[
\tilde{b}_{n}(\alpha,\mathcal{C}_{r})=\sqrt{\frac{32}{n}\left(  V_{\mathcal{C}%
_{r}}\log(n+1)+\log\frac{8}{\alpha}\right)  }.
\]
Then to choose an appropriate $r$, consider the best classifier $\hat{h}_{r}$
from each class $\mathcal{C}_{r}$, i.e. the one which minimizes the empirical
risk:%
\[
L_{emp}(\hat{h}_{r})=\inf_{h\in\mathcal{C}_{r}}L_{emp}(h).
\]
Then according to (\ref{last-inequ-(1)}) we have
\begin{equation}
L(\hat{h}_{r})\leq L_{emp}(\hat{h}_{r})+\tilde{b}_{n}(\alpha,\mathcal{C}%
_{r})\label{SRM-inequ(1)}%
\end{equation}
The structural risk minimization (SRM) idea is as follows: select the $r$
which minimizes the upper bound, i.e. the right side in (\ref{SRM-inequ(1)}).
For larger classes $\mathcal{C}_{r}$, the empirical risk $L_{emp}(\hat{h}%
_{r})$ will be smaller (fit to the data will be better) but the VC\ dimension
will be higher, so $\tilde{b}_{n}(\alpha,\mathcal{C}_{r})$ will be larger.
Minimizing the sum of both promises to find the best $L(\hat{h}_{r})$. That is
a heuristical principle, akin to other procedures of model choice. Various
justifications by rigourous results have been put forward in the literature,
cf. chapter 18 in \cite{DGL}.%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
\textit{The following section (commented out for now) needs thorough revision.
Massart/Blanchard paper on SVM gives understanding. First explain SRM for a
nested sequence, possibly find other example then SVM. The slides of Andrew
Moore, Carnegie Mellon "VC\ dimension for characterizing classifiers" (file
"Moore-tutorial-vcdim08") give a good understanding of the SRM\ principle,
which is just one of the many model choice methods. Then use Massart/Blanchard
paper to make a connection to SVM: since SVM is a penalized optimization
problem, it is RELATED\ TO\ A\ CONSTRAINED\ OPTIMIZATION\ PROBLEM (see
Massart/Blanchard paper, my handwritten pencil notes there, scanned as file
"blanchard-bousquet-massart-SVM-one-page-my-notes"). }

In this context, \textit{Vapnik's approach can be properly understood. It is
caused by the fact that SVM is penalized minimization, not minimization over a
fixed class. Explain that clearly. V. then heuristically uses the
correspondence penalized minimum / restricted minimum via Lagange multipliers.
On the latter topic, see also \cite{CST}, chap. 5, "optimization theory", a
good account of Lagrange, Kuhn-tucker etc. }
\end{mycomments-env}

\subsubsection{Structural risk minimization(2)}

We have discussed several methods of selecting a classifier from a given class
$\mathcal{C}$. E.g. $\mathcal{C}$ might be a a kernelized SVM with
(homogeneous) polynomial kernel $K(x,y)=\left(  \left\langle x,y\right\rangle
^{r}\right)  $ for some degree $r$. One is then confronted with the problem of
deciding what $\mathcal{C}$ to choose a priori, e. g. what degree $r$ to
choose for the polynomial kernel. Problems of this kind are generally called
\textit{model selection problems}. To address this problem in the context of
SVM, Vapnik proposed the principle of \textit{structural risk minimization}.
Suppose there is a nested structure on sets of classifiers: $\mathcal{C}_{r}$,
$r=1,2,\ldots$ such that $\mathcal{C}_{r_{1}}\subset\mathcal{C}_{r_{2}}$ if
$r_{1}<r_{2}$. Suppose that for each of these, there is an inequality
\begin{equation}
R(h)\leq R_{emp}(h)+\tilde{b}_{n}(\alpha,\mathcal{C}_{r})\text{ for all }%
h\in\mathcal{C}_{r}\label{last-inequ}%
\end{equation}
valid with probability at least $1-\alpha$, and a capacity bound $\tilde
{b}_{n}(\alpha,\mathcal{C}_{r})$ for the particular class $\mathcal{C}_{r}$.
If $V_{\mathcal{C}_{r}}$ is the VC dimension of $\mathcal{C}_{r}$ then as
shown above
\[
\tilde{b}_{n}(\alpha,\mathcal{C}_{r})=\sqrt{\frac{32}{n}\left(  V_{\mathcal{C}%
_{r}}\log(n+1)+\log\frac{8}{\alpha}\right)  }%
\]
Then to choose an appropriate $r$, consider the best classifier $\hat{h}_{r}$
from each class $\mathcal{C}_{r}$, i.e. the one which minimizes the empirical
risk:%
\[
R_{emp}(\hat{h}_{r})=\inf_{h\in\mathcal{C}_{r}}R_{emp}(h).
\]
Then according to (\ref{last-inequ}) we have
\begin{equation}
R(\hat{h}_{r})\leq R_{emp}(\hat{h}_{r})+\tilde{b}_{n}(\alpha,\mathcal{C}%
_{r})\label{SRM-inequ}%
\end{equation}
The structural risk minimization (SRM) idea is as follows: select the $r$
which minimizes the upper bound, i.e. the right side in (\ref{SRM-inequ}).

Recall that for support vector machines with high-dimensional feature space,
$R_{emp}(\hat{h}_{r})$ will usually be zero, since the data appear to be
separable in feature space. To obtain an upper bound on $R(\hat{h}_{r})$ for
SVM, it is of interest to consider the VC dimension $V_{\mathcal{C}}$ for
SVM's. Above in example 4 we noted that the VC\ dimension for the set of
\textit{all }linear classifiers in $\mathbb{R}^{k}$ (all hyperplanes) is $k+1
$. For high-dimensional feature space this $k$ would be large, e.g. for
polynomial degree $r$ kernels over $\mathbb{R}^{d}$ it is $\left(
%TCIMACRO{\QATOP{d+r-1}{r}}%
%BeginExpansion
\genfrac{}{}{0pt}{}{d+r-1}{r}%
%EndExpansion
\right)  $. For Gaussian kernels the dimension of feature space can be shown
to be infinite.

The following heuristic idea was proposed to explain the good performance of
SVM despite the high dimension of feature space. With SVM we consider only
separating hyperplanes for the particular data set, and also maximize the
margin ( which is similar to keeping the margin above a certain bound).
\textit{Thus the pertaining VC\ dimension "appears" to be smaller.} Let us
state another result of Vapnik and Chervonenkis, concerning a notion of or
"empirical" VC dimension, restricted to the particular training set at hand.

Let a hyperplane be defined on the training set
\[
D=\left\{  (x_{i},y_{i}),i=1,\ldots,n\right\}
\]
where $y_{i}\in\{1,-1\},$ and $x_{i}\in\mathbb{R}^{d}$, and we assume
\begin{equation}
\left\vert \left(  \left\langle x_{i},w\right\rangle +b\right)  \right\vert
\geq1\text{, }i=1,\ldots,n.\label{canon-form}%
\end{equation}
Recall that if we consider hyperplanes with unit vectors $w_{0},$ i.e.
$\left\Vert w_{0}\right\Vert =1,$ given by $\left\{  x:\left\langle
x,w_{0}\right\rangle +b_{0}=0\right\}  $ then there is a maximal margin
$\gamma$ with respect to the point set
\[
D^{\ast}=\left\{  (x_{i},i=1,\ldots,n\right\}  ,
\]
such that $\left\vert \left\langle x_{i},w_{0}\right\rangle +b_{0}\right\vert
\geq\gamma$, $i=1,\ldots,n.$ Here it is not assumed the hyperplane classifies
correctly, hence we do not require $y_{i}\left\langle x_{i},w_{0}\right\rangle
+b_{0}>0$, but consider the absolute value $\left\vert \left\langle
x_{i},w_{0}\right\rangle +b_{0}\right\vert .$ Then $\gamma$ is a lower bound
on the distance of all points $x_{i}$ to the hyperplane. Assuming $\gamma>0$
and setting $w=w_{0}/\gamma$, $b=b_{0}/\gamma$ we obtain (\ref{canon-form}).
The hyperplane is said to be in \textit{canonical form} with respect to
$D^{\ast}$ if
\[
\min_{x_{i}\in D^{\ast}}\left\vert \left\langle x_{i},w\right\rangle
+b\right\vert =1.
\]
Assume further that $D^{\ast}$ is bounded by a sphere of the radius $R$
\[
\left\vert x_{i}-a\right\vert \leq R
\]
($a$ is the center of the sphere). Note that the set of canonical hyperplanes
coincides with the set of all hyperplanes. It only specifies the normalization
of the parameters.

The idea of constructing a machine that fixes the empirical risk (i.e. makes
it zero by separation) and minimizes the confidence interval
(\ref{conf-interv}) is based on the existence of the following bound on the VC
dimension of canonical hyperplanes. Here we take a VC\ dimension of a set of
classifiers $\mathcal{C}$ \textit{restricted to the set }$D^{\ast}$, i.e. we
look for $k$-tuples of points which are shattered by $\mathcal{C}$ only among
subset of $D^{\ast}$.

\begin{theorem}
\label{Theor-margin-VC}A subset of canonical hyperplanes in $\mathbb{R}^{d},$
corresponding to classifiers
\[
f(x,w,b)=\mathrm{sign}\{\left\langle x,w\right\rangle +b\}
\]
defined on $D^{\ast}$ and satisfying the constraint
\[
\left\Vert w\right\Vert \leq A
\]
has VC dimension $V$ bounded by the inequality
\[
V\leq\min\left(  \left[  R^{2}A^{2}\right]  ,d\right)  +1.
\]

\end{theorem}

A proof for a closely related result, where the center of the sphere $a$ is
set $a=0$ and also $b=0$, can be found in [ScS], p. 142. Note that the
restriction on the set of hyperplanes makes the upper bound on the VC
dimension smaller, i.e. $\min\left(  \left[  R^{2}A^{2}\right]  ,d\right)  +1$
instead of the general $d+1$.

Let
\[
\Psi(x)=\left(  \psi_{1}(x),\ldots,\psi_{N}(x)\right)
\]
be a map into a finite dimensional feature space with dimension $N$. Recall
that SVM\ in feature space are classifiers
\[
f(x,w,b)=\mathrm{sign}\{\left\langle \Psi(x),w\right\rangle +b\}
\]
for $w\in\mathbb{R}^{N}$; the estimate of the theorem is appealing for high
$N$ where the general bound on VC dimension would be $N+1.$ However it has to
be stressed that here

\textbf{The idea of Vapnik's experiment with VC dimension (structural risk
minimization). }SVM's with varying degrees of polynomials are fitted to the
data. Then $N$ can be very large, e.g. $10^{9}$ for degree of polynomial $4$.
Degrees $1-7$ are all fitted, and for each of these an estimate of the VC
dimension of the pertaining set of classifiers is obtained from the above
theorem$,$ i.e. from
\[
V_{est}=\left[  R^{2}A^{2}\right]
\]
where $A$ is taken to be $\left\Vert \hat{w}\right\Vert =A$ ($\hat{w}$ is the
parameter of the optimal hyperplane in feature space). Note that in this
application of the theorem, $A$ is obtained from the data, so the procedure is
only heuristically motivated. Here $R_{emp}(\hat{h}_{r})=0$ since the data
appear to be separable in feature space.

However no rigorous justification of this particular use of the empirical VC
dimension has been put forward. As a consequence, research has moved from VC
bounds to other concepts of capacity of a set of classifiers (cf. chap 12 of [ScS]).%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\subsection{Bibliographic remarks}

\begin{enumerate}
\item The tutorial \cite{Burg} present a rigourous version of the structural
risk minimization principle, not for SVM but for related classifiers, the so
called gap tolerant classifiers.

\item In \cite{Herb}, section 4.2.3., p. 131 another mathematically rigorous
theory is developed around the structural risk minimization (SRM) principle,
using the concept of empirical VC dimension, p. 139, though the SRM is also
not justified in its original form.

\item Other concepts of capacity of a set of classifiers, which are more
suitable than the VC dimension for explaining the success of the kernelized
SVM, are discussed in chap 12 of \cite{ScS}.\ In particular, an exponential
inequality for the kernelized SVM with radial basis (Gaussian ) kernel is
proved in Corollary 12.6 (p. 364) there.

\item For oracle type inequalities for SVM, and connections to the SRM
principle, see Blanchard et al \cite{blanchard-massart-svm} and references
there. Tarigan and van de Geer \cite{Tarig-van-deGeer} give a result of this
type for SVM\ with an $\ell_{1}$ type penalty (as opposed to a smoothness
penalty). For oracle inequalities in a regression (not classification)
context, cf Sauv\'{e} \cite{sauv}.

\item Tsybakov \cite{Tsyb-margin} defines a concept of aggregation of
classifiers and discusses efficiency, in terms of the behaviour of the
regression function $r(x)$ near the decision boundary $r(x)=1/2$.
\end{enumerate}

\newpage

\appendix

\section{Appendix: Exercises}

\subsection{Homework Set 1\textbf{\ }}

\textbf{\bigskip Problem 1.1 (10 points) }Assume the points $x_{i}\in
\lbrack0,1]$ ordered and all distinct: $0<x_{1}<\ldots<x_{n}<1$. Consider the
matrix
\[
\text{ }\bar{K}=\left(  K\left(  x_{i},x_{j}\right)  \right)  _{i,j=1}^{n}%
\]
where $K\left(  u,t\right)  =\min(u,t)$. In the notes it is argued that the
symmetric matrix $\bar{K}$ is nonnegative definite (meaning $c^{\top}\bar
{K}c\geq0$ for every $c\in\mathbb{R}^{n})$. Prove the following result:

\textbf{Lemma.} \textit{The matrix }$\bar{K}$\textit{\ is nonsingular.}%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
\textbf{Solution. }Let $\mathbf{0}_{n}$ be the zero vector in $\mathbb{R}^{n}
$. It suffices to show that $Ka=\mathbf{0}_{n}$ implies $a=\mathbf{0}_{n}$.
For $a=\left(  \alpha_{1},\ldots,\alpha_{n}\right)  ^{\top}$ consider the
function
\[
f(t)=\sum_{i=1}^{n}\alpha_{i}K\left(  t,x_{i}\right)  ,
\]
then the vector $\bar{f}=\left(  f(x_{1}),\ldots,f(x_{n})\right)  ^{\top}$
satisfies $\bar{f}=Ka=\mathbf{0}_{n}$, hence $f(x_{i})=0,$ $i=1,\ldots,n$. But
$f$ is a continuous function, which satisfies $f(0)=0$, is linear on every
open interval $(x_{i-1},x_{i})$, $i=1,\ldots,n$ and constant on $(x_{n},1)$
(since it is a finite linear combination of such functions; here we set
$x_{0}=0$, $x_{n+1}=1$). It follows that $f(t)=0$ for $t\in\lbrack0,1]$ which
implies $f^{\prime}(t)=0$ for $t\in\lbrack0,1]$. For the derivative we find
\begin{align*}
f^{\prime}(t)  & =\sum_{i=1}^{n}\alpha_{i}=0\text{, }t\in(x_{0},x_{1}),\\
f^{\prime}(t)  & =\sum_{i=2}^{n}\alpha_{i}=0\text{, }t\in(x_{1},x_{2}),\\
& \ldots\\
f^{\prime}(t)  & =\alpha_{n}=0\text{, }t\in(x_{n-1},x_{n})
\end{align*}
and hence $\alpha_{i}=0,$ $i=1,\ldots,n$.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\bigskip

\textbf{Problem 1.2.} \textbf{(10 points) }In equation
(\ref{solution-lin-smooth-spline}), p. \pageref{solution-lin-smooth-spline} it
is argued that the solution of the minimization problem is
\[
\hat{f}=\hat{D}y\text{ for }\hat{D}=\left(  I_{n}+\lambda n\bar{K}%
^{-1}\right)  ^{-1}.
\]
where $\hat{D}$ is the so-called "hat matrix". Show that $\hat{D}$ exists for
any $\lambda>0$, i.e. the matrix $I_{n}+\lambda n\bar{K}^{-1}$ is nonsingular.
(Part of the reasoning is given in the paragraph below
(\ref{solution-lin-smooth-spline}), p. \pageref{solution-lin-smooth-spline}.
Nevertheless, write a proof as detailed as possible, including or not
including some of the arguments in the paragraph below
(\ref{solution-lin-smooth-spline})).%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
\textbf{Solution. }Let $\kappa_{1}\leq\kappa_{2}\leq\ldots\leq\kappa_{n}$ be
the ordered eigenvalues of $\bar{K}$ (all eigenvalues appearing with their
multiplicities). Since $\bar{K}$ is nonnegative definite, we have $\kappa
_{1}\geq0$. Since $\bar{K}$ is also nonsingular (Problem 1.1), we have
$\kappa_{1}>0$. The ordered eigenvalues of $\bar{K}^{-1}$ are $\kappa_{n}%
^{-1}\leq\kappa_{n-1}^{-1}\leq\ldots\leq\kappa_{1}^{-1}$ where $\kappa
_{n}^{-1}>0$. This implies that $\bar{K}^{-1}$ is positive definite. For
$\lambda>0$ the matrix $\lambda n\bar{K}^{-1}$ is also positive definite with
ordered eigenvalues $\lambda n\kappa_{n}^{-1}\leq\lambda n\kappa_{n-1}%
^{-1}\leq\ldots\leq\lambda n\kappa_{1}^{-1}$. This implies that the
eigenvalues of $I_{n}+\lambda n\bar{K}^{-1}$ are
\[
1+\lambda n\kappa_{n}^{-1}\leq1+\lambda n\kappa_{n-1}^{-1}\leq\ldots
\leq1+\lambda n\kappa_{1}^{-1}%
\]
where $1+\lambda n\kappa_{n}^{-1}>0.$ Hence $I_{n}+\lambda n\bar{K}^{-1}$ is nonsingular.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\bigskip

\subsection{Homework Set 2\textbf{\ }}

\textbf{Problem 2.1 (20 points) }Assume the points $x_{i}\in\lbrack0,1]$
ordered and all distinct. Consider the matrix
\begin{equation}
\text{ }\bar{K}=\left(  K\left(  x_{i},x_{j}\right)  \right)  _{i,j=1}%
^{n}\label{K-bar}%
\end{equation}
where $K\left(  u,t\right)  =\min(u,t)$. Use RKHS /spline theory to find the
inverse of $\bar{K}$ for the case of equispaced design points: $x_{i}=i/n$,
$i=1,\ldots,n$.%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
\textbf{Solution. }For $a=\left(  \alpha_{1},\ldots,\alpha_{n}\right)  ^{\top
}$ consider the function
\begin{equation}
f(t)=\sum_{i=1}^{n}\alpha_{i}K\left(  t,x_{i}\right)  ,\text{ }t\in\left[
0,1\right]  \text{. }\label{f-def}%
\end{equation}
then with a reasoning used on page \pageref{solution-lin-smooth-spline},
\begin{align}
\left\Vert f\right\Vert _{\ast}^{2}  & =\left\Vert \sum_{i=1}^{n}\alpha
_{i}K\left(  t,x_{i}\right)  \right\Vert _{\ast}^{2}=\sum_{i,j=1}^{n}%
\alpha_{i}\alpha_{j}\left\langle K\left(  \cdot,x_{j}\right)  ,K\left(
\cdot,x_{i}\right)  \right\rangle _{\ast}\nonumber\\
& =\sum_{i,j=1}^{n}\alpha_{i}\alpha_{j}K\left(  x_{i},x_{j}\right)  =a^{\top
}\bar{K}a.\label{quadr-form}%
\end{align}
Also, from (\ref{f-def}) we obtain
\[
f(x_{j})=\sum_{i=1}^{n}\alpha_{i}K\left(  x_{j},x_{i}\right)  \text{,
}j=1,\ldots,n,\text{ }%
\]
and with notation $\bar{f}=\left(  f(x_{1}),\ldots,f(x_{n})\right)  ^{\top}$%
\[
\bar{f}=\bar{K}a\text{.}%
\]
This implies
\[
a=\bar{K}^{-1}\bar{f}%
\]
(where existence of $\bar{K}^{-1}$ is shown in the same way as in Problem
1.1), hence from (\ref{quadr-form})%
\[
\left\Vert f\right\Vert _{\ast}^{2}=\bar{f}^{\top}\bar{K}^{-1}\bar{f}.
\]
But in this case the RKHS norm $\left\Vert f\right\Vert _{\ast}$ has the form
(p. \pageref{solution-lin-smooth-spline}), for $f\in H_{0}^{1}(0,1)$
\[
\left\Vert f\right\Vert _{\ast}^{2}=\left\Vert f^{\prime}\right\Vert _{2}%
^{2}=\int_{\left[  0,1\right]  }\left(  f^{\prime}(t)\right)  ^{2}dt.
\]
where $f^{\prime}$ is the generalized derivative (Radon-Nikodym-derivative
with respect to Lebesgue measure) of $f$. We obtain
\begin{equation}
\int_{\left[  0,1\right]  }\left(  f^{\prime}(t)\right)  ^{2}dt=\bar{f}^{\top
}\bar{K}^{-1}\bar{f}.\label{quadr-form-2}%
\end{equation}
Since $f$ given by (\ref{f-def}) is a piecewise linear spline, with knots in
the points $0,x_{1},\ldots,x_{n}$, satisfying $f(0)=0$, the derivative
$f^{\prime}$ is piecewise constant with the following values:%
\begin{align*}
f^{\prime}(t)  & =nf(x_{1})\text{, }t\in\left(  0,x_{1}\right) \\
f^{\prime}(t)  & =n\left(  f(x_{2})-f(x_{1})\right)  \text{, }t\in\left(
x_{1},x_{2}\right) \\
& \ldots\\
f^{\prime}(t)  & =n\left(  f(x_{n})-f(x_{n-1})\right)  \text{, }t\in\left(
x_{n-1},x_{n}\right)  .
\end{align*}
Hence we compute (setting $x_{0}=0$)
\begin{align*}
\int_{\left[  0,1\right]  }\left(  f^{\prime}(t)\right)  ^{2}dt  & =\left(
nf(x_{1})\right)  ^{2}\int_{(0,x_{1})}dt+\ldots+\left(  n\left(
f(x_{n})-f(x_{n-1})\right)  \right)  ^{2}\int_{(x_{n-1},x_{n})}dt\\
& =n\left(  \left(  f(x_{1})\right)  ^{2}+\sum_{j=2}^{n}\left(  f(x_{j}%
)-f(x_{j-1})\right)  ^{2}\right)
\end{align*}
Thus we obtain from (\ref{quadr-form-2})
\begin{equation}
\left(  f(x_{1})\right)  ^{2}+\sum_{j=2}^{n}\left(  f(x_{j})-f(x_{j-1}%
)\right)  ^{2}=n^{-1}\bar{f}^{\top}\bar{K}^{-1}\bar{f}.\label{quadr-form-3}%
\end{equation}
Both sides are quadratic form in the vector $\bar{f}$, that is they can be
written
\[
\bar{f}^{\top}B\bar{f}=\sum_{i,j=1}^{n}f(x_{i})f(x_{j})b_{ij}%
\]
for some symmetric matrix $B$. Here $\bar{f}$ can take all values $\bar{f}%
\in\mathbb{R}^{n}$ since $\bar{f}=\bar{K}a$ where $\bar{K}$ is nonsingular and
$a\in\mathbb{R}^{n}$ is arbitrary. Hence it suffices to find the matrix $B$ on
the left side of (\ref{quadr-form-3}). This matrix is unique if the symmetry
condition $B=B^{\top}$ is imposed; then we must have $B=\bar{K}^{-1}$.

We find
\begin{align}
\bar{f}^{\top}B\bar{f}  & =\left(  f(x_{1})\right)  ^{2}+\sum_{j=2}^{n}\left(
f(x_{j})-f(x_{j-1})\right)  ^{2}\label{quadr-form-4}\\
& =2\sum_{i=1}^{n-1}\left(  f(x_{i})\right)  ^{2}-2\sum_{i=2}^{n}%
f(x_{i-1})f(x_{i})+\left(  f(x_{n})\right)  ^{2}=n^{-1}\bar{f}^{\top}\bar
{K}^{-1}\bar{f}.\nonumber
\end{align}
It is now easy to see that the symmetric matrix $B$ takes the form
\begin{equation}
B=\left(  b_{ij}\right)  _{i=1,\ldots,n}^{j=1,\ldots,n}\text{, }%
b_{ij}=\left\{
\begin{array}
[c]{c}%
2\text{ if }i=j<n\\
1\text{ if }i=j=n\\
-1\text{ if }i=j+1\text{ or }i=j-1\\
0\text{ otherwise.}%
\end{array}
\right. \label{b-given}%
\end{equation}
This is a so called "band matrix" where only elements near the diagonal are
nonzero; as an example, for $n=5$ we have
\[
B=\left(
\begin{array}
[c]{ccccc}%
2 & -1 & 0 & 0 & 0\\
-1 & 2 & -1 & 0 & 0\\
0 & -1 & 2 & -1 & 0\\
0 & 0 & -1 & 2 & -1\\
0 & 0 & 0 & -1 & 1
\end{array}
\right)  .
\]
Finally, for $\bar{K}^{-1}$ we obtain
\[
\bar{K}^{-1}=nB\text{, }B\text{ given by (\ref{b-given}).}%
\]

\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\bigskip

\textbf{Problem 2.2.}

\textbf{a)} \textbf{(20 points) }Consider the general linear smoothing spline
as treated in Corollary \ref{coroll-lin-spline-general}, p.
\pageref{coroll-lin-spline-general}, where the design points $x_{i}\in
\lbrack0,1]$ are ordered and all distinct: $0<x_{1}<\ldots<x_{n}\leq1$. Let
$\hat{D}$ be the hat matrix for this problem, i.e. the matrix wich to any data
vector $y=\left(  y_{1},\ldots,y_{n}\right)  ^{\top}$ assigns the vector of
values in $x_{i}$ of the resulting linear smoothing spline solution $\check
{f}\in H^{1}(0,1)$
\[
\hat{f}=\left(  \check{f}(x_{1}),\ldots,\check{f}(x_{n})\right)  ^{\top}%
\]
according to $\hat{f}=\hat{D}y$. Here $\check{f}$ is the minimizer over
functions $f\in H^{1}(0,1)$ of
\begin{equation}
n^{-1}\sum_{i=1}^{n}(y_{i}-f(x_{i}))^{2}+\lambda\int_{0}^{1}\left(  f^{\prime
}(t)\right)  ^{2}dt.\label{start-minimize}%
\end{equation}

Show that
\begin{equation}
\hat{D}=P+Q\left(  Q+\lambda n\bar{K}^{-1}\right)  ^{-1}Q\label{hat-matrix}%
\end{equation}
where $P$ is the projection operator (projection matrix) onto the linear
subspace $\mathrm{span}(\boldsymbol{1})$ of $\mathbb{R}^{n}$ spanned by the
vector $\boldsymbol{1}=\left(  1,\ldots,1\right)  ^{\top}$ and $Q$ is the
projection matrix onto the orthogonal complement of $\mathrm{span}%
(\boldsymbol{1})$. The matrix $\bar{K}$ is given by (\ref{K-bar}).

\bigskip

\textbf{Hint. }\textit{Recall }\textbf{\ }\textit{projection operators in
}$\mathbb{R}^{n}$. Let $V$ be a $n\times k$ matrix, $k\leq n$ with linearly
independent columns, i.e. $\mathrm{rank}(V)=k.$ Let $\mathrm{span}(V)$ be the
linear subspace of $\mathbb{R}^{n}$ spanned by the columns of $V$. Then
$P:=V\left(  V^{\top}V\right)  ^{-1}V^{\top}$ is the projection operator onto
$\mathrm{span}(V)$ and $Q:=I_{n}-P$ is the projection operator onto the
orthogonal complement of $\mathrm{span}(V)$. We have $PP=P$, $P=P^{\top}$,
$PQ=0$ and there is an orthogonal $n\times k$ matrix $U$ (with $U^{\top
}U=I_{k}$) such that $P=UU^{\top}$. For any $x\in\mathbb{R}^{n},$ the vector
$Px\in\mathrm{span}(V)$ is the minimizer of $\left\Vert x-z\right\Vert $ over
$z\in\mathrm{span}(V)$.

\bigskip

\textbf{b)} \textbf{(optional)} Show that $\hat{D}$ also has the shrinkage
property as described on p. \pageref{inequ-pos-def}, that is all eigenvalues
of $\hat{D}$ are between $0$ and $1$.%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
\textbf{Solution.} \textbf{a)} As argued on p.
\pageref{coroll-lin-spline-general}, any minimizing function $\check{f}$ can
be written
\begin{equation}
\check{f}=\sum_{i=1}^{n}\alpha_{i}K\left(  \cdot,x_{i}\right)
+g\label{repres}%
\end{equation}
where $K\left(  t,x_{i}\right)  =\min\left(  t,x_{i}\right)  $ and $g$ is a
constant. Introduce notation $a=\left(  \alpha_{1},\ldots,\alpha_{n}\right)
^{\top};$ then we have from (\ref{repres})
\begin{equation}
\hat{f}=\bar{K}a+\boldsymbol{1}g.\label{repres-2}%
\end{equation}
Then (\ref{start-minimize}) transforms to (analogous to the reasoning after
relation (\ref{linear-spline-objective-func}), p.
\pageref{linear-spline-objective-func}),
\begin{align*}
& n^{-1}\left\Vert y-\hat{f}\right\Vert ^{2}+\lambda\left\Vert \sum_{i=1}%
^{n}\alpha_{i}K\left(  \cdot,x_{i}\right)  \right\Vert _{\ast}^{2}\\
& =n^{-1}\left\Vert y-\bar{K}a-\boldsymbol{1}g\right\Vert ^{2}+\lambda
a^{\top}\bar{K}a.
\end{align*}
To find the minimizing $a$ and $g$, we first find $g$ for fixed $a$. We have
\[
\left\Vert y-\bar{K}a-\boldsymbol{1}g\right\Vert ^{2}\geq\left\Vert y-\bar
{K}a-P\left(  y-\bar{K}a\right)  \right\Vert ^{2}=\left\Vert Q\left(
y-\bar{K}a\right)  \right\Vert ^{2}%
\]
and the minimizing $g$ satisfies
\begin{align}
\boldsymbol{1}g  & =P\left(  y-\bar{K}a\right)  ,\label{repres-2a}\\
g  & =n^{-1}\boldsymbol{1}^{\top}\left(  y-\bar{K}a\right)  .\label{repres-3}%
\end{align}
Set now $\bar{f}=\bar{K}a$; since $\bar{K}$ is nonsingular, the correspondence
is one-to-one. Note that
\begin{align}
\hat{f}  & =\bar{f}+\boldsymbol{1}g=\bar{f}+P\left(  y-\bar{K}a\right)
\nonumber\\
& =\bar{f}+P\left(  y-\bar{f}\right)  =Py+\left(  I-P\right)  \bar
{f}\nonumber\\
& =Py+Q\bar{f}.\label{repres-4}%
\end{align}
To find $\bar{f}$, we now have to minimize
\begin{align*}
& \left\Vert Q\left(  y-\bar{K}a\right)  \right\Vert ^{2}+n\lambda a^{\top
}\bar{K}a\\
& =\left\Vert Q\left(  y-\bar{f}\right)  \right\Vert ^{2}+n\lambda\bar
{f}^{\top}\bar{K}^{-1}\bar{f}\\
& =\left(  y-\bar{f}\right)  ^{\top}Q\left(  y-\bar{f}\right)  +n\lambda
\bar{f}^{\top}\bar{K}^{-1}\bar{f}%
\end{align*}
over $\bar{f}\in\mathbb{R}^{n}$. We differentiate and get a normal equation
\[
-2Q\left(  y-\bar{f}\right)  +2n\lambda\bar{K}^{-1}\bar{f}=0
\]
which transforms to
\[
\left(  Q+n\lambda\bar{K}^{-1}\right)  \bar{f}=Qy.
\]
Here $Q+n\lambda\bar{K}^{-1}$ is nonsingular since $\bar{K}^{-1}$ is positive
definite and $Q$ is nonnegative definite, hence $Q+n\lambda\bar{K}^{-1}$ is
positive definite. We obtain a solution
\[
\bar{f}=\left(  Q+n\lambda\bar{K}^{-1}\right)  ^{-1}Qy
\]
and in view of (\ref{repres-4})
\begin{align*}
\hat{f}  & =Py+Q\bar{f}\\
& =Py+Q\left(  Q+n\lambda\bar{K}^{-1}\right)  ^{-1}Qy
\end{align*}
as claimed.

\textbf{b)} Both $P$ and $Q\left(  Q+\lambda n\bar{K}^{-1}\right)  ^{-1}Q$ are
nonnegative definite symmetric matrices, so $\hat{D}$ is also nonnegative
definite and hence all eigenvalues are $\geq0$. To show that all eigenvalues
are $\leq1$, it suffices to show that for all $x\in\mathbb{R}^{n}$ satisfying
$\left\Vert x\right\Vert =1,$ we have $x^{\top}\hat{D}x\leq1$. Now
\[
x^{\top}\hat{D}x=x^{\top}Px+x^{\top}Q\left(  Q+\lambda n\bar{K}^{-1}\right)
^{-1}Qx,
\]
so if we show
\begin{equation}
x^{\top}Q\left(  Q+\lambda n\bar{K}^{-1}\right)  ^{-1}Qx\leq x^{\top
}Qx\label{show-1}%
\end{equation}
then in view of $P+Q=I_{n}$, the claim is established. To show (\ref{show-1}),
write $Q=UU^{\top}$ where $U$ is an orthogonal $n\times k$ matrix (with
$U^{\top}U=I_{k}$ and $k=n-1$). Furthermore, set $z=U^{\top}x$, then
$\left\Vert z\right\Vert ^{2}=x^{\top}Qx$ and it suffices to show
\begin{equation}
z^{\top}U^{\top}\left(  UU^{\top}+\lambda n\bar{K}^{-1}\right)  ^{-1}%
Uz\leq\left\Vert z\right\Vert ^{2}\label{show-2}%
\end{equation}
or equivalently, that for the maximal eigenvalue $\lambda_{\max}(\cdot)$ we
have
\[
\lambda_{\max}\left(  U^{\top}\left(  UU^{\top}+\lambda n\bar{K}^{-1}\right)
^{-1}U\right)  \leq1.
\]
We will use the following property of projection matrices $\Pi$ in
$m$-dimensional space $\mathbb{R}^{m}$: for any $\tilde{z}\in\mathbb{R}^{m}$
\begin{equation}
\tilde{z}^{\top}\Pi\tilde{z}\leq\left\Vert \tilde{z}\right\Vert ^{2}%
\label{proj-propert}%
\end{equation}
or equivalently: $\lambda_{\max}(\Pi)\leq1$ . Indeed, this follows from the
decomposition (Pythagoras theorem)
\[
\left\Vert \tilde{z}\right\Vert ^{2}=\left\Vert \Pi\tilde{z}\right\Vert
^{2}+\left\Vert \left(  I_{m}-\Pi\right)  \tilde{z}\right\Vert ^{2}%
\geq\left\Vert \Pi\tilde{z}\right\Vert ^{2}=\tilde{z}^{\top}\Pi\tilde{z}.
\]
To show (\ref{show-2}), note that every positive definite symmetric $n\times
n$ matrix $A$ we can find a positive definite symmetric $n\times n$ matrix
$A^{1/2}$ fulfilling $A^{1/2}A^{1/2}=A$ (take $A^{1/2}$ as having the same
eigenvectors as $A$, and as eigenvalues the square roots of those of $A$).
Accordingly we write $\lambda n\bar{K}^{-1}=DD$ for a positive definite
symmetric $n\times n$ matrix $D$. Furthermore for $m=n-1+n$ we define a
$m\times n$ matrix $\tilde{U}$ as consisting of submatrices $U^{\top}$, $D$ in
the following way
\[
\tilde{U}=\left(
\begin{array}
[c]{c}%
U^{\top}\\
D
\end{array}
\right)  ,
\]
and we define a vector $\tilde{z}\in\mathbb{R}^{m}$ by
\[
\tilde{z}=\left(
\begin{array}
[c]{c}%
z\\
\mathbf{0}_{n}%
\end{array}
\right)
\]
where $\mathbf{0}_{n}$ is the $n$-vector consisting of $0$'s. Then
\[
UU^{\top}+\lambda n\bar{K}^{-1}=UU^{\top}+DD=\tilde{U}^{\top}\tilde{U}%
\]
and
\[
z^{\top}U^{\top}\left(  UU^{\top}+\lambda n\bar{K}^{-1}\right)  ^{-1}%
Uz=\tilde{z}^{\top}\tilde{U}\left(  \tilde{U}^{\top}\tilde{U}\right)
^{-1}\tilde{U}^{\top}\tilde{z}.
\]
Now $\Pi=\tilde{U}\left(  \tilde{U}^{\top}\tilde{U}\right)  ^{-1}\tilde
{U}^{\top}$ is a projection matrix in $\mathbb{R}^{m}$, hence by
(\ref{proj-propert})
\[
z^{\top}U^{\top}\left(  UU^{\top}+\lambda n\bar{K}^{-1}\right)  ^{-1}%
Uz\leq\tilde{z}^{\top}\tilde{z}=\left\Vert z\right\Vert ^{2}%
\]
so (\ref{show-2}) is established.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\bigskip

\subsection{Homework Set 3\textbf{\ }}

\textbf{Problem 3.1. (15 points) }Consider the linear logistic regression
model for classification, discussed on p.
\pageref{predictive-probability-simplified}. For the predictive distribution
we obtained the expression (see (\ref{predictive-probability-simplified}) on
p. \pageref{predictive-probability-simplified})%
\[
P\left(  Y_{\ast}=1|y,x_{\ast}\right)  =\int l\left(  w^{\top}x_{\ast}\right)
p\left(  w|y\right)  dw.
\]
using a normal prior $w\sim N_{d}\left(  0,\Sigma\right)  $ for the parameter
$w\in\mathbb{R}^{d}$. Here one has a training set $\left(  x_{i},y_{i}\right)
$, $i=1,\ldots,n$ where the $y_{i}$ are assumed to be realizations of a random
variables $Y_{i}$ according to $P\left(  Y=1|X=x\right)  =l\left(  w^{\top
}x\right)  $, with $l$ the logistic function, and $x_{1},\ldots,x_{n},x_{\ast
}$ (all in $\mathbb{R}^{d}$) are considered nonrandom.

It is evident that in this model, the resulting posterior density $p\left(
w|y\right)  $ is nongaussian (cp. also (\ref{neg-log-posterior}) p.
\pageref{neg-log-posterior}). However in common approximation methods, a
Gaussian approximation $\tilde{p}\left(  w|y\right)  $ for $p\left(
w|y\right)  $ is postulated. Define the approximate predictive distribution%
\[
\tilde{P}\left(  Y_{\ast}=1|y,x_{\ast}\right)  =\int l\left(  w^{\top}x_{\ast
}\right)  \tilde{p}\left(  w|y\right)  dw.
\]
Show that if $\tilde{p}$ is any multivariate normal density on $\mathbb{R}%
^{d}$ then the decision boundary
\[
\left\{  x_{\ast}:\tilde{P}\left(  Y_{\ast}=1|y,x_{\ast}\right)  =1/2\right\}
\]
is a hyperplane in $\mathbb{R}^{d}$ passing through the origin.

\textbf{Hint:} use symmetries of the logistic function and normal densities.%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
\textbf{Solution.} Let $N_{d}(\mu,\Sigma)$ a multivariate normal distribution
and $W$ be a random vector with $\mathcal{L}(W)=N_{d}(\mu,\Sigma)$. It is
claimed that the set
\begin{equation}
\left\{  x:El\left(  x^{\top}W\right)  -1/2=0\right\} \label{solution-set}%
\end{equation}
is a hyperplane in $\mathbb{R}^{d}$ passing through the origin. The function
$l_{0}(t)=l(t)-1/2$, $t\in\mathbb{R}$ is antisymmetric (that is $l_{0}%
(-t)=-l_{0}(t)$); therefore, if the density of $x^{\top}W$ is symmetric around
the origin then $El_{0}\left(  x^{\top}W\right)  =0$. Now
\[
\mathcal{L}\left(  x^{\top}W\right)  =N\left(  x^{\top}\mu,x^{\top}\Sigma
x\right)  ,
\]
which has a symmetric density if and only if $x^{\top}\mu=0$. Therefore, the
hyperplane in $\mathbb{R}^{d}$%
\begin{equation}
\left\{  x:x^{\top}\mu=0\right\} \label{solution-set-2}%
\end{equation}
is contained in the set (\ref{solution-set}). To show that the two sets
coincide, assume $x^{\top}\mu>0$; then for a r.v. $V\sim N(0,x^{\top}\Sigma
x)$
\[
El\left(  x^{\top}W\right)  =El\left(  x^{\top}\mu+V\right)
\]
and since $l$ is strictly monotone increasing,
\[
El\left(  x^{\top}\mu+V\right)  >El\left(  V\right)  =1/2
\]
so $x$ is not in the set (\ref{solution-set}). Analogously it is shown that
any $x$ with $x^{\top}\mu<0$ is not in the set (\ref{solution-set}). Thus the
sets (\ref{solution-set}) and (\ref{solution-set-2}) coincide.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\bigskip

\textbf{Problem 3.2. (optional)}\textit{\ }Suppose that for Bayesian inference
in the above model, one adds the assumption that $x_{i}$, $i=1,\ldots,n$ are
also realizations of random variables with a specific distribution. Thus, each
$\left(  x_{i},y_{i}\right)  $, $i=1,\ldots,n$ is assumed to be an independent
realization of a random variable $(X,Y)$ where $P\left(  Y=1|X=x\right)
=l\left(  w^{\top}x\right)  $ and $X$ has a density
\[
q(x)=\frac{1}{2}\varphi(x-w/2)+\frac{1}{2}\varphi(x+w/2)\text{, }%
x\in\mathbb{R}^{d}%
\]
where $\varphi$ is the standard normal density on $\mathbb{R}^{d}$ (the
density of $N_{d}(0,I_{d})$). Again, assume a normal prior $w\sim N_{d}\left(
0,\Sigma\right)  $ for the parameter $w\in\mathbb{R}^{d}$ and let $p\left(
w|x,y\right)  $ be the posterior density of $w$ given $\left(  x,y\right)
=\left(  \left(  x_{i},y_{i}\right)  ,i=1,\ldots,n\right)  .$ Show that
$p\left(  w|x,y\right)  $ is normal.

(Note that the randomness assumption has not been extended to the value
$x_{\ast}$ from above.)%

%TCIMACRO{\TeXButton{begin-mycomments}{\begin{mycomments}}}%
%BeginExpansion
\begin{mycomments}%
%EndExpansion

\begin{mycomments-env}
\textbf{Solution.} According to Proposition
\ref{prop-logistic-is true-if-normal}, p.
\pageref{prop-logistic-is true-if-normal} the joint law of $(X,Y)$ can also be
described by two class-conditional normal laws for $X$:%
\begin{align}
\mathcal{L}\left(  X|Y=i\right)   & =N_{d}\left(  w/2,\Sigma\right)  \text{,
}i=0\\
\mathcal{L}\left(  X|Y=i\right)   & =N_{d}\left(  -w/2,\Sigma\right)  \text{,
}i=1
\end{align}
with equal probabilities
\begin{equation}
P\left(  Y=i\right)  =1/2,i=0,1.
\end{equation}
In this situation, where $\mathcal{L}(Y)$ does not depend on the parameter $w
$, the observed $y_{i},i=1,\ldots,n$ can be assumed nonrandom and the
calculation of the posterior of $w$ can proceed only on the basis of the prior
for $w$ and the conditional laws $\mathcal{L}\left(  x_{i}|y_{i}\right)  $. To
see this formally, we use the fact that the density for the \textrm{Bernoulli}%
$\mathrm{(}1/2$\textrm{) }law $Y$ with respect to the counting measure on the
set $\{0,1\}$ is given by
\[
\left(  \frac{1}{2}\right)  ^{y}\left(  \frac{1}{2}\right)  ^{1-y}=\frac{1}%
{2}.
\]
Let $\varphi_{\Sigma}$ be the density of $N_{d}\left(  0,\Sigma\right)  $;
then the joint density of $\left(  x_{i},y_{i}\right)  ,i=1,\ldots,n$ and $w$
is given by%
\[
\varphi_{\Sigma}(w)\left(  \frac{1}{2}\right)  ^{n}%
%TCIMACRO{\tprod \nolimits_{i=1}^{n}}%
%BeginExpansion
{\textstyle\prod\nolimits_{i=1}^{n}}
%EndExpansion
\varphi\left(  x_{i}+y_{i}w/2-\left(  1-y_{i}\right)  w/2\right)  .
\]
To obtain the posterior density of $w$, we have to divide this by the marginal
density of $\left(  x_{i},y_{i}\right)  ,i=1,\ldots,n$ which is
\[
\int_{\mathbb{R}^{d}}\varphi_{\Sigma}(w)\left(  \frac{1}{2}\right)  ^{n}%
%TCIMACRO{\tprod \nolimits_{i=1}^{n}}%
%BeginExpansion
{\textstyle\prod\nolimits_{i=1}^{n}}
%EndExpansion
\varphi\left(  x_{i}+y_{i}w/2-\left(  1-y_{i}\right)  w/2\right)  dw.
\]
We see that the factor $\left(  \frac{1}{2}\right)  ^{n}$ cancels out, and the
posterior density of $w$ is
\[
\frac{\varphi_{\Sigma}(w)%
%TCIMACRO{\tprod \nolimits_{i=1}^{n}}%
%BeginExpansion
{\textstyle\prod\nolimits_{i=1}^{n}}
%EndExpansion
\varphi\left(  x_{i}+y_{i}w/2-\left(  1-y_{i}\right)  w/2\right)  }%
{\int_{\mathbb{R}^{d}}\varphi_{\Sigma}(w)%
%TCIMACRO{\tprod \nolimits_{i=1}^{n}}%
%BeginExpansion
{\textstyle\prod\nolimits_{i=1}^{n}}
%EndExpansion
\varphi\left(  x_{i}+y_{i}w/2-\left(  1-y_{i}\right)  w/2\right)  dw}.
\]
Let $\xi_{i}$ denote independent $N_{d}\left(  0,I\right)  $ random vactors;
the above density is the posterior of $w$ in a regression model
\[
x_{i}=\left(  1-y_{i}\right)  w/2-y_{i}w/2+\xi_{i}\text{, }i=1,\ldots,n
\]
with prior $w\sim N_{d}\left(  0,\Sigma\right)  $ and nonrandom $y_{i}$. To
show that this is a normal density, it suffices to show that the given prior
on $w$ makes $w,x_{i},i=1,\ldots,n$ jointly normal (conditionally on the given
$y_{i},i=1,\ldots,n$). However $w,x_{i},i=1,\ldots,n$ represents a linear
transformation of $w,\xi_{i},i=1,\ldots,n$ which are jointly normal since they
are independent normal. Thus $w,x_{i},i=1,\ldots,n$ are indeed jointly normal
(conditionally on the given $y_{i},i=1,\ldots,n$), and hence the law
$\mathcal{L}(w|x_{i},y_{i},i=1,\ldots,n)$ is also normal.
\end{mycomments-env}%

%TCIMACRO{\TeXButton{end-mycomments}{\end{mycomments}}}%
%BeginExpansion
\end{mycomments}%
%EndExpansion

\section{References}

\begin{thebibliography}{9999}                                                                                %
\bibitem[Bish]{Bish}Bishop, C. M. \textsl{Pattern Recognition and Machine
Learning} (Information Science and Statistics). Springer 2006

\bibitem[BTA]{BTA}Berlinet, A., Thomas-Agnan, C., \textsl{Reproducing kernel
Hilbert spaces in Probability and Statistics,} Kluwer, 2004

\bibitem[BFOS]{BFOS}Breiman, L., Friedman, J., Olshen, R., Stone, C.
\textsl{Classification and Regression Trees}. Wadsworth, Belmont, CA., 1984.

\bibitem[Burg]{Burg}Burges, C. J. C., A Tutorial on Support Vector Machines
for Pattern Recognition. Data Mining and Knowledge Discovery, 2, 121-167 (1998)

\bibitem[CST]{CST}Cristianini, N. and Shawe-Taylor, J., \textsl{An
Introduction to Support Vector Machines and Other Kernel-based Learning
Methods}. Cambridge University Press, 2000.

\bibitem[D]{D}Durret, R., \textsl{\ Probability: Theory and Examples, } 2nd
Ed., Duxbury Press, 1996.

\bibitem[DGL]{DGL}Devroye, L, Gyorfi, L., Lugosi, G., \textsl{A Probabilistic
Theory of Pattern Recognition}, Springer 1997

\bibitem[Fine]{Fine}Fine, T. L., \textsl{Feedforward Neural Network
Methodology} (Springer Series in Statistics). Springer 1999

\bibitem[HTF]{HTF}Hastie, T, Tibshirani, R. J. H. Friedman, R.H., \textsl{The
Elements of Statistical Learning (Data Mining, Inference and Prediction)}%
\textit{, }Second Edition\textit{, }Springer, 2009.

\bibitem[Herb]{Herb}Herbrich, R. \textsl{Learning Kernel Classifiers: Theory
and Algorithms} (Adaptive Computation and Machine Learning). MIT Press 2002

\bibitem[Kl]{Kl}Klemel\"{a}, J., \textsl{Smoothing of Multivariate
Data.}Wiley, New York, 2009

\bibitem[RW]{RW}Rasmussen, C. E. and Williams, C. K. I., \textsl{Gaussian
Processes for Machine Learning} (Adaptive Computation and Machine Learning).
MIT Press 2006

\bibitem[ScS]{ScS}Sch\"{o}lkopf, B. and Smola, A. \textsl{Learning with
Kernels: Support Vector Machines, Regularization, Optimization, and Beyond}
(Adaptive Computation and Machine Learning). MIT Press, 2002

\bibitem[Vap1]{Vap1}Vapnik, V. , \textsl{Statistical Learning Theory}, Wiley, 1998..

\bibitem[Vap2]{Vap2}Vapnik, V. \textsl{The Nature of Statistical Learning
Theory}, 2nd ed, Springer 1999

\bibitem[Wah]{Wah}Wahba, Grace, \textsl{Spline models for observational data}.
SIAM [Society for Industrial and Applied Mathematics] (Philadelphia) 1990

\bibitem[W]{W}Wasserman, L., (2004). \textsl{All of Statistics. A Concise
Course in Statistical Inference}. Springer Verlag

\bibitem {blanchard-massart-svm}Blanchard, G., Bousquet, O., Massart, P.,
(2008). Statistical performance of support vector machines, Ann. Statist.
\textbf{36} 489-531.

\bibitem {bouch-et-al-survey}Boucheron, S, Bousquet, O. and Lugosi, G. (2005).
Theory of classification: A survey of some recent advances. ESAIM P\&S:
Probability and Statistics (online), 9, 323-375

\bibitem {Olsh}Olshen, R. (2007). Tree-structured regression and the
differentiation of integrals. Ann. Statist. \textbf{35} 1-12

\bibitem {sauv}Sauv\'{e}, M. (2009). Histogram selection in non Gaussian
regression. ESAIM: P\&S: Probability and Statistics (online), \textbf{13 }70--86

\bibitem {Tarig-van-deGeer}Tarigan, B. and van de Geer, S. A. (2006).
Classifiers of support vector machine type with $\ell_{1}$ complexity
regularization. Bernoulli \textbf{12} 1045--1076

\bibitem {Tsyb-margin}Tsybakov, A. B. (2004). Optimal aggregation of
classifiers in statistical learning, Ann. Statist. \textbf{32} 135-166.
\end{thebibliography}

\end{document}