% IRAM.tex
\documentclass[12pt]{article}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath, amsfonts, amsthm}
\usepackage{palatino}

% Stuff for the algorithm package
\usepackage[algo2e, ruled, linesnumbered]{algorithm2e}
\SetKwComment{Comment}{$\triangleright$ }{}
\dontprintsemicolon

\newcommand{\A}{\mathbf{A}}
\newtheorem{thm}{Theorem}
\newtheorem{lem}{Lemma}

\title{Implicitly Restarted Arnoldi's Method}
\author{Jeremy L. Conlin}

\begin{document}
\maketitle
\section{IRAM from ``The Source''}
Here I will try to reconstruct the description of Implicitly Restarted Arnoldi's Method from the original paper by D.C. Sorensen.

\subsection{The Arnoldi factorization}
The Arnoldi factorization may be viewed as a truncated reduction of an $n \times n$ matrix $\A$ to upper Hessenberg form.  After $k$ steps of the factorization one has
\begin{equation}
    \A V = VH + re_k^T,
    \label{eq:ArnoldiFactorization}
\end{equation}
where $V \in \mathbb{R}^{n \times k}$, $V^TV = I_k$, $H \in \mathbb{R}^{k \times k}$ is upper Hessenberg, $r \in \mathbb{R}^n$ with $\mathbf{0} = V^Tr$.  An alternative [and better in the views of the summarizer] way to write Eq. \ref{eq:ArnoldiFactorization} is
\begin{equation}
    \A V = \begin{pmatrix}V & v\end{pmatrix}\begin{pmatrix} H \\ \beta e_k^T \end{pmatrix} \mathrm{ where } \beta = \left| r \right| \mathrm{ and } v = \frac{1}{\beta}r.
    \label{}
\end{equation}

Approximate eigenvalues and eigenvectors are readily available through this factorization.  If $Hy = y\theta$ is an eigenpair for $H$, then the vector $x = Vy$ satisfies
\begin{equation}
    \left\|Ax-x\theta\right\| = \left\|\left(\A V - VH\right)y\right\| = \left|\beta e_k^Ty\right|.
    \label{}
\end{equation}
We call the vector $x$ a Ritz vector and the approximate eignvalue $\theta$ a Ritz value and note that the smaller $\left|\beta e_k^T\right|$ is, the better these approximations are.

The factorization Eq. \ref{eq:ArnoldiFactorization} may be advanced one step through the following recursion formulas:
\begin{subequations}\begin{gather}
    \beta = \|r\|, \hspace{0.25in} v = \frac{1}{\beta}r, \\ \label{eq:ArnoldiRecursion-1}
    V_+ = \begin{pmatrix}V & v\end{pmatrix}, \\
    w = \A v, \hspace{0.25in} \begin{pmatrix}H \\ \alpha \end{pmatrix} = V_+^Tw, \\
    H_+ = \begin{pmatrix} H & h \\ \beta e_k^T & \alpha \end{pmatrix}, \\
    r_+ = w - V_+\begin{pmatrix}h \\ \alpha \end{pmatrix} = \left(I-V_+V_+^T\right)w.  \label{eq:ArnoldiRecursion-4}
\end{gather}\end{subequations}
From this development it is easily seen that
\begin{equation}
    \A V_+ = V_+H_+ + r_+e_{k+1}^T,\hspace{.25in} V_+^TV_+ = I_{k+1}, V_+^Tr_+ = \mathbf{0}.
    \label{}
\end{equation}

\begin{thm}
    Let $\A V_k - V_kH_k = r_ke_k^T$ be a $k$-step Arnoldi factorization of $\A$, with $H$ unreduced (this is seen in Eq. \ref{eq:ArnoldiFactorization}).  Then $r_k = 0$ if and only if $v_1 = Qy$ where $\A Q = QR$ with $Q^HQ = I_k$ and $R$ upper triangular order of $k$.
    \label{thm:ArnoldiInvariant}
\end{thm}
Theorem \ref{thm:ArnoldiInvariant} provides the motivation for the algorithms we shall develop.  It suggests that one might find an invariant subspace by iteratively replacing the starting vector with a linear combination of approximate eigenvectors corresponding to eigenvalues of interest.

\subsection{Updating the Arnoldi factorization via QR-iterations}
In this section a direct analogue of th implicitly shifted QR-iteration will be derived in the context of the $k$-step Arnoldi factorization.

Throughout this discussion, the integer $k$ should be thought of as a fixed pre-specified integer of modest size.  Let $p$ be another positive integer, and consider the result of $k+p$ steps of the Arnoldi process appled to $\A$ , which has resulted in the construction of an orthogonal matrix $V_{k+p}$ such that
\begin{equation}
    \begin{split}
        \A V_{k+p} &= V_{k+p}H_{k+p} + r_{k+p}e_{k+p}^T \\
        &= \begin{pmatrix}V_{k+p} & v_{k+p+1} \end{pmatrix} \begin{pmatrix}H_{k+p} \\ \beta_{k+p}e_{k+p}^T \end{pmatrix}.
    \end{split}
    \label{eq:ArnoldiFactorization-k+p}
\end{equation}
An analogy of the explicitly shifte QR-algorithm may be applied to this truncated factorization of $\A$.  It consists of the following four steps.  Let $\mu$ be a shift and let $\left(H-\mu I\right) = QR$ with $Q$ orthogonal and $R$ upper triangular.  Then (putting $V = V_{k+p}$, $H = H_{k+p}$)
\begin{subequations}\begin{align}
    \left(\A - \mu I\right)V - V\left(H - \mu I\right) &= r_{k+p}e_{k+p}^T, \\
    \left(\A - \mu I\right)V - VQR &= r_{k+p}e_{k+p}^T, \\ \label{eq:IRAM-2}
    \left(\A - \mu I\right)\left(VQ\right) - \left(VQ\right)\left(RQ\right) &= r_{k+p}e_{k+p}^TQ, \\
    \A\left(VQ\right) - \left(VQ\right)\left(RQ + \mu I\right) &= r_{k+p}e_{k+p}^TQ, \\
\end{align}\end{subequations}
Let $V_+ = VQ$ and $H_+ = RQ + \mu I$.  Then $H_+$ is upper Hessenberg and applying the matrices in Eq. \ref{eq:IRAM-2} to the vector $e_1$ to expose the relationship of their first columns gives
\begin{equation}
    \left(\A - \mu I\right)v_1 = v_1^+\rho_{11}
    \label{}
\end{equation}
where $\rho_{11} = e_1^TRe_1$, $v_1^+ = V_+e_1$.

This idea may be extended for up to $p$ shifts being applied successively.  The application of a QR-iteration corresponding to an implicit shift $\mu$ produces an upper Hessenberg orthogonal  $Q \in \mathbb{R}^{k+p}$ such that
\begin{equation}
    \A V_{k+p}Q = \begin{pmatrix}V_{k+p}Q & v_{k+p+1}\end{pmatrix}\begin{pmatrix} Q^TH_{k+p}Q \\ \beta_{k+p}e_{k+p}^TQ \end{pmatrix}
    \label{}
\end{equation}
An application of $p$ implicit shifts therefore results in
\begin{equation}
    \A V_{k+p}^+ = \begin{pmatrix}V_{k+p}^+ & v_{k+p+1} \end{pmatrix}\begin{pmatrix}H_{k+p}^+ \\ \beta_{k+p}e_{k+p}^T\hat{Q} \end{pmatrix}
    \label{eq:pImplicitShifts}
\end{equation}
where $V_{k+p}^+ = V_{k+p}\hat{Q}$, $H_{k+p}^+ = \hat{Q}^TH_{k+p}\hat{Q}$, and $\hat{Q} = Q_1Q_2 \cdots Q_p$, with $Q_j$ the orthogonal matrix associated with the shift $\mu_j$.

Now, partition
\begin{equation}
    V_{k+p}^+ = \begin{pmatrix}V_k^+ & \hat{V}_p\end{pmatrix},\hspace{.25in} H_{k+p}^+ = \begin{pmatrix} H_k^+ & M \\ \hat{\beta}_{k+p}e_1e_k^T & \hat{H}_p \end{pmatrix},
    \label{}
\end{equation}
and note
\begin{equation}
    \beta_{k+p}e_{k+p}^T\hat{Q} = \left(\underbrace{0,0 \cdots \tilde{\beta}_{k+p} }_k,\underbrace{b^T}_p\right).
    \label{}
\end{equation}
Substituting into Eq. \ref{eq:pImplicitShifts} gives
\begin{equation}
    \A \left( V_k^+ , \hat{V}_p \right) = \left(V_k^+, \hat{V}_p, v_{k+p+1}\right)
    \begin{bmatrix}
        H_k^+ & M \\
        \hat{\beta}_ke_1e_k^T & \hat{H}_p \\
        \tilde{\beta}_{k+p}e_k^T & b^T
    \end{bmatrix}.
    \label{eq:Substituted}
\end{equation}
Equating the first $k$ columns on both sides of Eq. \ref{eq:Substituted} gives
\begin{equation}
    \A V_k^+ = V_k^+H_k^+ + r_k^+e_k^T
    \label{}
\end{equation}
so that
\begin{equation}
    \A V_k^+ = \left(V_k^+, v_{k+1}^+\right)
    \begin{pmatrix}
        H_k^+ \\
        \beta_k^+e_k^+
    \end{pmatrix}
    \label{eq:ArnoldiNewFactorization}
\end{equation}
where $v_{k+1}^+ = \left(1/\beta_K^+\right)r_k^+$, $r_k^+ \equiv \left(\hat{V}_pe_1\hat{\beta}_k + v_{k+p+1}\tilde{\beta}_{k+p} \right)$, and $\beta_k^+ = \left\|r_k^+\right\|$.  Note that $\left(V_k^+\right)^+\hat{V}_pe_1 = 0$ and $\left(V_k^+\right)^Tv_{k+p+1} = 0$, so $\left(V_k^+\right)^Tv_{k+1}^+ = 0$.  Thus Eq. \ref{eq:ArnoldiNewFactorization} is a legitimate Arnoldi factorization of $\A$.  Using this as a starting point it is possible to use $p$ additional steps of the Arnoldi recursions to Eqs. \ref{eq:ArnoldiRecursion-1}---\ref{eq:ArnoldiRecursion-4} to return to the original form Eq. \ref{eq:ArnoldiFactorization-k+p}.  This requires only $p$ evaluations of a matrix-vector product involving the matrix $\A$ and the $p$-new Arnoldi vectors.  This is to be contrasted with the [explicitly restarted Arnoldi's method by Saad] where the entire Arnoldi sequence is restarted.

\subsection{Algorithms}
In this section I will show the Arnoldi algorithm and the implicitly restarted Arnoldi's method for comparison.

\begin{algorithm2e}[H]
    \label{alg:Arnoldi}
    \SetVline
    \caption{Traditional Arnoldi's Method}

    \KwIn{$\A V - VH = re_k^T$ with $V^TV = I_k$, and $V^Tr = 0$.}
    \KwOut{$\A V - VH = re_{k+p}^T$ with $V^TV = I_{k+p}$, and $V^Tr = 0$.}

    $\left[H, V, r\right] = \mathtt{Arnoldi}\left( \A, H, V, r, k, p \right)$

    \BlankLine
    \For{$j=1,2,\ldots,p$}{
    $\beta \leftarrow \|r\|$; \hspace{.25in} if $\left( \beta < \mathrm{Tol} \right)$ stop;\;
    $H \leftarrow \begin{pmatrix}
        H \\ \beta e_{k+j+1}^T
    \end{pmatrix}$   \hspace{0.15in} \tcp{Add row to $H$}\;
    $v \leftarrow \frac{1}{\beta}r$  \hspace{0.75in} \tcp{Normalize $v$}\;
    $V \leftarrow \left(V, v\right)$ \hspace{0.5in} \tcp{Add column to $V$}\;
    $w \leftarrow \A v$\;
    $h \leftarrow V^Tw$\;
    $H \leftarrow \left(H, h\right)$\;
    $r \leftarrow w - Vh$\;
    Reorthogonalize if necessary\;
    }
\end{algorithm2e}

The basic or traditional Arnoldi's (Algorithm \ref{alg:Arnoldi}) method is used in the restarted version as shown in Algorithm \ref{alg:IRAM}.

\begin{algorithm2e}[H]
    \label{alg:IRAM}
    \SetVline
    \caption{Implicitly Restarted Arnoldi's Method}

    $\left[V, H, r\right] = \mathtt{IRAM}\left(\A, k, p, tol\right)$

    \BlankLine
    \emph{initialize} $V(:,1) = v_1$\;
    \hspace{0.6in} $H \leftarrow\left(v_1^T\A v_1\right)$\;
    \hspace{0.6in} $r \leftarrow \A v_1 - v_1H$\;
    $\left[H, V, r\right] = \mathtt{Arnoldi}\left( \A, H, V, r, 1, k \right)$ \hspace{0.32in} \tcp{Take $k$ steps of Arnoldi}\;
    \For{$m = 1,2,\cdots$}{
        if $\left(\|r\| < tol\right)$ stop;\;
        $\left[H, V, r\right] = \mathtt{Arnoldi}\left( \A, H, V, r, k, p \right)$ \hspace{0.05in} \tcp{Take $p$ more steps of Arnoldi}\;
        \BlankLine
        \tcp{Shifted QR algorithm}
        $u = \mathtt{Shifts}\left(H,p\right)$\;
        $Q \leftarrow I_{k+p}$ \hspace{0.75in} \tcp{Initialize $Q$}\;
        \For{$j=1,2,\cdots, p$}{
            $H \leftarrow Q_j^T H Q_j$\;
            $Q \leftarrow Q Q_j$\;
        }
        $v\leftarrow \left(VQ\right)e{k+1}$\;
        $V \leftarrow \left(VQ\right)\begin{pmatrix} I_k \\ 0 \end{pmatrix}$\;
            $r \leftarrow \left(v\beta_k + r\sigma_k\right)$ where $\beta_k = e_{k+1}^THe_k$, $\sigma_k = e_{k+p}^TQe_k$\;
    }
\end{algorithm2e}

Each application of an implicit shift $\mu_j$ will replace the starting vector $v_1$ with $\left(\A-\mu_jI\right)v_1$.  Thus after completion of each loop in Algorithm \ref{alg:IRAM}:
\begin{equation}
    Ve_1 = v_1  \leftarrow \psi(\A)v_1;
    \label{}
\end{equation}
where $\psi(\lambda) = \left(1/\tau\right) \prod_{j=1}^p\left(\lambda - \mu_j\right)$ with $\tau$ a normalization factor.  Numerous choices are possible for the selection of these $p$ shifts.  One possibility is to choose $p$ ``exact'' shifts with respect to $H$.  One could find the eigenvalues of $H$---this is simple because $H$ is small---and sort them according to largest real part or largest modulus.  The shifts would then be the $p$ smallest eigenvalues.  Selecting these exact shifts has interesting consequences in the iteration.

\begin{lem}
    Let $\lambda(H) = \left\{\theta_1,\cdots,\theta_k\right\}\cup\left\{\mu_1, \cdots, \mu_p\right\}$ be a disjoint partition of the spectrum of $H$ and let
    \begin{equation}
        H_+ = Q^THQ,
        \label{}
    \end{equation}
    where $Q = Q_1Q_2\cdots Q_p$ with $Q_j$ implicitly determined by the shift $\mu_j$.  If $\beta_j\neq 0, 1 \leq j \leq k-1$, then $\beta_k = 0$ and
    \begin{equation}
        H_+ \begin{pmatrix}
            H_k^+ & M^+ \\
            0 R_p
        \end{pmatrix},
        \label{}
    \end{equation}
    where $\lambda\left(H_k^+\right) = \left\{\theta_1,\cdots,\theta_k\right\}$, $\lambda\left(R_p\right) = \left\{\mu_1, \mu_2,\cdots, \mu_p\right\}$. Moreover,
    \begin{equation}
        v_1^+ = VQe_1 = \sum x_j,
        \label{}
    \end{equation}
    where each $x_j$ is a Ritz vector corresponding to the Ritz value $\theta_j$, i.e., $x_j = Vy_j$ where $Hy_j = y_j\theta_j, 1 \leq j \leq k$.
\end{lem}

This lemma provides a very nice interpretation of the iteration when exact shifts are chosen.  Casting out the unwanted set of eigenvalues using exact shifts is mathematically equivalent to restarting the Arnoldi factorization from the beginning after updating $v_1 \leftarrow \sum x_j\xi_j$, a linear combination of Ritz vectors associated with the ``wanted'' eigenvalues.  Thus the updated starting vector has been implicitly replaced by the sum of $k$ approximate eigenvectors.  Approximate eigenvectors from a Krylov subspace of dimension $k+p$ are available at each iteration for a cost of $p$ rather than $k+p$ matrix-vector products per iteration.

\section{IRAM from ``Fundamental of Matrix Computaions''}
After $m$ steps of the Arnoldi process we have generated $Q_m = \left[q_1 \cdots q_m\right]$.  The columns of $Q_m$ are orthonormal.  An upper Hessenberg matrix, $H_m$, is also generated.  The relationship between $Q_m$, $H_m$, and the linear operator $\A$ is formed:
\begin{equation}
    \A Q_m = Q_mH_m + q_{m+1}h_{m+1,m}e_m^T.
    \label{eq:start}
\end{equation}
The eigenvalues of $H_m$ are the Ritz values of $\A$ associated with the subspace spanned by the columns of $Q_m$.  These eigenvalues can be ordered with the smallest $j$ eigenvalues used as shifts in the QR algorithm.  These eigenvalues approximate the region of the spectrum we are not interested in and want to suppress.

IRAM performs $j$ iterations of the QR algorithm on the upper Hessenberg matrix $H_m$ using the $j$ smallest eigenvalues.  This is computationally cheap because $m$, the size of $H_m$ is small.  The effect of the QR iterations is a unitary similarity transformation
\begin{equation}
    \hat{H_m} = V^{-1}H_mV_m,
    \label{eq:simTransform}
\end{equation}
where
\begin{equation}
    p(H_m) = V_mR_m,
\end{equation}
$V_m$ is the unitary matrix usually called $Q$ in the QR algorithm; $R_m$ is upper triangular and $p$ is a polynomial of degree $j$ with zeros equal to the shifts we previously chose.\footnote{I need to be able to show this.}

The QR algorithm preserves upper Hessenberg form so $\hat{H}_m$ is also upper Hessenberg.  Let $\hat{Q}_m = Q_mV_m$, and let $\hat{q}_1$ be the first column of $\hat{Q}_m$.

The next iteration of IRAM consists of $m$ more Arnoldi steps, but these don't have to be started from scratch.  To see this, multiply Eq. \ref{eq:start} by $V_m$ on the right side and use Eq. \ref{eq:simTransform} we obtain
\begin{subequations}\begin{align}
    \A Q_mV_m &= Q_mH_mV_m + q_{m+1}h_{m+1,m}e_m^TV_m \\
    \A \left(Q_mV_m\right) &= Q_m\left(V_m\hat{H}_m\right) + q_{m+1}h_{m+1,m}e_m^TV_m \\
    \A \hat{Q}_m &= \left(Q_mV_m\right)\hat{H}_m + q_{m+1}h_{m+1,m}e_m^TV_m \\
    \A \hat{Q}_m &= \hat{Q}_m\hat{H}_m + q_{m+1}h_{m+1,m}e_m^TV_m  \label{eq:middle}
\end{align}\end{subequations}
One can show that $e_m^TV_m$ has $m-j-1$ leading zeros (after performing $j$ exact shifts).  If we drop the last $j$ entries from this vector we obtain $\beta e_k^T$ where $\beta$ is some non-zero scalar.  Dropping the last $j$ columns from Eq. \ref{eq:middle} we obtain
\begin{equation}
    \A \hat{Q}_k = \hat{Q}_k\hat{H}_k + \check{q}_{m+1}\check{h}_{k+1,k}e_k^T + q_{m+1}h_{m+1,m}\beta e_k^T.
    \label{eq:midCheck}
\end{equation}
Here $\check{q}_{k+1}$ is the $(k+1)$st column of $\hat{Q}_m$ and $\check{h}_{k+1,k}$ is the $(k+1,k)$ entry of $\hat{H}_{l+1,k}$.

Now we define
\begin{equation}
    \hat{q}_{k+1} = \gamma\left(\check{q}_{k+1}\check{h}_{k+1,k} + q_{m+1}h_{m+1,m}\beta\right)
\end{equation}
with $\gamma$ such that $\|\hat{q}_{k+1}\|_2 = 1$.

If we let $\hat{h}_{k+1,k} = 1/\gamma$ then Eq. \ref{eq:midCheck} becomes
\begin{equation}
    \A \hat{Q}_k = \hat{Q}_k\hat{H}_k + \hat{q}_{k+1}\hat{h}_{k+1,k}e_k^T,
\end{equation}
which is identical to Eq. \ref{eq:start}, except for the hats on the symbols and $m$ being replaced by $k$.  Since we know that the Arnoldi vectors (the orthonormal columns of $Q$) are uniquely determined by the first column, we can conclude that this is the same equation that would have been created if Arnoldi's method had been run as usual.  Thus IRAM can start at step $k$ saving $k-1$ steps, a potentially dramatic decrease in runtime.

\subsection{Why IRAM works}
In this subsection I will describe how the shifts suppress that region of the spectrum of $\A$.
\end{document}


\documentclass[22pt,a4paper,notitlepage]{article}

%% Use your favourite packages here
\usepackage[margin=0.8in]{geometry}
\usepackage[T1]{fontenc}
\usepackage{times}
\usepackage{amsmath}
\usepackage{braket}
\usepackage{cancel}
\usepackage{hyperref}

\begin{document}
\title{Title of your quick note}
\author{Your Name}
\date{\today} % Can also use a custom date
\maketitle

%%=================Start====================%%

\noindent % Start line without indentation
Write sentences using words and equations such as $x^3 - y = 0$. We can also write standalone equations like below

$$ H \ket{\Psi} = E \ket{\Psi}.
$$
\\ % Line break without indentation
More words.

%% Use sections to organize
\section{Section with numbers}

\subsection{Subsection with numbers}

\section*{Section without numbers}

\subsection*{Subsection without numbers}

Use \textit{align} to write multiple lines of equations as below

\begin{align*}
\ket{\Psi}
&:= \sum_{\mu} \: c_{\mu} \ket{\mu},
\\
E
&= \frac{\braket{\Psi | H | \Psi}}{\braket{\Psi|\Psi}},
\end{align*}
more words just below equations.
\\ \\ % Line break without indentation
We can also include equation numbers.

\begin{align}
\ket{\Psi}
&:= \sum_{\mu} \: c_{\mu} \ket{\mu},
\\
E
&= \frac{\braket{\Psi | H | \Psi}}{\braket{\Psi|\Psi}}.
\end{align}

\noindent
We can create hyperlinks using \href{https://www.overleaf.com/learn/latex/Hyperlinks}{hyperref} and also \cancel{strikethrough} using the \textit{cancel} package.

\end{document}


%\newpage
In this chapter, we will mainly consider two statistic models, namely
logistic regression for linearly separable sets,
and support vector machine (SVM).
These two linear models form the foundation of deep learning, since
the final fully connected output layer of a deep neural network is
often given by one of these two linear classifiers.  One main objective in this chapter is that  these linear classification models can be used to classify a collection of
linearly separable classes.

In the presentation of this chapter, we treat both logistic regression
and support vector machine as pure mathematical techniques for
linearly separable sets.  In the later chapters, we will relate these
techniques in the context of machine learning, especially deep
learning.
\section{Definition of linearly separable sets}
In this section, we consider a special class of $k$ linearly separable sets for $k\ge 2$.  Let us first introduce the following
definition for binary classification.

For $k=2$, there is a very simple geometric interpretation of two
linearly separable sets.
\begin{definition}\label{lem:2class}
  The two sets $A_1$, $A_2\subset \mathbb{R}^d$ are linearly separable
   if there exists a hyperplane
  \begin{equation}
    \label{2classH}
H_0=\{x:wx+b=0\},
  \end{equation}
 such that $wx+b>0$ if $x\in A_1$ and $wx+b<0$ if $x\in A_2$.
  \end{definition}
\begin{figure}
\centering
\includegraphics[width=1.5in]{LinearS1.png}  \quad  %\includegraphics[width=1.5in]{LinearS2.png}
\caption{One linearly separable set}
\label{twoclassification}
\end{figure}

\begin{figure}
\centering
\includegraphics[width=1.5in]{NLinearS1.png}  \quad  \includegraphics[width=1.5in]{NLinearS2.png}
\caption{Two non-linearly separable sets}
\label{twoclassification}
\end{figure}

%\blankpage
%\newbreak
\begin{lemma}\label{lem:2class}
  The two sets $A_1$, $A_2\subset \mathbb{R}^d$ are linearly separable
 if there exists
  \begin{equation}
    \label{Wb}
W=
\begin{pmatrix}
  w_1\\
w_2
\end{pmatrix}
\in \mathbb{R}^{2\times d},
b=
\begin{pmatrix}
  b_1\\
b_2
\end{pmatrix}
\in \mathbb{R}^{2\times d},
\end{equation}
such that,  % for each $1\le i\le 2$ and $ j \neq i$
\begin{equation}
\label{eq:3}
 w_1x+b_1 > w_2x+b_2,\ \forall x\in A_1,
 \end{equation}
and
\begin{equation}
\label{eq:3}
 w_1x+b_1 < w_2x+b_2,\ \forall x\in A_2.
 \end{equation}
\end{lemma}

\begin{proof}
	Here, we can just take $w = w_1 - w_2$ and $b = b_1 - b_2$,
	then we can check that the hyperplane $wx + b$ satisfies the
	definition as presented before.
\end{proof}

%\blankpage
%\newbreak
Now let us consider multi-class classification.
To begin with the definition, let us assume that the data space is
divided into $k$ classes represented by $k$ disjoint sets $A_1,A_2,\cdots,A_k\subset \mathbb{R}^d$, which means
\begin{equation}
A = A_1\cup A_2\cup \cdots \cup A_k, ~A_i\cap A_j = \emptyset, \forall i \neq j.
\end{equation}

\begin{definition}[Linearly Separable]
  A collection of subsets $A_1,...,A_k\subset \mathbb{R}^d$ are
  linearly separable if there exist
  \begin{equation}
    \label{Wb}
W=
\begin{pmatrix}
  w_1\\
\vdots\\
w_k
\end{pmatrix}
\in \mathbb{R}^{k\times d},
b=
\begin{pmatrix}
  b_1\\
\vdots\\
b_k
\end{pmatrix}
\in \mathbb{R}^{k\times d},
\end{equation}
such that,   for each $1\le i\le k$ and $ j \neq i$
\begin{equation}
\label{eq:3}
 w_ix+b_i > w_jx+b_j,\ \forall x\in A_i.
 \end{equation}
namely, each pairs of $A_i$ and $A_j$ are linearly separable by the plane
\begin{equation}
  \label{Hij}
H_{ij}=\{(w_i-w_j)\cdot x+(b_i-b_j) = 0\}, \quad \forall j\neq i.
\end{equation}
\end{definition}
The geometric interpretation for linearly separable sets is less
obvious when $k>2$.
\begin{lemma}{\label{Interplation}}
Assume that $A_1,...,A_k$ are linearly separable and $ W\in
\mathbb{R}^{k\times d} $ and $b\in\mathbb{R}^k $ satisfy \eqref{Wb}.  Define
\begin{equation}
\label{Gammai}
\Gamma_i(W,b) = \{x\in\mathbb R^d: (Wx+b)_i > (Wx+b)_j,\ \forall j \neq i\}
\end{equation}
Then for each $i$,
\begin{equation}
  \label{AiGamma}
A_i \subset \Gamma_i(W,b)
\end{equation}
\end{lemma}
We note that  each $\Gamma_i(W,b)  $ is a polygon whose boundary consists of hyperplanes given by
\eqref{Hij}.
\begin{figure}
\centering
	\includegraphics[width=1.5in]{./figures/3-class.PNG}
	\caption{Linearly separable sets in 2-d space (k = 3)}
	\label{twoclassification}
\end{figure}

We next introduce two more definitions of linearly separable sets that
have more clear geometric interpretation.

\begin{definition}[All-vs-One Linearly Separable]
 A collection of subsets $A_1,...,A_k\subset \mathbb{R}^d$ is
 all-vs-one linearly separable
if for each $i = 1,...,k$,
$A_i$ and $\displaystyle \cup_{j\neq i} A_j$ are linearly separable.
\end{definition}

\begin{figure}[H]
\centering
	\includegraphics[width=1.5in]{./figures/MulLClassfication.PNG}
	\caption{All-vs-One linearly separable sets (k = 3)}
	\label{twoclassification}
\end{figure}

\begin{definition}[Pairwise Linearly Separable]
  A collection of subsets $A_1,...,A_k\subset \mathbb{R}^d$ is
  pairwise linearly separable if for each pair of indices $1\leq i <
  j\leq k$,
$A_i$ and $A_j$ are linearly separable.
\end{definition}

\begin{figure}[H]
\centering
	\includegraphics[width=1.5in]{./figures/pairwise_linearly_separable.png}
	\caption{Pairwise linearly separable sets in 2-d space (k = 3)}
	\label{pairwise_separable_example}
\end{figure}

We begin by comparing our notion of linearly separable to the two
other previously introduced geometric definitions of all-vs-one
linearly separable and pairwise lineaerly separable.  Obviously, in
the case of two classes, they are all equivalent, however, with more
than two classes this is no longer the case. We do have the following
implications, though.

\begin{lemma}
  If $A_1,...,A_k\subset \mathbb{R}^d$ are all-vs-one linearly
  separable, then they are linearly separable as well.
\end{lemma}
\begin{proof}
 Assume that $A_1,...,A_k$ are all-vs-one linearly separable. For each $i$, let $w_i$, $b_i$ be such that $w_ix + b_i$ separates
 $A_i$ from $\cup_{j\neq i} A_j$, i.e. $w_ix + b_i > 0$ for $x\in A_i$ and $w_ix + b_i < 0$ for $x \in \cup_{j\neq i} A_j$.

 Set $W = (w_1^T,w_2^T,\cdots,w_k^T)^T$, $b = (b_1,b_2,\cdots,b_k)^T$ and observe that if $x\in A_i$, then
 $(Wx + b)_i > 0$ while $(Wx + b)_j < 0$ for all $j\neq i$.
\end{proof}

\begin{lemma}
  If $A_1,...,A_k\subset \mathbb{R}^n$ are linearly separable, then
  they are pairwise linearly separable as well.
\end{lemma}

\begin{proof}
  If $A_1,...,A_k\subset \mathbb{R}^d$ are linearly separable, suppose
  that $W = (w_1^T,w_2^T,\cdots,w_k^T)^T$, $b =
  (b_1,b_2,\cdots,b_k)^T$. So we have
	\begin{equation}
	\begin{cases}
	w_i x+ b_i > w_j x + b_j & x\in A_i \\
	w_i x+ b_i < w_j x + b_j& x\in A_j \\
	\end{cases}
	\end{equation}
	Take $w_{i,j} = w_i - w_j, b_{i,j} = b_i-b_j$, then we have
	\begin{equation}
	w_{i,j}x + b_{i,j}\begin{cases}
	> 0 & x\in A_i \\
	< 0 & x\in A_j \\
	\end{cases}
	\end{equation}
	So $A_1,...,A_k$ are pairwise linearly separable.
\end{proof}

However, the converses of both of these statements are false, as the following examples show.
\begin{example}[Linearly separable but not all-vs-one linearly separable]
 Consider the sets $A_1, A_2, A_3\subset \mathbb{R}$ given by $A_1 = [-4,-2]$, $A_2 = [-1,1]$, and $A_3 = [2,4]$. These
 sets are clearly not one-vs-all linearly separable because $A_2$ cannot be separated from both $A_1$ and $A_3$ by a single
 plane (in $\mathbb{R}$ this is just cutting the real line at a given number, and $A_2$ is in the middle).

 However, these sets are linearly separated by $W = [-2,0,2]^T$ and $b = [-3,0,-3]^T$, for example.
\end{example}
\begin{example}[Pairwise linearly separable but not linearly separable]
Consider the sets $A_1, A_2, A_3\subset \mathbb{R}^2$ shown in figure \ref{pairwise_separable_example}.
Note that $A_i$ and $A_j$ are separated by hyperplane $H_{i,j}$ (drawn in the figure) and so these sets are
pairwise linearly separable. We will show that they are not linearly separable.

Assume to the contrary that $W\in \mathbb{R}^{3\times 2}$ and $b\in \mathbb{R}^2$ separate $A_1$, $A_2$, and $A_3$. Then
$(w_i - w_j)x + (b_i - b_j)$ must be a plane which separates $A_i$ and $A_j$. Now consider a point $z$ bounded by $A_1$, $A_2$ and $A_3$ in figure
\ref{pairwise_separable_example}. We see from the figure that given any plane separating $A_1$ from $A_2$, $z$ must
be on the same side as $A_2$, given any plane separating $A_2$ from $A_3$, $z$ must be on the same side as
$A_3$, and given any plane separating $A_3$ from $A_1$, $z$ must be on the same side as $A_1$.

This means that $(w_2 - w_1)z + (b_2 - b_1) > 0$, $(w_3 - w_2)z + (b_3 - b_2) > 0$, and
$(w_1 - w_3)z + (b_1 - b_3) > 0$. Adding these together, we obtain $0 > 0$, a contradiction.

The essence behind this example is that although the sets $A_1$, $A_2$, and $A_3$ are pairwise linearly separable,
no possible pairwise separation allows us to consistently classify arbitrary new points. However, a linear separation
would give us a consistent scheme for classifying new points.

\end{example}

So the notion of linear separability is sandwiched in between the more
intuitive notions of all-vs-one and pairwise separability. It turns
out that linear separability is the notion which is most useful for
the $k$-class classification problem and so we focus on this notion of
separability from now on.

\newpage
\section{Some simple linear classifiers}
We begin with the simplest situation, where there are only two classes.
\begin{lemma}
If $A_1$, $A_2\subset\mathbb{R}^n$ are linearly separable, then there exists $w\in\mathbb{R}^{1\times n}$, $b\in\mathbb{R}$ such that
        \begin{equation}
          f(x):=h\left( \begin{array}{cc}
              wx+b \\
              -(wx+b)
            \end{array}
          \right)
          =
          \begin{cases}
            e_1 \quad x\in A_1 \\
            e_2 \quad x\in A_2,
          \end{cases}
        \end{equation}
      where $e_1=\left( \begin{array}{cc} 1\\0 \end{array} \right)$,
      $e_2=\left( \begin{array}{cc} 0\\1 \end{array} \right)$ and
        $h$ is the Heaviside function defined by:
        \begin{equation}
        \label{heaviside}
        h(t):=\begin{cases}
        0 \quad t < 0, \\
        1 \quad t \ge 0 .
        \end{cases}
        \end{equation}
      \end{lemma}
      The Heaviside function $h$ defined in \eqref{heaviside} is a
      classic activation function.
      \begin{figure}
      	\centering
      	\includegraphics[width=2in]{figures/Heaviside.png}
      	\caption{Heaviside activation function }
      \end{figure}

      But its discontinuity at $t=0$
      makes it difficult to use in practice. Instead we consider the
      following linear-step function:
  \begin{equation}\label{linear-step}
  \sigma(t)=
    \begin{cases}
       0 \quad t<0, \\
      t\quad 0\le t\le 1, \\
     1 \quad t> 1. \\
    \end{cases}
  \end{equation}
\begin{figure}
\centering
\includegraphics[width=2in]{./figures/zig}
\caption{Linear-step function}
\end{figure}
This $\sigma(t)$ is closely related to the so-called Rectified Linear Unit function:
\begin{equation}\label{ReLU}
ReLU(t):=\begin{cases}
0 \quad t< 0, \\
t \quad t\ge 0.
\end{cases}
\end{equation}
\begin{figure}
\centering
\includegraphics[width=2in]{./figures/ReLU}
\caption{ReLU activation function }
\end{figure}
Namely
\begin{equation}
  \label{sigma-ReLU}
\sigma(t)=ReLU(t)-ReLU(t-1).
\end{equation}

\begin{lemma}
	If $A_1$, $A_2\subset\mathbb{R}^n$ are linearly separable, then there exist $w\in\mathbb{R}^{1\times n}$, $b\in\mathbb{R}$ such that
	\begin{equation}
	f(x):=\left( \begin{array}{cc}
	\sigma(wx+b) \\
	\sigma(-(wx+b))
	\end{array}
	\right)
	= e_i,\quad\mathrm{if}\ x\in A_i,\ i=1,2.
	\end{equation}
\end{lemma}
\begin{proof}
  By the definition of linearly separable sets, there exist
  $w_0\in\mathbb{R}^{1\times n}$, $b_0\in\mathbb{R}$ such that
  $w_0x+b_0>0$ if $x\in A_1$ and $w_0x+b_0<0$ if $x\in A_2$. We can
  find $\varepsilon>0$ such that
\begin{equation}
w_0x+b_0\begin{cases}
>\varepsilon \qquad x\in A_1, \\
<-\varepsilon \quad x\in A_2 .
\end{cases}
\end{equation}
Let $w=w_0/\varepsilon$, $b=b_0/\varepsilon$, we have:
\begin{equation}
wx+b\begin{cases}
>1 \qquad x\in A_1, \\
<-1 \quad x\in A_2 .
\end{cases}
\end{equation}
which implies $f(x)=e_i$ if $x\in A_i$.
\end{proof}

The extension of the above lemma to $k$-classes is straightforward.
\begin{lemma}
  If $A_i \subset \mathbb{R}^n(1\le i \le k)$ are linearly separable,
  there exist $W\in\mathbb{R}^{k\times n}$, $b\in\mathbb{R}^k$ s.t.
\begin{equation}
  \label{simple-f}
	f(x):=\sigma(Wx+b) = e_i,\quad\mathrm{for\ }x\in A_i
\end{equation}
\end{lemma}

Oftentimes, we use the following notation
\begin{equation}
  \label{theta}
\theta=(W,b)\in\mathbb{R}^{k\times(n+1)} \mbox{ and } \theta[x]=Wx+b.
\end{equation}


%%
\documentclass[11pt]{article}
\usepackage[english]{babel}
\usepackage[a4paper]{geometry}
\usepackage{amssymb,amsmath,amsthm}
\usepackage{graphicx,wrapfig}
\usepackage{latexsym}
\usepackage{mathtools}
\usepackage{hyperref}
\usepackage{fontspec}
\setmainfont{Cochin}
\usepackage[italic]{mathastext}
%
\usepackage{cancel}
%
\usepackage[backend=bibtex,sorting=none,giveninits=true]{biblatex}
\bibliography{../../../../Latex/Bibliotheque/bibliotheque.bib}
%
%---------------------------------------------------------
%\newcommand{\Set}[1]{\left\{#1\right\}}
%\newcommand{\transpose}[1]{#1^{\mathsf{T}}}
%\renewcommand{\div}{\operatorname{div}}
%\newcommand{\dn}[1]{\frac{\partial #1}{\partial n}}
%\newcommand{\GammaD}{\Gamma_{\rm D}}
%\newcommand{\GammaN}{\Gamma_{\rm N}}
%\newcommand{\udir}{u^{\rm D}}
\newcommand{\qd}{q^{\rm D}}
\input{../../../../Latex/Articles/macros.tex}
%---------------------------------------------------------
%
%---------------------------------------------------------
%
%
%==========================================
\title{SimFem}
\author{Roland Becker}
%
%==========================================
\begin{document}
%==========================================
\maketitle
\setcounter{tocdepth}{3}
\tableofcontents
%
%
%==========================================
\section{Geometry and finite elements}\label{sec:}
%==========================================
%
%
%-------------------------------------------------------------------------
\subsection{Simplices}\label{subsec:}
%-------------------------------------------------------------------------
%
We consider an arbitrary non-degenerate simplex $K=(x_0,x_1,\ldots, x_{d})$. The (signed) volume of $K$ is given by
%
\begin{equation}\label{eq:}
|K| = \frac{1}{d!} \det(x_1-x_{0},\ldots, x_{d}-x_{0})= \frac{1}{d!} \det(1,x_{0},x_1\ldots, x_{d})\quad 1=\transpose{(1,\ldots,1)}.
\end{equation}
%
The $d+1$ sides $S_k$ (co-dimension one, $d-1$-simplices or facets) are defined by
$S_k=(x_0,\ldots, \cancel{x_k}, \ldots, x_{d})$. The height is $d_k=|P_{S_k}x_k - x_k|$, where $P_S$ is the orthogonal projection on the hyperplane associated to $S$. We have
%
\begin{align*}
d_k = d\frac{|K|}{|S_k|} \qquad\mbox{(and for $d=3 \; |S_k| = \frac12 |u\times v| $)}
\end{align*}
%
%
%-------------------------------------------------------------------------
\subsection{Finite elements}\label{subsec:}
%-------------------------------------------------------------------------
%
%
The $d+1$ basis functions of the Courant element are the barycentric coordinates
$\lambda_i$ defined as being affine with respect to the coordinates and $\lambda_i(x_j)=\delta_{ij}$. The constant gradient is given by
%
\begin{align*}
\nabla \lambda_i = - \frac{1}{d_i}\vec{n_i}.
\end{align*}
%
The relation with the $d+1$ Crouzeix-Raviart basis functions $\psi_i$ is given by
%
\begin{align*}
\psi_i = 1 - d\lambda_i
\end{align*}
%
Finally the $d+1$ Raviart-Thomas basis functions $\xi_i$, associated to side $S_i$, i.e. the opposite node $x_i$, are given by
%
\begin{align*}
\xi_i = \frac{x-x_i}{d_i} = \frac{1}{d_i}\sum_{j=0\atop i\ne j}^d x_j \lambda_j
\end{align*}
%

%
%-------------------------------------------------------------------------
\subsection{Numerical integration}\label{subsec:}
%-------------------------------------------------------------------------
%
Any polynomial in the barycentric coordinates can be integrates exactly.
%
\begin{equation}\label{eq:}
\int_K \prod_{i=1}^{d+1}\lambda_i^{n_i} \,dv = d!|K|\frac{\prod\limits_{i=1}^{d+1} n_i!}{\left( \sum\limits_{i=1}^{d+1} n_i + d\right)!}
\end{equation}
%
see \cite{EisenbergMalvern73}, \cite{VermolenSegal18}.
%

Let $V=\vect{\phi}$. For a smooth function $f$ and $u=\sum_j u_j \phi_j$ we use and approximation based on $\vect{\psi}$ such that $\psi_l(x_k-=\delta_{kl}$ and
%
\begin{align*}
f(u) \approx \sum_k f(u(x_k))\psi_k = \sum_k f(\sum_j u_j \phi_j(x_k))\psi_k
\end{align*}
%
Then
%
\begin{align*}
\int_K f(u) \phi_i \approx \sum_k f_k\int_K \psi_k\phi_i,\quad f_k = f(\sum_l u_l \phi_l(x_k))\\
\int_K f'(u)(\phi_j) \phi_i \approx \sum_k f'_{k,j}\int_K  \psi_k\phi_i,\quad f'_{k,j} = f'(\sum_l u_l \phi_l(x_k))\phi_j(x_k)
\end{align*}
%
For $\psi=\phi$ this becomes considerably cheaper:
%
\begin{align*}
\int_K f(u) \phi_i \approx \sum_k f_k\int_K \phi_k\phi_i,\quad f_k = f(u_k)\\
\int_K f'(u)(\phi_j) \phi_i \approx f'_j\int_K  \phi_j \phi_i,\quad f'_k = f'(u_k)
\end{align*}
%

%
%-------------------------------------------------------------------------
\subsection{Element matrices for $C^1$}\label{subsec:}
%-------------------------------------------------------------------------
%
%
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\subsubsection{Mass matrix}
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%
%
\begin{align*}
M_{ij} = \begin{cases}
\frac{2d!|K|}{(2+d)!}& $i=j$\\
\frac{d!|K|}{(2+d)!}& $i\ne j$
\end{cases}\quad
M^{\rm 1D} =
|K|
\begin{bmatrix}
\frac13 & \frac16\\
\frac16 & \frac13
\end{bmatrix}, \;
M^{\rm 2D} =
|K|
\begin{bmatrix}
\frac16 & \frac{1}{12} & \frac{1}{12}\\
\frac{1}{12} & \frac16 & \frac{1}{12}\\
\frac{1}{12} & \frac{1}{12} & \frac16
\end{bmatrix}, \;
M^{\rm 3D} =
|K|
\begin{bmatrix}
\frac{1}{10} & \frac{1}{20} & \frac{1}{20} & \frac{1}{20}\\
\frac{1}{20} & \frac{1}{10} & \frac{1}{20} & \frac{1}{20}\\
\frac{1}{20} & \frac{1}{20} & \frac{1}{10} & \frac{1}{20}\\
\frac{1}{20} & \frac{1}{20} & \frac{1}{20} & \frac{1}{10}
\end{bmatrix}
\end{align*}
%

Lumped mass
\begin{align*}
\tilde M_{ij} = \begin{cases}
\frac{|K|}{(1+d)} & $i=j$\\
0& $i\ne j$
\end{cases}
\end{align*}

%
%==========================================
\section{Test problems}\label{sec:}
%==========================================
%
%
%-------------------------------------------------------------------------
\subsection{Advection-Diffusion-Reaction}\label{subsec:}
%-------------------------------------------------------------------------
%
%
\begin{equation}\label{eq:}
\div(\beta u) - \div(k \nabla u) + \psi(u) = f\quad\mbox{in $\Omega$},\qquad u=\udir \mbox{on $\GammaD$},\qquad k\dn{u}=\qd \mbox{on $\GammaN$}
\end{equation}
%
%
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\subsubsection{Courant element}
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%
%
\begin{align*}
\int_{\Omega}\psi(u)v +  \int_{\Omega}k \nabla u\cdot \nabla v -\int_{\Omega}u\beta\cdot\nabla v+\int_{\partial\Omega}\beta_n^+u v = \int_{\Omega}fv +\int_{\partial\Omega}|\beta_n^-|\udir v
\end{align*}
%
Piecewise constant approximation for $k$
%
\begin{align*}
\int_K \nabla \lambda_i \cdot \nabla \lambda_j = |K| \frac{n_i}{d_i}\cdot\frac{n_j}{d_j} = |K| \frac{|S_i|}{d|K|}\frac{|S_j|}{d|K|} n_i\cdot n_j
= \frac{1}{d^2|K|}\tilde n_i\cdot \tilde n_j\\
\\
-\int_K u \beta\cdot\nabla \lambda_i = u(x_K)|K| \beta\cdot\frac{n_i}{d_i} = u(x_K)\frac{1}{d}\beta\cdot\tilde n_i \\
- \int_S f \dn{v_i} = -f(x_S) |S| \nabla v_i \cdot n = f(x_S) |S| \frac{1}{d_i} n_i \cdot n= f(x_S) |S| \frac{|S_i|}{d|K|} n_i \cdot n
= f(x_S) \frac{\tilde n_i \cdot \tilde n}{d|K|}
\end{align*}
%

%
%-------------------------------------------------------------------------
\subsection{Turing}\label{subsec:}
%-------------------------------------------------------------------------
%
We consider a reaction-diffusion system on $\Omega=]-1,+1[^d$
%
\begin{equation}\label{eq:reacdiff}
\left\{\;
\begin{split}
\frac{\partial u}{\partial t}  -k_u\Delta u =& f(u,v)\quad\mbox{in $\Omega$},\qquad \frac{\partial u}{\partial n}=0\quad\mbox{on $\partial\Omega$},\\
\frac{\partial v}{\partial t}  -k_v \Delta v=&  g(u,v)\quad\mbox{in $\Omega$},\qquad \frac{\partial v}{\partial n}=0\quad\mbox{on $\partial\Omega$},
\end{split}
\right.
\end{equation}
%
Alan Turing discovered that the astonishing effect of destabilization by diffusion \cite{Turing52}, which leads to pattern formation
\footnote{For an introduction and references see \url{https://en.wikipedia.org/wiki/Reaction–diffusion_system}.}
%
\begin{equation}\label{eq:}
f(u,v) = (a-u) - \psi(u,v),\qquad  g(u,v)=\psi(u,v).
\end{equation}
%
An equilibrium point satisfies
%
\begin{align*}
u^* = a,\qquad \psi(a,v^*)=0
\end{align*}
%
The linear stability analysis is based on the Jacobian
%
\begin{align*}
\begin{bmatrix}
-1 -\psi'_u & -\psi'_v\\ \psi'_u & \psi'_v
\end{bmatrix}
\quad\Rightarrow\quad \operatorname{tr} = \psi'_v-\psi'_u-1,\quad \det= -\psi'_v
\end{align*}

The brusselator is given by
%
\begin{equation}\label{eq:brusselator}
\psi(u,v) = bu - u^2v\qquad ( \psi'_u = b-2uv,\quad \psi'_v=-u^2).
\end{equation}

An equilibrium point of (\ref{eq:brusselator}) necessarily satisfies $u^*=a$ and $v^*=b/a$. We now have
%
\begin{align*}
\operatorname{tr} = -b + 2uv -u^2-1 ,\quad \det = u^2
\end{align*}
%
and for the equilibrium point
%
\begin{align*}
\operatorname{tr}^* = -1 - b +2b -a^2 = b-1-a^2,\quad {\det}^* = a^2
\end{align*}
%
We conclude that a  Hopf bifurcation appears if $b>a^2+1$.

%
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\subsubsection{The influence of diffusion}
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%
We consider an expansion into eigenfunctions of the Laplace operator withe eigenvalues $l\ge0$. The for the frequency $l$ we have the Jacopian
%
\begin{align*}
\begin{bmatrix}
-1 -\psi'_u - k_ul& -\psi'_v\\ \psi'_u & \psi'_v - k_vl
\end{bmatrix}
\quad\Rightarrow\quad \operatorname{tr} = \psi'_v-\psi'_u-1- (k_u+ k_v)l,\\
\det=  (1+k_ul)(k_vl-\psi'_v) +k_vl\psi'_u  = k_uk_v l^2 +(k_v(\psi'_u+1)-k_u\psi'_v)l - \psi'_v
\end{align*}
For the brusselator we have at the eqilubrium
%
\begin{align*}
\det= k_uk_v l^2 +(k_u a^2-k_v(b-1))l +a^2.
\end{align*}
%
The discriminant is
%
\begin{align*}
\Delta = (k_u a^2-k_v(b-1))^2 - 4 a^2 k_uk_v
\end{align*}
%
which is positive ($a=1$) for
%
\begin{equation}\label{eq:}
b \le b^* = 1 + \frac{k_u}{k_v} + 2 \frac{\sqrt{k_u}}{\sqrt{k_v}}
\end{equation}
%
The critical frequency is
%
\begin{align*}
l^*  = \frac{k_v(b^*-1)-k_u}{2k_uk_v} = 2 k_u^{-1/2}k_v{-3/2}
\end{align*}
%

The data for our test problems are
%
\begin{align*}
&\mbox{(cas 1)}\qquad a = 1.0\;,\quad b=2.1, \quad k_u = 0.0\;, \quad k_v = 0.0\;,\quad T=20.0\\
&\mbox{(cas 2)}\qquad a = 1.0\;,\quad b=1.9, \quad k_u = 0.0001\;, \quad k_v = 0.01\;,\quad T=20.0\\
&u_0(x) = \begin{cases}
1 & \mbox{if for all $i$ $x_i \in [-0.4,0.0]$}\\
0 & \mbox{else}
\end{cases}
\qquad
v_0(x) = \begin{cases}
1 & \mbox{if for all $i$ $x_i \in [-0.2,0.2]$}\\
0 & \mbox{else}
\end{cases}
\end{align*}
%

%==========================================
\printbibliography
%==========================================
\end{document}
%==========================================


\subsection{Analytic Functions}
\begin{definition}
    An analytic function on a Riemann surface $R$ is an analytic map $R\to\mathbb C$.
\end{definition}
We can put analytic functions into a nice form by our study of this structure of Riemann surfaces.
\begin{theorem}[Inverse Function Theorem]
    Let $f$ be an analytic function on a domain $S\subset\mathbb C$.
    If $f^\prime(z_0)\neq 0$ for $z_0\in D$, then there are open neighbourhoods $U$ of $z_0$ and $V$ of $f(z_0)$ such that $f$ restricts to a biholomorphism $U\to V$.
\end{theorem}
\begin{proof}
    Omitted.
\end{proof}
\begin{proposition}\label{local_p_k}
    Let $f$ be a non-constant analytic function on a Riemann surface $R$ and $p\in R$ be a zero of $f$.
    There is a chart $(\phi,U)$ about $p$ with $\phi(p)=0$ such that $f\circ\phi^{-1}(z)=z^m$ for some integer $m>0$.
\end{proposition}
\begin{proof}
    Let $(\psi,V)$ be a chart with $\psi(p)=0$ as adding a constant does not change anything.
    $f$ is not globally constant, so it is not locally constant by the identity principle for Riemann surfaces.
    \footnote{Proved in example sheet.}
    Therefore is an analytic $g$ in a neighbourhood $W\subset\psi(V)$ of $0$ such that $f\circ\psi^{-1}(z)=z^mg(z)$ with $g(0)\neq 0$.\\
    But then since $g$ is continuous, there is $\delta>0$ such that $D(0,\delta)\subset W$ and $g(D(0,\delta))\subset D(g(0),|g(0)|)$ does not contain $0$.
    So there is an analytic branch cut of $\sqrt[m]{\cdot}$ on $g(D(0,\delta))$.
    Define $h(z)=z\cdot\sqrt[m]{g(z)}$ on $D(0,\delta)$, then $f\circ\psi^{-1}(z)=(h(z))^n$.
    Differentiating $h$ gives $h^\prime(0)=\sqrt[m]{g(0)}\neq 0$, so $h$ has an analytic inverse on $D(0,\epsilon)$ for some $0<\epsilon\le\delta$.
    Then $\phi=h\circ\psi$ and $U=\phi^{-1}(D(0,\epsilon))$ gives the required chart as
    $$f\circ\phi^{-1}(z)=f\circ\psi^{-1}\circ h^{-1}(z)=(h(h^{-1}(z)))^m=z^m$$
    which is what we wanted.
\end{proof}


\documentclass[12pt]{article}
\usepackage[margin=2cm]{geometry}
\usepackage{amsmath}
\usepackage{slashed}
\usepackage{tikz}

\begin{document}

\noindent
A high energy electron and positron collision can create two muons.
\begin{center}
\begin{tikzpicture}
\draw[dashed] (0,0) circle (0.5cm);
\draw[thick,->] (2,0) node[anchor=west] {$e^+$} -- (0.6,0);
\draw[thick,->] (-2,0) node[anchor=east] {$e^-$} -- (-0.6,0);
\draw[thick,->] (0.40,0.40) -- (1.3,1.3) node[anchor=south west] {$\mu^-$};
\draw[thick,->] (-0.4,-0.4) -- (-1.3,-1.3) node[anchor=north east] {$\mu^+$};
\draw (1,0.5) node {$\theta$};
\end{tikzpicture}
\end{center}

\noindent
Here is the same diagram with momentum and spinor labels.
\begin{center}
\begin{tikzpicture}
\draw[dashed] (0,0) circle (0.5cm);
\draw[thick,->] (2,0) node[anchor=west] {$p_2, v_2$} -- (0.6,0);
\draw[thick,->] (-2,0) node[anchor=east] {$p_1, u_1$} -- (-0.6,0);
\draw[thick,->] (0.40,0.40) -- (1.3,1.3) node[anchor=south west] {$p_3, u_3$};
\draw[thick,->] (-0.4,-0.4) -- (-1.3,-1.3) node[anchor=north east] {$p_4, v_4$};
\draw (1,0.5) node {$\theta$};
\end{tikzpicture}
\end{center}

\noindent
In a typical collider experiment the momentum vectors are
$$
p_1=\begin{pmatrix}E\\0\\0\\p\end{pmatrix}\qquad
p_2=\begin{pmatrix}E\\0\\0\\-p\end{pmatrix}\qquad
p_3=\begin{pmatrix}
E\\
\rho\sin\theta\cos\phi\\
\rho\sin\theta\sin\phi\\
\rho\cos\theta
\end{pmatrix}
\qquad
p_4=\begin{pmatrix}
E\\
-\rho\sin\theta\cos\phi\\
-\rho\sin\theta\sin\phi\\
-\rho\cos\theta
\end{pmatrix}
$$

\noindent
where $E$ is beam energy, $p=\sqrt{E^2-m^2}$, $\rho=\sqrt{E^2-M^2}$,
$m$ is electron mass $0.51\,\text{MeV}$,
and $M$ is muon mass $106\,\text{MeV}$.
The spinors are
\begin{gather*}
u_{11}=\begin{pmatrix}E+m\\0\\p\\0\end{pmatrix}\quad
v_{21}=\begin{pmatrix}-p\\0\\E+m\\0\end{pmatrix}\quad
u_{31}=\begin{pmatrix}E+M\\0\\p_3^z\\p_3^x+ip_3^y\end{pmatrix}\quad
v_{41}=\begin{pmatrix}p_4^z\\p_4^x+ip_4^y\\E+M\\0\end{pmatrix}
\\
u_{12}=\begin{pmatrix}0\\E+m\\0\\-p\end{pmatrix}\quad
v_{22}=\begin{pmatrix}0\\p\\0\\E+m\end{pmatrix}\quad
u_{32}=\begin{pmatrix}0\\E+M\\p_3^x-ip_3^y\\-p_3^z\end{pmatrix}\quad
v_{42}=\begin{pmatrix}p_4^x-ip_4^y\\-p_4^z\\0\\E+M\end{pmatrix}
\end{gather*}

\noindent
The last digit in a spinor subscript is 1 for spin up and 2 for spin down.
Note that the spinors are not individually normalized.
Instead, a combined spinor normalization constant $N=(E+m)^2(E+M)^2$
will be used where needed.

\bigskip
\noindent
This is the probability density for muon production.
Symbol $s=(p_1+p_2)^2=4E^2$,
symbol $s_j$ selects the spin of spinor $j$,
and $e$ is electron charge.
\begin{equation*}
|\mathcal{M}(s_1,s_2,s_3,s_4)|^2
=\frac{e^4}{s^2}\frac{1}{N}\left|(\bar{u}_3\gamma_\mu v_4)(\bar{v}_2\gamma^\mu u_1)\right|^2
\end{equation*}

\noindent
The expected probability density $\langle|\mathcal{M}|^2\rangle$
is computed by summing $|\mathcal{M}|^2$ over all spin states
and dividing by the number of inbound states.
There are four inbound states.
\begin{align*}
\langle|\mathcal{M}|^2\rangle
&=\frac{1}{4}\sum_{s_1=1}^2\sum_{s_2=1}^2\sum_{s_3=1}^2\sum_{s_4=1}^2|\mathcal{M}(s_1,s_2,s_3,s_4)|^2
\\
&=\frac{e^4}{4s^2}\sum_{s_1=1}^2\sum_{s_2=1}^2\sum_{s_3=1}^2\sum_{s_4=1}^2
\frac{1}{N}\left|(\bar{u}_3\gamma_\mu v_4)(\bar{v}_2\gamma^\mu u_1)\right|^2
\end{align*}

\noindent
Another way to compute $\langle|\mathcal{M}|^2\rangle$ is to use the Casimir trick.
\begin{equation*}
\langle|\mathcal{M}|^2\rangle
=\frac{e^4}{4s^2}
\mathop{\rm Tr}\left((\slashed{p}_3+M)\gamma^\mu(\slashed{p}_4-M)\gamma^\nu\right)
\mathop{\rm Tr}\left((\slashed{p}_2-m)\gamma_\mu(\slashed{p}_1+m)\gamma_\nu\right)
\end{equation*}

\noindent
Here is a third way to compute $\langle|\mathcal{M}|^2\rangle$.
\begin{equation*}
\langle|\mathcal{M}|^2\rangle
=\frac{e^4}{4s^2}
\left(
32 (p_1\cdot p_3) (p_2\cdot p_4) +
32 (p_1\cdot p_4) (p_2\cdot p_3) +
32 m^2 (p_3\cdot p_4) +
32 M^2 (p_1\cdot p_2) +
64 m^2 M^2
\right)
\end{equation*}

\noindent
For the momentum vectors given above the result is
\begin{equation*}
\langle|\mathcal{M}|^2\rangle
=e^4\left(1+\cos^2\theta+\frac{m^2+M^2}{E^2}\sin^2\theta+\frac{m^2M^2}{E^4}\cos^2\theta\right)
\end{equation*}

\noindent
The Stanford Linear Collider
had a collision energy of $2E=91$~GeV.
For beam energies such as SLC where $E\gg M$ the above equation can be approximated as
$$
\langle|\mathcal{M}|^2\rangle=e^4(1+\cos^2\theta)
$$

\noindent
The differential cross section is
$$
\frac{d\sigma}{d\Omega}
=\frac{\langle|\mathcal{M}|^2\rangle}{64\pi^2s}
=\frac{e^4}{256\pi^2E^2}(1+\cos^2\theta)
$$

\noindent
Recall that $e^2=4\pi\alpha$ hence
\begin{equation*}
\frac{d\sigma}{d\Omega}=\frac{\alpha^2}{16E^2}(1+\cos^2\theta)
\end{equation*}

\noindent
The total cross section calculation requires the following definite integral.
$$
\int_\Omega(1+\cos^2\theta)\,d\Omega
=\int_0^{2\pi}\int_0^\pi(1+\cos^2\theta)\sin\theta\,d\theta\,d\phi
=\frac{8}{3}\int_0^{2\pi}d\phi
=\frac{16\pi}{3}
$$

\noindent
Hence the total cross section is
$$
\sigma
=\int_\Omega d\sigma
=\int_\Omega\frac{\alpha^2}{16E^2}(1+\cos^2\theta)\,d\Omega
=\frac{\alpha^2}{16E^2}\frac{16\pi}{3}
=\frac{\pi\alpha^2}{3E^2}
$$

\noindent
We can integrate the differential cross section to obtain a cumulative distribution function.

\bigskip
\noindent
Let
\begin{equation*}
I(\xi)=2\pi\int_0^\xi\frac{d\sigma}{d\Omega}\,\sin\theta\,d\theta,
\qquad0\le\xi\le\pi
\end{equation*}

\noindent
The result is
\begin{equation*}
I(\xi)=2\pi\left(\frac{\alpha^2}{16E^2}\right)
\left(-\frac{1}{3}\cos^3\xi-\cos\xi+\frac{4}{3}\right)
\end{equation*}

\noindent
The cumulative distribution function is
\begin{equation*}
F(\theta)=\frac{I(\theta)}{I(\pi)},\qquad0\le\theta\le\pi
\end{equation*}

\noindent
Hence
\begin{equation*}
P(\theta_1\le\theta\le\theta_2)=F(\theta_2)-F(\theta_1)
\end{equation*}

\noindent
The normalized probability density is
\begin{equation*}
f(\theta)=\frac{dF(\theta)}{d\theta}=\frac{3}{8}(1+\cos^2\theta)\sin\theta,
\qquad0\le\theta\le\pi
\end{equation*}

\noindent
Run ``muon-production-5.txt'' to draw the probability density function.

\begin{center}
\includegraphics[scale=0.5]{muon-production.png}
\end{center}

\noindent
Run ``muon-production-1.txt'' to verify that
$$
\frac{1}{N}\sum_{s_1=1}^2\sum_{s_2=1}^2\sum_{s_3=1}^2\sum_{s_4=1}^2
\left|(\bar{u}_3\gamma_\mu v_4)(\bar{v}_2\gamma^\mu u_1)\right|^2
=
\mathop{\rm Tr}\left((\slashed{p}_3+M)\gamma^\mu(\slashed{p}_4-M)\gamma^\nu\right)
\mathop{\rm Tr}\left((\slashed{p}_2-m)\gamma_\mu(\slashed{p}_1+m)\gamma_\nu\right)
$$

\bigskip
\noindent
Run ``muon-production-2.txt'' to verify that
\begin{multline*}
\frac{1}{64E^4}
\mathop{\rm Tr}\left((\slashed{p}_3+M)\gamma^\mu(\slashed{p}_4-M)\gamma^\nu\right)
\mathop{\rm Tr}\left((\slashed{p}_2-m)\gamma_\mu(\slashed{p}_1+m)\gamma_\nu\right)
\\
=1+\cos^2\theta+\frac{m^2+M^2}{E^2}\sin^2\theta+\frac{m^2M^2}{E^4}\cos^2\theta
\end{multline*}

\noindent
and to verify that
\begin{multline*}
\mathop{\rm Tr}\left((\slashed{p}_3+M)\gamma^\mu(\slashed{p}_4-M)\gamma^\nu\right)
\mathop{\rm Tr}\left((\slashed{p}_2-m)\gamma_\mu(\slashed{p}_1+m)\gamma_\nu\right)
\\=
32 (p_1\cdot p_3) (p_2\cdot p_4) +
32 (p_1\cdot p_4) (p_2\cdot p_3) +
32 m^2 (p_3\cdot p_4) +
32 M^2 (p_1\cdot p_2) +
64 m^2 M^2
\end{multline*}

\subsection*{Data from SLAC PEP experiment}
See www.hepdata.net/record/ins216031, Table 1, 29.0 GeV.

\begin{center}
\begin{tabular}{|c|c|}
\hline
$x$ & $y$\\
\hline
$-0.925$ & 67.08\\
$-0.85\phantom{0}$ & 58.67\\
$-0.75\phantom{0}$ & 54.66\\
$-0.65\phantom{0}$ & 51.72\\
$-0.55\phantom{0}$ & 43.70\\
$-0.45\phantom{0}$ & 41.12\\
$-0.35\phantom{0}$ & 39.71\\
$-0.25\phantom{0}$ & 35.34\\
$-0.15\phantom{0}$ & 33.35\\
$-0.05\phantom{0}$ & 34.69\\
$\phantom{+}0.05\phantom{0}$ & 34.05\\
$\phantom{+}0.15\phantom{0}$ & 34.48\\
$\phantom{+}0.25\phantom{0}$ & 34.66\\
$\phantom{+}0.35\phantom{0}$ & 35.23\\
$\phantom{+}0.45\phantom{0}$ & 35.60\\
$\phantom{+}0.55\phantom{0}$ & 40.13\\
$\phantom{+}0.65\phantom{0}$ & 42.56\\
$\phantom{+}0.75\phantom{0}$ & 46.37\\
$\phantom{+}0.85\phantom{0}$ & 49.28\\
$\phantom{+}0.925$ & 55.70\\
\hline
\end{tabular}
\end{center}

\noindent
Data $x$ and $y$ have the following relationship with cross section parameters.
\begin{equation*}
x=\cos\theta
\qquad
y=(2E)^2\frac{d\sigma}{d\cos\theta}
\end{equation*}

\noindent
The differential cross section for muon production is
\begin{equation*}
\frac{d\sigma}{d\Omega}=\frac{\alpha^2}{16E^2}(1+\cos^2\theta)
\end{equation*}

\noindent
Let us compute predicted values $\hat{y}$ from the cross section formula.
Start by finding the relationship between $d\Omega$ and $d\cos\theta$.
Since $1+\cos^2\theta$ has no dependence on $\phi$ we have
\begin{equation*}
\int_\Omega(1+\cos^2\theta)\,d\Omega
=\int_0^{2\pi}\int_0^{\pi}(1+\cos^2\theta)\sin\theta\,d\theta\,d\phi
=2\pi\int_0^\pi(1+\cos^2\theta)\sin\theta\,d\theta
\end{equation*}

\noindent
Hence
\begin{equation*}
d\Omega=2\pi\sin\theta\,d\theta=-2\pi\,d\cos\theta
\end{equation*}

\noindent
We want positive cross sections so drop the minus sign and set
\begin{equation*}
\frac{d\sigma}{d\cos\theta}=2\pi\frac{d\sigma}{d\Omega}
\end{equation*}

\noindent
We can now write
\begin{align*}
y&=(2E)^2\frac{d\sigma}{d\cos\theta}\\
&=(2E)^2(2\pi)\frac{d\sigma}{d\Omega}\\
&=(2E)^2(2\pi)\frac{\alpha^2}{16E^2}(1+\cos^2\theta)\\
&=\frac{\pi\alpha^2}{2}(1+\cos^2\theta)
\end{align*}

\noindent
Multiply by $(\hbar c)^2$ to convert to SI
and multiply by $10^{37}$ to convert square meters to nanobarns.
\begin{equation*}
y=\frac{\pi\alpha^2}{2}(1+\cos^2\theta)\times(\hbar c)^2\times10^{37}
\end{equation*}

\noindent
Replace $\cos\theta$ with explanatory variable $x$ to obtain $\hat{y}$.
\begin{equation*}
\hat{y}=\frac{\pi\alpha^2}{2}(1+x^2)\times(\hbar c)^2\times10^{37}
\end{equation*}

\noindent
Here are the predicted values $\hat{y}$ based on the above formula.

\begin{center}
\begin{tabular}{|c|c|c|}
\hline
$x$ & $y$ & $\hat{y}$ \\
\hline
$-0.925$ & 67.08 & 60.44\\
$-0.85\phantom{0}$ & 58.67 & 56.10\\
$-0.75\phantom{0}$ & 54.66 & 50.89\\
$-0.65\phantom{0}$ & 51.72 & 46.33\\
$-0.55\phantom{0}$ & 43.70 & 42.42\\
$-0.45\phantom{0}$ & 41.12 & 39.17\\
$-0.35\phantom{0}$ & 39.71 & 36.56\\
$-0.25\phantom{0}$ & 35.34 & 34.61\\
$-0.15\phantom{0}$ & 33.35 & 33.30\\
$-0.05\phantom{0}$ & 34.69 & 32.65\\
$\phantom{+}0.05\phantom{0}$ & 34.05 & 32.65\\
$\phantom{+}0.15\phantom{0}$ & 34.48 & 33.30\\
$\phantom{+}0.25\phantom{0}$ & 34.66 & 34.61\\
$\phantom{+}0.35\phantom{0}$ & 35.23 & 36.56\\
$\phantom{+}0.45\phantom{0}$ & 35.60 & 39.17\\
$\phantom{+}0.55\phantom{0}$ & 40.13 & 42.42\\
$\phantom{+}0.65\phantom{0}$ & 42.56 & 46.33\\
$\phantom{+}0.75\phantom{0}$ & 46.37 & 50.89\\
$\phantom{+}0.85\phantom{0}$ & 49.28 & 56.10\\
$\phantom{+}0.925$ & 55.70 & 60.44\\
\hline
\end{tabular}
\end{center}

\noindent
The coefficient of determination $R^2$ measures how well predicted values fit the real data.
\begin{equation*}
R^2=1-\frac{\sum(y-\hat{y})^2}{\sum(y-\bar{y})^2}=0.87
\end{equation*}

\noindent
The result indicates that the model $d\sigma$ explains 87\% of the variance in the data.

\bigskip
\noindent
Run ``muon-production-3.txt'' to compute the above results.

\subsection*{Electroweak model}
The following differential cross section formula from electroweak
theory results in a better fit to the
data.\footnote{F. Mandl and G. Shaw, {\it Quantum Field Theory Revised Edition,} 316.}

\begin{equation*}
\frac{d\sigma}{d\Omega}=F(s)(1+\cos^2\theta)+G(s)\cos\theta
\end{equation*}

\noindent
where
\begin{align*}
F(s)&=\frac{\alpha^2}{4s}
\left(
1+\frac{g_V^2}{\sqrt{2}\pi}\left(\frac{m_Z^2}{s-m_Z^2}\right)\left(\frac{sG}{\alpha}\right)
+\frac{(g_A^2+g_V^2)^2}{8\pi^2}\left(\frac{m_Z^2}{s-m_Z^2}\right)^2\left(\frac{sG}{\alpha}\right)^2
\right)
\\
G(s)&=\frac{\alpha^2}{4s}
\left(
\frac{\sqrt{2}g_A^2}{\pi}\left(\frac{m_Z^2}{s-m_Z^2}\right)\left(\frac{sG}{\alpha}\right)
+\frac{g_A^2g_V^2}{\pi^2}\left(\frac{m_Z^2}{s-m_Z^2}\right)^2\left(\frac{sG}{\alpha}\right)^2
\right)
\end{align*}

\noindent
and
\begin{align*}
g_A&=-0.5
\\
g_V&=-0.0348
\\
m_Z&=91.17\,\text{GeV}
\\
G&=1.166\times10^{-5}\,\text{GeV}^{-2}
\end{align*}

\noindent
The corresponding formula for $\hat{y}$ is
\begin{equation*}
\hat{y}=2\pi\left[F(s)(1+x^2)+G(s)x\right]\times(\hbar c)^2\times10^{37}
\end{equation*}

\noindent
where $\sqrt{s}=29\,\text{GeV}$ is the center of mass collision energy.
Here are the predicted values $\hat{y}$ based on the above formula.

\begin{center}
\begin{tabular}{|c|c|c|}
\hline
$x$ & $y$ & $\hat{y}$ \\
\hline
$-0.925$ & 67.08 & 65.59\\
$-0.85\phantom{0}$ & 58.67 & 60.84\\
$-0.75\phantom{0}$ & 54.66 & 55.07\\
$-0.65\phantom{0}$ & 51.72 & 49.96\\
$-0.55\phantom{0}$ & 43.70 & 45.49\\
$-0.45\phantom{0}$ & 41.12 & 41.69\\
$-0.35\phantom{0}$ & 39.71 & 38.53\\
$-0.25\phantom{0}$ & 35.34 & 36.02\\
$-0.15\phantom{0}$ & 33.35 & 34.17\\
$-0.05\phantom{0}$ & 34.69 & 32.97\\
$\phantom{+}0.05\phantom{0}$ & 34.05 & 32.42\\
$\phantom{+}0.15\phantom{0}$ & 34.48 & 32.53\\
$\phantom{+}0.25\phantom{0}$ & 34.66 & 33.28\\
$\phantom{+}0.35\phantom{0}$ & 35.23 & 34.69\\
$\phantom{+}0.45\phantom{0}$ & 35.60 & 36.75\\
$\phantom{+}0.55\phantom{0}$ & 40.13 & 39.47\\
$\phantom{+}0.65\phantom{0}$ & 42.56 & 42.83\\
$\phantom{+}0.75\phantom{0}$ & 46.37 & 46.85\\
$\phantom{+}0.85\phantom{0}$ & 49.28 & 51.52\\
$\phantom{+}0.925$ & 55.70 & 55.45\\
\hline
\end{tabular}
\end{center}

\noindent
The coefficient of determination $R^2$ is
\begin{equation*}
R^2=1-\frac{\sum(y-\hat{y})^2}{\sum(y-\bar{y})^2}=0.98
\end{equation*}

\noindent
The result indicates that electroweak theory explains 98\% of the variance in the data.

\bigskip
\noindent
Run ``muon-production-4.txt'' to verify.

\subsection*{Notes}
Here are a few notes about how the scripts work.

\bigskip
\noindent
In component notation the traces become sums over the repeated index $\alpha$.
\begin{align*}
\mathop{\rm Tr}\left((\slashed{p}_3+M)\gamma^\mu(\slashed{p}_4-M)\gamma^\nu\right)
&=
(\slashed{p}_3+M)^\alpha{}_\beta
\gamma^{\mu\beta}{}_\rho
(\slashed{p}_4-M)^\rho{}_\sigma
\gamma^{\nu\sigma}{}_\alpha
\\
\mathop{\rm Tr}\left((\slashed{p}_2-m)\gamma_\mu(\slashed{p}_1+m)\gamma_\nu\right)
&=
(\slashed{p}_2-m)^\alpha{}_\beta
\gamma_\mu{}^\beta{}_\rho
(\slashed{p}_1+m)^\rho{}_\sigma
\gamma_\nu{}^\sigma{}_\alpha
\end{align*}

\noindent
To convert the above formulas to Eigenmath code,
the $\gamma$ tensors need to be transposed
so that repeated indices are adjacent to each other.
Also, multiply $\gamma^\mu$ by the metric tensor to lower the index.
\begin{align*}
\gamma^{\beta\mu}{}_\rho\quad&\rightarrow\quad
\text{\tt gammaT = transpose(gamma)}\\
\gamma^\beta{}_{\mu\rho}\quad&\rightarrow\quad
\text{\tt gammaL = transpose(dot(gmunu,gamma))}
\end{align*}

\noindent
Define the following $4\times4$ matrices.
\begin{align*}
(\slashed{p}_1+m)\quad&\rightarrow\quad\text{\tt X1 = pslash1 + m I}\\
(\slashed{p}_2-m)\quad&\rightarrow\quad\text{\tt X2 = pslash2 - m I}\\
(\slashed{p}_3+M)\quad&\rightarrow\quad\text{\tt X3 = pslash3 + M I}\\
(\slashed{p}_4-M)\quad&\rightarrow\quad\text{\tt X4 = pslash4 - M I}
\end{align*}

\noindent
Then
\begin{align*}
(\slashed{p}_3+M)^\alpha{}_\beta
\gamma^{\mu\beta}{}_\rho
(\slashed{p}_4-M)^\rho{}_\sigma
\gamma^{\nu\sigma}{}_\alpha
\quad&\rightarrow\quad
\text{\tt T1 = contract(dot(X3,gammaT,X4,gammaT),1,4)}
\\
(\slashed{p}_2-m)^\alpha{}_\beta
\gamma_\mu{}^\beta{}_\rho
(\slashed{p}_1+m)^\rho{}_\sigma
\gamma_\nu{}^\sigma{}_\alpha
\quad&\rightarrow\quad
\text{\tt T2 = contract(dot(X2,gammaL,X1,gammaL),1,4)}
\end{align*}

\noindent
Next, multiply matrices and sum over repeated indices.
The dot function sums over $\nu$ then the contract function
sums over $\mu$. The transpose makes the $\nu$ indices adjacent
as required by the dot function.
$$
\mathop{\rm Tr}(\cdots\gamma^\mu\cdots\gamma^\nu)\mathop{\rm Tr}(\cdots\gamma_\mu\cdots\gamma_\nu)
\quad\rightarrow\quad
\text{\tt contract(dot(T1,transpose(T2)))}
$$

\end{document}


\section{Simple examples}\label{examples}
%=====================================

Without loss of generality, let us start with the B-rep of a unit cube (see Figure~\ref{}a) whith topology given by two arrays of arrays \texttt{EV} and \texttt{FV} providing indices of vertices \texttt{V} on boundary of edges \texttt{E} or faces \texttt{F}.
{\small\begin{verbatim}
EV = [[1,2],[3,4],[5,6],[7,8],[1,3],[2,4],[5,7],[6,8],[1,5],[2,6],[3,7],[4,8]]
FV = [[1,2,3,4],[5,6,7,8],[1,2,5,6],[3,4,7,8],[1,3,5,7],[2,4,6,8]]
\end{verbatim}}
The $[\partial_1]$ matrix is ready to compute from \texttt{EV}:
{\small\begin{lstlisting}
n = length(EV);
Is,Js,Vs = map(cat,[EV, [[i,i] for i=1:n], [[1,1] for i=1:n]]);
$\partial_1$ = sparse(Is,Js,Vs);
\end{lstlisting}}
{\small\begin{lstlisting}
Matrix(convert(SparseMatrixCSC{Int8,Int64}, $\partial_1$))
8x12 Array{Int8,2}:
 1  0  0  0  1  0  0  0  1  0  0  0
 1  0  0  0  0  1  0  0  0  1  0  0
 0  1  0  0  1  0  0  0  0  0  1  0
 0  1  0  0  0  1  0  0  0  0  0  1
 0  0  1  0  0  0  1  0  1  0  0  0
 0  0  1  0  0  0  0  1  0  1  0  0
 0  0  0  1  0  0  1  0  0  0  1  0
 0  0  0  1  0  0  0  1  0  0  0  1
\end{lstlisting}}

The $\partial_2$ matrix is computed by filtering the elements from the product $[\partial_1^\top] * \texttt{fv'}$, where the sparse matrix \texttt{fv} is generated from \texttt{FV} array.

{\small\begin{lstlisting}
m = length(FV);
Is,Js,Vs = map(cat,[[[i for k=1:length(f)] for (i,f) in enumerate(FV)],
    [FV[i] for i=1:m], [ones(Int8, length(FV[i])) for i=1:m]]);
fv = sparse(Is,Js,Vs);
Matrix(convert(SparseMatrixCSC{Int8,Int64}, fv))
6x8 Array{Int8,2}:
 1  1  1  1  0  0  0  0
 0  0  0  0  1  1  1  1
 1  1  0  0  1  1  0  0
 0  0  1  1  0  0  1  1
 1  0  1  0  1  0  1  0
 0  1  0  1  0  1  0  1
\end{lstlisting}}

{\small\begin{lstlisting}
triples = map(tuple,SparseArrays.findnz($\partial_1$' * fv')...);
mat3xm = hcat([ [i,j,1] for (i,j,v) in triples if v==2]...);
Is,Js,Vs = [mat3xm[1,:], mat3xm[2,:], convert(Array{Int8,1},mat3xm[3,:])];
$\partial_2$ = sparse(Is,Js,Vs)
Matrix($\partial_2$)
12x6 Array{Int64,2}:
 1  0  1  0  0  0
 1  0  0  1  0  0
 0  1  1  0  0  0
 0  1  0  1  0  0
 1  0  0  0  1  0
 1  0  0  0  0  1
 0  1  0  0  1  0
 0  1  0  0  0  1
 0  0  1  0  1  0
 0  0  1  0  0  1
 0  0  0  1  1  0
 0  0  0  1  0  1
\end{lstlisting}}

VV = [[k] for k=1:size(V,2)]
model = (V, [VV,EV,FV])::Lar.LARmodel

meshes = GL.numbering(1.5)(model, GL.COLORS[1], 0.1)
%#push!(meshes, GL.GLFrame)
GL.VIEW(meshes);


\subsection{Topological Data Analysis}
\label{sec:si_tda}
Since the exact manifold (or distribution) of the input space is not
known in general and the SOM algorithms only approximate it, we
simplify these manifolds by retaining their original topological
structure. Here we approach the manifolds of input and neural spaces
using the Alpha complex. Before diving into more details regarding
TDA, we provide here a few definitions and some notation. A
$k$-simplex $\sigma$ is the convex hull of $k+1$ affinely independent
points (for instance a $0$-simplex is a point, a $1$-simplex is an
edge, a $2$-simplex is a triangle, etc). A simplicial complex with
vertex set $\mathcal{V}$ is a set $\mathcal{S}$ of finite subsets of
$\mathcal{V}$ such that the elements of $\mathcal{V}$ belong to
$\mathcal{S}$ and for any $\sigma \in \mathcal{S}$ any subset $\sigma$
belongs to $\mathcal{S}$. Said differently, a simplicial complex is a
space that has been constructed out of intervals, triangles, and other
higher dimensional simplices.

In our analysis we let $\mathcal{S}(\mathcal{M}, \alpha)$ be a Alpha
simplicial complex with $\mathcal{M}$ being a point cloud, either the
input space or the neural one, and $\alpha$ is the ``persistence''
parameter.  More specifically, $\alpha$ is a threshold (or radius as
we will see later) that determines if the set $X$ spans a $k$-simplex
if and only if $d(x_i, x_j) \leq \alpha$ for all $0 \leq i, j \leq
k$. From a practical point of view, we first define a family of
thresholds $\alpha$ (or radius) and for each $\alpha$, we center a
ball of radius $\alpha$ on each data point and look for possible
intersections with other balls. This process is called filtration of
simplicial complexes. We start from a small $\alpha$ where there are
no intersecting balls (disjoint set of balls) and steadily we increase
the size of $\alpha$ up to a point where a single connected blob
emerges. As $\alpha$ varies from a low to a large value, holes open
and close as different balls start intersecting. Every time an
intersection emerges we assign a {\em birth} point $b_i$ and as the
$\alpha$ increases and some new intersections of larger simplicies
emerge some of the old simplicies die (since they merge with other
smaller simplicies to form larger ones). Then we assign a {\em death}
point $d_i$. A pair of a birth and death points $(b_i, d_i)$ is
plotted on a Cartesian two-dimensional plane and indicates when a
simplicial complex was created and when it died. This two-dimensional
diagram is called persistent diagram and the pairs (birth, death) that
last longer reflect significant topological properties.  The longevity
of birth-death pairs is more clear in the persistent barcodes where
the lifespan of such a pair is depicted as a straight line.

In other words, for each value of $\alpha$ we obtain new simplicial
complexes and thus new topological properties such as homology are
revealed. Homology encodes the number of points, holes, or voids in a
space.  For more thorough reading we refer the reader to
\citep{Chazal:2017,Ghrist:2008,Zomorodian:2005}. In this work, we used
the Gudhi library~\citep{Maria:2014} to compute the Alpha simplicial
complexes, the filtrations and the persistent diagrams and
barcodes. Therefore, we compute the persistent diagram and persistent
barcode of the input space and of the maps and we calculate the
Bottleneck distance between the input and SOM and RSOM maps
diagrams. The bottleneck distance provides a tool to compare two
persistent diagrams in a quantitative way.  The Bottleneck distance
between two persistent diagrams $\text{dgm}_1$ and $\text{dgm}_2$ as
it is described in~\cite{Chazal:2017}
%%
\begin{align}S
    \label{eq:bottle}
    d_b(\text{dgm}_1, \text{dgm}_2) &= \inf_{\text{matching }m}\{ \max_{(p, q) \in m} \{||p - q||_{\infty} \} \},
\end{align}
%%
where $p \in \text{dgm}_1 \backslash \Delta$, $q \in \text{dgm}_2
\backslash \Delta$, $\Delta$ is the diagonal of the persistent diagram
(the diagonal $\Delta$ represents all the points that they die the
very moment they get born, $b = d$). A matching between two diagrams
$\text{dgm}_1$ and $\text{dgm}_2$ is a subset $m \subset \text{dgm}_1
\times \text{dgm}_2$ such that every point in $\text{dgm}_1 \backslash
\Delta$ and $\text{dgm}_2 \backslash \Delta$ appears exactly once in $m$.

\subsection{Eigenvalues distribution}
\label{sec:dist}

One way to investigate if there is any significant difference between the regular and random SOMs is to compare their neural responses to the same random stimuli. Therefore, we measure the neural activity and build a covariance matrix out of it. Then, we compute the eigenvalues of the covariance matrix (or Gram matrix) and we estimate a probability distribution. Thus, we can compare the eigenvalues distributions of the two maps and compare them to each other. If the distributions are close enough in the sense of Wasserstein distance then the two SOMs are similar in terms of neural activation.  A Gram matrix is an $n \times n$ matrix given by where $n$ is the number of neurons of the map and ${\bf Y} \in \mathbb{R}^{n \times m}$ is a matrix for which each column is the activation of all $n$ neurons to a random stimulus.

From a computational point of view we construct the matrix ${\bf Y}$ by applying a set of stimuli to the self-organized map and computing the activity of each neuron within the map. This implies that ${\bf Y} \in \mathbb{R}^{m \times n}$, where $m=1024$ (the number of neurons) and $n={2, 3}$ (two- or three-dimensional input samples). Then we compute the covariance or Gram matrix as ${\bf M} = {\bf Y}{\bf Y}^T \in \mathbb{R}^{n \times n}$, where $n$ is the number of neurons. Then we compute the eigenvalues and obtain their distribution by sampling the activity of neurons of each experiment for $200$ different initial conditions using $50$ input sample each time. At the end of sampling we get an \emph{ensemble} of $200$ Gram matrices and finally we estimate the probability density of the eigenvalues on each \emph{ensemble} by applying a Kernel Density Estimation method~\citep{Parzen:1962} (KDE) with a Gaussian kernel and bandwidth $h=0.4$. This allows us to quantify any differences on the distributions of the regular and randomized SOMs by calculating the Earth-Mover or Wasserstein-1 distance over the two distributions (regular ($P$) and random SOM ($Q$)). The Wasserstein distance is computed as $W(P, Q) = \inf_{\gamma \in \Pi(P, Q)}\{\mathbb{E}_{(x, y) \sim \gamma}\Big[||x - y||\Big]\}$, where $\Pi(P, Q)$ denotes the set of all joint distributions $\gamma (x, y)$, whose marginals are $P$ and $Q$, respectively. Intuitively, $\gamma (x,y)$ indicates  how  much ``mass'' must be transported from $x$ to $y$ to transform the distribution $P$ into the distribution $Q$.

The distributions of the eigenvalues of the RSOM and the regular SOM are shown on figure~\ref{fig:eigenvalues}. We can conclude that the two distributions are alike and do not suggest any significant difference between the two maps in terms of neural activity. This implies that the RSOM and the regular SOM have similar statistics of their neural activities. This means that the loss of information and the \emph{stretch} to the input data from both RSOM and regular SOM are pretty close and the underlying  topology of the two maps do not really affect the neural activity. This is also confirmed
by measuring the Wasserstein distance between the two distributions. The blue curve shows the regular SOM or distribution $P$ and the black curve the RSOM or distribution $Q$. The Wasserstein distance between the two distributions $P$ and $Q$ indicates that the two distributions are nearly identical on all datasets. The Wasserstein distances in Table~\ref{table:distances}
confirm that the eigenvalues distributions of SOM and RSOM are almost identical indicating that both maps retain the
same amount of information after learning the representations of input spaces.

\begin{table}[!ht]
  \begin{center}
    \begin{tabular}{ll}
        \textbf{Experiment} & \textbf{Wasserstein Distance} \\
        \hline
        $2$D ring dataset               & $0.0000323$\\
        $2$D uniform dataset with holes & $0.0000207$  \\
        $3$D uniform dataset            & $0.0001583$ \\
        MNIST dataset                   & $0.0015$ \\
    \end{tabular}
      \caption{\textbf{Wasserstein distances of eigenvalues distributions.} We report here the Wasserstein
      distances between eigenvalues distributions of SOM and RSOM for each of the four major experiments we
      ran. The results indicate that the distributions are close pointing out that the SOM and RSOM capture
      a similar level of information during training. For more information regarding how we computed the
      eigenvalues distributions and the Wasserstein distance please see Section~\ref{sec:dist}.}
      \label{table:distances}
  \end{center}
\end{table}

\begin{figure}
  \includegraphics[width=\columnwidth]{eig-distributions-new.pdf}
  %
  \caption{Eigenvalues distribution for \textbf{A} 2D Ring dataset \textbf{B} 2D uniform dataset with holes \textbf{C} 3D uniform dataset and \textbf{D} MNIST Dataset
  }%
  \label{fig:eigenvalues}
 \end{figure}


\documentclass{article}
%\usepackage{fullpage}
%\usepackage{nopageno}
\usepackage[margin=1.5in]{geometry}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage[normalem]{ulem}
\usepackage{fancyhdr}
%\renewcommand\headheight{12pt}
\pagestyle{fancy}
\lhead{February 26, 2014}
\rhead{Jon Allen}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\section*{Chapter 5}
\subsection*{5.}
Expand $(2x-y)^7$ using the binomial theorem.
\begin{align*}
  (2x-y)^7&=\sum\limits_{k=0}^7{\binom{7}{k}(2x)^{7-k}(-y)^k}\\
  &=\binom{7}{0}2^7x^7y^0-\binom{7}{1}2^6x^6y^1+\binom{7}{2}2^5x^5y^2-\binom{7}{3}2^4x^4y^3\\
  &\qquad+\binom{7}{4}2^3x^3y^4-\binom{7}{5}2^2x^2y^5+\binom{7}{6}2^1x^1y^6-\binom{7}{7}2^0x^0y^7\\
  &=128x^7-7\cdot64x^6y+21\cdot32x^5y^2-35\cdot16x^4y^3\\
  &\qquad+35\cdot8x^3y^4-21\cdot4x^2y^5+7\cdot2xy^6-y^7\\
  &=128x^7-448x^6y+672x^5y^2-560x^4y^3+280x^3y^4-84x^2y^5+14xy^6-y^7
\end{align*}
\subsection*{7.}
Use the binomial theorem to prove that
\begin{align*}
  3^n&=\sum\limits_{k=0}^n{\binom{n}{k}2^k}.
\end{align*}
\begin{align*}
  \sum\limits_{k=0}^n{\binom{n}{k}2^k}&=\sum\limits_{k=0}^n{\binom{n}{k}1^{n-k}2^k}\\
  &=(1+2)^n\\
  &=3^n
\end{align*}
Generalize to find the sum
\begin{align*}
  \sum\limits_{k=0}^n{\binom{n}{k}r^k}&=\sum\limits_{k=0}^n{\binom{n}{k}1^{n-k}r^k}=(1+r)^n
\end{align*}
\subsection*{8.}
Use the binomialtheorem to prove that
\begin{align*}
  2^n&=\sum\limits_{k=0}^n{(-1)^k\binom{n}{k}3^{n-k}}.
\end{align*}
\begin{align*}
  \sum\limits_{k=0}^n{(-1)^k\binom{n}{k}3^{n-k}}&=(3-1)^n=2^n
\end{align*}
\subsection*{10.}
Use \emph{combinatorial} reasoning to prove the identity (5.2).
\begin{align*}
  k\binom{n}{k}&=n\binom{n-1}{k-1}
\end{align*}
Imagine an ordered pair $(A,a)$. The first item in the pair is a set $A$ with $k$ elements chosen from a set $S$ with $n$ elements. So $\lvert A\rvert=k, \lvert S\rvert=n,$ and $A\subset S$. The second item in the pair is one of the elements from the first item ($a\in A$). Then we have $\binom{n}{k}$ ways to choose $A$ and $k$ ways to choose $a$. This gives us $k\binom{n}{k}$ possible ordered pairs.

Now lets choose $a$ first. Because $a\in S$ and $\lvert S\rvert=n$ we have $n$ possibilities $a$. Since we know that $a\in A$, we have already chosen one element of $A$. We need now only choose the $k-1$ elements of $A\setminus a$ from the $n-1$ sized set $S\setminus a$. We can do this in $\binom{n-1}{k-1}$ ways. This gives us $n\binom{n-1}{k-1}$ possible ordered pairs. And we have our identity.$\Box$
\subsection*{11.}
Use \emph{combinatorial} reasoning to prove the identity (in the form given)
\begin{align*}
  \binom{n}{k}-\binom{n-3}{k}&=\binom{n-1}{k-1}+\binom{n-2}{k-1}+\binom{n-3}{k-1}
\end{align*}
(\emph{Hint:} Let $S$ be a set with three distinguished elements $a, b,$ and $c$ and count certain $k$-subsets of $S$.)

Imagine we have a set $S=\{x_1,x_2,\dots,x_{n-3},a,b,c\}$. We can grab $k$ elements from $S$ in $\binom{n}{k}$ ways. If we want to grab $k$ elements out $S$, but none of them are $a,b,c$ we can do so in $\binom{n-3}{k}$ ways. So $\binom{n}{k}-\binom{n-3}{k}$ is the number of ways to choose $k$ elements from the set where at least one of the elements is $a,b$, or $c$.

Now lets count this quantity from the other end. First lets find all the sets that have $a$ in them. Since we've already chosen $a$ we can choose $k-1$ elements out of the $n-1$ sized set $S\setminus a$, or $\binom{n-1}{k-1}$. Now we want to find all the sets that contain $b$. When we counted the sets with $a$ in them that also included the sets that have both $a$ and $b$ in them. So we choose $b$ and then we need to grab all the sets of $k-1$ size from the $n-2$ sized set $S\setminus\{a,b\}$. The ways we can do this is $\binom{n-2}{k-1}$. And similarly the sets that contain $c$ but neither $a$ nor $b$ count $\binom{n-3}{k-1}$. So we see that the number ways to choose $k$ elements from $\{x_1,x_2,\dots,x_{n-3},a,b,c\}$ where at least one of the elements is $a,b,$ or $c$ is $\binom{n-1}{k-1}+\binom{n-2}{k-1}+\binom{n-3}{k-1}$. And we have our identity.$\Box$
\subsection*{14.}
Prove that
\[\binom{r}{k}=\frac{r}{r-k}\binom{r-1}{k}\]
for $r$ a real number and $k$ an integer with $r\ne k$.
\subsection*{proof}
The binomial coefficient is defined by
\begin{align*}
  \binom{r}{k}&=
  \begin{cases}
    \frac{r(r-1)\cdots(r-k+1)}{k!}&\text{if }k\ge 1\\
    1&\text{if }k=0\\
    0&\text{if }k\le-1
  \end{cases}
\end{align*}
We have three cases to prove then. We'll do the trivial cases first, and then the more involved case.
\subsubsection*{case $k\le-1$}
In this case $\binom{r}{k}=0$ and $\binom{r-1}{k}=0$. We see that both sides of our equality are $0$ and so the hypothesis holds in this case.
\subsubsection*{case $k=0$}
In this case we have $\binom{r}{k}=\binom{r-1}{k}=1$ and $\frac{r}{r-k}=\frac{r}{r-0}=1$. And both sides of our equality are $1$ so the hypothesis holds for this case as well.
\subsubsection*{case $k\ge1$}
We will let the algebra do the talking for this case.
\begin{align*}
  \binom{r}{k}&=\frac{r(r-1)\cdots(r-k+1)}{k!}\\
  &=r\frac{(r-1)\cdots(r-k+1)(r-k)}{k!(r-k)}\\
  &=\frac{r}{(r-k)}\frac{(r-1)\cdots(r-k+1)(r-k)}{k!}\\
  \intertext{Now we add and subtract one from the terms that contain $k$ and group the result for our convenience.}
  &=\frac{r}{(r-k)}\frac{(r-1)((r-1)-1)\cdots((r-1)-k+2)((r-1)-k+1)}{k!}\\
  &=\frac{r}{r-k}\binom{r-1}{k}
\end{align*}
So the equality holds for all cases and our hypothesis is proven.$\Box$
\subsection*{15.}
Prove, that for every integer $n>1,$
\[\binom{n}{1}-2\binom{n}{2}+3\binom{n}{3}+\dots+(-1)^{n-1}n\binom{n}{n}=0\]
\subsection*{proof}
First we we will convert our equation to use the summation notation. Then we will apply identity 5.2 followed by the application of the binomial theorem. This should give us our result in short order.
\begin{align*}
  0&=\binom{n}{1}-2\binom{n}{2}+3\binom{n}{3}+\dots+(-1)^{n-1}n\binom{n}{n}\\
  &=\sum\limits_{k=1}^n{(-1)^{k-1}k\binom{n}{k}}\\
  &=\sum\limits_{k=1}^n{(-1)^{k-1}n\binom{n-1}{k-1}}\\
  &=n\sum\limits_{k=0}^{n-1}{(-1)^{k}\binom{n-1}{k}}\\
  &=n\sum\limits_{k=0}^{n-1}{\binom{n-1}{k}1^{(n-1)-k}(-1)^{k}}\\
  &=n(1-1)^{n-1}=n\cdot0^{n-1}=0
\end{align*}
And as predicted we have our result.$\Box$
\subsection*{16.}
By integrating the binomial expansion, prove that, for a positive integer $n$,
\[1+\frac{1}{2}\binom{n}{1}+\frac{1}{3}\binom{n}{2}+\dots+\frac{1}{n+1}\binom{n}{n}=\frac{2^{n+1}-1}{n+1}.\]
\begin{align*}
  \int_0^1{\sum\limits_{k=0}^n{\binom{n}{k}x^k}\,\mathrm{d}x}&=\int_0^1{(1+x)^n\,\mathrm{d}x}\\
  \sum\limits_{k=0}^n{\binom{n}{k}\int_0^1{x^k\,\mathrm{d}x}}&=\int_0^1{(1+x)^n\,\mathrm{d}x}\\
  \sum\limits_{k=0}^n{\binom{n}{k}\left[\frac{x^{k+1}}{k+1}\right]_0^1}&=\left.\frac{(1+x)^{n+1}}{n+1}\right\rvert_0^1\\
  \sum\limits_{k=0}^n{\binom{n}{k}\left[\frac{1^{k+1}}{k+1}-\frac{0^{k+1}}{k+1}\right]}&=\frac{(1+1)^{n+1}}{n+1}-\frac{(1+0)^{n+1}}{n+1}\\
  \sum\limits_{k=0}^n{\binom{n}{k}\left[\frac{1}{k+1}-0\right]}&=\frac{2^{n+1}}{n+1}-\frac{1}{n+1}\\
  \sum\limits_{k=0}^n{\frac{1}{k+1}\binom{n}{k}}&=\frac{2^{n+1}-1}{n+1}\\
  1+\frac{1}{2}\binom{n}{1}+\frac{1}{3}\binom{n}{2}+\dots+\frac{1}{n+1}\binom{n}{n}&=\frac{2^{n+1}-1}{n+1}
\end{align*}
\subsection*{17.}
Prove the identity in the previous exercise by  using (5.2) and (5.3).
\begin{align*}
  \frac{2^{n+1}-1}{n+1}&=1+\frac{1}{2}\binom{n}{1}+\frac{1}{3}\binom{n}{2}+\dots+\frac{1}{n+1}\binom{n}{n}\\
  (n+1)\frac{2^{n+1}-1}{n+1}&=(n+1)\sum\limits_{k=0}^n{\frac{1}{k+1}\binom{n}{k}}\\
  2^{n+1}-1&=\sum\limits_{k=0}^n{\frac{1}{k+1}(n+1)\binom{n}{k}}\\
  \intertext{Apply identity 5.2}
  &=\sum\limits_{k=0}^n{\frac{1}{k+1}(k+1)\binom{n+1}{k+1}}\\
  &=\sum\limits_{k=0}^n{\binom{n+1}{k+1}}=\sum\limits_{k=1}^{n+1}{\binom{n+1}{k}}\\
  &=-\binom{n+1}{0}+\sum\limits_{k=0}^{n+1}{\binom{n+1}{k}}\\
  \intertext{simplify and apply 5.3}
  &=-1+2^{n+1}
\end{align*}
\subsection*{21.}
Prove that, for all real nmbers $r$ and all integers $k$,
\[\binom{-r}{k}=(-1)^k\binom{r+k-1}{k}.\]
If $k=0$ $\binom{-r}{k}=1$ and $(-1)^0\binom{r+k-1}{k}=1$. If $k\le-1$ then $\binom{-r}{k}=0$ and $(-1)^k\binom{r+k-1}{k}=(-1)^k\cdot0=0$. If $k\ge1$ then:
\begin{align*}
  \binom{-r}{k}&=\frac{-r(-r-1)\cdots(-r-k+1)}{k!}\\
  &=\frac{1}{k!}\prod_{n=0}^{k-1}{(-r)-n}=\frac{(-1)^k}{k!}\prod_{n=0}^{k-1}{r+n}\\
  &=\frac{(-1)^k}{k!}\prod_{n=0}^{k-1}{(r+k-1)+(n-(k-1))}\\
  &=\frac{(-1)^k}{k!}\prod_{n=-(k-1)}^{0}{(r+k-1)+n}=\frac{(-1)^k}{k!}\prod_{n=0}^{k-1}{(r+k-1)-n}\\
  &=(-1)^k\frac{(r+k-1)(r+k-1-1)\cdots(r+k-1-k+1)}{k!}\\
  \binom{-r}{k}&=(-1)^k\binom{r+k-1}{k}
\end{align*}
\subsection*{27.}
Let $n$ and $k$ be positive integers. Give a combinatorial proof of the identity (5.15):
\[n(n+1)2^{n-2}=\sum\limits_{k=1}^n{k^2\binom{n}{k}}.\]
\subsection*{proof}
Imagine we have a 3-tuple $(A,a,b)$. We'll say $A$ is a subset of some set $S$ where $\lvert A\rvert=k$ and $\lvert S\rvert=n$ for some integers $k,n$ such that $0<k\le n$. Let $a,b\in A$. We then have $k^2\binom{n}{k}$ possible 3-tuples where $A$ is of size $k$. If we want to count how many tuples are possible for all $k$ then we arrive at the formula $\sum\limits_{k=1}^n{k^2\binom{n}{k}}$.

Now let us instead choose the last two items in the 3-tuple first. We have two cases here. Either the last two items are the same, or they are different. If they are the same then we have $n$ choices for these items. If they are different we have $n(n-1)$ choices. Now if imagine that each element of $S$ can either be in $A$ or not. Since we have already chosen $a,b\in A$ we have $2^{n-1}$ posibilities if $a=b$ and $2^{n-2}$ possibilities if $a\ne b$. This gives us the following formula to count all the possible 3-tuples:
\begin{align*}
  n2^{n-1}+n(n-1)2^{n-2}&=2n2^{n-2}+n(n-1)2^{n-2}\\
  &=2^{n-2}(2n+n^2-n)\\
  &=2^{n-2}(n^2+n)\\
  &=n(n+1)2^{n-2}
\end{align*}
And we see both sides of our hypothesis count these 3-tuples and are therefore equal.$\Box$
\subsection*{28.}
Let $n$ and $k$ be positive integers. Give a combinatorial proof that
\[\sum\limits_{k=1}^n{k\binom{n}{k}^2}=n\binom{2n-1}{n-1}\]
\subsection*{proof}
We want to form an ordered pair $(C,a)$ consisting firstly a set $C$, and secondly, an element $a$ from that set. The set has $n$ elements ($\lvert C\rvert=n$) and is a subset of the union of two subsets of some disjoint sets $A$ and $B$ which contain $n$ elements each. So $\lvert A\rvert=\lvert B\rvert=n$ and $C\subset(A\cup B)$. The second item in the pair is in $A$. So $a\in A$ and $a\in C$. Let's count how many ordered pairs we can make.

First let's choose $a\in C$ from $A$. We can do this in $n$ different ways. Now we choose the rest of $C$ (or $C\setminus a$) by grabbing $n-1$ elements from $(A\cup B)\setminus a$. We can do this in $\binom{2n-1}{n-1}$ ways. So we can build the ordered pairs in $n\binom{2n-1}{n-1}$ ways.

Now lets count in a different way. First we take some $k$ elements from $A$ where $1\le k\le n$. We can do this in $\binom{n}{k}$ ways. We can choose $a$ in $k$ different ways from these elements. We then choose $n-k$ elements from $B$ to complete set $C$. We can do this in $\binom{n}{n-k}$. We can then form an ordered pair where $k$ elements come from $A$ and $n-k$ elements come from $B$ in $k\binom{n}{k}\binom{n}{n-k}$. It is trivial to show that $\binom{n}{k}=\binom{n}{n-k}$. So adding together the number of ways we can form the ordered pairs in this way for all possible values of $k$ we have $\sum\limits_{k=1}^n{k\binom{n}{k}^2}$.

And we see then that $\sum\limits_{k=1}^n{k\binom{n}{k}^2}=n\binom{2n-1}{n-1}$.$\Box$
\end{document}