\documentclass{article}
\usepackage{amsmath, amsfonts}
\title{An ideal non-extended formulation for leaky ReLU}
\begin{document}
\maketitle
\section{Objective}
For a leaky ReLU unit
\begin{subequations}
\begin{align}
	y = \max(c(w^Tx+b), w^Tx+b)\\
	L \le x \le U
\end{align}
\end{subequations}
where $0\le c < 1$, we will find an ideal non-extended formulation. Please refer to \cite{Anderson2020} for an explanation on the ideal formulation. In \cite{Anderson2020} they derived the ideal formulation for a ReLU neuron $y = \max(0, w^Tx+b)$, here we extend their approach to leaky ReLU neuron.

\section{Big-M formulation}
If we denote $m^+ = \sum_i \max(w_iU_i, w_iL_i) + b_i, m^- = \sum_i \min(w_iU_i, w_iL_i) + b_i$, then big-M formulation is
\begin{subequations}
\begin{align}
	y \ge c(w^Tx+b)\\
	y \ge w^Tx+b\\
    y \le c(w^Tx+b) + (1-c)m^+\beta\\
    y \le w^Tx+b - (1-c)m^-(1-\beta)\\
    L \le x \le U\\
\end{align}
\end{subequations}
This formulation is not ideal, as explained in Fig 1 of \cite{Anderson2020}.

\section{Ideal extended formulation}
We consider an ideal extended formulation with new slack continuous variables
\begin{subequations}
\begin{align}
	x = x^0 + x^1\\
	y = y^0 + y^1\\
	y^0 = c(w^Tx^0 + b(1-\beta)) \le 0\\
	y^1 = w^Tx^1 + b\beta\ge 0\\
	L(1-\beta) \le x^0 \le U(1-\beta)\\
	L\beta \le x^1 \le U\beta
\end{align}
\end{subequations}
This ideal formulation is extended as it introduces new slack variables $x^0, x^1, y^0, y^1$.

\section{Ideal non-extended formulation}
To derive the ideal non-extended formulation, we want to remove the slack variables $x^0, x^1, y^0, y^1$ in the ideal extended formulation. We first write $x^1, y^0, y^1$ as function of $x, \beta, x^0$
\begin{subequations}
\begin{align}
	x^1 = x - x^0\\
	y^0 = c(w^Tx^0 + b(1-\beta))\\
	y^1 = w^Tx^1 + b\beta = w^Tx - w^Tx^0 + b\beta\\
	y = y^0 + y^1 = w^Tx - (1-c)w^Tx^0 + cb + b(1-c)\beta
\end{align}
\end{subequations}
Hence we can express $(1-c)w^1x^0_1$ as
\begin{align}
	(1-c)w_1x^0_1 = w^Tx - (1-c)\sum_{i>1}w_ix^0_i + cb  + b(1-c)\beta - y\label{eq:1_c_w1_x01}
\end{align}
We define
\begin{align}
	\bar{L}_i = \begin{cases}L_i\text{ if } w_i > 0\\U_i\text{ if } w_i < 0\end{cases}
	\bar{U}_i = \begin{cases}U_i\text{ if } w_i > 0\\L_i\text{ if } w_i < 0\end{cases}
\end{align}
We have
\begin{subequations}
\begin{align}
	(1-c)w_1\bar{L}_1(1-\beta) \le (1-c)w_1x^0_1\le (1-c)w_1\bar{U}_1(1-\beta)\\
	(1-c)w_1(x_1-\bar{U}_1\beta)\le (1-c)w_1x^0_1\le(1-c)w_1(x_1-\bar{L}_1\beta)
\end{align}
\end{subequations}
replacing $(1-c)w_1x^0_1$ with the right-hand side of \eqref{eq:1_c_w1_x01} we get
\begin{subequations}
	\begin{align}
		y \le w^Tx + cb + b(1-c)\beta - (1-c)w_1\bar{L}_1(1-\beta)-(1-c)\sum_{i>1}w_ix^0_i\\
		y \le cw_1x_1 + \sum_{i>1}w_ix_i + cb + b(1-c)\beta + (1-c)w_1\bar{U}_1\beta - (1-c)\sum_{i>1}w_ix^0_i\\
		y \ge w^Tx + cb + b(1-c)\beta - (1-c)w_1\bar{U}_1(1-\beta)-(1-c)\sum_{i>1}w_ix^0_i\\
		y \ge cw_1x_1 + \sum_{i>1}w_ix_i + cb + b(1-c)\beta + (1-c)w_1\bar{L}_1\beta - (1-c)\sum_{i>1}w_ix^0_i
	\end{align}
\end{subequations}
Now we proceed to get rid of $x^0_2$, we have
\begin{align}
	\begin{split}
	\begin{cases}
		w^Tx + cb + b(1-c)\beta - (1-c)w_1\bar{U}_1(1-\beta) - (1-c)\sum_{i>2}w_ix^0_i - y\\
		cw_1x_1 + \sum_{i>1}w_ix_i + cb + b(1-c)\beta + (1-c)\bar{L}_1\beta - (1-c)\sum_{i>2}w_ix^0_i-y\\
		(1-c)w_2\bar{L}_2(1-\beta)\\
		(1-c)w_2(x_2-\bar{U}_2\beta)
	\end{cases}\\
	\le (1-c)w_wx^0_2\le\\
	\begin{cases}
		w^Tx + cb + b(1-c)\beta - (1-c)w_1\bar{L}_1(1-\beta) - (1-c)\sum_{i>2}w_ix^0_i - y\\
		cw_1x_1 + \sum_{i>1}w_ix_i+cb+b(1-c)\beta + (1-c)\bar{U}_1\beta - (1-c)\sum_{i>2}w_ix^0_i-y\\
		(1-c)w_2\bar{U}_2(1-\beta)\\
		(1-c)w_2(x_2-\bar{L}_2\beta)
	\end{cases}
\end{split}
\end{align}
So we end up with the constraints
\begin{subequations}
\begin{align}
	y\ge cb + b(1-c)\beta + \sum_{i\in\mathcal{I}}w_ix_i - (1-c)(1-\beta)\sum_{i\in\mathcal{I}}w_i\bar{U}_i + c\sum_{i\notin\mathcal{I}}w_ix_i + (1-c)\beta\sum_{i\notin\mathcal{I}}w_i\bar{L}_i \label{eq:leaky_relu_nonextended_ideal_ge}\\
	y \le cb + b(1-c)\beta + \sum_{i\in\mathcal{I}}w_ix_i -(1-c)(1-\beta)\sum_{i\in\mathcal{I}}w_i\bar{L}_i + c\sum_{i\notin\mathcal{I}}w_ix_i + (1-c)\beta\sum_{i\notin\mathcal{I}}w_i\bar{U}_i \label{eq:leaky_relu_nonextended_ideal_le}
\end{align}
\end{subequations}
where $\mathcal{I}$ is a subset of $\{1, 2, ..., n\}$. Moreover, \eqref{eq:leaky_relu_nonextended_ideal_ge} is redundant.

In the end, we obtain the following non-extended ideal formulation
\begin{subequations}
\begin{align}
	y \ge c(w^Tx+b)\label{eq:leaky_relu_nonextended_ideal1}\\
	y \ge w^Tx+b\label{eq:leaky_relu_nonextended_ideal2}\\
	\begin{split}
		y \le cb + b(1-c)\beta + \sum_{i\in\mathcal{I}}\left(w_ix_i -(1-c)(1-\beta)w_i\bar{L}_i\right) +\\ \sum_{i\notin\mathcal{I}}\left(cw_ix_i + (1-c)\beta w_i\bar{U}_i\right)\label{eq:leaky_relu_nonextended_ideal3}
	\end{split}\\
	L\le x \le U\label{eq:leaky_relu_nonextended_ideal4}
\end{align}
\end{subequations}
Notice that there are exponential number of subsets $\mathcal{I}$. To find $\mathcal{I}$, we consider to take a point $\hat{x}, \hat{y}, \hat{\beta}$ satisfying the constraints in the big-M formulation, with $\hat{\beta}$ being the continuous relaxation $0 \le \hat{\beta} \le 1$. Then we take $\mathcal{I}$ as
\begin{align}
	\mathcal{I} = \{i|w_i\hat{x}_i \le (1-\hat{\beta})w_i\bar{L}_i + \hat{\beta}w_i\bar{U}_i\}\label{eq:find_index_set}
\end{align}

We take an iterative procedure to strengthen the big-M formulation. Say in the current iterations, the strengthened big-M formulation already contains the constraint with $\mathcal{I}_1,\hdots, \mathcal{I}_k$, then for an $\hat{x}$ at the boundary of the box $L \le x \le U$, we first compute its range of $\hat{\beta}$ under the existing constraint
\begin{align}
	\max(c(w^T\hat{x}+b), w^T\hat{x}+b) \le cb + b(1-c)\hat{\beta} + \sum_{i\in\mathcal{I}_k}(w_ix_i - (1-c)(1-\hat{\beta})w_i\bar{L}_i) + \sum_{i\notin\mathcal{I}_k}(cw_ix_i+(1-c)\hat{\beta}w_i\bar{U}_i)
\end{align}
Then for each extreme $\hat{x},\hat{\beta}$, we compute the corresponding index set $\mathcal{I}$ from \eqref{eq:find_index_set}. We then decide whether to enforce this cutting plane, by whether this new cutting plane shrinks the righ-hand side of \eqref{eq:leaky_relu_nonextended_ideal3} at $\hat{x}, \hat{\beta}$.

\section{Adding bounds on ReLU input}
If the lower and upper bound of the ReLU input $m^+/m^-$ is not computed as $\max/\min_{L\le x\le U} w^Tx+b$ (for example, we compute the bounds by solving an LP/MIP), then the leaky ReLU constraint $y = \max(cw^Tx+b, w^Tx+b)$ has the following ideal extended formulation
\begin{subequations}
	\begin{align}
	x = x^0 + x^1\\
	y = y^0 + y^1\\
	y^0 = c(w^Tx^0 + b(1-\beta))\le 0\\
	y^1 = w^Tx^1+b\beta \ge 0\\
	(m^--b)(1-\beta) \le w^Tx^0 \le(m^+-b)(1-\beta)\\
	(m^--b)\beta\le w^Tx^1\le(m^+-b)\beta\\
	L(1-\beta)\le x^0\le U(1-\beta)\\
	L\beta\le x^1\le U\beta
\end{align}
\end{subequations}
Hence we have the condition
\begin{align}
	\begin{split}
	\begin{cases}
		(1-c)(m^--b)(1-\beta) - (1-c)\sum_{i>1}w_ix^0_i\\
		(1-c)w^Tx - (1-c)(m^+-b)\beta - (1-c)\sum_{i>1}w_ix^0_i\\
		(1-c)w_1\bar{L}_1(1-\beta)\\
		(1-c)w_1(x_1-\bar{U}_1\beta)
	\end{cases}\\
	\le (1-c)w_1x^0_1\le\\
	\begin{cases}
		(1-c)(m^+-b)(1-\beta) - (1-c)\sum_{i>1}w_ix^0_i\\
		(1-c)w^Tx - (1-c)(m^--b)\beta - (1-c)\sum_{i>1}w_ix^0_i\\
		(1-c)w_1\bar{U}_1(1-\beta)\\
		(1-c)w_1(x_1-\bar{L}_1\beta)
	\end{cases}
\end{split}
\end{align}
Using the relationship
\begin{align}
	(1-c)w_1x^0_1 = w^Tx - (1-c)\sum_{i>1}w_ix^0_i + cb  + b(1-c)\beta - y\label{eq:1_c_w1_x01}
\end{align}
we get
\begin{align}
	\begin{split}
	\begin{cases}
		w^Tx + cb + b(1-c)\beta - (1-c)w_1\bar{U}_1(1-\beta)-(1-c)\sum_{i>1}w_ix^0_i\\
		cw_1x_1 + \sum_{i>1}w_ix_i + cb + b(1-c)\beta + (1-c)w_1\bar{L}_1\beta - (1-c)\sum_{i>1}w_ix^0_i\\
		w^Tx+b - (1-\beta)(1-c)m^+\\
		c(w^Tx+b) + m^-\beta(1-c)
	\end{cases}\\
	\le y \le\\
	\begin{cases}
		w^Tx + cb + b(1-c)\beta - (1-c)w_1\bar{L}_1(1-\beta)-(1-c)\sum_{i>1}w_ix^0_i\\
		cw_1x_1 + \sum_{i>1}w_ix_i + cb + b(1-c)\beta + (1-c)w_1\bar{U}_1\beta - (1-c)\sum_{i>1}w_ix^0_i\\
		w^Tx+b - (1-\beta)(1-c)m^-\\
		c(w^Tx+b) + m^+\beta(1-c)
	\end{cases}
\end{split}
\end{align}

\section{Appendix}
In this appendix, we try to derive the ideal non-extended formulation for ReLU unit (not the leaky ReLU) unit
\begin{subequations}
\begin{align}
	y = \max(0, w^Tx + b)\\
	L \le x \le U
\end{align}
\end{subequations}
\subsection{big-M formulation}
We denote $m^+ = \sum_i \max(w_iL_i, w_iU_i), m^- = \sum_i \min(w_iL_i, w_iU_i)$, and get the following big-M formulation
\begin{subequations}
\begin{align}
	y \ge 0\\
	y \ge w^Tx + b\\
	y \le m^+\beta\\
	y \le w^Tx+b - m^-(1-\beta)\\
	L \le x \le U
\end{align}
\end{subequations}
\subsection{Ideal extended formulation}
Using the multiple-choice trick, we get the following ideal formulation with slack variables
\begin{subequations}
\begin{align}
	x = x^0 + x^1\\
	y = y^0 + y^1\\
	y^0 = 0 \ge w^Tx^0 + b(1-\beta)\\
	y^1 = w^Tx^1 + b\beta\ge 0\\
	L(1-\beta)\le x^0\le U(1-\beta)\\
	L\beta\le x^1\le U\beta
\end{align}
\end{subequations}

\subsection{Ideal non-extended formulation}
To derive the ideal non-extended formulation, we will need to remove the slack variable $x^0, x^1, y^0, y^1$ from the ideal extended formulation. To do so, we first remove $x^1, y^0, y^1$ as
\begin{align}
	x^1 = x - x^0\\
	y^0 = 0\\
	y^1 = w^T(x-x^0) + b\beta = y
\end{align}
Hence we have
\begin{align}
	w^Tx^0 = w^Tx + b\beta-y\\
	\Leftrightarrow w_1x^0_1 = w^Tx + b\beta -\sum_{i>1}w_ix^0_i - y\label{eq:w1x01}
\end{align}
Without loss of generality, we assume $w_i > 0\; \forall i$, hence
\begin{subequations}
\begin{align}
	w_1L_1(1-\beta)\le w_1x^0_1\le w_1U_1(1-\beta)\\
	w_1x_1 - w_1U_1\beta \le w_1x^0_1 \le w_1x_1 - w_1L_1\beta
\end{align}
\label{eq:w1x01_bounds}
\end{subequations}
Substituting $w_1x^0_1$ in the inequalities \eqref{eq:w1x01_bounds} with the right hand side of \eqref{eq:w1x01}, we get the following inequalities
\begin{subequations}
	\begin{align}
		y \ge w^Tx + b\beta - \sum_{i>1}w_ix^0_i - w_1U_1(1-\beta)\\
		y \ge \sum_{i>1}w_ix_i + b\beta - \sum_{i>1}w_ix^0_i + w_1L_1\beta\\
		y \le w^Tx + b\beta - \sum_{i>1}w_ix^0_i - w_1L_1(1-\beta)\\
		y \le \sum_{i>1}w_ix_i + b\beta - \sum_{i>1}w_ix^0_i  + w_1U_1\beta
	\end{align}
\end{subequations}
these inequalities remove the slack variable $x^0_1$.

Now we proceed to remove the slack variable $x^0_2$, we have
\begin{align}
	\begin{split}
	\max\begin{cases}
		w^Tx + b\beta -\sum_{i>2}w_ix^0_i - y - w_1U_1(1-\beta)\\
		\sum_{i>1}w_ix_i + b\beta - \sum_{i>2}w_ix^0_i -y + w_1L_1\beta\\
		w_2L_2(1-\beta)\\
		w_2x_2 - w_2U_2\beta
	\end{cases}\\
	\le w_2x^0_2\le\\
	\min\begin{cases}
		w^Tx + b\beta - \sum_{i>2}w_ix^0_i-y - w_1L_1(1-\beta)\\
		\sum_{i>1}w_ix_i + b\beta - \sum_{i>2}w_ix^0_i-y+w_1U_1\beta\\
		w_2U_2(1-\beta)\\
		w_2x_2-w_2L_2\beta
	\end{cases}
\end{split}
\end{align}
we get the inequalities
\begin{subequations}
\begin{align}
	y \ge w^Tx + b\beta - \sum_{i>2}w_ix^0_i - \sum_{i \le 2}w_iU_i(1-\beta)\\
	y \ge \sum_{i\neq 2}w_ix_i + b\beta - \sum_{i>2}w_ix^0_i - w_1U_1(1-\beta) + w_2L_2\beta\\
	y \ge \sum_{i>1}w_ix_i + b\beta - \sum_{i>2}w_ix^0_i + w_1L_1\beta - w_2U_2(1-\beta)\\
	y \ge \sum_{i>2}w_ix_i + b\beta - \sum_{i>2}w_ix^0_i  + \sum_{i\le 2}w_iL_i\beta\\
	y \le w^Tx + b + b\beta - \sum_{i>2}w_ix^0_i - \sum_{i\le 2}w_iL_i(1-\beta)\\
	y \le \sum_{i>1}w_ix_i + b\beta - \sum_{i>2}w_ix^0_i  + w_1U_1\beta - w_2L_2(1-\beta)\\
	y \le \sum_{i\neq 2}w_ix_i + b\beta - \sum_{i>2}w_ix^0_i - w_1L_1(1-\beta) + w_2U_2\beta\\
	y \le \sum_{i > 2}w_ix_i + b\beta - \sum_{i>2}w_ix^0_i + \sum_{i\le 2}w_iU_i\beta
\end{align}
\end{subequations}
If we follow this procedure to remove the remaining slack variables, we end up with the inequalities
\begin{subequations}
	\begin{align}
		y \ge \sum_{i\in\mathcal{I}}w_ix_i + b\beta - \sum_{i\in\mathcal{I}}w_iU_i(1-\beta) + \sum_{i\notin\mathcal{I}}w_iL_i\beta\\
		y \le \sum_{i\in\mathcal{I}}w_ix_i + b\beta - \sum_{i\in\mathcal{I}}w_iL_i(1-\beta) + \sum_{i\notin\mathcal{I}}w_iU_i\beta
	\end{align}
\end{subequations}
where $I$ is a subset of the set $\{1, 2, \hdots, n\}$.

\begin{thebibliography}{9}
	\bibitem{Anderson2020}
	Ross Anderson, Joey Huchette, Christian Tjandraatmadja and Juan Pablo Vielma \textit{Strong mixed-integer programming formulations for trained neural networks}, Mathematical Programming, 2020
\end{thebibliography}
\end{document}


\section{Multi-class}

\textbf{1-vs-all}: $c$ models, confidence $f^{(i)} (x) = w^{(i )\top} x$

\textbf{1-vs-1}: $c\frac{c-1}{2}$ models, voting scheme

% ===
\emph{Multi-class Hinge loss}\\
\vspace{-12pt}
$\ell(w^{(1:c)};x,y) = \max\{ 0, 1\! + \overbracket[.7pt][.7pt]{\max\limits_{i:\,j\neq y} w^{(j)\top}x}^{(\ast\ast)} - \overbracket[.7pt][.7pt]{w^{(y)\top}x}^{(\ast\ast\ast)} \}$

\textbf{Confidence:} $w^{(y)\top}x \geq \max\limits_{j\neq y} w^{(i)\top}x + 1$ {\footnotesize$(\ast)$}

{\setstretch{0.9}
$\nabla\!_{w^{(j)}} \ell =
\begin{cases}
	0	& (\ast)\text{ satisfied \textit{or} } j\notin\{y,\hat y\}\\
	-x	& \neg(\ast) \text{ and } j=y		\rightsquigarrow (\ast\ast)\\
	+x	& \neg(\ast) \text{ and } j=\hat y	\rightsquigarrow (\ast\ast\ast)
\end{cases}$
}


\subsection{Covers}

A space \(X\) is covered by a set of subsets of \(X\), \(C\), if the union of \(C\) is \(X\).

\subsection{Subcover}

A subset of \(C\) which still covers \(X\) is a a subcover.

\subsection{Open cover}

\(C\) is an open cover if each member is an open set.

\subsection{Universal cover}


%!TEX root = ClementiCooperBarba2018.tex

The original implementation of \pygbe used continuum electrostatic theory to compute
the solvation energy of biomolecular systems. In that setting, biomolecules are modeled as
dielectric cavities inside an infinite continuum solvent,
leading to a Poisson equation inside the molecules and Laplace or Poisson-Boltzmann in the solvent medium (with appropriate boundary conditions).
This set of partial differential equations can be
expressed with the corresponding boundary integral equation along the molecular interface,
which \pygbe solves using a boundary element method \cite{CooperBardhanBarba2013,CooperClementiBarba2015}.

The present work extends \pygbe to the LSPR biosensing application.
In the long-wavelength limit, Maxwell's equations can be approximated by a Laplace equation,
which permits using the methods implemented in \pygbe, with modifications
to allow for complex-valued permittivities, and to include the
effect of an external electric field.
This section describes the mathematical formulation for computing electromagnetic scattering
in the long-wavelength setting, and develops the associated boundary integral equations
and their discretized form.

\subsection{Scattering of small particles} \label{sec:scattering_small}

Electromagnetic scattering is usually modeled with Maxwell's equations.
When the wavelength of the incoming wave is much larger than the
scatterer, these can be reduced to a \emph{quasi-static}
first-order approximation \cite{MayergoyzZhang2007}:
%
\begin{align} \label{eq:electrostatic_scatter_E}
\nabla \cdot \mathbf{E}_{1s} &= 0 \qquad \nabla \times \mathbf{E}_{1s} = 0, \nonumber \\
\nabla \cdot \mathbf{E}_{2s} &= 0 \qquad \nabla \times \mathbf{E}_{2s} = 0, \nonumber \\
\text{with interface conditions, } \nonumber \\
(\epsilon_1\mathbf{E}_{1s} - \epsilon_2\mathbf{E}_{2s})\cdot\mathbf{n} &= (\epsilon_2-\epsilon_1)\mathbf{E}_i\cdot \mathbf{n}.
\end{align}
%
In Equation \eqref{eq:electrostatic_scatter_E}, $\mathbf{E}_{1s}$ and $\mathbf{E}_{2s}$
are the electric fields of the scattered wave in the nanoparticle and host regions, respectively
(see Figure \ref{fig:part_wave}),
$\mathbf{E}_{i}$ is the field of the incoming wave, and $\epsilon_1$
and $\epsilon_2$ are the permittivities.
This approximation decouples the electric and magnetic fields, neglects the magnetic field,
and describes the electric field as a curl-free vector field.
Hence, we can reformulate Equation \eqref{eq:electrostatic_scatter_E} with a scalar potential
($-\nabla \phi_{js} = \mathbf{E}_{js}$), as follows:
%
\begin{align} \label{eq:electrostatic_scatter}
\nabla^2 \phi_{1s} &= 0 \qquad \nabla^2 \phi_{2s} = 0 \qquad\text{on $\Omega_1$, $\Omega_2$} \nonumber \\
\epsilon_1\frac{\partial\phi_{1s}}{\partial \mathbf{n}} - \epsilon_2\frac{\partial\phi_{2s}}{\partial\mathbf{n}} &= (\epsilon_2-\epsilon_1)\frac{\partial\phi_i}{\partial\mathbf{n}} \quad \phi_{1s} = \phi_{2s} \quad \text{on $\Gamma$}.
\end{align}
%
Equation \eqref{eq:electrostatic_scatter} is an electrostatic equation
with an imposed electric field $\mathbf{E}_i=-\nabla\phi_i$, where $\Gamma$
is the boundary between regions $\Omega_1$ and $\Omega_2$.

\begin{figure}%[h] %  figure placement: here, top, bottom, or page
   \centering
   \includegraphics[width=0.45\textwidth]{particle_wave.pdf}
   \caption{Nanoparticle interacting with an electromagnetic wave.}
   \label{fig:part_wave}
\end{figure}

\subsection{Far-field scattering} \label{sec:ff_scattering}

In LSPR, the scattered electromagnetic wave is measured by a detector located far away
from the scatterer (nanoparticle), and plasmon resonance is identified when the energy
detected is minimum. In the far-field limit, the scattered field
in the outside region ($\Omega_2$) is given by:

\begin{equation} \label{eq:scat_efield_long_range}
    \mathbf{E}_{2s} = \frac{1}{4\pi\epsilon_2}k^2\frac{e^{ikr}}{r} (\mathbf{\hat{r}} \times \mathbf{p})\times\mathbf{\hat{r}}.
\end{equation}

\noindent where $k=2\pi/\lambda$ is the wave number and $\lambda$ the wavelength, $\mathbf{\hat{r}}$
is a unit vector in the direction of the observation point, and $\mathbf{p}$ is
the dipole moment.
We can obtain the scattered field using the
scattering amplitude \cite{Jackson}:

\begin{equation} \label{eq:scat_efield_fwa}
    \mathbf{E}_{2s}(\mathbf{r})_{r\to\infty} = \frac{e^{ikr}}{r} \mathbf{F}(\mathbf{k},\mathbf{k}_0),
\end{equation}

\noindent where $\mathbf{F}$ is the scattering amplitude, $\mathbf{k}$ is the
scattered wave vector in the direction of propagation, and $\mathbf{k}_0$ the
wave vector of the incident field.

\subsection{Extinction cross-section and optical theorem} \label{sec:cext_ot}

The extinction cross-section ($C_\text{ext}$) is a measure of the energy that
does not reach the detector, either because of scattering in other directions,
or absorption. This quantity is defined as the ratio between the lost energy and
the intensity of the incoming wave, and has units of area.
The extinction cross-section peaks at resonance of plasmons.

The extinction cross-section is related to the forward-scattering amplitude via the optical theorem.
The traditional expression for this relationship applies for non-absorbing media
\cite{MayergoyzZhang2007, Jackson};
Mishchenko \cite{Mishchenko2007} corrected it for absorbing media,
giving an expression that can be re-written using Jackson's notation \cite{Jackson} as follows:

\begin{equation} \label{eq:cext_fwa}
    C_\text{ext} = \frac{4\pi}{k^\prime} \operatorname{Im} \left[ \frac{\mathbf{\hat{e}}_i}{|\mathbf{E}_i|}\mathbf{F}(\mathbf{k}=\mathbf{k}_0, \mathbf{k}_0) \right].
\end{equation}

\noindent Here, $k^\prime$ is the real part of the complex wave number,

\begin{equation}
    k = k^\prime + ik^{\prime\prime} = \frac{2\pi}{\lambda} n,
\end{equation}

\noindent and $n$ is the refraction index of the host medium.

Combining Equations \eqref{eq:scat_efield_long_range} and \eqref{eq:scat_efield_fwa},
we can compute the scattering amplitude to then obtain the extinction cross-section
with Equation \eqref{eq:cext_fwa}.

\subsection{The boundary element method} \label{sec:lspr_bem}

\subsubsection{Electrostatic potential of a nanoparticle under an electric field} \label{sec:pot_elec_field}

\paragraph{Integral formulation}

Using Green's second identity, the system of partial differential equations
in Equation \eqref{eq:electrostatic_scatter} can be rewritten as a system
of boundary integral equations \cite{BrebbiaDominguez1992}. Evaluating on the surface $\Gamma$, this
becomes
%
\begin{align} \label{eq:integral_eq_lspr_nobc}
\frac{\phi_{1s,\Gamma}}{2}+ K_{L}^{\Gamma}(\phi_{1s,\Gamma}) - V_{L}^{\Gamma} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{1s,\Gamma} \right) = 0&  \nonumber \\
\frac{\phi_{2s,\Gamma}}{2} - K_{L}^{\Gamma}(\phi_{2s,\Gamma}) + V_{L}^{\Gamma} \left( \frac{\partial}{\partial \mathbf{n}} \phi_{2s,\Gamma} \right) = 0&,
\end{align}
%
where $V$ and $K$ are the single- and double-layer operators, respectively:
%
\begin{equation}\label{eq:single_layer}
V^{\Gamma}_L (\psi(\mathbf{r}_\Gamma)) = \oint_\Gamma \psi(\mathbf{r}'_\Gamma) G_L(\mathbf{r}_\Gamma, \mathbf{r}'_\Gamma) \text{d} \Gamma',
\end{equation}
%
\begin{equation}\label{eq:double_layer}
K^{\Gamma}_L (\psi(\mathbf{r}_\Gamma)) = \oint_\Gamma \psi(\mathbf{r}'_\Gamma) \frac{\partial}{\partial \mathbf{n}}G_L(\mathbf{r}_\Gamma, \mathbf{r}'_\Gamma) \text{d} \Gamma'.
\end{equation}
%
\noindent Here, $G_L$ is the free-space Green's function of the Laplace equation:
%
\begin{equation}
G_L(\mathbf{r},\mathbf{r}') = \frac{1}{4\pi|\mathbf{r}-\mathbf{r}'|}
\end{equation}

\noindent Applying the interface conditions of Equation \eqref{eq:electrostatic_scatter},
leads to:
%
\begin{align} \label{eq:integral_eq_lspr}
\frac{\phi_{1s,\Gamma}}{2}+ K_{L}^{\Gamma}(\phi_{1s,\Gamma}) - V_{L}^{\Gamma} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{1s,\Gamma} \right) &= 0  \nonumber \\
\frac{\phi_{1s,\Gamma}}{2} - K_{L}^{\Gamma}(\phi_{1s,\Gamma}) + \frac{\epsilon_1}{\epsilon_2}V_{L}^{\Gamma} \left( \frac{\partial}{\partial \mathbf{n}} \phi_{1s,\Gamma}  \right) &= \nonumber \\
 \frac{\epsilon_2-\epsilon_1}{\epsilon_2}V_{L}^{\Gamma}\left( \frac{\partial}{\partial \mathbf{n}} \phi_{i,\Gamma} \right)\quad \text{on $\Gamma$.}
\end{align}

%The weak formulation of Laplace equation with test function $w$:

%\begin{equation} \label{eq:lap_weak}
%\int_\Omega \nabla^2 \phi(\mathbf{r}_\Omega') w(\mathbf{r}_\Omega') \text{d} \Omega^\prime= 0.
%\end{equation}

%\noindent where the evaluation point is $\mathbf{r}_\Omega$ a location in the domain $\Omega$.

%If we use the Laplace's free-space Green's function as the test function $w$ we
%get:

%\begin{equation} \label{eq:lap_weak2}
%\int_\Omega \nabla^2 \phi(\mathbf{r}'_\Omega) G_L(\mathbf{r}_\Omega,\mathbf{r}'_\Omega) \text{d} \Omega^\prime= 0.
%\end{equation}

%Manipulating the integrand using the product rule and later the divergence
%theorem, we get:

%\begin{equation} \label{eq:lap_bie_dom}
%\phi(\mathbf{r}_\Omega) = \oint_\Gamma G_L(\mathbf{r}_\Omega,\mathbf{r}'_\Gamma)  \frac{\partial} {\partial \mathbf{n}} \phi(\mathbf{r}'_\Gamma)  \text{d} \Gamma^\prime - \oint_\Gamma \phi(\mathbf{r}'_\Gamma)  \frac{\partial}{\partial \mathbf{n}} G_L(\mathbf{r}_\Omega,\mathbf{r}'_\Gamma) \text{d} \Gamma^\prime
%\end{equation}

%\noindent where \eqref{eq:lap_bie_dom}, $\mathbf{r}$ can be anywhere in the domain $\Omega$,
%and $\mathbf{r}'$ runs only on the boundary $\Gamma$. This equation has a
%singularity when $\mathbf{r}=\mathbf{r}'$. To handle this problem, we perform the
%integral on a surface $\Gamma'$ that is like $\Gamma$ but with a hemisphere of
%radius $\varepsilon$ center at $\mathbf{r}$. We split the integrals into the part
%that we have no singularity and the part that has the hemisphere. After solving these
%equations when $\varepsilon \to 0$, equation \eqref{eq:lap_bie_dom} results in:

%\begin{equation} \label{eq:lap_bie}
%\frac{\phi(\mathbf{r}_\Gamma)}{2} +  \oint_\Gamma \phi(\mathbf{r}'_\Gamma)  \frac{\partial}{\partial \mathbf{n}} G_L(\mathbf{r}_\Gamma,\mathbf{r}'_\Gamma) \text{d} \Gamma^\prime = \oint_\Gamma G_L(\mathbf{r}_\Gamma,\mathbf{r}'_\Gamma)  \frac{\partial} {\partial \mathbf{n}} \phi(\mathbf{r}'_\Gamma)  \text{d} \Gamma^\prime,
%\end{equation}

%\noindent where these are Cauchy principal value integrals.

%Using the single and double layer operators:

%{\color{red} Chris, V is single layer operator but is it K the double layer one?
%In your thesis you have that the double layer is W and you refers as K as
%"an operator" eq 2.108 and 2.112 in your thesis. Also shouldn't the $\text{d} \Gamma$
%be $\text{d} \Gamma'$? }

%\begin{equation}\label{eq:single_layer}
%V^{\mathbf{r}_\Gamma}_L (\psi(\mathbf{r}_\Gamma)) = \oint_\Gamma \psi(\mathbf{r}'_\Gamma) G_L(\mathbf{r}_\Gamma, \mathbf{r}'_\Gamma) \text{d} \Gamma.
%\end{equation}

%\begin{equation}\label{eq:double_layer}
%K^{\mathbf{r}_\Gamma}_L (\psi(\mathbf{r}_\Gamma)) = \oint_\Gamma \psi(\mathbf{r}'_\Gamma) \frac{\partial}{\partial \mathbf{n}}G_L(\mathbf{r}_\Gamma, \mathbf{r}'_\Gamma) \text{d} \Gamma.
%\end{equation}

%We can rewrite equation \eqref{eq:lap_bie} using the operator notation, as:

%\begin{equation} \label{eq:lap_operator}
%\left[ \frac{\mathbb{I}}{2} + K_L^{\mathbf{r}_\Gamma} \right] \left( \phi_\Gamma \right) = V_L^{\mathbf{r}_\Gamma} \left( \frac{\partial}{\partial \mathbf{n}} \phi_\Gamma \right),
%\end{equation}

%\noindent where $\mathbb{I}$ is the identity operator.

%Following the same steps, we can rewrite the Laplace equations in equation
%\eqref{eq:electrostatic_scatter} as:

%
%\begin{align} \label{eq:integral_eq_lspr_nobc}
%\frac{\phi_{1s,\Gamma}}{2}+ K_{L}^{\Gamma}(\phi_{1s,\Gamma}) - V_{L}^{\Gamma} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{1s,\Gamma} \right) = 0&  \nonumber \\
%\frac{\phi_{2s,\Gamma}}{2} - K_{L}^{\Gamma}(\phi_{2s,\Gamma}) + V_{L}^{\Gamma} \left( \frac{\partial}{\partial \mathbf{n}} \phi_{2s,\Gamma} \right) = 0& \quad \text{on $\Gamma$,}
%\end{align}

%Applying the interface conditions of Equation \eqref{eq:electrostatic_scatter},
%we get:

%\begin{align} \label{eq:integral_eq_lspr}
%\frac{\phi_{1s,\Gamma}}{2}+ K_{L}^{\Gamma}(\phi_{1s,\Gamma}) - V_{L}^{\Gamma} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{1s,\Gamma} \right) = 0&  \nonumber \\
%\frac{\phi_{1s,\Gamma}}{2} - K_{L}^{\Gamma}(\phi_{1s,\Gamma}) + \frac{1}{\epsilon_2}V_{L}^{\Gamma} \left( \epsilon_1 \frac{\partial}{\partial \mathbf{n}} \phi_{1s,\Gamma} - (\epsilon_2-\epsilon_1) \frac{\partial}{\partial \mathbf{n}} \phi_{i,\Gamma} \right) = 0& \quad \text{on $\Gamma$.}
%\end{align}

\newpage
\subsubsection{Analyte-sensor electrostatic potential under an electric field}

The sketch in Figure \ref{fig:analyte-sensor} shows a metallic nanoparticle ($\Omega_1$) interacting with an analyte ($\Omega_3$), under an external electric field.
Mathematically, this situation can be modeled as

\begin{align}\label{eq:electrostatic_scatter_prot_sen}
\nabla^2 \phi_{1s} &= 0, \qquad \nabla^2 \phi_{2s} = 0 \qquad\text{on $\Omega_1$, $\Omega_2$} \nonumber\\
\nabla^2 \phi_{3s} &= -\frac{1}{\epsilon_3} \sum_{k=0}^{N_q} \delta(|\mathbf{r}-\mathbf{r}_k|) q_k \qquad\text{on $\Omega_3$} \nonumber \\
\epsilon_1\frac{\partial\phi_{1s}}{\partial \mathbf{n}} - \epsilon_2\frac{\partial\phi_{2s}}{\partial\mathbf{n}} &= (\epsilon_2-\epsilon_1)\frac{\partial\phi_i}{\partial\mathbf{n}} \quad \phi_{1s} = \phi_{2s} \quad \text{on $\Gamma_1$}. \nonumber\\
\epsilon_3\frac{\partial\phi_{3s}}{\partial \mathbf{n}} - \epsilon_2\frac{\partial\phi_{2s}}{\partial\mathbf{n}} &= (\epsilon_2-\epsilon_3)\frac{\partial\phi_i}{\partial\mathbf{n}} \quad \phi_{3s} = \phi_{2s} \quad \text{on $\Gamma_2$}.
\end{align}
%
where $q_k$ are the point charges of the atoms inside the protein, located at $\mathbf{r}_k$.

\paragraph{Integral formulation}

Similar to Equation \eqref{eq:integral_eq_lspr}, we can write the system of partial differential equations in \eqref{eq:electrostatic_scatter_prot_sen} as

\begin{widetext} % to get the full-pagewidth equations in the two-column layout

\begin{align} \label{eq:integral_eq_lspr_nobc_system}
\frac{\phi_{1s,\Gamma_1}}{2}+ K_{L,\Gamma_1}^{\Gamma_1}(\phi_{1s,\Gamma_1}) - V_{L,\Gamma_1}^{\Gamma_1} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{1s,\Gamma_1} \right) &= 0  \nonumber \\
\frac{\phi_{2s,\Gamma_1}}{2} - K_{L,\Gamma_1}^{\Gamma_1}(\phi_{2s,\Gamma_1}) + V_{L,\Gamma_1}^{\Gamma_1} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{2s,\Gamma_1} \right)
 - K_{L,\Gamma_2}^{\Gamma_1}(\phi_{2s,\Gamma_2}) + V_{L,\Gamma_2}^{\Gamma_1} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{2s,\Gamma_2} \right) &= 0  \nonumber \\
\frac{\phi_{2s,\Gamma_2}}{2} - K_{L,\Gamma_1}^{\Gamma_2}(\phi_{2s,\Gamma_1}) + V_{L,\Gamma_1}^{\Gamma_2} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{2s,\Gamma_1} \right)
- K_{L,\Gamma_2}^{\Gamma_2}(\phi_{2s,\Gamma_2}) + V_{L,\Gamma_2}^{\Gamma_2} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{2s,\Gamma_2} \right) &= 0  \nonumber \\
\frac{\phi_{3s,\Gamma_2}}{2} + K_{L,\Gamma_2}^{\Gamma_2}(\phi_{3s,\Gamma_2}) - V_{L,\Gamma_2}^{\Gamma_2} \left( \frac{\partial}{\partial \mathbf{n}} \phi_{3s,\Gamma_2} \right) &= \frac{1}{4\pi\epsilon_3} \sum_{k=0}^{N_q} \frac{q_k}{|\mathbf{r}_{\Gamma_2} - \mathbf{r}_k|} ,
\end{align}
%
\noindent where $V$ and $K$ are the single- and double-layer operators in equations
\eqref{eq:single_layer} and \eqref{eq:double_layer}. In this case, we distinguish between the
surface where the integrals run (subindex), and the surface that contains the evaluation point (superindex).

Applying the interface conditions of equation \eqref{eq:electrostatic_scatter_prot_sen},
leads to:

\begin{align} \label{eq:integral_eq_lspr_system}
\frac{\phi_{1s,\Gamma_1}}{2}&+ K_{L,\Gamma_1}^{\Gamma_1}(\phi_{1s,\Gamma_1}) - V_{L,\Gamma_1}^{\Gamma_1} \left(\frac{\partial}{\partial \mathbf{n}}\phi_{1s,\Gamma_1} \right) = 0  \nonumber \\
 \frac{\phi_{1s,\Gamma_1}}{2}& - K_{L,\Gamma_1}^{\Gamma_1}(\phi_{1s,\Gamma_1}) + V_{L,\Gamma_1}^{\Gamma_1} \left(\frac{\epsilon_1}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{1s,\Gamma_1} \right) - V_{L,\Gamma_1}^{\Gamma_1} \left(\frac{\epsilon_2-\epsilon_1}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{i,\Gamma_1} \right) \nonumber\\
 & - K_{L,\Gamma_2}^{\Gamma_1}(\phi_{3s,\Gamma_2}) + V_{L,\Gamma_2}^{\Gamma_1} \left(\frac{\epsilon_3}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{3s,\Gamma_2} \right)  - V_{L,\Gamma_2}^{\Gamma_1} \left(\frac{\epsilon_2 -\epsilon_3}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{i,\Gamma_2} \right) = 0   \nonumber \\
 \frac{\phi_{3s,\Gamma_1}}{2}& - K_{L,\Gamma_1}^{\Gamma_2}(\phi_{1s,\Gamma_1}) + V_{L,\Gamma_1}^{\Gamma_2} \left(\frac{\epsilon_1}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{1s,\Gamma_1} \right) - V_{L,\Gamma_1}^{\Gamma_2} \left(\frac{\epsilon_2-\epsilon_1}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{i,\Gamma_1} \right) \nonumber \\
& - K_{L,\Gamma_2}^{\Gamma_2}(\phi_{3s,\Gamma_2}) + V_{L,\Gamma_2}^{\Gamma_2} \left(\frac{\epsilon_3}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{3s,\Gamma_2} \right)  - V_{L,\Gamma_2}^{\Gamma_2} \left(\frac{\epsilon_2 -\epsilon_3}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{i,\Gamma_2} \right) = 0  \nonumber \\
\frac{\phi_{3s,\Gamma_2}}{2}& + K_{L,\Gamma_2}^{\Gamma_2}(\phi_{3s,\Gamma_2}) - V_{L,\Gamma_2}^{\Gamma_2} \left( \frac{\partial}{\partial \mathbf{n}} \phi_{3s,\Gamma_2} \right) = \frac{1}{4\pi\epsilon_3} \sum_{k=0}^{N_q} \frac{q_k}{|\mathbf{r}_{\Gamma_2} - \mathbf{r}_k|}
\end{align}
\end{widetext}

\begin{figure}%[b] %  figure placement: here, top, bottom, or page
    \centering
    \includegraphics[width=0.25\textwidth]{protein_sensor_regions.pdf}
    \caption{Analyte-sensor system under electric field.}
    \label{fig:analyte-sensor}
 \end{figure}

\paragraph{Discretization and linear system}

We discretize the surface into flat triangles, and assume that  $\phi$ and
$\partial \phi/\partial \mathbf{n}$ are constant within each element. We can
then write the layer operators in their discretized form as follows:
%
\begin{align} \label{eq:layers_disc}
V_{L,\text{disc}}^{\mathbf{r}_\Gamma} \left( \frac{\partial}{\partial \mathbf{n}} \phi(\mathbf{r}_{\Gamma}) \right) &= \sum_{j=1}^{N_p} \frac{\partial}{\partial \mathbf{n}} \phi(\mathbf{r}_{\Gamma_j}) \int_{\Gamma_j} G_L(\mathbf{r}_\Gamma,\mathbf{r}_{\Gamma_j})  \mathrm{d} \Gamma_j  \nonumber \\
K_{L,\text{disc}}^{\mathbf{r}_\Gamma}(\phi(\mathbf{r}_{\Gamma})) &=  \sum_{j=1}^{N_p}\phi(\mathbf{r}_{\Gamma_j})\int_{\Gamma_j} \frac{\partial}{\partial \mathbf{n}} \left[ G_L(\mathbf{r}_\Gamma,\mathbf{r}_{\Gamma_j}) \right]\mathrm{d} \Gamma_j
\end{align}
%
\noindent where $N_p$ is the number of discretization elements on $\Gamma$,
and $\phi(\mathbf{r}_{\Gamma_j})$ and $\frac{\partial}{\partial \mathbf{n}}
\phi(\mathbf{r}_{\Gamma_j})$ are the values of $\phi$ and
$\frac{\partial \phi}{\partial \mathbf{n}}$ on panel $\Gamma_j$.
Using centroid collocation, we can write equation \eqref{eq:integral_eq_lspr} in matrix form as:
%
 \begin{equation} \label{eq:matrix_lspr}
 \left[
    \begin{matrix}
       \frac{1}{2} + K_{L}^{\Gamma} & -V_{L}^{\Gamma}  \vspace{0.2cm} \\
       \frac{1}{2} - K_{L}^{\Gamma} &  \frac{\epsilon_1}{\epsilon_2} V_{L}^{\Gamma}  \vspace{0.2cm}
    \end{matrix}
    \right] \left[
    \begin{matrix}
       \phi_{1s,\Gamma} \vspace{0.2cm} \\
       \frac{\partial}{\partial \mathbf{n}} \phi_{1s,\Gamma} \vspace{0.2cm}
    \end{matrix}
     \right] =
    \left[
    \begin{matrix}
       0 \\
       V_{L}^{\Gamma} \left(\frac{\epsilon_2-\epsilon_1}{\epsilon_2} \frac{\partial\phi_i}{\partial\mathbf{n}}\right) \vspace{0.2cm}
    \end{matrix}
    \right]
 \end{equation}
%
Equation \eqref{eq:integral_eq_lspr_system} can be represented as:
%
\begin{align} \label{eq:matrix_multi}
 \left[
    \begin{matrix}
       \frac{1}{2}+K_{L, \Gamma_1}^{\Gamma_1} & -V_{L, \Gamma_1}^{\Gamma_1} & 0 &  0   \vspace{0.2cm} \\
       \frac{1}{2}-K_{L, \Gamma_1}^{\Gamma_1} & \frac{\epsilon_1}{\epsilon_2} V_{L, \Gamma_1}^{\Gamma_1} & -K_{L, \Gamma_2}^{\Gamma_1} & \frac{\epsilon_3}{\epsilon_2} V_{L, \Gamma_2}^{\Gamma_1} \vspace{0.2cm}  \\
        -K_{L, \Gamma_1}^{\Gamma_2}&\frac{\epsilon_1}{\epsilon_2} V_{L, \Gamma_1}^{\Gamma_2} & \frac{1}{2}-K_{L, \Gamma_2}^{\Gamma_2}  &  \frac{\epsilon_3}{\epsilon_2} V_{L, \Gamma_2}^{\Gamma_2} \vspace{0.2cm} \\
       0 & 0 & \frac{1}{2}+K_{L, \Gamma_2}^{\Gamma_2}&  - V_{L, \Gamma_2}^{\Gamma_2}   \vspace{0.2cm} \\
    \end{matrix}
    \right]
\cdot
 \left[
    \begin{matrix}
    \phi_{1,\Gamma_1} \vspace{0.2cm} \\
    \frac{\partial}{\partial \mathbf{n}} \phi_{1,\Gamma_1} \vspace{0.2cm} \\
    \phi_{3,\Gamma_2} \vspace{0.2cm} \\
    \frac{\partial}{\partial \mathbf{n}} \phi_{3,\Gamma_2} \vspace{0.2cm} \\
    \end{matrix}
\right]&
 \nonumber \\
 = \left[
    \begin{matrix}
    0 \vspace{0.2cm} \\
    V_{L,\Gamma_1}^{\Gamma_1} \left(\frac{\epsilon_2-\epsilon_1}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{i,\Gamma_1} \right)
    + V_{L,\Gamma_2}^{\Gamma_1} \left(\frac{\epsilon_2 -\epsilon_3}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{i,\Gamma_2} \right)
    \vspace{0.2cm}\\
    V_{L,\Gamma_1}^{\Gamma_2} \left(\frac{\epsilon_2-\epsilon_1}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{i,\Gamma_1} \right)
    + V_{L,\Gamma_2}^{\Gamma_2} \left(\frac{\epsilon_2 -\epsilon_3}{\epsilon_2}\frac{\partial}{\partial \mathbf{n}}\phi_{i,\Gamma_2} \right)
    \vspace{0.2cm}\\
    \frac{1}{4\pi\epsilon_3}\sum_{k=0}^{N_q} \frac{q_k}{|\mathbf{r}_{\Gamma_2} - \mathbf{r}_k|} \vspace{0.2cm}  \\
    \end{matrix}
\right]&
\end{align}
%
\noindent where the elements of the matrix are
%
\begin{align} \label{eq:layers_element}
V_{L,ij}^{\Gamma} &= \int_{\Gamma_j} G_L(\mathbf{r}_{\Gamma_i},\mathbf{r}_{\Gamma_j})  \mathrm{d} \Gamma_j, \nonumber \\
K_{L,ij}^{\Gamma} &= \int_{\Gamma_j} \frac{\partial}{\partial \mathbf{n}} \left[ G_L(\mathbf{r}_{\Gamma_i},\mathbf{r}_{\Gamma_j}) \right]\mathrm{d} \Gamma_j,
\end{align}
%
\noindent with $\mathbf{r}_{\Gamma_i}$ being at the center of panel $\Gamma_i$.

\paragraph{Integral evaluation}

We evaluate the integrals in Equation \eqref{eq:layers_element} with Gauss quadrature
rules. The $1/r$ singularity of the Green's function poses a
problem to obtaining good accuracy when the integral is
singular or near-singular. Therefore, we define three different regions, as follows.
\begin{description}
\item[Singular integrals:] If the collocation point is in the integration element,
the singularity is difficult to resolve with standard
Gauss integration schemes. In this case, we use a semi-analytical technique
\cite{HessSmith1967,ZhuHuangSongWhite2001} that places $N_k$ quadrature nodes on the
edges of the triangle.

\item[Near-singular integrals:] If the collocation point is close to the integration element,
the integrand has a high gradient, and high-order quadrature rules are required.
We use the representative length of the integrated triangle ($L = \sqrt{2\cdot\text{Area}}$)
to define a threshold of the \emph{nearby} region, for example, when the integration panel
is $2L$ or less away from the collocation point. For near-singular integrals, we use
$K_{fine}=19, 25  \text{ or }  37$ points per triangle.

\item[Far-away integrals:] When the distance between the collocation point and the integration
element is beyond the threshold, they are considered to be far-away.
At this point, the integrand is smooth enough that we obtain good
accuracy with low-order integration, for example, with
$K=1, 3  \text{ or } 4$ Gauss quadrature points per boundary element.
\end{description}

\subsubsection{Boundary integral expression of the dipole moment}

As shown in Equation \eqref{eq:scat_efield_long_range}, the scattered electric
field in the far-away limit depends on the dipole moment. The dipole moment is
defined as
%
\begin{equation} \label{eq:dipole_def}
\mathbf{p} = \int_\Omega \mathbf{r} \rho \text{d}\Omega,
\end{equation}
%
and rewriting this equation using Gauss' law, we obtain
%
\begin{equation} \label{eq:dipole_def_gauss}
\mathbf{p} = -\epsilon_2\int_\Omega \mathbf{r} \nabla^2 \phi_{2s} \text{d}\Omega.
\end{equation}
%
For component $i$, this becomes:
%
\begin{equation} \label{eq:dipole_def_gauss_i}
{p_i} = -\epsilon_2\int_\Omega {x_i} \nabla^2 \phi_{2s} \text{d}\Omega.
\end{equation}
%
Using the identity
%
\begin{equation} \label{eq:identity_grad}
  \nabla \cdot \left(f \mathbf{v}\right) = \left( \nabla f \right)\cdot \mathbf{v} + f\left(\nabla \cdot \mathbf{v}\right)
\end{equation}
%
with $f=x_i$ and $\mathbf{v} = \nabla\phi_{2s}$, we can rewrite Equation \eqref{eq:dipole_def_gauss_i}
as
%
\begin{equation}
- \frac{p_i}{\epsilon_2} = \int_\Omega \nabla \cdot \left( x_i \nabla \phi_{2s} \right) \; \text{d}\Omega - \int_\Omega \nabla x_i \cdot \nabla\phi_{2s} \; \text{d}\Omega, \nonumber
\end{equation}
\noindent and applying the divergence theorem
\begin{equation} \label{eq:dip_gauss_interm_1}
- \frac{p_i}{\epsilon_2}= \oint_\Gamma  x_i  \nabla \phi_{2s} \cdot \mathbf{n} \; \text{d}\Gamma - \int_\Omega \nabla x_i \cdot \nabla\phi_{2s} \; \text{d}\Omega.
\end{equation}
%
Using the identity \eqref{eq:identity_grad} again in Equation \eqref{eq:dip_gauss_interm_1}, this time
taking $f=\phi_{2s}$ and $\mathbf{v} = \nabla x_i$, we get:
%
\begin{align} \label{eq:dip_gauss_interm_2}
 - \frac{p_i}{\epsilon_2} =& \oint_\Gamma  x_i  \frac{\partial \phi_{2s}}{\partial \mathbf{n}} \text{d}\Gamma - \nonumber \\
 & \left[ \int_\Omega \nabla \cdot \left( \phi_{2s} \nabla x_i \right)\;\text{d}\Omega - \int_\Omega  \phi_{2s} \nabla^2 x_i \;\text{d}\Omega\right] \nonumber\\
%&\text{and applying the divergence theorem} \nonumber \\
=& \oint_\Gamma  x_i  \frac{\partial \phi_{2s}}{\partial \mathbf{n}} \; \text{d}\Gamma - \oint_\Gamma \phi_{2s} \nabla x_i \cdot \mathbf{n} \; \text{d}\Gamma \nonumber \\
=& \oint_\Gamma  x_i  \frac{\partial \phi_{2s}}{\partial \mathbf{n}} \; \text{d}\Gamma - \oint_\Gamma \phi_{2s} n_i \;\text{d}\Gamma
\end{align}
%
Throughout this derivation, the normals are pointing into $\Omega_1$. However, in our implementation
all normals are pointing outwards, and we need to include an extra negative sign, yielding:
%
\begin{equation} \label{eq:dipole_def_gauss_i_final}
{p_i} = \epsilon_2 \left[ \oint_\Gamma  x_i  \frac{\partial \phi_{2s}}{\partial \mathbf{n}} \text{d}\Gamma - \oint_\Gamma \phi_{2s} n_i \; \text{d}\Gamma \right].
\end{equation}

Using BEM, we obtain the electrostatic potential and its normal derivative, on the surface of the nanoparticle,
which we use in Equation \eqref{eq:dipole_def_gauss_i_final} to get the dipole
moment, and in Equation \eqref{eq:scat_efield_long_range} to obtain the scattered
electric field. We can then use Equation \eqref{eq:scat_efield_fwa} and Equation
\eqref{eq:cext_fwa} to get the extinction cross section.

\subsection{Acceleration strategies} \label{sec:acc_strategies}

One disadvantage of the Boundary Element Method (BEM) is that it generates dense matrices
after discretization. Solving the resulting linear system using
Gaussian elimination would require $\O{N^3}$ computations and $\O{N^2}$ storage, whereas for a
Krylov-subspace iterative solver, like the Generalized Minimal Residual Method (GMRES),
computations drop to $\O{N^2}$ because they are dominated by dense matrix-vector
products. This makes BEM inefficient with more than a few thousand boundary elements,
which are the mesh sizes required for real applications.

In our formulation with Gaussian quadrature and collocation, the matrix-vector product
becomes an $N$-body problem, with Gauss nodes acting as centers of mass (\emph{sources}),
and the collocation points acting as evaluation points for the potential (\emph{targets}).
To overcome the unfavorable scaling,
we accelerate the matrix-vector product using a treecode algorithm \cite{BarnesHut1986,DuanKrasny2001},
which is a fast-summation algorithm capable of reducing $\O{N^2}$
computational patterns like
%
\begin{equation} \label{eq:summation}
V(\mathbf{x}_i) = \sum_{j=1}^{N} q_j \psi(\mathbf{x}_i, \mathbf{y}_j)
\end{equation}
%
\noindent to a computational complexity of $\O{N \log N}$. In Equation \eqref{eq:summation}
$q_j$ is the weight, $\psi$ the kernel, $\mathbf{y}_j$ the locations of sources and
$\mathbf{x}_i$ the locations of targets.

The treecode groups sources geometrically in boxes of an octree, built ensuring
that no box in the lowest level has more than $N_\text{crit}$ sources. If a group of
sources is far away from a target, their influence is aggregated at an expansion center,
and the target interacts with the box, rather than with each source independently.
If the group of targets is  close, the treecode queries the child
boxes. If the box has no children and still is not far enough, the interaction is
performed directly via \eqref{eq:summation}.
 The threshold to decide if a box is far enough is called the multipole-
acceptance criterion (MAC), defined as:
%
\begin{equation}
\theta > \frac{r_b}{r},
\end{equation}
%
\noindent where $r_b$ is the box size and $r$ the distance between the box center and the target.
Common values of $\theta$ are $1/2$ and $2/3$.
To approximate the contribution of the sources, we use Taylor expansions
of order $P$.
The treecode allows us to control the accuracy of the approximation by modifying $\theta$ and $P$.
Further details of the treecode implementation in \pygbe can be found in \cite{CooperBarba-share154331,CooperBardhanBarba2013}.

\subsection{Code modifications and added features} \label{sec:code_imp}

As mentioned at the beginning of this section, the present work extends the \pygbe code
to allow its application to nano-plasmonics.
The code required the following modifications and added features:

\begin{itemize}
    \item Re-writing the GMRES solver to accept complex numbers.
    \item Splitting treecode calculations into real and imaginary parts.
    \item Re-formatting configuration files to include electric field intensity and  wavelength.
    \item Adding the new function \texttt{read\_electric\_field}, to read the electric field intensity and its wavelength from configuration files.
    \item Adding the new function \texttt{dipole\_moment} to compute numerically the dipole moment by Equation \eqref{eq:dipole_def_gauss_i_final}.
    \item Adding a new function to compute the  extinction cross section (\texttt{extinction\_cross\_section}).
    \item Organizing LSPR computations on a different main script (called \texttt{lspr.py}).
\end{itemize}

\noindent For information about how to use the code, run examples and tests, see the
\pygbe documentation at \url{http://barbagroup.github.io/pygbe/docs/}

\subsection{Protein mesh preparation}
In Figure \ref{fig:analyte-sensor}, $\Omega_3$ is a region that represents the analyte molecule, which contains a point charge distribution of the partial charges, and is interfaced with the solvent by $\Gamma_2$, the solvent excluded surface (\texttt{SES}).
The \texttt{SES} is generated by rolling a spherical probe of the size of a water molecule ($1.4$\AA~ radius) around the analyte, and tracking the points where the probe and molecule make contact.
The open-source software Nanoshaper \cite{Nanoshaper} uses the molecular structure to produce a triangulation of the \texttt{SES}, which can be read by our software.
In particular, Nanoshaper takes as inputs the atomic coordinates, obtained from the Protein Data Bank, and radii, which were
extracted from a \texttt{pqr} file generated with \texttt{pdb2pqr} \cite{Dolinsky04}.
We obtained the charge and van der Waals parameters of the analyte from \texttt{pdb2pqr} using the built-in \texttt{amber} force field.
In support of the reproducibility of our results, we deposited the final meshes in the Zenodo data repository.
See section \ref{sec:repro} for details.


\documentclass[letterpaper,12pt,leqno]{article}
\usepackage{paper,math,notes}

\begin{document}

\title{Mathematical Methods for Macroeconomics: One-Hour Exam}
\author{Pascal Michaillat}
\date{}

\begin{titlepage}
\maketitle
\end{titlepage}

\section*{Question 1. (50 pts)}

Let $\a \in (0,1)$, $\d  \in (0,1)$, $\rho \in (0,1)$, and $\s>0$. Impose that $\rho+\d<1$. Given $k(0)$, we want to find the function $c(t) $ to maximize
\begin{equation*}
\int_{0}^{+\infty }e^{-\rho\cdot  t}\cdot \frac{c(t)^{1-\s}-1}{1-\s} dt,
\end{equation*}
subject to the law of motion
\begin{equation*}
\dot{k}(t) =k(t)^{\a}-c(t)-\d \cdot k(t).
\end{equation*}

\begin{enumerate}

\item (20 pts) Which variable do you choose as a state variable? Which variable do you choose as a control variable? Write down the current-value Hamiltonian and derive the optimality conditions.

\item (5 pts) The Euler equation is the first-order differential equation that characterizes the optimal function $c(t)$. Determine the Euler equation.

\item (10 pts) Suppose $\a =1$ and $\s =1$. Show that the system describing the optimal functions $\{k(t),c(t)\}$ reduces to a linear, homogenous system of first-order differential equations. Show that the system is unstable by computing the eigenvalues.

\item (15 pts) Suppose $\a <1$ and $\s >0$. Show that the system describing the optimal functions $\{k(t),c(t)\}$ reduces to a nonlinear system of first-order differential equations. Use a phase-diagram to show that the steady state of the system is a saddle point. Explain how you draw the phase diagram.

\end{enumerate}

\section*{Question 2. (50 pts)}

Let $\b \in (0,1)$ and $r>0$. Given $k_{0}>0$, we want to find a collection of sequences $\{c_{t},k_{t+1}\}_{t=0}^{+\infty}$ to maximize
\begin{equation*}
\sum_{t=0}^{\infty }\b^{t} \cdot \ln(c_{t}),
\end{equation*}
subject to the constraints
\begin{equation*}
k_{t+1}=(1+r)\cdot k_{t}-c_{t}
\end{equation*}
for all $t\geq 0$.

\paragraph{Lagragian} We first solve the maximization problem using the Lagrangian method.

\begin{enumerate}
\item (5 pts) Write down the Lagrangian of the problem.
\item (5 pts) Derive the first-order condition(s) of the maximization problem.
\item (5 pts) Derive the Euler equation.
\end{enumerate}

\paragraph{Dynamic Programming} Next we solve the maximization problem using the dynamic programming method.

\begin{enumerate}\setcounter{enumi}{3}

\item (5 pts) Which variable do you choose as a state variable? Which variable do you choose as a control variable? Write down the Bellman equation.

\item (5 pts)  Derive the first-order condition associated with the Bellman equation.

\item (5 pts)  Derive the Benveniste-Scheinkman equation.

\item (5 pts)  Derive the Euler equation. Compare it with the Euler equation obtained with the Lagrangian method and discuss.

\item (5 pts) Suppose that the policy function takes the form $h(k)=A\cdot (1+r)\cdot k$ where $A\in (0,1) $. Derive $A$.

\item (10 pts) Suppose that the value function takes the form $V(k)=B+D\cdot \ln(k),$ where $B$ and $D$ are constants. Using the expression for the policy function that you derived in the previous question, derive $B$ and $D$.

\end{enumerate}
\end{document}


%!TEX root = head-full.tex

\section{Metropolis-Hastings} \label{sec:metropolis_hastings}
In the Introduction, we have shown the main idea of an MCMC sampling method. In this section, we will introduce the Metropolis Hastings Algorithm and conduct an experiment.

\subsection{Algorithm\protect\footnote{Available at \protect\url{https://github.com/lzhbrian/MCMC/blob/master/src/MH/metropolis_hasting.R} in R\cite{R}}}

\subsubsection{Detailed Balance Condition}
Before stepping further into the MH Algorithm, We would first introduce a theorem called the Detailed Balance Condition.

In the introduction, we said that we want to construct a Markov Chain which its stationary distribution $\pi(x)$ just equals to the required probability distribution $p(x)$.

At first, a theorem is needed.
\begin{theorem}[Detail Balance Condition] \label{theo:detail_balance_condition}
Given a non periodic Markov Chain, if
\begin{equation}
\pi(i) P_{ij} = \pi(j) P_{ji}~~for~all~i,j
\end{equation}
then $\pi(x)$ is the stationary distribution of this Markov Chain.
\end{theorem}
So, the key question will be how to construct a Markov Chain which satisfy this Detail Balance Condition.

\subsubsection{MCMC sampling method}
Suppose we already have a transition matrix $Q$ for a Markov Chain, $q(i,j)$ denote the probabilty of transition from state $i$ to state $j$. For the general case,
$$p(i) q(i,j) \neq p(j) q(j,i)$$
That is to say, we do not have the detailed balance condition (Theorem~\ref{theo:detail_balance_condition})
So we introduce an $\alpha(i,j)$ s.t.
\begin{equation}
p(i) q(i,j)\alpha(i,j) = p(j) q(j,i)\alpha(j,i)
\end{equation}
By sysmetrical characteristic, we choose:
\begin{equation}
\alpha(i,j)= p(j) q(j,i)， \quad \alpha(j,i) = p(i) q(i,j)
\end{equation}
So the new Markov Chain $Q'$ would have the property of which its stationary distribution is $p(x)$
\begin{equation}
\label{detailed-balance}
p(i) \underbrace{q(i,j)\alpha(i,j)}_{Q'(i,j)}
= p(j) \underbrace{q(j,i)\alpha(j,i)}_{Q'(j,i)}
\end{equation}

We call the $\alpha(i,j)$ we introduced, accepting ratio. It means that, in the original Markov Chain $Q$, when state $i$ transits to state $j$ with a probability of $q(i,j)$, we accept this transition with a probabilty of $\alpha(i,j)$

Now, we have derived the MCMC sampling method.

\subsubsection{Metropolis-Hastings Algorithm}
The MCMC sampling method is a marvellous work. However, it has a critical drawback that if $\alpha(i,j)$ \& $\alpha(j,i)$ are too small, we would seldom accept the transition.

A solution is that we multiply both $\alpha(i,j)$ \& $\alpha(j,i)$ with a constant to make sure that the larger one between them equals $1$. By doing so, we change the accepting ratio to
\begin{equation}
\alpha(i,j) = \min\left\{\frac{p(j)q(j,i)}{p(i)q(i,j)},1\right\}
\end{equation}
and now, we get Metropolis-Hastings Algorithm\cite{hastings1970monte}.

I would like to further introduce one more concept called accepting rate(not accepting ratio), which denotes the statistic ratio of accepting the transition. i.e. If we request 10 transition and we accept 8 times, then the accepting rate would be 0.8. This concept is crucial when we are dealing with a continual Markov Chain to use the MH algorithm.

\subsubsection{Symmetric Case}
In a Markov Chain whose transition matrix is symmetric, we have
$$q(i,j)=q(j,i)$$
so the accepting ratio could be simplified to
\begin{equation}
\alpha(i,j) = \min\left\{\frac{p(j)}{p(i)},1\right\}
\end{equation}
which is also known as the Metropolis Algorithm\cite{metropolis1953equation}.

\subsubsection{Continual Case}
In a continual Markov Chain, such as the experiment we are going to do in the next subsection, we have a vague definition of transition matrix $Q$. So we introduce a concept called the proposal jump size, $sd.T$.

The method we get $x_{k+1}$ from $x_{k}$ is to add a sampled point of a normal distribution with a variance of the jump size and $\mu=0$. For a two dimension example, we have:
\begin{equation}
x_{k+1} = x_{k} + sd.T \left( \begin{array}{ccc}
n_{1} \\
n_{2} \end{array} \right)
\end{equation}

	\begin{algorithm}
        \caption{Metropolis-Hastings}
        \begin{algorithmic}
        	\Require Required distribution $p$, Transfer Matrix $Q$
        	\State Initialize $x_{1}$
            \For{$t = 1 \to \inf$}
                \State Sample $y \sim q(x|x_{t})$
	            \State Sample $u \sim U{[0,1]}$
	            \If {$u < \alpha(x_{t},y)=min\{\frac{p(y)q(x_{t}|y)}{p(x_{t})q(y|x_{t})},1\}$}
	                \State $x_{t+1} \gets y$
	            \Else
	            	\State $x_{t+1} \gets x_{t}$
	            \EndIf
			\EndFor
        \end{algorithmic}
    \end{algorithm}

\subsection{Sampling Experiment}
For our experiment, we use an example of a bivariate Normal distribution, with
$$ \mu = \left( \begin{array}{ccc}
5 \\
10 \end{array} \right),
\Sigma = \left( \begin{array}{ccc}
1 & 1\\
1 & 4\end{array} \right)$$

By theoretical computation, we can easily compute the the pearson correlation between the two dimensional value is 0.5.
$$ \rho = 0.5 $$

We then generate 10,000 samples using the MH algorithm and take the second half (i.e. the last 5,000 points), setting the standard deviation of proposal to 3.0. We can see from the result (Figure~\ref{fig:sample_result}) that we have derived 5,000 sampled points whose pearson correlation value $\rho=0.50009$, which matches the theoretical value.

\begin{figure}[tb]
\vspace{-0.5in}
  	\centering
  	\includegraphics[width=0.4\textwidth]{figure/sample_result.eps}
\vspace{-0.2in}
	\caption{Sampling result of 5,000 points \protect\\ correlation = 0.50009 , set sd.T = 3.0}
	\label{fig:sample_result}
\end{figure}

\subsection{Performance Analysis}

\subsubsection{Choice of proposal jump size}
MH algorithm is an effective MCMC method for many diverse problems. However, for a continual case in MH algorithm, its performance somewhat depends on the selection of the proposal density. With the proposal jump size being small, the accepting rate would be very low and eventually stick to only one point(eg. the initial point); When the proposal jump size is too big, the accepting rate would be too high.

Roberts et al. have shown in previous work\cite{roberts1997weak} that the optimal accepting rate of the MH algorithm should approximately be at 0.234 for the case of an N-dimensional Gaussian target distribution. We test the accepting rates in different proposal jump size(Figure~\ref{fig:acc_sdt}) and find that the optimal value should be at appoximately 3.0 to acquire a model with accepting rate being close to 0.234. That is the reason why we choose 3.0 as our proposal jump size.

\begin{figure}[tb]
\vspace{-0.2in}
  	\centering
  	\includegraphics[width=0.4\textwidth]{figure/acc_sdt.eps}
\vspace{-0.2in}
	\caption{Accepting rate on different proposal jump size.}
	\label{fig:acc_sdt}
\end{figure}

\subsubsection{Efficiency}
Due to the limit of the accepting rate, for a high dimensional condition, using the MH sampling methods may spend more time in traverse all of the possible states, which could sometimes be be less satisfying. Thus, many would switch to Gibbs Sampling Algorithm.

\subsection{Gibbs Sampling}
Gibbs Sampling is a special case of Metropolis Hastings Algorithm, by letting the accepting rate = 1, we will get a Gibbs Sampler. As the length \& time limit, we will not specify more here.
But it is worthy to notice that Gibbs sampling method is used more often than Metropolis-Hastings method in the real practice, probably because it has a slightly simpler process.


\documentclass[10pt]{article}

\usepackage[utf8]{inputenc}
\usepackage[margin=1in]{geometry}
\usepackage[]{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}

\newcommand{\RR}{\mathbb{R}}
\newcommand{\abs}[1]{\vert#1\vert}
\newcommand{\set}[1]{\{#1\}}
\DeclareMathOperator*{\argmin}{\textrm{arg\,min}}

\theoremstyle{definition}
\newtheorem{definition}{Definition}[section]

\title{Fatness of Simple Polygons}
\author{Sharmila Duppala \and David Kraemer}
\date{April 2018}

\begin{document}

\maketitle

\section{Problem Statement}

\paragraph{Professor Mitchell's description.}
Computing the Niceness of a Polygonal Shape. There are various notions of
quantifying how “nice” or how “fat” a simple polygon $P$ is. A “nicest” polygon
might be a regular $n$-gon, which most closely approximates a circular disk. This
project seeks to implement some precise metrics for niceness, and compare them
on simple polygons (possibly moused in by a user or read in from a file, etc).
To make it simple and discrete, I propose that you discretize the boundary of
$P$ with discrete points $p_i$ (additional vertices), with a prescribed spacing,
$\delta$. (That is, for edges of $P$ longer than $\delta$, add vertices along
  the edge so that the new sub-segments are of length at most $\delta$, for a
user-specified parameter $\delta$.)

Then, consider discrete choices of radii $r = \rho, 2\rho, 3\rho, \ldots$ for
disks, $B(p_i,r)$, centered at the discrete boundary points $p_i$, for each
radius $r$. The (discrete) fatness of $P$ is given by the smallest value of the
ratio $\lambda (B(p_i,r)\cap P)/\lambda (B(p_i, r))$, over all choices of $p_i$
and $r = \rho, 2\rho, 3\rho, \ldots$ such that $P$ is not contained fully inside
$B(p_i,r)$.

  Implement and experiment with this fatness measure. It may be possible to use
  the algorithm to assist a project at Harvard on Gerrymandering, where the goal
  is to quantify how “compact” polygonal election districts are. (So I hope that
  at least a couple of people choose to do this project! We can discuss
further.)

(Theoretically, we are interested in finding an algorithm to compute the “exact”
  fatness of $P$ (which allows disks centered anywhere inside $P$, of any radius
  such that the disk does not contain all of $P$), without resorting to the
  simple discretization. Or, can we compute a provable approximation to the
exact fatness?)

\section{Definitions}

Our present objective is to develop rigorous measures of the ``niceness'' of
simple bounded closed planar polygons with the intuitive hierarchy that regular
$n$-gons are ``nicest'' followed by convex $n$-gons and proceeded by ``fat''
polygons---i.e., polygons which retain their structure under smoothing filters.

Throughout we assume that $P \subseteq \RR^2$ is a simple bounded closed
polygon. We shall denote by $\partial P$ the boundary of $P$, with
$\abs{\partial P}$ denoting the perimeter and $\lambda(P)$ denoting the area.
For given points $x,y \in \RR$, denote the closed line segment bounded by $x$
and $y$ by $[x,y]$.  The class of such polygons is denoted by $\mathcal{P}$.

One class of ``fatness'' measures arises through partitioning $P$ into two
(interior disjoint) polygons $P = P' \cup P''$ via chords.  A chord is a pair
$x,y \in \partial P$ such that the segment $[x,y]$ is contained by the relative
interior of $P$ except at the endpoints $x$ and $y$.  Such a chord $[x,y]$
defines a partition of $P = P' \cup P''$ by orientation, so that $\partial P'$
is the arc from $x$ to $y$ with $[y,x]$ and that $\partial P''$ is the arc from
$y$ to $x$ with $[x,y]$.

\begin{definition}
  Let $f : \mathcal{P} \to \RR$. For a given $P \in \mathcal{P}$,
  its \emph{chord-$f$ score} is given by
  \begin{equation*}
    s_f(P) = \inf_{x,y \in \partial P} \max(f(P'), f(P''))
  \end{equation*}
  where the chord $[x,y]$ partitions $P = P' \cup P''$.
\end{definition}

Intuitively, $f$ represents a polygon's cost with respect to a certain
measurement.  For example, if $f$ indicates the perimeter of $P$, then $s_f$ is
associated with a partition of $P$ such that the maximum perimeter of either
subpolygon is minimized. Indeed, measures such as perimeter or area are the
typical choices of $f$, but any suitable property of the polygon may be
employed.

The visibility kernel of $P$, denoted by $\ker P$, is the subset of $P$ which
``sees'' all of $P$. That is, if $x \in \ker P$ and $y \in P$, then $[x,y]
\subseteq P$. Another proposed measure of polygonal ``niceness'' utilizes the
visibility kernel.

\begin{definition}
  The \emph{visibility kernel score} is given by
  \begin{equation*}
    s_{\textrm{vis}}(P) = \frac{\lambda(\ker P)}{\lambda(P)}.
  \end{equation*}
\end{definition}

If $P$ is a regular $n$-gon or convex $s_{\textrm{vis}}(P) = 1$, since trivially
$\ker P = P$. On the other hand, if $P$ is not star shaped, then
$s_{\textrm{vis}}(P) = 0$.

Equipped with an $L_p$ norm ($p \in [1,\infty]$), denote by $B(x,\rho)$ the
closed ball of radius $\rho > 0$ centered at $x \in \RR^2$.  The natural choice
is simply the Euclidean norm, but $L_{\infty}$ provides a setting for which
computing $B(x,\rho) \cap P$ resolves to polygonal intersection. By topological
equivalence, we lose no specificity. A final measure of ``fatness'' measures
local nonconvexities of $P$ and identifies the most offending ratio.
\begin{definition}
  The \emph{$\alpha$-fatness score} is given by
  \begin{equation*}
    \alpha(P) = \inf\set{\frac{\lambda(B(x,\rho) \cap
    P)}{\lambda(B(x,\rho))} : x \in \partial P, \rho > 0, P \not\subseteq B(x,\rho)}
  \end{equation*}
\end{definition}

For a given $S \subseteq \RR^2$, let $\mathcal{C}(x,S)$ denote the connected
component of $S$ containing $x$. An alternative formulation of $\alpha$-fatness
replaces $B(x,\rho) \cap P$ with its connected component: $\mathcal{C}(x,
B(x,\rho) \cap P)$. In principle, $\alpha(P)$ is computed by sweeping
continuously, but for simplicity we can merely discretize $\partial P$ with a
parameter $\delta > 0$ indicating the maximal distance between points in
$\partial P$.

\section{Implementation}

We plan to implement these measures of polygonal ``fatness'' and compare them
for assorted collections of polygons, from theoretical and
randomly-generated examples to historical electoral map data. We hope to
determine the relative strengths of these measures as heuristics for polygonal
``fatness'' individually or in an ensemble on this data.

\end{document}


\section{Elementary number theory}

One of the things type theory is great for, is for the formalization of mathematics in a computer proof assistant. Those are programs that can compile any type theoretical construction to check that this construction indeed has the type it was claimed it has.

At this point in our development of type theory there are two areas of mathematics that would be natural to try to do in type theory: discrete mathematics and elementary number theory. Indeed, how does one define in type theory the greatest common divisor of two natural numbers, or how does one show that there are infinitely many primes? How does one even formalize that every non-empty subset of the natural numbers has a least element?

To answer these questions we will run into questions of decidability. How do we write a term that decides wheter a number is prime or not? Or indeed, is it even true that every non-empty subset of the natural numbers has a least element? What about the subset of $\N$ that contains $1$, and it contains $0$ if and only if Goldbach's conjecture holds? Finding the least element of this subset is equivalent to settling the conjecture!

Therefore, we will prove the well-foundedness of the natural numbers for decidable subsets of $\N$. In fact, we will show it for decidable families, because sometimes we don't know in advance whether a family of types is in fact a subtype. A consequence of involving decidability in the well-foundedness of the natural numbers is that for many properties one has to prove that they are decidable. Luckily this is the case: many of the familiar properties that one encounters in number theory are indeed decidable.

\subsection{Decidability}

A common way of reasoning in mathematics is via a proof by contradiction: ``in order to show that $P$ holds we show that it cannot be the case that $P$ doesn't hold". There are no inference rules in type theory that allow us to obtain a term of type $P$ from a term of type $\neg\neg P$. However, for some propositions $P$ one can construct a function $\neg\neg P \to P$. The \emph{decidable propositions} from a class of such propositions $P$ for which we can show $\neg\neg P \to P$.

\begin{defn}
  A type $A$ is said to be decidable if it comes equipped with a term of type
  \begin{equation*}
    \mathsf{is\usc{}decidable}(A)\defeq A+\neg A.
  \end{equation*}
  Decidable propositions are called \define{classical}. We will write
  \begin{equation*}
    \mathsf{classical\usc{}Prop} \defeq \sm{P:\mathsf{Prop}}\mathsf{is\usc{}decidable}(P)
  \end{equation*}
  for the type of all classical propositions (with respect to a universe $\UU$).
\end{defn}

\begin{eg}
  The types $\unit$ and $\emptyt$ are decidable. Indeed, we have
  \begin{align*}
    \inl(\ttt) & :\mathsf{is\usc{}decidable}(\unit) \\
    \inr(\idfunc) & : \mathsf{is\usc{}decidable}(\emptyt).\qedhere
  \end{align*}
  Any type $A$ equipped with a point $a:A$ is decidable.
\end{eg}

\begin{lem}
  For each $m,n:\N$, the types $\EqN(m,n)$, $m\leq n$ and $m<n$ are decidable.
\end{lem}

\begin{proof}
  The proofs in each of the three cases is similar, so we only show that $\EqN(m,n)$ is decidable for each $m,n:\N$. This is done by induction on $m$ and $n$. Note that the types
  \begin{align*}
    \EqN(\zeroN,\zeroN) & \jdeq \unit \\
    \EqN(\zeroN,\succN(n)) & \jdeq \emptyt \\
    \EqN(\succN(m),\zeroN) & \jdeq \emptyt
  \end{align*}
  are all decidable. Moreover, the type $\EqN(\succN(m),\succN(n))\jdeq \EqN(m,n)$ is decidable by the inductive hypothesis.
\end{proof}

Typically we are mostly interested in decidability of propositions. However, we have defined the notion of decidability for general types, because the condition on an arbitrary type that its identity types are decidable is of some interest. We now study such types.

\begin{defn}
  We say that a type $A$ has decidable equality if the identity type $x=y$ is decidable for every $x,y:A$.
\end{defn}

\begin{cor}
  Equality on the natural numbers is decidable.
\end{cor}

\begin{proof}
  This follows immediately from the equivalences $(m=n)\simeq \EqN(m,n)$, and the fact that $\EqN(m,n)$ is decidable.
\end{proof}

\begin{lem}
  Suppose that $A$ and $B$ are types with decidable equality. Then the coproduct $A+B$ also has decidable equality.
\end{lem}

\begin{proof}
  Our goal is to construct a dependent function
  \begin{equation*}
    d_{A+B} : \prd{z,z':A+B}\mathsf{is\usc{}decidable}(z=z').
  \end{equation*}
  This function is constructed by coproduct induction on both $z$ and $z'$, so we have four cases to consider. Recall from \cref{thm:id-coprod-compute} that we have equivalences
  \begin{align*}
    (\inl(x)=\inl(x')) & \simeq (x=x') \\
    (\inl(x)=\inr(y')) & \simeq \emptyt \\
    (\inr(y)=\inl(x')) & \simeq \emptyt \\
    (\inr(y)=\inr(y')) & \simeq (y=y').
  \end{align*}
  Therefore the type $z=z'$ is equivalent to a decidable type in each of the four cases.
\end{proof}

\begin{cor}
  The type $\Z$ has decidable equality.
\end{cor}

\begin{cor}
  For any $n:\N$ the type $\mathsf{Fin}(n)$ has decidable equality.
\end{cor}

We have already shown in \cref{thm:eq_nat} that the type of natural numbers is a set. In fact, any type with decidable equality is a set. This fact is known as Hedberg's theorem. Our proof of Hedberg's theorem might appear to be slightly complicated compared to the proof in \cite{hottbook}. This is due to the fact that we haven't introduced function extensionality yet, which is used there to observe that $\neg\neg(x=y)$ is a proposition for any $x,y:A$.

\begin{thm}[Hedberg]
  Any type with decidable equality is a set.
\end{thm}

\begin{proof}
  Let $A$ be a type, and let
  \begin{equation*}
    d:\prd{x,y:A}(x=y)+\neg(x=y).
  \end{equation*}
  Our proof is an application of \cref{lem:prop_to_id}. In order to construct a binary relation $R$, we first consider the type family $D(x,y):((\id{x}{y})+\neg(\id{x}{y}))\to\UU$ given by
  \begin{align*}
    D(x,y,\inl(p)) & \defeq \unit \\
    D(x,y,\inr(p)) & \defeq \emptyt.
  \end{align*}
  Now we define the binary relation $R:A\to (A\to\UU)$ by
  \begin{equation*}
    R(x,y)\defeq D(x,y,d(x,y)).
  \end{equation*}
  It follows that $R(x,y)$ is a proposition for every $x,y:A$, since $D(x,y,z)$ is a proposition for every $z:(x=y)+\neg(x=y)$.

  To see that $R$ is reflexive it suffices to construct a term of type
  \begin{equation*}
    \rho':\prd{x:A}{p:(x=x)+\neg(x=x)}D(x,x,p)
  \end{equation*}
  For any $x:A$ we define $\rho'(x)$ by case analysis on $p:(x=x)+\neg(x=x)$. Given $p:x=x$, we take $\rho'(x,\inl(p))\defeq\ttt$. Given $p:\neg(x=x)$, we obtain $p(\refl{x}):\emptyt$, so there is nothing to define in this case. This completes the definition of $\rho'$.

  It remains to show that $R(x,y)\to (x=y)$ for any $x,y:A$. Note that it suffices to construct a function
  \begin{equation*}
    \alpha:D(x,y,p)\to (x=y)
  \end{equation*}
  for any $x,y:A$ and $p:(x=y)+\neg(x=y)$. This function is constructed by case analisys on $p$. Given $p:x=y$, we simply define $\alpha(x,y,\inl(p))\defeq p$. Given $p:\neg(x=y)$ the type $D(x,y,p)$ is empty, so there is nothing to define.
\end{proof}

\subsection{The well-ordering principle for decidable families over \texorpdfstring{$\N$}{ℕ}}

\begin{defn}
  A family $P$ over a type $A$ is said to be decidable if $P(x)$ is decidable for every $x:A$. A \define{decidable subset} of a type $A$ is a map
  \begin{equation*}
    P:A\to\mathsf{classical\usc{}Prop}.
  \end{equation*}
\end{defn}

\begin{defn}
  Let $P$ be a decidable family over $\N$, and let $n:\N$ be a natural number equipped with $p:P(n)$. We say that $n$ is a \define{minimal $P$-element} if it comes equipped with a term of type
  \begin{equation*}
    \mathsf{is\usc{}minimal}_P(n,p)\defeq \Big(\prd{m:\N}P(m)\to (n\leq m)\Big)
  \end{equation*}
\end{defn}

Note that the type $\mathsf{is\usc{}minimal}_P(n,p)$ doesn't depend on $p$. However, it doesn't make much sense that $n$ is a minimal element of $P$ unless we already know that $n$ is in $P$. Indeed, if we would omit the hypothesis that $n$ is in $P$, it would be more accurate to say that $n$ is a \emph{lower bound} of $P$. The following theorem is the well-ordering principle of $\N$.

\begin{thm}
  Let $P$ be a decidable family over $\N$. Then there is a function
  \begin{equation*}
    \Big(\sm{n:\N}P(n)\Big)\to\Big(\sm{m:\N}{p:P(m)}\mathsf{is\usc{}minimal}_P(m,p)\Big).
  \end{equation*}
\end{thm}

\begin{proof}
  Consider a universe $\UU$ that contains $P$. We show by induction on $n:\N$ that there is a function
  \begin{equation*}
    Q(n)\to \Big(\sm{m:\N}{p:Q(m)}\mathsf{is\usc{}minimal}_Q(m,p)\Big)
  \end{equation*}
  for every decidable family $Q:\N\to\UU$. Note that we performed a swap in the order of quantification, using the universe that contains $P$. This slightly strengthens the inductive hypothesis, which we will be able to exploit.

  The base case is trivial, since $\zeroN$ is the least natural number. For the inductive step, suppose that $Q(\succN(n))$ holds. Note that $Q(\zeroN)$ is assumed to be decidable, so we proceed by case analysis on $Q(\zeroN)+\neg Q(\zeroN)$. Given $q:Q(\zeroN)$, it follows immediately that $\zeroN$ must be minimal. In the case where $\neg Q(\zeroN)$, we consider the decidable subset $Q'$ of $\N$ given by
  \begin{equation*}
    Q'(n)\defeq Q(\succN(n)).
  \end{equation*}
  Since we have $q:Q'(n)$, we obtain a minimal element in $Q'$ by the inductive hypothesis. Of course, by the assumption that $Q(\zeroN)$ doesn't hold, the minimal element of $Q'$ is also the minimal element of $Q$.
\end{proof}

\subsection{The pigeonhole principle}

The pigeonhole principle states that if we place more than $n$ balls in $n$ bags, then at least one bag will contain more than one ball. In this section we will give a type theoretical proof of the pigeonhole principle.

First we give a definition of a function that counts the number of elements in a decidable subset of $\mathsf{Fin}(n)$.

\begin{defn}
  Let $P$ be a decidable subset of $\mathsf{Fin}(n)$. We define the number $|P|:\N$ of elements in $P$.
\end{defn}

\begin{proof}[Construction]
  We give the construction of $|P|$ by induction on $n:\N$. In the base case we note that $\mathsf{Fin}(\zeroN)$ has no elements, so we define $|P|\defeq\zeroN$.

  For the inductive step, we define $|P|$ by case analysis on $P(\inr(\ttt))+\neg P(\inr(\ttt))$. Let $P'$ be the family over $\mathsf{Fin}(n)$ given by $P'(i)\defeq P(\inl(i))$. In the case where $P(\inr(\ttt))$ holds, then we define $|P|\defeq \succN |P'|$. In the case where $P(\inr(\ttt))$ doesn't hold we define $|P|\defeq |P'|$.
\end{proof}

\begin{defn}
  For any $i:\mathsf{Fin}(\succN(n))$ we define a function
  \begin{equation*}
    \mathsf{skip}(i):\mathsf{Fin}(n)\to\mathsf{Fin}(\succN(n)).
  \end{equation*}
\end{defn}

\begin{proof}[Construction]
  The function $\mathsf{skip}(i)$ is defined by induction on $n:\N$. In the base case, the function
  \begin{equation*}
    \mathsf{skip}(i):\mathsf{Fin}(\zeroN)\to\mathsf{Fin}(\succN(\zeroN))
  \end{equation*}
  is defined to be the unique map out of the empty type. In the successor case we define
  \begin{equation*}
    \mathsf{skip}(i) : \mathsf{Fin}(\succN(n))\to\mathsf{Fin}(\succN(\succN(n)))
  \end{equation*}
  by induction on $i:\mathsf{Fin}(\succN(\succN(n)))$. The function
  \begin{equation*}
    \mathsf{skip}(\inl(i)):\mathsf{Fin}(\succN(n))\to\mathsf{Fin}(\succN(\succN(n)))
  \end{equation*}
  is a map between coproducts, so it can be defined using the functorial action of coproducts of \cref{ex:coproduct_functor}. We take
  \begin{equation*}
    \mathsf{skip}(\inl(i))\defeq \mathsf{skip}(i)+\idfunc.
  \end{equation*}
  The function
  \begin{equation*}
    \mathsf{skip}(\inr(i)):\mathsf{Fin}(\succN(n))\to\mathsf{Fin}(\succN(\succN(n)))
  \end{equation*}
  is just the function $\inl$.
\end{proof}

\begin{lem}
  For each $i:\mathsf{Fin}(\succN(n))$, the function
  \begin{equation*}
    \mathsf{skip}(i):\mathsf{Fin}(n)\to\mathsf{Fin}(\succN(n))
  \end{equation*}
  is an embedding.
\end{lem}

\begin{proof}
  This assertion is proven by induction on $n$. In the base case, we note that any map out of the empty type is an embedding, by \cref{ex:is-emb-empty}. In the inductive step we proceed by case analysis on $i:\mathsf{Fin}(\succN(\succN(n)))$. In the case of $\inl(i)$ we note that
  \begin{equation*}
    \mathsf{skip}(\inl(i))\jdeq \mathsf{skip}(i)+\idfunc
  \end{equation*}
  is the functorial action of coproducts on two embeddings. Therefore we conclude by \cref{ex:is-emb-coprod} that this map is an embedding. In the case of $\inr(i)$ we note that $\inl$ is an embedding by \cref{ex:is-emb-inl-inr}.
\end{proof}

\begin{lem}
  Consider a map $g:\mathsf{Fin}(m)\to\mathsf{Fin}(\succN(n))$. Furthermore, suppose that $i:\mathsf{Fin}(\succN(n))$ is not in the image of $g$, i.e.~that $\neg(\fib{g}{i})$. Then we can construct a commuting triangle
  \begin{equation*}
    \begin{tikzcd}
      & \mathsf{Fin}(n) \arrow[d,"\mathsf{skip}(i)"] \\
      \mathsf{Fin}(m) \arrow[r,swap,"g"] \arrow[ur,densely dotted,"f"] & \mathsf{Fin}(\succN(n)).
    \end{tikzcd}
  \end{equation*}
\end{lem}

Finally, we prove the pigeonhole principle.

\begin{thm}\label{thm:pigeonhole}
  For any $m,n:\N$ and any function $f:\mathsf{Fin}(m)\to\mathsf{Fin}(n)$, if $m>n$, then there is an $i:\mathsf{Fin}(n)$ which is in the image of more than one point in $\mathsf{Fin}(m)$.
\end{thm}

\begin{proof}
  The pigeonhole principle is proven by induction on $m,n:\N$. In the base case for $m$ there is nothing to show because $m>n$ is empty. For the inductive step on $m$ and the base case for $n$, we note that $\mathsf{Fin}(\succN(m))\jdeq \mathsf{Fin}(m)+\unit$ and $\mathsf{Fin}(\zeroN)\jdeq \emptyt$. Therefore $f:\mathsf{Fin}(\succN(m))\to\mathsf{Fin}(\zeroN)$ is a function from a pointed type to the empty type, which gives us a contradiction.

  It remains to give the inductive step for $n$. Let $i\defeq f(\inr(\ttt)):\mathsf{Fin}(\succN(n))$. Since the ordering relation $<$ on $\N$ is decidable, we can decide whether $i$ is in the image of more than one point in $\mathsf{Fin}(m)$ by deciding whether or not $1<|P|$ holds for
  \begin{equation*}
    P(j)\defeq (f(j)=i)
  \end{equation*}
  If this is the case, this completes the proof.

  Now suppose that $1\not<|P|$. Since $P(\inr(\ttt))$ holds it follows that $|P|=1$. Now we observe that $i$ is not in the image of $f\circ \inl$. Therefore we obtain a commuting square
  \begin{equation*}
    \begin{tikzcd}
      \mathsf{Fin}(m) \arrow[r,densely dotted,"{f'}"] \arrow[d,swap,"\inl"] & \mathsf{Fin}(n) \arrow[d,"{\mathsf{skip}(i)}"] \\
      \mathsf{Fin}(\succN(m)) \arrow[r,swap,"f"] & \mathsf{Fin}(\succN(n)).
    \end{tikzcd}
  \end{equation*}

  Note that the induction hypothesis the pigeonhole principle applies to the function $f':\mathsf{Fin}(m)\to\mathsf{Fin}(n)$. Since $m>n$ it follows that there is an element $j:\mathsf{Fin}(n)$ that is in the image of $f'$ of more than one element of $\mathsf{Fin}(m)$. Now we observe that there is an equivalence
  \begin{equation*}
    \fib{f'}{j}\simeq \fib{f}{\mathsf{skip}(i,j)}
  \end{equation*}
  because both the left and right maps in the commuting square are embeddings. Therefore we conclude that $\mathsf{skip}(i,j)$ is in the image of $f$ of more than one element of $\mathsf{Fin}(\succN(m))$.
 \end{proof}

\begin{cor}\label{cor:pigeonhole}
  Given $m>n$, no function $\mathsf{Fin}(m)\to\mathsf{Fin}(n)$ is an embedding.
\end{cor}

It is straightforward to see that the statements of \cref{thm:pigeonhole,cor:pigeonhole} are equivalent, and one might argue that the statement of \cref{cor:pigeonhole} is the more `type theoretical way' of phrasing the pigeonhole principle. However, the relation to counting the number of points that get mapped to

\begin{thm}\label{thm:generalized-pigeonhole}
  For any $m,n:\N$ and any function $f:\mathsf{Fin}(m)\to\mathsf{Fin}(n)$, if $m>kn$ for some $k:\N$, then there is an $i:\mathsf{Fin}(n)$ which is in the image of more than $k$ points in $\mathsf{Fin}(m)$.
\end{thm}

\subsection{Defining the greatest common divisor}

\begin{lem}
  For any $d,n:\N$, the type $d\mid n$ is decidable.
\end{lem}

\begin{proof}
  We give the proof by case analysis on $(d=\zeroN)+(d\neq\zeroN)$. If $d=\zeroN$, then $d\mid n$ holds if and only if $\zeroN=n$, which is decidable.

  If $d\neq\zeroN$, then it follows that $n\leq nd$. Therefore we obtain by the well-ordering principle of the natural numbers a minimal $m:\N$ that satisfies the decidable property $n\leq md$. Now we observe that $d\mid n$ holds if and only if $n=md$, which is decidable.
\end{proof}

\begin{defn}
  A type family $P$ over $\N$ is said to be \define{bounded from above} by $m$ for some natural number $m$, if it comes equipped with a term of type
  \begin{equation*}
    \mathsf{is\usc{}bounded}_m(P) \defeq \prd{n:\N}P(n)\to (n\leq m).
  \end{equation*}
\end{defn}

\begin{defn}
  Let $P$ be a type family over $\N$, and consider $p:P(n)$. We say that $n$ is the maximal $P$-number if it comes equipped with a term of type
  \begin{equation*}
    \mathsf{is\usc{}maximal}_P(n,p) \defeq \prd{m:\N} P(m)\to m\leq n.
  \end{equation*}
\end{defn}

In the following lemma we show that if a decidable family $P$ is bounded from above and inhabited, then it possesses a maximal element.

\begin{lem}\label{lem:maximal}
  Consider a decidable type family $P$ over $\N$ which is bounded from above by $m$. Then there is a function
  \begin{equation*}
    \mathsf{maximum}_P:\Big(\sm{n:\N}P(n)\Big)\to\Big(\sm{n:\N}{p:P(n)}\mathsf{is\usc{}maximal}_P(n,p)\Big).
  \end{equation*}
\end{lem}

\begin{proof}
  We define the asserted function by induction on $m$. In the base case, if we have $p:P(n)$, then it follows from $n\leq 0$ that $n=0$. It follows by the boundedness of $P$ that $(n,p)$ is maximal.

  In the inductive step we proceed by case analysis on $P(\succN(m))$. This is allowed because $P$ is decidable. If we have $q:P(\succN(m))$, then it follows by the boundedness of $P$ that $(\succN(m),q)$ is maximal. If $\neg P(\succN(m))$, then it follows that $P$ is bounded by $m$, which allows us to proceed by recursion.
\end{proof}

\begin{defn}
  For any two natural numbers $m,n$ we define the \define{greatest common divisor} $\gcd(m,n)$, which satisfies the following two properties:
  \begin{enumerate}
  \item We have both $\gcd(m,n)\mid m$ and $\gcd(m,n)\mid n$.
  \item For any $d:\N$ we have $d\mid \gcd(m,n)$ if and only if both $d\mid m$ and $d\mid n$ hold.
  \end{enumerate}
\end{defn}

\begin{proof}[Construction]
  Consider the type family $P(d)\defeq (d\mid m)\times (d\mid n)$. Then $P$ is bounded from above by $m$. Moreover, $P(1)$ holds since $1\mid n$ for any natural number $n$. Furthermore, the divisibility relation is decidable, so it follows that $P$ is a family of decidable types. Now the greatest common divisor is defined as the maximal $P$-element, which is obtained by \cref{lem:maximal}
\end{proof}

\subsection{The trial division primality test}

\begin{thm}
  For any $n:\N$, the proposition $\mathsf{is\usc{}prime}(n)$ is decidable.
\end{thm}

It is important to note that, even when we prove that a type such as $\mathsf{is\usc{}prime}(n)$ is decidable, it is only after we \emph{evaluate} the proof term that we know whether the type under consideration has a term or not. In other words, for any given $n$ we don't know right away whether it is prime or not. Evaluating whether $n$ is prime can be computationally costly, so it may be desirable in any specific situation to give a separate mathematical \emph{argument} that decides whether or not the number is prime.

\subsection{Prime decomposition}

We will show now that any natural number $n>0$ can be written as a product of primes
\begin{equation*}
  n=p_1^{k_1}\cdots p_{m}^{k_m}
\end{equation*}
This prime decomposition is unique if we require that the primes $p_i<p_{i+1}$ for each $0<i<m$. In order to establish these facts in type theory, we first have to define finite products.

\subsection{The infinitude of primes}

Recall that Euclid's proof

\begin{thm}
  There are infinitely many primes.
\end{thm}

\begin{proof}
  We will show that for every $n:\N$ there is a prime number that is larger than $n$. In other words, we will construct a term of type
  \begin{equation*}
    \prd{n:\N}\sm{p:\N}\mathsf{is\usc{}prime}(p)\times (n\leq p).
  \end{equation*}
  To show this, let $P$ be the type family over $\N$ given by
  \begin{equation*}
    P(m)\defeq \mathsf{is\usc{}prime}(m)\times (m\leq n).
  \end{equation*}
  Since $P$ is decidable and bounded from above by $n$, we can take the product of all $m:\N$ for which $P$ holds. Write $k$ for this product. Then $k+1$ is coprime to every prime less than $n$. Therefore the primes in its prime factorization must all be larger than $n$.
\end{proof}

\begin{cor}
  There is a function
  \begin{equation*}
    \mathsf{prime} : \N \to \sm{p:\N}\mathsf{is\usc{}prime}(p)
  \end{equation*}
  that sends $n$ to the $n$-th prime. This function is strictly monotone, so it is an embedding.
\end{cor}

Some further ideas to include in this chapter:
\begin{enumerate}
\item If $2^n-1$ is prime, then $n$ is prime.
\item Fermat's little theorem.
\end{enumerate}

\begin{exercises}
\item Show that for any $f:\mathsf{Fin}(m)\to\mathsf{Fin}(n)$ and any $i:\mathsf{Fin}(n)$, the type $\fib{f}{i}$ is decidable.
\item Consider a decidable type $P(i)$ indexed by $i:\mathsf{Fin}(n)$.
  \begin{subexenum}
  \item Show that the type
    \begin{equation*}
      \prd{i:\mathsf{Fin}(n)}P(i)
    \end{equation*}
    is decidable.
  \item Show that the type
    \begin{equation*}
      \sm{i:\mathsf{Fin}(n)}P(i)
    \end{equation*}
    is decidable.
  \end{subexenum}
\item
  \begin{subexenum}
  \item Show that $\nat$ and $\bool$ have decidable equality. Hint: to show that $\mathbb{N}$ has decidable equality, show first that the successor function is injective.
  \item Show that if $A$ and $B$ have decidable equality, then so do $A+B$ and $A\times B$. Conclude that $\Z$ has decidable equality.
  \item Show that if $A$ is a retract of a type $B$ with decidable equality, then $A$ also has decidable equality.
  \end{subexenum}
\item Define the prime-counting function $\pi:\N\to\N$.
\item (The Cantor-Schr\"oder-Bernstein theorem) Let $X$ and $Y$ be two sets with decidable equality, and consider two maps $f:X\to Y$ and $g:Y\to X$, both of which we assume to be injective. Construct an equivalence $X\simeq Y$.
\item For any $k:\Z$, define a function $i\mapsto i+k \mod n$ of type $\mathsf{Fin}(n)\to\mathsf{Fin}(n)$. Show that this function is an equivalence.
\item For any $k:\Z$, define a function $i\mapsto i\cdot k \mod n$ of type $\mathsf{Fin}(n)\to\mathsf{Fin}(n)$. Show that this function is an equivalence if and only if $\gcd(n,k)=1$.
\item Show that
  \begin{equation*}
    \sum_{i=0}^n \binom{n-i}{i}=F_{n+1}
  \end{equation*}
\end{exercises}


\subsubsection{Cross-Validation}
\label{cv}

Because ML models are trained by minimizing a loss function $L$, the
    resulting value of $L$ underestimates the true error we see when
    predicting into the actual future by design.
To counter that, one popular and model-agnostic approach is cross-validation
    (CV), as summarized, for example, by \cite{hastie2013}.
CV is a resampling technique, which ranomdly splits the samples into a
    training and a test set.
Trained on the former, an ML model makes forecasts on the latter.
Then, the value of $L$ calculated only on the test set gives a realistic and
    unbiased estimate of the true forecasting error, and may be used for one
    of two distinct aspects:
First, it assesses the quality of a fit and provides an idea as to how the
    model would perform in production when predicting into the actual future.
Second, the errors of models of either different methods or the same method
    with different parameters may be compared with each other to select the
    best model.
In order to first select the best model and then assess its quality, one must
    apply two chained CVs:
The samples are divided into training, validation, and test sets, and all
    models are trained on the training set and compared on the validation set.
Then, the winner is retrained on the union of the training and validation
    sets and assessed on the test set.

Regarding the splitting, there are various approaches, and we choose the
    so-called $k$-fold CV, where the samples are randomly divided into $k$
    folds of the same size.
Each fold is used as a test set once and the remaining $k-1$ folds become
    the corresponding training set.
The resulting $k$ error measures are averaged.
A $k$-fold CV with $k=5$ or $k=10$ is a compromise between the two extreme
    cases of having only one split and the so-called leave-one-out CV
    where $k = m$: Computation is still relatively fast and each sample is
    part of several training sets maximizing the learning from the data.
We adapt the $k$-fold CV to the ordinal stucture in $\mat{X}$ and $\vec{y}$ in
    Sub-section \ref{unified_cv}.


% ------------------------------------------------------------------
\chapter{Geometry}\label{s:geometry}
% ------------------------------------------------------------------

This chapter looks at the geometry of the CNN input-output mapping.

% ------------------------------------------------------------------
\section{Receptive fields and transformations}\label{s:receptive}
% ------------------------------------------------------------------

Very often it is interesting to map a convolutional operator back to image space. Due to various downsampling and padding operations, this is not entirely trivial and analysed in this section. Considering a 1D example will be enough; hence, consider an input signal $x_i$, a filter $f_{i'}$ and an output signal $y_{i''}$, where
\[
   1 - P_w^- \leq i \leq W + P_w^+,  \qquad 1 \leq i' \leq W'.
\]
where $P_w^-,P_w^+ \geq 0$ is the amount of padding applied to the input signal to the left and right respectively. The output $y_i$ is obtained by applying a filter of size $W'$ in the following range
\[
   - P_w^- + S (i'' - 1) + [1, W'] = [1 - P_w^- + S(i'' - 1), - P_w^- + S(i''-1) + W']
\]
where $S$ is the subsampling factor. For $i'' = 1$, the leftmost sample of this window falls at the leftmost sample of the padded input signal. The rightmost sample $W'' \geq i''$ of the output signal is obtained by guaranteeing that the filter window stays within the padded input signal:
\[
- P_w^- + S(i''-1) + W' \leq W + P_w^+
\qquad\Rightarrow\qquad
i'' \leq \frac{W - W' + P_w^- + P_w^+}{S} + 1
\]
Since $i''$ is an integer quantity, the maximum value is:
\begin{equation}\label{e:width}
W'' =
\left\lfloor \frac{W - W' + P_w^- + P_w^+}{S} \right\rfloor
+ 1.
\end{equation}
Note that $W''$ is smaller than 1 if $W' > W + P_w^- + P_w^+$; in this case the filter $f_{i'}$ is wider than the padded input signal $x_i$ and the domain of $y_{i''}$ becomes empty (this generates an error in most MatConvNet functions).

Now consider a sequence of convolutional operators. One starts from an input signal $x_0$ of width $W_0$ and applies a sequence of  operators of parameters
$
(P_{w1}^-,  P_{w1}^+, S_1, W'_1),
$
$
(P_{w2}^-,  P_{w2}^+, S_2, W'_2),
$
$\dots,
$
$
(P_{wL}^-,  P_{wL},^+ S_L, W'_L)
$
to obtain signals $x_1,x_2,\dots,x_L$ of width $W_1,W_2,\dots,W_L$ respectively. First, note that the widths of these signals are obtained from $W_0$ and the operator parameters by a recursive application of~\eqref{e:width}. Due to the flooring operation, unfortunately it does not seem possible to simplify significantly the resulting expression. However, disregarding this operation one obtains the approximate expression
\[
W_l \approx
\frac{W_0}{\prod_{p=1}^l S_q}
-
\sum_{p=1}^l
\frac
{W'_p - P_{wp}^- - P_{wp}^+  - S_{p}}
{\prod_{q=p}^l S_q}.
\]
This expression is exact when $S_1 = S_2 =\dots = S_l =1$:
\[
W_l = W_0 - \sum_{p=1}^l (W'_p - P_{wp}^- - P_{wp}^+ - 1).
\]
Note in particular that without padding and filters $W_p' >1$ the widths decrease with depth.

Next, we are interested in computing the samples $x_{0,i_0}$ of the input signal that affect a particular sample $x_{L,i_L}$ at the end of the chain (this we call the \emph{receptive field} of the operator). Suppose that at level $l$ we have determined that samples $i_l \in [I_l^-(i_L), I_l^+(i_L)]$ affect $x_{L,i_L}$ and compute the samples at the level below. These are given by the union of filter applications:
\[
 \cup_{I_l^-(i_L) \leq i_l \leq I_l^+(i_L)} \left(-P_{wl}^- + S_l (i_l-1)  + [1, W_l'] \right).
\]
Hence we find the recurrences:
\begin{align*}
 I_{l-1}^-(i_L) &= - P_{wl}^- + S_l (I_l^-(i_L)-1) + 1,
 \\
 I_{l-1}^+(i_L) &= - P_{wl}^- + S_l (I_l^+(i_L)-1) + W_l'.
\end{align*}
Given the base case $I_L^-(i_L) = I_L^+(i_L) = i_L$,  one gets:
\begin{align*}
I_{l}^-(i_L)
&=
1
+ \left(\prod_{p=l+1}^L S_p\right) (i_L - 1)
- \sum_{p =l+1}^L  \left(\prod_{q=l+1}^{p-1} S_q\right)P_{wp}^-,
\\
I_{l}^-(i_L)
&=
1
+ \left(\prod_{p=l+1}^L S_p\right) (i_L - 1)
+ \sum_{p = l+1}^L  \left(\prod_{q=l+1}^{p-1} S_q\right)(W_p' - 1 - P_{wp}^-).
\end{align*}
We can now compute several quantities of interest. First, the receptive field width on the image is:
\[
 \Delta
 = I_0^+(i_L) - I_0^-(i_L) + 1
 = 1+ \sum_{p = 1}^L  \left(\prod_{q=1}^{p-1} S_q\right)(W_p' - 1).
\]
Second, the leftmost sample of the input signal $x_{0,i_0}$ affecting output sample $x_{l,i_L}$ is
\[
i_0(i_L)  =
1
+ \left(\prod_{p=1}^L S_p\right) (i_L - 1)
- \sum_{p =1}^L  \left(\prod_{q=1}^{p-1} S_q\right)P_{wp}^-
\]
Note that the effect of padding at different layers accumulates, resulting in an overall padding potentially larger than the padding $P_{w1}^-$ specified by the first operator (i.e. $i_0(1) \leq - P_{w1}^-$).

Finally, it is interesting to reparametrise these coordinates as if the discrete signal were continuous and if (as it is usually the case) all the convolutional operators are ``centred''. For example, the operators could be filters with an odd size $W_l'$ equal to the delta function (e.g. for $W_l'=5$ then $f_l = (0,0,1,0,0)$). Let $u_L = i_L$ be the ``continuous'' version of index $i_L$, where each discrete sample corresponds, as per MATLAB's convention, to a tile of extent $u_L \in [-1/2,1/2] + i_L$ (hence $u_L$ is the centre coordinate of a sample). As before, $x_L(u_L)$ is obtained by applying an operator to the input signal $x_0$; the centre of the support of this operator falls at coordinate:
\[
  u_0(u_L) = \alpha\, (u_L - 1) + \beta = i_0(u_L) + \frac{\Delta-1}{2} = \alpha\, (u_L - 1) + \beta.
\]
Hence
\[
\alpha = \prod_{p=1}^L S_p,
\qquad
\beta
 =
1
+
\sum_{p = 1}^L  \left(\prod_{q=1}^{p-1} S_q\right)
\left(
\frac{W_p' - 1}{2}  - P_{wp}^-
\right).
\]
Note in particular that the offset is zero if $S_1 =\dots=S_L=1$ and $P_{wp}^- = (W_p'-1)/2$ as in this case the padding is just enough such that the leftmost application of each convolutional operator has the centre that falls on the first sample of the corresponding input signal.


\section{Derivatives of Inverse Functions}\label{sec:DerivativesofInverse}
Suppose we wanted to find the \ifont{derivative of the inverse}, but do not have an actual formula for the inverse function?
Then we can use the following derivative formula for the inverse evaluated at $a$.

%\begin{formulabox}[Derivative of $f^{-1}(x)$]
%Given an invertible function $f(x)$, the derivative of its inverse function $f^{-1}(x)$ evaluated at $x=a$ is:
%$$\left[f^{-1}\right]'(a)=\frac{1}{f'\left[ f^{-1}(a) \right]}$$
%\end{formulabox}

\begin{theorem}{Derivatives of Inverse Functions}{deriv_inverse_functions}
{Let $f$ be differentiable and one to one on an open interval $I$, where $\fp(x) \neq 0$ for all $x$ in $I$, let $J$ be the range of $f$ on $I$, let $g$ be the inverse function of $f$, and let $f(a) = b$ for some $a$ in $I$. Then $g$ is a differentiable function on $J$, and in particular,

%	\begin{center}
\hskip-7pt	\begin{tabular}{ccc}
	1. $\ds \left(f\primeskip^{-1}\right)'(b)=g\primeskip'(b) = \frac{1}{\fp(a)}$ &\hskip 4pt and \hskip 4pt&  2. $\ds \left(f\primeskip^{-1}\right)'(x)=g\primeskip'(x) = \frac{1}{\fp(g(x))}$
	\end{tabular}
%	\end{center}
}
\end{theorem}

To see why this is true, start with the function $y=f^{-1}(x)$.
Write this as $x=f(y)$  and differentiate both sides implicitly with respect to $x$ using the chain rule:
$$1=f'(y)\cdot \frac{dy}{dx}.$$
Thus,
$$\frac{dy}{dx}=\frac{1}{f'(y)},$$
but $y=f^{-1}(x)$, thus,
$$\left[f^{-1}\right]'(x)=\frac{1}{f'\left[ f^{-1}(x) \right]}.$$
At the point $x=a$ this becomes:
$$\left[f^{-1}\right]'(a)=\frac{1}{f'\left[ f^{-1}(a) \right]}$$

In Section \ref{sec:DerivativeExpLog}, we saw that $\ds \frac{d}{dx}\big(\ln x\big) = \frac{1}{x}$. We can justify that now using Theorem \ref{thm:deriv_inverse_functions}, as shown in the example.\\

\begin{example}{Finding the derivative of $y=\ln x$}{ex_deriv_lnx}
{
Use Theorem \ref{thm:deriv_inverse_functions} to compute $\ds \frac{d}{dx}\big(\ln x\big)$.}
{View $y= \ln x$ as the inverse of $y = e^x$. Therefore, using our standard notation, let $f(x) = e^x$ and $g(x) = \ln x$. We wish to find $g\primeskip'(x)$. Theorem \ref{thm:deriv_inverse_functions} gives:
		\begin{align*}
		g\primeskip'(x) &= \frac{1}{\fp(g(x))} \\
					&=	\frac{1}{e^{\ln x}}\rule{0pt}{15pt} \\
					&= \frac{1}{x}.\rule{0pt}{17pt}
		\end{align*}
\vskip-\baselineskip
}
\end{example}

\begin{example}{Derivatives of Inverse Functions}{DerivativesInverseFunctions}
Suppose $f(x)=x^5+2x^3+7x+1$.
Find $\left[f^{-1}\right]'(1)$.
\end{example}

\begin{solution}
First we should show that $f^{-1}$ exists (i.e. that $f$ is one-to-one). In this case the derivative $f'(x)=5x^4+6x^2+7$ is strictly greater than 0 for all $x$, so $f$ is strictly increasing and thus one-to-one.

It's difficult to find the inverse of $f(x)$ (and then take the derivative).
Thus, we use the above formula evaluated at $1$:
$$\left[f^{-1}\right]'(1)=\frac{1}{f'\left[ f^{-1}(1) \right]}.$$
Note that to use this formula we need to know what $f^{-1}(1)$ is, and the derivative $f'(x)$.
To find $f^{-1}(1)$ we make a table of values (plugging in $x=-3,-2,-1,0,1,2,3$ into $f(x)$) and see what value of $x$ gives $1$.
We omit the table and simply observe that $f(0)=1$.
Thus,
$$f^{-1}(1)=0.$$
Now we have:
$$\left[f^{-1}\right]'(1)=\frac{1}{f'\left( 0 \right)}.$$
And so, $f'(0)=7$.
Therefore, $$\left[f^{-1}\right]'(1)=\frac{1}{7}.$$
\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Subsections to include
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\input{4-derivatives/4-9-1-der-inv-trig}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:DerivativesofInverse}}

\begin{enumialphparenastyle}

%%%%%%%%%%
\begin{ex}
	Given $f(x)=1+\ln(x-2)$, first show that $f^{-1}$ exists, then compute $\left[f^{-1}\right]'(1)$.
\begin{sol}
	$1$
\end{sol}
\end{ex}

\begin{ex}
The \dfont{inverse cotangent function}, denoted by $\cot^{-1}(x)$, is defined to be the inverse of the restricted cotangent function: $\cot (x)$, $0<x<\pi$. Find the derivative of $\cot^{-1}(x)$.
\end{ex}

\begin{ex}
The \dfont{inverse secant function}, denoted by $\sec^{-1}(x)$, is defined to be the inverse of the restricted secant function: $\sec(x)$, $x\in[0,\pi/2)\cup[\pi,3\pi/2)$. Find the derivative of $\sec^{-1}(x)$.
\end{ex}

\begin{ex}
The \dfont{inverse cosecant function}, denoted by $\csc^{-1}(x)$, is defined to be the inverse of the restricted cosecant function: $\csc(x)$, $x\in(0,\pi/2]\cup(\pi,3\pi/2]$. Find the derivative of $\csc^{-1}(x)$.
\end{ex}

\begin{ex}
Suppose $f(x)=x^3+4x+2$. Find the slope of the tangent line to the graph of $g(x)=xf^{-1}(x)$ at the point where $x=7$.
\end{ex}

\begin{ex}
Find the derivatives of $\sin^{-1}(x)+\cos^{-1}(x)$ and $(x^2+1)\tan^{-1}(x)$.
\end{ex}

\begin{ex}
Differentiate $y=\sin^{-1}(x^2)$ and $y=\tan^{-1}(3x)$.
\end{ex}

\end{enumialphparenastyle}