\chapter{Finite elements in 2D}

In Chapter~\ref{chap: FEM 1d} we considered a formally self-adjoint, linear,
second-order differential operator~\eqref{eq: L self-adjoint}.  The 2D
equivalent has the form
\begin{equation}\label{eq: L self-adjoint 2d}
\begin{aligned}
\mathcal{L}u&=-\nabla\cdot\bigl(a\nabla u\bigr)+cu\\
	&=-\frac{\partial}{\partial x}\biggl(a\,\frac{\partial u}{\partial x}\biggr)
	-\frac{\partial}{\partial y}\biggl(a\,\frac{\partial u}{\partial y}\biggr)
	+cu,
\end{aligned}
\end{equation}
where the coefficients $a$~and $c$ must be smooth functions of $x$~and $y$, and
there is a constant~$a_{\min}$ such that
\[
a(x,y)\ge a_{\min}>0\quad\text{for $(x,y)\in\Omega$,}
\]
ensuring that $\mathcal{L}$ is \emph{uniformly elliptic}.  Here, as in
Chapter~\ref{chap: finite diff 2d}, $\Omega$ is a bounded open subset
of~$\mathbb{R}^2$ with a piecewise smooth boundary~$\Gamma=\partial\Omega$, but
we will now suppose that
\[
\Gamma=\Gamma_{\mathrm{D}}\cup\Gamma_{\mathrm{N}},
\]
where $\Gamma_{\mathrm{D}}$~and $\Gamma_{\mathrm{N}}$ are non-overlapping,
relatively closed subsets of~$\partial\Omega$ consisting of finitely many
smooth curves.  (Thus, the intersection
$\Gamma_{\mathrm{D}}\cap\Gamma_{\mathrm{N}}$ consists of finitely many
\emph{collision points}.) Our aim is to use the finite element method to compute
numerical solutions to a \emph{mixed boundary-value problem} of the form
\begin{equation}\label{eq: self-adjoint bvp 2d}
\begin{aligned}
\mathcal{L}u&=f&&\text{in~$\Omega$,}\\
u&=g_{\mathrm{D}}&&\text{on~$\Gamma_{\mathrm{D}}$,}\\
a\,\frac{\partial u}{\partial n}&=g_{\mathrm{N}}&&
	\text{on~$\Gamma_{\mathrm{N}}$.}
\end{aligned}
\end{equation}
Here, $\partial u/\partial n$ is the derivative of~$u$ in the direction of the
\emph{outward unit normal}~$\boldsymbol{n}$ for~$\Omega$, that is,
\[
\frac{\partial u}{\partial n}(x,y)
	=\boldsymbol{n}(x,y)\cdot\nabla u(x,y)
	\quad\text{for $(x,y)\in\partial\Omega$.}
\]
We refer to~$\Gamma_{\mathrm{D}}$~and $\Gamma_{\mathrm{N}}$ as the
\emph{Dirichlet}~and \emph{Neumann} parts of the boundary, respectively, since
we specify a Dirichlet boundary condition~$u=g_{\mathrm{D}}$
on~$\Gamma_{\mathrm{D}}$ and a Neumann boundary
condition~$a\partial u/\partial n=g_{\mathrm{N}}$ on~$\Gamma_{\mathrm{N}}$.

In the special case of a \emph{pure Dirichlet problem}, the Neumann part of the
boundary is empty and so $u$ is specified on the whole of~$\Gamma$ (as
in Chapter~\ref{chap: finite diff 2d}).  In the opposite case of a
\emph{pure Neumann problem}, the Dirichlet part of the boundary is empty and so
$a\,\partial u/\partial n$ is specified on the whole of~$\Gamma$.

\section{First Green identity}

Recall the \emph{divergence theorem} from vector calculus; on the right-hand
side, the integral over~$\Gamma$ is with respect to \

\begin{theorem}\label{thm: divergence}
If the vector field $\boldsymbol{F}:\Omega\cup\Gamma\to\mathbb{R}^2$ is $C^1$,
then
\[
\int_\Omega\nabla\cdot\boldsymbol{F}
	=\int_\Gamma\boldsymbol{F}\cdot\boldsymbol{n}.
\]
\end{theorem}

Written out more explicitly, if
\[
\boldsymbol{F}(x,y)=P(x,y)\,\boldsymbol{i}+Q(x,y)\,\boldsymbol{j}
\quad\text{and}\quad
\boldsymbol{n}=n_x\,\boldsymbol{i}+n_y\,\boldsymbol{j},
\]
then the divergence theorem says that
\[
\iint_\Omega\biggl(\frac{\partial P}{\partial x}+\frac{\partial Q}{\partial y}
	\biggr)\,dx\,dy=\int_\Gamma\bigl(P\,n_x+Q\,n_y)\,ds,
\]
where $ds$ is the element of arc length along~$\Gamma$.  We also recall the
following vector field identity.

\begin{lemma}\label{lem: div phi F}
For a $C^1$ scalar field~$\phi$ and a $C^1$ vector field~$\boldsymbol{F}$,
\[
\nabla\cdot(\phi\boldsymbol{F})=(\nabla\phi)\cdot\boldsymbol{F}
	+\phi\,\nabla\cdot\boldsymbol{F}.
\]
\end{lemma}

Together, Theorem~\ref{thm: divergence}~and Lemma~\ref{lem: div phi F} may be
used to prove a 2D version of~\eqref{eq: int by parts}.

\begin{theorem}[First Green Identity]\label{thm: first Green}
If $u:\Omega\cup\Gamma\to\mathbb{R}$ is $C^2$, and if
$v:\Omega\cup\Gamma\to\mathbb{R}$ is $C^1$, then
\[
\int_\Omega(\mathcal{L}u)\,v
	=\int_\Omega\bigl(a\nabla u\cdot\nabla v+cuv\bigr)
	-\int_\Gamma a\,\frac{\partial u}{\partial n}\,v.
\]
\end{theorem}
\begin{proof}
Taking $\phi=v$~and $\boldsymbol{F}=a\nabla u$ in Lemma~\ref{lem: div phi F},
we have
\[
\nabla\cdot(va\nabla u)=(\nabla v)\cdot(a\nabla u)+v\nabla\cdot(a\nabla u),
\]
so
\begin{align*}
(\mathcal{L}u)v&=\bigl(-\nabla\cdot(a\nabla u)+cu\bigr)v
	=-v\nabla\cdot(a\nabla u)+cuv\\
	&=(\nabla v)\cdot(a\nabla u)-\nabla\cdot(va\nabla u)+cuv
	=\bigl(a\nabla u\cdot\nabla v+cuv)-\nabla\cdot(va\nabla u).
\end{align*}
Applying Theorem~\ref{thm: divergence} with~$\boldsymbol{F}=va\nabla u$, it
follows that
\[
\int_\Omega(\mathcal{L}u)v=\int_\Omega\bigl(a\nabla u\cdot\nabla v+cuv\bigr)
	-\int_\Gamma\boldsymbol(va\nabla u)\cdot\boldsymbol{n},
\]
which gives the desired identity because
$(\nabla u)\cdot\boldsymbol{n}=\partial u/\partial n$.
\end{proof}

Since
\[
\int_\Gamma a\,\frac{\partial u}{\partial n}\,v
	=\int_{\Gamma_{\mathrm{D}}} a\,\frac{\partial u}{\partial n}\,v
	+\int_{\Gamma_{\mathrm{N}}} a\,\frac{\partial u}{\partial n}\,v,
\]
we see that if $u$ is a $C^2$ solution of~\eqref{eq: self-adjoint bvp 2d}, and
if $v$ is $C^1$, then
\begin{equation}\label{eq: Lu=f weak 2d}
\int_\Omega\bigl(a\nabla u\cdot\nabla v+cuv\bigr)=\int_\Omega fv
	+\int_{\Gamma_{\mathrm{N}}}g_{\mathrm{N}}v
	\quad\text{provided $v=0$ on $\Gamma_{\mathrm{D}}$.}
\end{equation}
Compare this property with its 1D equivalent~\eqref{eq: Lu=f weak 1d}.

\section{Triangulation and nodal basis}\label{sec: triangulation}

\begin{figure}
\caption{A regular triangulation.}\label{fig: good Th}
\begin{center}
\includegraphics[scale=0.7]{../src/chap6/good_triangulation.pdf}
\end{center}
\end{figure}

\begin{figure}
\caption{A triangulation that fails to be regular.}\label{fig: bad Th}
\begin{center}
\includegraphics[scale=0.7]{../src/chap6/bad_triangulation.pdf}
\end{center}
\end{figure}

Assume now that $\Omega$ is a polygon.  It follows by induction on the number
of vertices that there exists a \emph{triangulation} of~$\Omega$, that is, a
finite set~$\mathcal{T}$ of \emph{non-overlapping} closed triangles whose
union is the closure of~$\Omega$. A triangulation~$\mathcal{T}$ is
\emph{regular} if the following two conditions are satisfied:
\begin{enumerate}
\item No triangle in~$\mathcal{T}$ is \emph{degenerate}, that is,
no~$K\in\mathcal{T}$ has collinear vertices.
\item The intersection $K_1\cap K_2$ of any two distinct triangles $K_1$,
$K_2\in\mathcal{T}$ is either empty, a common edge or a common vertex.
\end{enumerate}
For example, Figure~\ref{fig: good Th} shows a regular triangulation with
7~vertices, or \emph{nodes}, numbered in red, and 6~triangles, or
\emph{elements}, numbered in blue.  (The six boundary edges are
numbered in green.) However, the triangulation in
Figure~\ref{fig: bad Th} is not regular: the intersection of triangles 2~and 6
is an edge of triangle~6 but is not (the whole of) an edge of triangle~2.
Vertex~7 in Figure~\ref{fig: bad Th} is said to be a \emph{hanging node}.

A simple data structure to store a triangulation consists of two arrays,
$\boldsymbol{P}$~and $\boldsymbol{T}$.  The first stores the coordinates of the
$j$th node in its $j$th column, and the second stores the node numbers of the
$k$th triangle in its $k$th column. A third matrix~$\boldsymbol{E}$ is required
to deal with boundary conditions: the $j$th column stores the node numbers of
the $j$th boundary edge.  Thus, the triangulation in
Figure~\ref{fig: good Th} may be described by
\[
\boldsymbol{P}=\begin{bmatrix}
-2& 2& 2& 1&-1&-2& 0\\
-2&-2& 0& 2& 2& 0& 0
\end{bmatrix}
\quad\text{and}\quad
\boldsymbol{T}=\begin{bmatrix}
0&0&1&2&3&4\\
6&1&2&3&4&5\\
5&6&6&6&6&6\end{bmatrix},
\]
with
\[
\boldsymbol{E}=\begin{bmatrix}
1&2&3&4&5&6\\
2&3&4&5&6&1\end{bmatrix}.
\]
To simplify some geometric computations, it is best to ensure that the node
numbering within each triangle proceeds counterclockwise, and that the edge
node numbering follows the induced orientation of~$\partial\Omega$. (Even with
this restriction, there are $3$~possibilities for each column
of~$\boldsymbol{T}$.)

Denote the maximum element diameter
by~$h=\max_{K\in\mathcal{T}}\operatorname{diam}(K)$, and let $V_h$ denote the
vector space consisting of those functions~$v:\Omega\cup\Gamma\to\mathbb{R}$
that are continuous and piecewise-linear with respect to~$\mathcal{T}$.  Thus,
if $v\in V_h$ then for each~$K\in\mathcal{T}$ there are coefficients
$c\brak{K}_0$, $c\brak{K}_1$~and $c\brak{K}_2$ such that
\begin{equation}\label{eq: v K 1 x y}
v(x,y)=c\brak{K}_0+c\brak{K}_1x+c\brak{K}_2y
	\quad\text{for $(x,y)\in K$.}
\end{equation}
Let $\mathsf{n}\brak{K}_1$, $\mathsf{n}\brak{K}_2$, $\mathsf{n}\brak{K}_3$
denote the vertices of the triangle~$K$, and let $\psi\brak{K}_1$,
$\psi\brak{K}_2$, $\psi\brak{K}_3$ denote the unique linear functions
satisfying
\begin{equation}\label{eq: psi node triangle}
\psi\brak{K}_j(\mathsf{n}\brak{K}_i)=\delta_{jk}
	\quad\text{for $i$, $j\in\{1,2,3\}$.}
\end{equation}
In Section~\ref{sec: barycentric}, we will derive explicit representations of
these functions.  The property~\eqref{eq: psi node triangle} implies that if
$v\in V_h$ then
\[
v(x,y)=\sum_{i=1}^3v(\mathsf{n}\brak{K}_i)\psi\brak{K}_i(x,y)
	\quad\text{for $(x,y)\in K$,}
\]
showing that $v$ is uniquely determined by its values at the nodes
of~$\mathcal{T}$.

\begin{figure}
\caption{A piecewise-linear ``tent function'', equal to~$1$ at one node, and
$0$ at all other nodes.}\label{fig: tent func}
\begin{center}
\includegraphics[scale=0.6]{../src/chap6/tent_func.pdf}
\end{center}
\end{figure}

In fact, suppose that $\mathsf{n}_1$, $\mathsf{n}_2$, \dots, $\mathsf{n}_M$ is
an enumeration of the nodes of~$\mathcal{T}$.  For~$1\le r\le M$, we
define~$\chi_r\in V_h$ by requiring
\begin{equation}\label{eq: chi 2d}
\chi_r(\mathsf{n}_s)=\delta_{rs}\quad\text{for $r$, $s\in\{1, 2, \dots, M\}$.}
\end{equation}
Figure~\ref{fig: tent func} shows an example of such a ``tent function''.
If $v\in V_h$, then
\[
v(x,y)=\sum_{r=1}^M v(\mathsf{n}_r)\chi_r(x,y)
	\quad\text{for $(x,y)\in\Omega\cup\Gamma$,}
\]
and we call $\{\chi_1,\chi_2,\ldots,\chi_M\}$ the \emph{nodal basis} for~$V_h$
(Exercise~\ref{ex: nodal basis}).

\section{Finite element method}

Suppose that a regular triangulation~$\mathcal{T}$ is \emph{aligned} with
the decomposition $\Gamma=\Gamma_{\mathrm{D}}\cup\Gamma_{\mathrm{N}}$ of the
boundary of~$\Omega$.  This assumption means that $\Gamma_{\mathrm{D}}$
is a union of edges of triangles in~$\mathcal{T}$ (in which
case, the same must be true of~$\Gamma_{\mathrm{N}}$).  The vertices lying on
the Dirichlet boundary~$\Gamma_{\mathrm{D}}$ are called the \emph{fixed nodes},
because the values of the solution~$u$ are fixed at these points.  The
remaining vertices are called the \emph{free nodes}; these belong
to~$\Omega\cup\Gamma_{\mathrm{N}}$, but note that the collision points, where
$\Gamma_{\mathrm{D}}$~and $\Gamma_{\mathrm{N}}$ meet, are among the fixed nodes.

Suppose that there are $M\free$~free nodes and $M\fix$~fixed nodes, giving a
total of $M=M\free+M\fix$~nodes. It is convenient to number the nodes so that
free ones come first, followed by the fixed ones. That is, $\mathsf{n}_1$,
$\mathsf{n}_2$, \dots, $\mathsf{n}_{M\free}$ are free, and
$\mathsf{n}_{M\free+1}$, $\mathsf{n}_{M\free+2}$, \dots, $\mathsf{n}_M$ are
fixed.

Let $g_{\mathrm{D},h}:\Gamma_{\mathrm{D}}\to\mathbb{R}$ be a piecewise-linear
approximation to~$g_{\mathrm{D}}$.  An obvious choice is the interpolant, so
that
\[
g_{\mathrm{D},h}(\mathsf{n}_r)=g(\mathsf{n}_r)
    \quad\text{for $M\free+1\le r\le M$.}
\]
We define the trial set
\[
S_h=\{\,v\in V_h:\text{$v=g_{\mathrm{D},h}$ on $\Gamma_{\mathrm{D}}$}\,\}
\]
and the test space
\[
T_h=\{\,v\in V_h:\text{$v=0$ on $\Gamma_{\mathrm{D}}$}\,\}.
\]
Recalling \eqref{eq: Lu=f weak 2d}, the finite element
solution~$u_h\in S_h$ is then defined by requiring that
\begin{equation}\label{eq: FEM 2d}
\int_\Omega\bigl(a\nabla u_h\cdot\nabla v+cu_hv\bigr)=\int_\Omega fv
	+\int_{\Gamma_{\mathrm{N}}}g_{\mathrm{N}}v
	\quad\text{for all $v\in T_h$.}
\end{equation}

We expand the finite element solution in the nodal basis, and enforce the
Dirichlet boundary condition to obtain the representation
\begin{equation}\label{eq: uh U 2d}
u_h(x,y)=\sum_{s=1}^MU_s\chi_s(x,y)=\sum_{s=1}^{M\free}U_s\chi_s(x,y)
    +\sum_{s=M\free+1}^M g_{D,h}(\mathsf{n}_s)\chi_s(x,y).
\end{equation}
Since $\{\chi_1,\chi_2,\ldots,\chi_{M\free}\}$ is a basis for the trial
space~$T_h$, the requirement~\eqref{eq: FEM 2d} is equivalent to
\begin{equation}\label{eq: FEM 2d alt}
\int_\Omega\bigl(a\nabla u_h\cdot\nabla\chi_r+cu_h\chi_r\bigr)
    =\int_\Omega f\chi_r
    -\int_{\Gamma_{\mathrm{N}}}g_{\mathrm{N}}\chi_r
    \quad\text{for $1\le r\le M\free$,}
\end{equation}
and so, after inserting the representation~\eqref{eq: uh U 2d}, we obtain an
$M\free\times M\free$~linear system
\begin{equation}\label{eq: FEM 2D linear system}
\sum_{s=1}^{M\free}\bigl(a_{rs}+c_{rs}\bigr)U_s
    =f_r+g_{\mathrm{N},r}-\sum_{s=M\free+1}^M(a_{rs}+c_{rs})g_{\mathrm{D},s}
    \quad\text{for $1\le r\le M\free$,}
\end{equation}
where
\[
a_{rs}=\int_\Omega a\nabla\chi_s\cdot\nabla\chi_r,\qquad
c_{rs}=\int_\Omega c\chi_s\chi_r,\qquad
f_r=\int_\Omega f\chi_r,\qquad
g_{\mathrm{N},r}=\int_{\Gamma_{\mathrm{N}}}g_{\mathrm{N}}\chi_r,
\]
and $g_{\mathrm{D},s}=g_{\mathrm{D},h}(\boldsymbol{n}_s)$. Let
\[
\boldsymbol{U}=[U_s]_{s=1}^M
    =\begin{bmatrix}\boldsymbol{U}\free\\ \boldsymbol{U}\fix \end{bmatrix}
\quad\text{where}\quad
\boldsymbol{U}\free=\begin{bmatrix}U_1\\ U_2\\ \vdots\\ U_{M\free}\end{bmatrix}
\quad\text{and}\quad
\boldsymbol{U}\fix=\begin{bmatrix}U_{M\free+1}\\ U_{M\free+2}\\ \vdots\\ U_M
\end{bmatrix},
\]
and
\[
\boldsymbol{A}=[a_{rs}]_{\substack{1\le r\le M\free\\ 1\le s\le M}}
=[\,\boldsymbol{A}\free\quad\boldsymbol{A}\fix\,]
\]
where
\[
\boldsymbol{A}\free=\begin{bmatrix}
a_{11}&\cdots&a_{1M\free}\\
\vdots&\ddots&\vdots\\
a_{M\free1}&\cdots&a_{M\free M\free}
\end{bmatrix}
\quad\text{and}\quad
\boldsymbol{A}\fix=\begin{bmatrix}
a_{1,M\free+1}&\cdots&a_{1M}\\
\vdots&\ddots&\vdots\\
a_{M\free,M\free+1}&\cdots&a_{M\free,M}
\end{bmatrix}.
\]
Similarly, $\boldsymbol{C}=[\boldsymbol{C}\free\quad\boldsymbol{C}\fix]$, and
we can write the linear system~\eqref{eq: FEM 2D linear system} as
\[
\bigl(\boldsymbol{A}\free+\boldsymbol{C}\free\bigr)\boldsymbol{U}\free
    =\boldsymbol{f}+\boldsymbol{g}_{\mathrm{N}}
-\bigl(\boldsymbol{A}\fix+\boldsymbol{C}\fix\bigr)\boldsymbol{g}_{\mathrm{D}}
\quad\text{and}\quad
\boldsymbol{U}\fix=\boldsymbol{g}_{\mathrm{D}},
\]
with
\[
\boldsymbol{f}=\begin{bmatrix}f_1\\ f_2\\ \vdots\\ f_{M\free}\end{bmatrix},
\qquad
\boldsymbol{g}_{\mathrm{N}}=\begin{bmatrix}
g_{\mathrm{N},1}\\ g_{\mathrm{N},2}\\ \vdots\\
g_{\mathrm{N},M\free}\end{bmatrix},
\qquad
\boldsymbol{g}_{\mathrm{D}}=\begin{bmatrix}
g_{\mathrm{D},M\free+1}]\\ g_{\mathrm{D},M\free+2}]\\ \vdots\\
g_{\mathrm{D},M}]
\end{bmatrix}.
\]

\section{Barycentric coordinates and element matrices}\label{sec: barycentric}

Consider a triangle~$K$ with vertices $\boldsymbol{a}_1$,
$\boldsymbol{a}_2$~and $\boldsymbol{a}_3$.  The \emph{barycentric coordinates}
$(\xi_1,\xi_2,\xi_3)$ of a point~$\boldsymbol{x}$ with respect to~$K$ are
defined by the relations
\[
\boldsymbol{x}=\xi_1\boldsymbol{a}_1+\xi_2\boldsymbol{a}_2
    +\xi_3\boldsymbol{a}_3
    \quad\text{and}\quad
\xi_1+\xi_2+\xi_3=1.
\]
Thus, $\boldsymbol{a}_1$, $\boldsymbol{a}_2$, $\boldsymbol{a}_3$ have
barycentric coordinates $(1,0,0)$, $(0,1,0)$, $(0,0,1)$, respectively. We will
see below that each~$\xi_p$ is a linear function of~$\boldsymbol{x}$, so
\begin{equation}\label{eq: xi psi}
\xi_p=\psi_p\brak{K}(\boldsymbol{x})
    \quad\text{for $1\le p\le 3$,}
\end{equation}
where the $\psi_p\brak{K}$ are the linear linear shape functions introduced in
Section~\ref{sec: triangulation}.  Thus, a level set of any
barycentric coordinate is a straight line, as illustrated
in~\ref{fig: barycentric}.

\begin{figure}
\caption{Level sets of the barycentric coordinates
$(\xi_1,\xi_2,\xi_3)$, and the centroid~$\boldsymbol{c}$.}
\label{fig: barycentric}
\begin{center}
\includegraphics[scale=0.75]{../src/chap6/barycentric.pdf}
\end{center}
\end{figure}

Let $\boldsymbol{B}$ denote the inverse transpose
(Exercise~\ref{ex: inv transpose}) of the matrix with columns
$\boldsymbol{a}_1-\boldsymbol{a}_3$~and $\boldsymbol{a}_1-\boldsymbol{a}_3$,
that is,
\[
\boldsymbol{B}=[\,(\boldsymbol{a}_1-\boldsymbol{a}_3)\quad
    (\boldsymbol{a}_1-\boldsymbol{a}_3)\,]^{-T},
\]
and write
\begin{equation}\label{eq: bp vector def}
\boldsymbol{B}=[\,\boldsymbol{b}_1\quad\boldsymbol{b}_2\,]
\quad\text{with}\quad\boldsymbol{b}_3=-(\boldsymbol{b}_1+\boldsymbol{b}_2).
\end{equation}
The \emph{centroid} of~$K$ is the point
\[
\boldsymbol{c}=\tfrac13\boldsymbol{a}_1+\tfrac13\boldsymbol{a}_2
+\tfrac13\boldsymbol{a}_3
\]
with barycentric coordinates~$(\tfrac13,\tfrac13,\tfrac13)$. The barycentric
coordinates of a given point~$\boldsymbol{x}$ can be computed as follows.

\begin{theorem}\label{thm: barycentric}
With the notation above,
\[
\xi_j=\tfrac13+\boldsymbol{b}_j\cdot(\boldsymbol{x}-\boldsymbol{c})
\quad\text{for $j\in\{1,2,3\}$.}
\]
\end{theorem}
\begin{proof}
Since $\xi_1+\xi_2+\xi_3=1$ we can express $\boldsymbol{x}$ in terms of
$\xi_1$~and $\xi_2$ only,
\[
\boldsymbol{x}=\xi_1\boldsymbol{a}_1+\xi_2\boldsymbol{a}_2
    +(1-\xi_1-\xi_2)\boldsymbol{a}_3
    =\boldsymbol{a}_3+\xi_1(\boldsymbol{a}_1-\boldsymbol{a}_3)
    +\xi_2(\boldsymbol{a}_2-\boldsymbol{a}_3).
\]
Thus,
\[
\boldsymbol{x}-\boldsymbol{a}_3
=[\,(\boldsymbol{a}_1-\boldsymbol{a}_3)\quad
    (\boldsymbol{a}_2-\boldsymbol{a}_3\,]
\begin{bmatrix}\xi_1\\ \xi_2 \end{bmatrix}
=B^{-T}\begin{bmatrix}\xi_1\\ \xi_2 \end{bmatrix}
\]
and so
\[
\begin{bmatrix}\xi_1\\ \xi_2 \end{bmatrix}=B^T(\boldsymbol{x}-\boldsymbol{a}_3)
=\begin{bmatrix}\boldsymbol{b}_1^T\\ \boldsymbol{b}_2^T\end{bmatrix}
    (\boldsymbol{x}-\boldsymbol{a}_3)
=\begin{bmatrix}\boldsymbol{b}_1\cdot(\boldsymbol{x}-\boldsymbol{a}_3)\\
\boldsymbol{b}_2\cdot(\boldsymbol{x}-\boldsymbol{a}_3)\end{bmatrix},
\]
that is,
\[
\xi_1=\boldsymbol{b}_1\cdot(\boldsymbol{x}-\boldsymbol{a}_3)
\quad\text{and}\quad
\xi_2=\boldsymbol{b}_2\cdot(\boldsymbol{x}-\boldsymbol{a}_3),
\]
Since $\boldsymbol{a}_1$, $\boldsymbol{a}_2$, $\boldsymbol{a}_3$ have
barycentric coordinates $(1,0,0)$, $(0,1,0)$, $(0,0,1)$, respectively,
\[
\boldsymbol{b}_1\cdot(\boldsymbol{a}_1-\boldsymbol{a}_3)
=1=\boldsymbol{b}_2\cdot(\boldsymbol{a}_2-\boldsymbol{a}_3)
\]
whereas
\[
\boldsymbol{b}_1\cdot(\boldsymbol{a}_2-\boldsymbol{a}_3)
=0=\boldsymbol{b}_2\cdot(\boldsymbol{a}_1-\boldsymbol{a}_3),
\]
so, for $j\in\{1,2\}$,
\[
\boldsymbol{b}_j\cdot(\boldsymbol{c}-\boldsymbol{a}_3)
    =\boldsymbol{b}_j\cdot(
\tfrac13\boldsymbol{a}_1+\tfrac13\boldsymbol{a}_2+\tfrac13\boldsymbol{a}_3
    -\boldsymbol{a}_3)
    =\tfrac13\boldsymbol{b}_j\cdot(\boldsymbol{a}_1-\boldsymbol{a}_3)
    +\tfrac13\boldsymbol{b}_j\cdot(\boldsymbol{a}_2-\boldsymbol{a}_3)
    =\tfrac13
\]
and hence
\[
\xi_j=\boldsymbol{b}_j\cdot(
    \boldsymbol{x}-\boldsymbol{c}+\boldsymbol{c}-\boldsymbol{a}_3)
    =\boldsymbol{b}_j\cdot(\boldsymbol{x}-\boldsymbol{c})+\tfrac13.
\]
Finally,
\[
\xi_3=1-\xi_2-\xi_3=(1-\tfrac13-\tfrac13)
    -(\boldsymbol{b}_1+\boldsymbol{b}_2)\cdot
    (\boldsymbol{x}-\boldsymbol{a}_3)
    =\tfrac13+\boldsymbol{b}_3\cdot(\boldsymbol{x}-\boldsymbol{a}_3).
\]
\end{proof}

We define the triangular reference element
\[
K_{\mathrm{ref}}=\{\,(\xi_1,\xi_2):
    \text{$0\le\xi_1\le1$ and $0\le\xi_2\le\xi_1$}\,\}
\]
with reference nodes $\boldsymbol{n}_p=\boldsymbol{a}_p$ for~$1\le p\le 3$,
and observe that the affine transformation $K_{\mathrm{ref}}\to K$ defined by
\[
(\xi_1,\xi_2)\mapsto\boldsymbol{x}(\xi_1,\xi_2)=\boldsymbol{a}_3
    +\xi_1(\boldsymbol{a}_1-\boldsymbol{a}_3)
    +\xi_2(\boldsymbol{a}_2-\boldsymbol{a}_3)
\]
is one-one and onto, with Jacobian determinant
\[
\frac{\partial(x_1,x_2)}{\xi_1,\xi_2})
    =\det[\,(\boldsymbol{a}_1-\boldsymbol{a}_3)\quad
            (\boldsymbol{a}_2-\boldsymbol{a}_3)\,].
\]
Since $\bigl|\det[\,(\boldsymbol{a}_1-\boldsymbol{a}_3)\quad
(\boldsymbol{a}_2-\boldsymbol{a}_3)\,]\bigr|$ equals the area of the
parallelogram spanned by the vectors $\boldsymbol{a}_1-\boldsymbol{a}_3$~and
$\boldsymbol{a}_2-\boldsymbol{a}_3$, which in turns equals $2\mathrm{area}(K)$,
it follows that
\begin{equation}\label{eq: int K f}
\int_Kf=2\,\mathrm{area}(K)\int_0^1\int_0^{1-\xi_1}
    f\bigl(\boldsymbol{x}(\xi_1,\xi_2)\bigr)\,d\xi_2\,d\xi_1.
\end{equation}
The formula in the next theorem allows us to compute $\int_Kf$ for any
polynomial~$f$.

\begin{theorem}\label{thm: int xi}
For all non-negative integers $n_1$, $n_2$, $n_3$,
\[
\int_K\xi_1^{n_1}\xi_2^{n_2}\xi_3^{n_3}=2\,\mathrm{area}(K)\,
    \frac{n_1!\,n_2!\,n_3!}{(n_1+n_2+n_3+2)!}.
\]
\end{theorem}
\begin{proof}
By~\eqref{eq: int K f},
\begin{align*}
\int_K\xi_1^{n_1}\xi_2^{n_2}\xi_3^{n_3}&=2\,\mathrm{area}(K)
    \int_0^1\int_0^{1-\xi_1}\xi_1^{n_1}\xi_2^{n_2}(1-\xi_1-\xi_2)^{n_3}
        \,d\xi_2\,d\xi_1\\
    &=2\,\mathrm{area}(K)
    \int_0^1\xi_1^{n_1}\int_0^{1-\xi_1}\xi_2^{n_2}(1-\xi_1-\xi_2)^{n_3}
        \,d\xi_2\,d\xi_1.
\end{align*}
Integrating by parts $m$~times, we see that for any~$a>0$,
\begin{align*}
\int_0^a\,\frac{\xi^n}{n!}\,\frac{(a-\xi)^m}{m!}\,d\xi
&=\int_0^a\,\frac{\xi^{n+1}}{(n+1)!}\,\frac{(a-\xi)^{m-1}}{(m-1)!}\,d\xi\\
&=\cdots
=\int_0^a\,\frac{\xi^{n+m}}{(n+m)!}\,d\xi=\frac{a^{n+m+1}}{(n+m+1)!},
\end{align*}
so
\begin{multline*}
\int_0^1\xi_1^{n_1}\int_0^{1-\xi_1}\xi_2^{n_2}(1-\xi_1-\xi_2)^{n_3}
        \,d\xi_2\,d\xi_1
    =n_2!\,n_3!\int_0^1\xi_1^{n_1}\int_0^{1-\xi_1}
    \frac{\xi_2^{n_2}}{n_2!}\,\frac{(1-\xi_1-\xi_2)^{n_3}}{n_3!}
        \,d\xi_1\,d\xi_1\\
    =n_1!\,n_2!\,n_3!\int_0^1\frac{\xi_1^{n_1}}{n_1!}\,
        \frac{(1-\xi_1)^{n_2+n_3+1}}{(n_2+n_3+1)!}\,d\xi_1
    =\frac{n_1!\,n_2!\,n_3!}{(n_1+n_2+n_3+2)!},
\end{multline*}
giving the desired formula.
\end{proof}

\begin{figure}
\caption{The vectors $\boldsymbol{b}_1$, $\boldsymbol{b}_2$~and
$\boldsymbol{b}_3$ defined in~\eqref{eq: bp vector def}.}\label{fig: b vectors}
\begin{center}
\includegraphics[scale=0.8]{../src/chap6/b_vectors.pdf}
\end{center}
\end{figure}

Recalling \eqref{eq: xi psi}~and Theorem~\ref{thm: barycentric}, the linear
shape functions are given by
\[
\psi_j(\boldsymbol{x})=\xi_j
    =\tfrac13+\boldsymbol{b}_j\cdot(\boldsymbol{x}-\boldsymbol{a}_j)
    \quad\text{for $j\in\{1,2,3\}$,}
\]
and so
\[
\nabla\psi_j=\boldsymbol{b}_j\quad\text{for $j\in\{1,2,3\}$.}
\]
Thus, $\boldsymbol{b}_j$ is orthogonal to the level sets of~$\psi_j$ and in
particular to the side of~$K$ opposite~$\boldsymbol{a}_j$, as shown in
Figure~\ref{fig: b vectors}.

Using the same approach as in the 1D case in
Section~\ref{sec: matrix assembly 1d}, we will assemble the
$M\times(M+R)$ global stiffness
matrix~$\boldsymbol{A}=[\,\boldsymbol{A}'\quad\boldsymbol{A}'']$ from
the $3\times3$ element stiffness matrices
\[
\boldsymbol{A}\brak{K}=\bigl[a\brak{K}_{ij}\bigr]_{i,j=1}^3
\quad\text{where}\quad
a\brak{K}_{ij}=\int_Ka\nabla\psi_j\brak{K}\cdot\nabla\psi\brak{K}_i.
\]
Likewise, the $M\times(M+R)$ global mass
matrix~$\boldsymbol{C}=[\,\boldsymbol{C}'\quad\boldsymbol{C}'']$ will be
assembled from the $3\times3$ element mass matrices,
\[
\boldsymbol{C}\brak{K}=\bigl[c\brak{K}_{ij}\bigr]_{i,j=1}^3
\quad\text{where}\quad
c\brak{K}_{ij}=\int_Kc\psi_j\brak{K}\psi\brak{K}_i.
\]
Explicitly, the entries of the element stiffness matrix are
\[
a\brak{K}_{ij}=\boldsymbol{b}_j\cdot\boldsymbol{b}_i\int_K a
    =\boldsymbol{b}_j\cdot\boldsymbol{b}_i\int_0^1\int_0^{1-\xi_1}
        a\bigl(\boldsymbol{x}(\xi_1,\xi_2)\bigr)\,d\xi_2\,d\xi_1,
\]
and those of the element mass matrix are
\[
c\brak{K}_{ij}=\int_Kc\xi_j\xi_i=\int_0^1\int_0^{1-\xi_1}
        c\bigl(\boldsymbol{x}(\xi_1,\xi_2)\bigr)\xi_j\xi_i\,d\xi_2\,d\xi_1,
\]
remembering that $\xi_3=1-\xi_1-\xi_2$.
In particular, if $a(\boldsymbol{x})=1$~and $c(\boldsymbol{x})=1$, then
Theorem~\ref{thm: int xi} shows that
\[
\newcommand{\bb}{\boldsymbol{b}}
\boldsymbol{A}\brak{K}=\mathrm{area}(K)\begin{bmatrix}
\bb_1\cdot\bb_1&\bb_2\cdot\bb_1&\bb_3\cdot\bb_1\\
\bb_1\cdot\bb_2&\bb_2\cdot\bb_2&\bb_3\cdot\bb_2\\
\bb_1\cdot\bb_3&\bb_2\cdot\bb_3&\bb_3\cdot\bb_3
\end{bmatrix}
\quad\text{and}\quad
\boldsymbol{C}\brak{K}=\frac{\mathrm{area}(K)}{12}\begin{bmatrix}
2&1&1\\ 1&2&1\\ 1&1&2 \end{bmatrix}.
\]

The 3-dimensional element load vector is defined by
\[
\boldsymbol{f}\brak{K}=[f\brak{K}_i]_{i=1}^3
\quad\text{where}\quad
f\brak{K}_i=\int_K f\psi\brak{K}_i
	=\int_0^1\int_0^{1-\xi_1}f\bigl(\boldsymbol{x}(\xi_1,\xi_2)\bigr)\xi_i
	\,d\xi_2 \,d\xi_1
\]

\section{Quadratic elements}

\begin{figure}
\caption{The midpoint $\boldsymbol{m}_j$ of the side opposite to the
vertex~$\boldsymbol{a}_j$ for $j\in\{1,2,3\}$.}
\label{fig: midpoints}
\begin{center}
\includegraphics[scale=0.75]{../src/chap6/midpoints.pdf}
\end{center}
\end{figure}

\begin{figure}
\caption{Quadratic shape functions}
\label{fig: quad shape funcs}
\hfil
\includegraphics[scale=0.8]{../src/chap6/quad_shape_funcs/psi1-crop.pdf}
\hfil
\includegraphics[scale=0.8]{../src/chap6/quad_shape_funcs/psi2-crop.pdf}
\hfil
\includegraphics[scale=0.8]{../src/chap6/quad_shape_funcs/psi3-crop.pdf}
\hfil
\\[2\jot]
\includegraphics[scale=0.8]{../src/chap6/quad_shape_funcs/psi4-crop.pdf}
\hfil
\includegraphics[scale=0.8]{../src/chap6/quad_shape_funcs/psi5-crop.pdf}
\hfil
\includegraphics[scale=0.8]{../src/chap6/quad_shape_funcs/psi6-crop.pdf}
\hfil
\end{figure}

A general quadratic polynomial in two variables $x_1$~and $x_2$ has the form
\[
v(x_1,x_2)=a_{00}+a_{10}x_1+a_{01}x_2+a_{20}x_1^2+a_{11}x_1x_2+a_{02}x_2^2.
\]
Thus, the space of all such polynomials has dimension~$6$ and we therefore
require $6$ nodes in our triangular element~$K$. For $j\in\{1,2,3\}$, let
$\boldsymbol{m}_j$ denote the midpoint of the side of the triangle~$K$ opposite
the vertex~$\boldsymbol{a}_j$, as illustrated in
\cref{fig: midpoints}.  The barycentric coordinates of $\boldsymbol{m}_1$,
$\boldsymbol{m}_2$, $\boldsymbol{m}_3$ are then $(0,\tfrac12,\tfrac12)$,
$(\tfrac12,0,\tfrac12)$, $(\tfrac12,\tfrac12,0)$, respectively.  We then define
our 6~nodes in the element~$K$ by
\[
\boldsymbol{n}\brak{K}_1=\boldsymbol{a}_1,\quad
\boldsymbol{n}\brak{K}_2=\boldsymbol{a}_2,\quad
\boldsymbol{n}\brak{K}_3=\boldsymbol{a}_3,\quad
\boldsymbol{n}\brak{K}_4=\boldsymbol{m}_1,\quad
\boldsymbol{n}\brak{K}_5=\boldsymbol{m}_2,\quad
\boldsymbol{n}\brak{K}_6=\boldsymbol{m}_3,
\]
and the corresponding quadratic shape functions~$\psi\brak{K}_j$
for~$1\le \le 6$ by requiring that
\[
\psi\brak{K}_j(\boldsymbol{n}\brak{K}_i)=\delta_{ij}
\quad\text{for $i$, $j\in\{1,2,3,4,5,6\}$.}
\]
As functions of the barycentric coordinates, we easily verify that
\[
\psi\brak{K}_1(\boldsymbol{x})=2\xi_1(\xi_1-\tfrac12),\qquad
\psi\brak{K}_2(\boldsymbol{x})=2\xi_2(\xi_2-\tfrac12),\qquad
\psi\brak{K}_3(\boldsymbol{x})=2\xi_3(\xi_3-\tfrac12),
\]
and
\[
\psi\brak{K}_4(\boldsymbol{x})=4\xi_2\xi_3,\qquad
\psi\brak{K}_5(\boldsymbol{x})=4\xi_3\xi_1,\qquad
\psi\brak{K}_6(\boldsymbol{x})=4\xi_1\xi_2.
\]
\cref{fig: quad shape funcs} plots these functions for a typical triangle.
Since $\nabla\xi_j=\boldsymbol{b}_j$,
\[
\nabla\psi\brak{K}_j(\boldsymbol{x})=(4\xi_j-1)\boldsymbol{b}_j
\quad\text{for $1\le j\le 3$,}
\]
and
\begin{align*}
\nabla\psi\brak{K}_4(\boldsymbol{x})
    &=4(\xi_2\boldsymbol{b}_3+\xi_3\boldsymbol{b}_2),\\
\nabla\psi\brak{K}_5(\boldsymbol{x})
    &=4(\xi_3\boldsymbol{b}_1+\xi_1\boldsymbol{b}_3),\\
\nabla\psi\brak{K}_6(\boldsymbol{x})
    &=4(\xi_1\boldsymbol{b}_2+\xi_2\boldsymbol{b}_1).
\end{align*}
Using \cref{thm: int xi}

\section{Optimality property}

\begin{Exercises}

\exercise
Derive the weak formulation of the boundary-value
problem~\eqref{eq: self-adjoint bvp 2d} if we replace the Neumann boundary
condition with the more general Robin boundary condition
\[
a\,\frac{\partial u}{\partial n}+bu=g_{\mathrm{N}}
    \quad\text{on $\Gamma_{\mathrm{N}}$.}
\]
\begin{ans}
The weak solution $u$ satisfies
\[
\int_\Omega(a\nabla u\cdot\nabla v+cuv)+\int_{\Gamma_{\mathrm{N}}}buv
    =\int_\Omega fv+\int_{\Gamma_{\mathrm{N}}}g_{\mathrm{N}}v
    \quad\text{whenever $v=0$ on $\Gamma_{\mathrm{N}}$,}
\]
with $u=g_{\mathrm{D}}$ on~$\Gamma_{\mathrm{D}}$
\end{ans}

\exercise
Derive the weak formulation for the elliptic eigenproblem
\[
\begin{aligned}
\mathcal{L}\phi&=\lambda b\phi&&\text{in $\Omega$},\\
\phi&=0&&\text{on $\Gamma_{\mathrm{D}}$,}\\
a\,\frac{\partial\phi}{\partial n}&=0&&\text{on $\Gamma_{\mathrm{N}}$,}
\end{aligned}
\]
where, as usual, $\mathcal{L}\phi=-\nabla\cdot(a\nabla\phi)+c\phi$.
\begin{ans}
The eigenpair $(\phi,\lambda)$ satisfies
\[
\int_\Omega(a\nabla\phi\cdot\nabla v+c\phi v)=\lambda\int_\Omega b\phi v
\quad\text{whenever $v=0$ on $\Gamma_{\mathrm{D}}$,}
\]
with $\phi=0$ on~$\Gamma_{\mathrm{D}}$.
\end{ans}

\exercise\label{ex: drawn triang}
Consider the triangulation~$\mathcal{K}$ determined by the nodal coordinate
matrix
\[
\boldsymbol{N}=\begin{bmatrix}
1&0&1&0&-1& 0&-1\\
1&1&0&0& 0&-1&-1\end{bmatrix}
\]
and the triangle connectivity matrix
\[
\boldsymbol{T}^{\mathcal{K}}=\begin{bmatrix}
1&4&4&3&4&5\\
2&3&2&4&5&7\\
3&2&5&6&6&6\end{bmatrix}.
\]
\begin{description}
\item{(i)} Draw $\mathcal{K}$, numbering the nodes and triangles.
\item{(ii)} Enumerate the outer edges, given that the edge connectivity
matrix is
\[
\boldsymbol{T}^{\mathcal{E}}=\begin{bmatrix}
6&3&1&2&5&7\\
3&1&2&5&7&6\end{bmatrix}.
\]
\item{(iii)} Determine $\Gamma_{\mathrm{D}}$~and $\Gamma_{\mathrm{N}}$ assuming
$M^{\textrm{free}}=4$.
\end{description}
\begin{ans}
(i) See \cref{fig: drawn triang}\quad (ii) Edges are numbered in italic\quad
(iii) $\Gamma_{\mathrm{N}}$ consists of edges
\emph{1}--\emph{4}; $\Gamma_{\mathrm{D}}$ consists of edges \emph{5}~and
\emph{6}.
\begin{figure}
\caption{Triangulation for Exercise~\ref{ex: drawn triang}}
\label{fig: drawn triang}
\begin{center}
\includegraphics[scale=1.0]{../src/chap6/ex1_triangulation.pdf}
\end{center}
\end{figure}
\end{ans}

\exercise
Suppose that $\Omega=\Omega_1\cup\Gamma_{\mathrm{i}}\cup\Omega_2$ where the
\emph{interface}~$\Gamma_{\mathrm{i}}$ is a piecewise smooth curve, and that
\[
a(x,y)=\begin{cases}
a_1(x,y)&\text{for $(x,y)\in\Omega_1$,}\\
a_2(x,y)&\text{for $(x,y)\in\Omega_2$.}
\end{cases}
\]
Define corresponding partial differential operators
$\mathcal{L}_ku=-\nabla\cdot(a_k\nabla u)$ on $\Omega_k$ for~$k\in\{1,2\}$.
Suppose that
\[
\mathcal{L}_ku_k=f_k\quad\text{on $\Omega_k$ for $k\in\{1,2\}$,}
\]
and define
\[
u(x,y)=\begin{cases}
u_1(x,y)&\text{for $(x,y)\in\Omega_1$,}\\
u_2(x,y)&\text{for $(x,y)\in\Omega_2$,}
\end{cases}
\qquad\text{and}\qquad
f(x,y)=\begin{cases}
f_1(x,y)&\text{for $(x,y)\in\Omega_1$,}\\
f_2(x,y)&\text{for $(x,y)\in\Omega_2$.}
\end{cases}
\]
Under what condition(s) on $u_1$~and $u_2$ do $u$~and $f$ satisfy
\[
\int_\Omega a\nabla u\cdot\nabla v
    =\int_\Omega fv
    -\int_{\partial\Omega}a\,\frac{\partial u}{\partial n}\,v
\]
for any test function~$v$?  Hint: let $\boldsymbol{n}_{\mathrm{i}}$ denote the
unit normal along~$\Gamma_{\mathrm{i}}$, outward to~$\Omega_1$ and inward
to~$\Omega_2$.
\begin{ans}
Both $u$ and its flux~$a\nabla u$ must be continuous across the interface, that
is, $u_1=u_2$ and
$a_1\boldsymbol{n}_{\mathrm{i}}u_1=a_2\boldsymbol{n}_{\mathrm{i}}u_2$
on~$\Gamma_{\mathrm{i}}$.
\end{ans}

\exercise\label{ex: nodal basis}
Prove that the functions $\chi_j\in V_h$ satisfying~\eqref{eq: chi 2d} form a
basis for the piecewise-linear, finite element space~$V_h$, that is, prove that
the nodal basis really is a basis.

\exercise\label{ex: inv transpose}
Let $\boldsymbol{A}$ be a nonsingular matrix.
Show that $(\boldsymbol{A}^{-1})^T=(\boldsymbol{A}^T)^{-1}$; we denote this
\emph{inverse transpose} matrix by~$\boldsymbol{A}^{-T}$.

\exercise
Consider a three-point quadrature rule of the form
\begin{equation}\label{eq: 3 quad}
\int_Kf\approx\frac{\mathrm{area}(K)}{3}\sum_{p=1}^3
    f(\boldsymbol{x}\brak{K}_p)
\end{equation}
where, for some choice of the parameter~$\lambda\in(0,1)$,
\begin{equation}\label{eq: 3 quad points}
\begin{aligned}
\boldsymbol{x}\brak{K}_1=(1-2\lambda)\boldsymbol{a}_1+\lambda\boldsymbol{a}_2
    +\lambda\boldsymbol{a}_3,\\
\boldsymbol{x}\brak{K}_2=\lambda\boldsymbol{a}_1+(1-2\lambda)\boldsymbol{a}_2
    +\lambda\boldsymbol{a}_3,\\
\boldsymbol{x}\brak{K}_3=\lambda\boldsymbol{a}_1+\lambda\boldsymbol{a}_2
    +(1-2\lambda)\boldsymbol{a}_3.
\end{aligned}
\end{equation}
Verify that this rule integrates all quadratic polynomials exactly iff
$\lambda=1/6$~or $1/2$.  See Figure~\ref{fig: quadrature points}.
\begin{figure}
\caption{Quadrature points \eqref{eq: 3 quad points} for the
rule~\eqref{eq: 3 quad} with $\lambda=1/6$ (left) and $\lambda=1/2$ (right).}
\label{fig: quadrature points}
\begin{center}
\includegraphics[scale=0.85]{../src/chap6/quadrature_points-crop.pdf}
\end{center}
\end{figure}

\begin{exercise}
To achieve higher accuracy, we can consider a six-point quadrature rule
\[
\int_K f\approx\frac{\mathrm{area}(K)}{3}\biggl(
	w\sum_{p=1}^3 f\bigl(\boldsymbol{x}\brak{K}_p(\lambda_1)\bigr)
	+(1-w)\sum_{p=1}^3 f\bigl(\boldsymbol{x}\brak{K}_p(\lambda_2)\bigr)\biggr),
\]
depending on the parameters $w$, $\lambda_1$~and $\lambda_2$, with
$\boldsymbol{x}\brak{K}_p=\boldsymbol{x}\brak{K}_p(\lambda)$ defined as
in~\eqref{eq: 3 quad points}.
\end{exercise}

\begin{figure}
\caption{Triangulation for Exercise~\ref{ex: 2019 exam problem}.}
\label{fig: 2019 exam problem}
\begin{center}
\begin{tikzpicture}[scale=0.70]
\draw[-,very thick] (-6,0) -- (0,-6);
\draw[-] (0,-6) -- (6,0) -- (6,6) -- (3,9) -- (0,6) -- (-6,0);
\draw[-] (-6,0) -- (6,0);
\draw[-] (0,-6) -- (0,6) -- (6,6) -- (0,0);
\node at (3,7.5) {\textbf{1}};
\node at (2,4) {\textbf{2}};
\node at (-2,2) {\textbf{3}};
\node at (4,2) {\textbf{4}};
\node at (-2,-2) {\textbf{5}};
\node at (2,-2) {\textbf{6}};
\draw[fill,white] (3,9)  circle (0.30); \draw (3,9) circle (0.30);
\node at (3, 9) {$1$};
\draw[fill,white] (0,6)  circle (0.30); \draw (0,6)  circle (0.30);
\node at (0, 6) {$2$};
\draw[fill,white] (6,6)  circle (0.30); \draw (6,6)  circle (0.30);
\node at  (6, 6) {$3$};
\draw[fill,white] (0,0)  circle (0.30); \draw (0,0)  circle (0.30);
\node at  (0, 0) {$4$};
\draw[fill,white] (6,0)  circle (0.30); \draw (6,0)  circle (0.30);
\node       at  (6, 0) {$5$};
\draw[fill,white] (-6,0) circle (0.30); \draw (-6,0) circle (0.30);
\node at (-6, 0) {$6$};
\draw[fill,white] (0,-6) circle (0.30); \draw (0,-6) circle (0.30);
\node       at  (0,-6) {$7$};
\node at (5.4,6.3) {*};
\node at (5.4,5.7) {*};
\node at (-0.3,0.5) {*};
\node at (0.7,0.3) {*};
\node at (-0.3,-5.4) {*};
\node at (5.3,-0.4) {*};
\node[below left] at (-3,-3) {$\Gamma_{\mathrm{D}}$};
\end{tikzpicture}
\end{center}
\end{figure}

\exercise\label{ex: 2019 exam problem}
Consider the triangulation shown in Figure~\ref{fig: 2019 exam problem}.
The global node numbers are circled and the element numbers are in bold.
The choice of the first node in each element is indicated with an asterisk,
after which the second and third follow \textbf{counterclockwise}. The
part~$\Gamma_{\mathrm{D}}$ of the boundary where a Dirichlet boundary
condition applies is shown in a thicker line (between nodes $6$~and $7$).  Let
$\boldsymbol{f}=[f_r]$ and $\boldsymbol{A}=[a_{rs}]$ denote the global load
vector and the global stiffness matrix, and let
$\boldsymbol{f}\brak{p}=[f\brak{p}_j]$~and
$\boldsymbol{A}\brak{p}=[a\brak{p}_{jk}]$ denote
the element load vector and element stiffness matrix for the $p$th element
($1\le p\le6$).
\begin{description}
\item{(a)} Write out the $3\times6$ connectivity matrix.
\item{(b)} Express $f_4$ as a sum over entries~$f\brak{p}_j$ of the element
load vectors.
\item{(c)} Express $a_{22}$, $a_{35}$~and $a_{47}$ as sums over
entries~$a\brak{p}_{jk}$ of the element matrices.
\end{description}

\begin{figure}
\caption{Triangulation for Exercise~\ref{ex: FEM triang}.}
\label{fig: FEM triang}
\begin{center}
\includegraphics[scale=1.0]{../src/chap6/ex2_triangulation-crop.pdf}
\end{center}
\end{figure}

\exercise\label{ex: FEM triang}
Consider the finite element method for a boundary-value
problem~\eqref{eq: self-adjoint bvp 2d} using the triangulation shows in
\cref{fig: FEM triang}.  Note that $\Gamma_{\mathrm{D}}$ consists of the bottom
and right sides of~$\Omega$ (that is, the thicker edges numbered $6$--$8$.)
\begin{description}
\item{(i)} What are $M\free$~and $M\fix$, the numbers of free and fixed nodes?
\item{(ii)} What is $Q_{\mathrm{N}}$, the number of edges
along~$\Gamma_{\mathrm{N}}$?
\item{(iii)} Write down the triangle connectivity
matrix~$\boldsymbol{T}^{\mathcal{K}}$.
\item{(iv)} Write down the edge connectivity
matrix~$\boldsymbol{T}^{\mathcal{E}}$.
\item{(v)} What are the dimensions of the global load
vector~$\boldsymbol{f}=[f_r]$, the global stiffness
matrix~$\boldsymbol{A}=[a_{rs}]$ and the global Neumann
vector~$\boldsymbol{g}_{\mathrm{N}}=[g_{\mathrm{N},r}]$?
\item{(vi)} Express each $f_r$ as a sum of entries~$f\brak{p}_i$ from the
element load vectors $\boldsymbol{f}\brak{1}$, $\boldsymbol{f}\brak{2}$, \dots,
$\boldsymbol{f}\brak{10}$.
\item{(vii)} Express each nonzero~$a_{rs}$ as a sum of entries~$a\brak{p}_{ij}$
from the element stiffness matrices $\boldsymbol{A}\brak{1}$,
$\boldsymbol{A}\brak{2}$, \dots, $\boldsymbol{A}\brak{10}$.  (From symmetry, it
suffices to list the cases with $r\le s$.)
\item{(viii)} Express each $g_{\mathrm{N},r}$ as a sum of
entries~$g^{[q]}_{\mathrm{N},r}$ of the edge Neumann vectors
$\boldsymbol{g}^{[1]}_{\mathrm{N}}$, $\boldsymbol{g}^{[2]}_{\mathrm{N}}$, \dots
$\boldsymbol{g}^{[5]}_{\mathrm{N}}$.
\end{description}

\end{Exercises}


\section{Universal kernels(Li Jiang)}

\begin{definition}
	Let $X$ be a non-empty set. Then  a function $k:X\times X \rightarrow \mathbb{K}$ is called a \textbf{kernel} on $X$ if there exists a $\mathbb{K}$-Hilbert space $H$ and a map $\Phi:X\rightarrow H$ such that for all $x,x' \in X$ we have
	\begin{equation}
	k(x,x') = \langle \Phi(x'),\Phi(x) \rangle.
	\end{equation}
	We call $\Phi$ a \textbf{feature map} and $H$ a \textbf{feature space} of $k$.
\end{definition}

Denote
\begin{equation}
K_X = {\rm span} \{k(x,\cdot), x\in X\}.
\end{equation}

\begin{definition}[Universal Kernels]
	A continuous kernel $k$ on a compact metric space $X$ is called \textbf{universal} if $K_X$ is dense in $C(X)$, i.e., for every function $g\in C(X)$ and all $\epsilon>0$ there exists an $f\in K_X$ such that
	\begin{equation}
	\|f-g\|_{\infty} \leq \epsilon.
	\end{equation}
\end{definition}

Given any \textbf{feature map} $\Phi$ and  a \textbf{feature space} $H$ of $k$, we denote
\begin{equation}
S_{\Phi,H} = {\rm span} \{\Phi(x): x\in X\},
\end{equation}
and
\begin{equation}
F_{\Phi,H} = \{f:  \exists w\in H, f(x) = \langle w, \Phi(x) \rangle_H, \forall x\in X\}.
\end{equation}

\begin{lemma}
	\begin{equation}
		K_X = \{f:  \exists w\in S_{\Phi,H}, f(x) = \langle w, \Phi(x) \rangle_H, \forall x\in X\}.
	\end{equation}
\end{lemma}

\begin{lemma}
	\begin{equation}
	F_{\Phi,H} = \{f:  \exists w\in \overline{S_{\Phi,H}}, f(x) = \langle w, \Phi(x) \rangle_H, \forall x\in X\}.
	\end{equation}
\end{lemma}

\begin{proof}
	We only need to notice that
	\begin{equation}
	H = \overline{S_{\Phi,H}} \oplus \overline{S_{\Phi,H}}^{\perp}.
	\end{equation}
	Then for any $w\in H$, there exists a $w_1 \in \overline{S_{\Phi,H}}$, and a $w_2 \in \overline{S_{\Phi,H}}^{\perp}$ such that $w = w_1 + w_2$. Notice that $\langle w_2, \Phi(x) \rangle_H = 0,~\forall x\in X$. So we have
	\begin{equation}
		\langle w, \Phi(x) \rangle_H = \langle w_1, \Phi(x) \rangle_H,~\forall x \in X.
	\end{equation}
\end{proof}
%Easy to observe that there is a surjective between $F_{\Phi,H}$ and $\overline{S_{\Phi,H}}$.

%\begin{lemma}
%	For any two pairs $(\Phi,H)$ and $(\Phi',H')$, there is an isomorphism between $\overline{S_{\Phi,H}}$ and $\overline{S_{\Phi',H'}}$.
%\end{lemma}

%\begin{lemma}
%	Let $X\neq \emptyset$ and $k$ be a continuous kernel over $X$ with feature space $H$ and feature map $\Phi: X\rightarrow H$, then $\Phi$ must be continuous.
%\end{lemma}
%
%\begin{proof}
%	\begin{equation}
%	\|\Phi(x) - \Phi(x')\|_H^2 = k(x,x) + k(x',x') - k(x,x') - k(x',x).
%	\end{equation}
%	So the continuity of kernel implies the continuity of feature map $\Phi$.
%\end{proof}

\begin{lemma}
	Let $X$ be a compact metric space and $k$ be a continuous kernel over $X$ with feature space $H$ and feature map $\Phi: X\rightarrow H$, then we have
	\begin{equation}
	K_X \subset F_{\Phi,H} \subset \overline{K_X}.
	\end{equation}
\end{lemma}

\begin{proof}
	$K_X \subset F_{\Phi,H} $ is a direct corollary of the preceding 2 lemmas. So we only need to show that $F_{\Phi,H} \subset \overline{K_X}$.\\
	Given any $f\in F_{\Phi,H}$, there exists a $w\in \overline{S_{\Phi,H}}$ such that $f(\cdot) = \langle w,\Phi(\cdot) \rangle_H$. Choose a sequence $\{w_n\}$ in $S_{\Phi,H}$ which satisfies $w_n$ converges to $w$ in $H$, then easy to observe that $f_n(\cdot) = \langle w_n,\Phi(\cdot) \rangle_H\in K_X$. Notice that
	\begin{equation}
		|f_n(x) - f(x)| \leq \|w_n - w\|_H \sqrt{k(x,x)}
	\end{equation}
	Denote $C = \sup_{x\in X} \sqrt{k(x,x)} < \infty$, then
	\begin{equation}
		\|f_n - f\|_{u} \leq C \|w_n - w\|_H,
	\end{equation}
	so $f_n$ converges to $f$ in $C(X)$, which implies $F_{\Phi,H} \subset \overline{K_X}$.
\end{proof}

\begin{corollary}
	$K_X$ is dense in $C(X)$ if and only if there exists a pair $\Phi,H$ such that $F_{\Phi,H}$ is dense in $C(X)$.
\end{corollary}

According to the last theorem, we know that given $H$ and $\Phi$ as the feature space and feature map of kernel $k$, then we only need to prove that for every function $g\in C(X)$ and all $\epsilon>0$ there exists an $\omega\in H$ such that
\begin{equation}
\|\langle \omega,\Phi(\cdot) \rangle - g\|_{\infty} \leq \epsilon.
\end{equation}

\begin{theorem}
	Let $X$ be a compact metric space and $k$ be a continuous kernel on $X$. Suppose that we have a feature map $\Phi: X\rightarrow l_2$ of $k$. We write $\Phi_n: X\rightarrow \mathbb{R}$ for its n-th component, i.e., $\Phi(x) = (\phi_n(x))_{n\in \mathbb{N}}$, $x\in X$. If $\mathcal{A}:= {\rm span} \{\phi_n: n\in \mathbb{N}\}$ is dense in $C(X)$, then $k$ is universal.
\end{theorem}

\begin{proof}
	We only need to notice that  $\mathcal{A}\subset F_{\Phi,H} $.
\end{proof}

\begin{corollary}[Universal Taylor kernels].
	Fix an $r\in (0,+\infty]$ and a $C^\infty$ function $f:(-r,r) \rightarrow \mathbb{R}$ that can be expanded into its Taylor series at 0, i.e.,
	\begin{equation}
	f(t) = \sum_{n = 0}^\infty a_n t^n, ~~t\in (-r,r).
	\end{equation}
	Let $X:= \{x\in \mathbb{R}^d: \|x\|_2 < \sqrt{r}\}$. If we have $a_n>0$ for all $n\geq 0$, then $k$ given by
	\begin{equation}
	k(x,x') := f(\langle x,x'\rangle),~~x,x'\in X,
	\end{equation}
	is a universal kernel on every compact subset of $X$.
\end{corollary}

\begin{proof}
	Suppose that $X_0$ is an arbitrary compact subset of $X$.\\
	Notice that
	\begin{align*}
	k(x,x') &= \sum_{n = 0} a_n (\sum_{j = 1}^d x_j x_j')^n\\
	&= \sum_{n =0}^\infty a_n \sum_{\sum_{i = 1}^d j_i= n,~j_i\geq0} \frac{n!}{\prod_{i = 1}^d j_i!} \prod_{i = 1}^d x_i^{j_i}\prod_{i = 1}^d (x'_i)^{j_i}\\
	&= \sum_{j_1,\cdots,j_d\geq0} a_{j_1+\cdots+j_d} \frac{(j_1+\cdots+j_d)!}{\prod_{i = 1}^d j_i!} \prod_{i = 1}^d x_i^{j_i}\prod_{i = 1}^d (x'_i)^{j_i}
	\end{align*}
	Denote that $c_{j_1,\cdots,j_d} = \sqrt{a_{j_1+\cdots+j_d} \frac{(j_1+\cdots+j_d)!}{\prod_{i = 1}^d j_i!}}$, $\phi_{j_1,\cdots,j_d} = c_{j_1,\cdots,j_d} \prod_{i = 1}^d x_i^{j_i}$, and define feature map $\Phi: X_0 \rightarrow l_2$ as
	\begin{equation}
	\Phi(x) := (\phi_{j_1,\cdots,j_d}(x))_{j_1,\cdots,j_d\geq 0}
	\end{equation}
	Because $c_{j_1,\cdots,j_d} > 0$ for all $j_1,\cdots,j_d\geq0$, so $\mathcal{A}:= {\rm span} \{\phi_{j_1,\cdots,j_d}: j_1,\cdots,j_d\geq0\}$ is the d-variable polynomial space. According to the Stone-Weierstrass approximation theorem, we know $\mathcal{A}$ is dense in $C(X)$.
\end{proof}

\begin{corollary}
	Exponential kernel $k_{\gamma}(x,x') = e^{\langle x,x' \rangle}$ is universal.
\end{corollary}

\begin{lemma}
	Let $X$ be a compact metric space and $k$ be a universal kernel on $X$. Then $k(x,x)>0$ for all $x\in X$, and the \textbf{normalized kernel} $k^*$: $X\times X \rightarrow \mathbb{R}$ defined by
	\begin{equation}
	k^*(x,x') := \frac{k(x,x')}{\sqrt{k(x,x)k(x',x')}},~~x,x'\in X,
	\end{equation}
	is universal.
\end{lemma}

\begin{corollary}
	Gaussian RBF kernel $k_{\gamma}(x,x') = e^{-\frac{\|x-x'\|_2^2}{\gamma^2}}$ is universal.
\end{corollary}

\begin{theorem}[Stone-Weierstra$\beta$]
	Let $(X,d)$ be a compact metric space and $\mathcal{A}\subset C(X)$ be an algebra. Then $\mathcal{A}$ is dense in $C(X)$ if both $\mathcal{A}$ does not vanish, i.e., for all $x\in X$, there exists an $f\in \mathcal{A}$ with $f(x)\neq 0$, and $\mathcal{A}$ separates points, i.e., for all $x,y\in X$ with $x\neq y$, there exists an $f\in \mathcal{A}$ with $f(x)\neq f(y)$.
\end{theorem}

\begin{theorem}
	Let $X$ be a compact metric space and $k$ be a continuous kernel on $X$ with $k(x,x)>0$ for all $x\in X$. Suppose that we have an injective feature map $\Phi: X\rightarrow l_2$ of $k$. We write $\phi_n: X\rightarrow \mathbb{R}$ for its n-th component, i.e., $\Phi(x) = (\phi_n(x))_{n\in \mathbb{N}}$, $x\in X$. If $\mathcal{A}:= {\rm span} \{\phi_n: n\in \mathbb{N}\}$ is an algebra, then $k$ is universal.
\end{theorem}

\begin{proof}
	We only need to verify that $\mathcal{A}$ is dense in $C(X)$, thus we only need to verify $\mathcal{A}$ satisfies the conditions in Stone-Weierstra$\beta$ theorem. \\
	Notice that for all $x\in X$, we have
	\begin{equation}
	\|\Phi(x)\|_{l2}^2 = \sum_{n=1}^{\infty} \phi_n^2(x) = k(x,x) >0.
	\end{equation}
	So for all $x\in X$, there is at least one $\phi_n$ such that $\phi_n(x) \neq 0$.\\
	Also, because $\Phi$ is injective, we know for any $x\neq y$, $\Phi(x)\neq \Phi(y)$, which implies there exists a $\phi_n$ such that $\phi_n(x) \neq \phi_n(y)$.

\end{proof}


\subsubsection{Overdamped ($\Delta > 0$)}
This is the simplest and easiest case to deal with because our two roots, $r_1$ and $r_2$, are real and distinct. So, out solution is
\begin{equation*}
	y = C_1e^{r_1 t} + C_2e^{r_2 t}
\end{equation*}
\begin{center}
	\includegraphics[width=0.5\textwidth]{./higherOrder/freeVibrs/overdamped.png}
\end{center}
We know that $r_1, r_2 < 0$, so
\begin{equation*}
	\lim\limits_{t \to 0}{C_1e^{r_1 t} + C_2e^{r_2 t}} = 0
\end{equation*}
meaning the mass's oscillation decays over time.


\subsection{Peak-to-Average Ratio}\label{subsec:peakratio}

In addition to examining traffic demands across the entire four-hour
prime-time window, we also explored how subscribers in the treatment
group exhibited different behavior for the 15-minute interval of highest \green{(95th percentile)
demand in a day, regardless of prime-time hours.} We measure the disparity
between a subscriber's daily 95th percentile and
the mean usage as the \emph{peak-to-average ratio} \red{(PAR)}\sgfoot{replace all instances of peak-ratio or peak-to-mean ratio with peak-to-average ratio or PAR, which is a standard measurement stat for waveform analysis (also called crest ratio)}. (This metric
shows the ratio of peak values to the effective value, and extends those used
in conventional studies of user traffic patterns, such as the Sandvine
Reports' peak traffic analysis ~\cite{sandvine20141h}.)

\begin{figure}[t]
\centering
\begin{minipage}{\linewidth}
\centering
\includegraphics[width=.75\linewidth]{figures/peakratio_cdf_mean-devices.pdf}
\caption{\green{Distribution of the daily peak-to-average ratio per subscriber, averaged for each subscriber over the measurement period in the treatment and  control groups.}}
\label{fig:CDF-peak-ratio-mean}
\end{minipage}
\end{figure}

Figure~\ref{fig:CDF-peak-ratio-mean} plots the PAR for each
subscriber in the treatment and control groups. The median PAR
 for subscribers from the treatment group is 4.64, compared to 4.51
for the control group.
We found that 40\% of the subscribers in both groups have PAR
greater than 5; the PAR of subscribers in the treatment group is higher than those in the
control group, perhaps indicating that users in both higher service tiers do
in fact use the additional capacity for short periods of time. The
notable difference occurs for peak-to-average ratios of less-than 5: as
we observed in Section~\ref{subsec:behavior}, \red{subscribers with more
moderate (median) traffic demands tend to increase their peak demand more in
response to the increased service tier.}  Again, we believe these trends
appear not because users are necessarily eager to fill the additional
capacity of a higher service tier, but rather may be occurring because the upgrade
results in better performance, and that this improved user experience in
turn causes these subscribers to make more use of the Internet.

\green{The lower prime-time ratio by volume, and a consistently higher
peak-to-average ratio per subscriber} indicates the following:
subscribers in the treatment group have higher peak-to-average ratio than
those in the control group. \green{However, these subscribers tend to still have
low absolute demand, so the relatively higher PAR for the treatment group does not significantly
affect total traffic during prime-time and, when it is high, the demand
tends to be in non-prime-time hours}.  Consistent with the results in
Section~\ref{subsec:primetime}, we also found that on weekdays, the
peak-to-average ratios in the treatment group are higher than the control
group, whereas on weekends peak-to-average ratios for both the control and
treatment groups are similar.


\documentclass[letterpaper, twoside, 12pt]{book}
\usepackage{packet}

\begin{document}

\setcounter{chapter}{3}

\chapter{Packet 4.1: Sections 16.1-16.4}

\setcounter{chapter}{16}
\setcounter{section}{0}

\section{Vector Fields} %16.1

\begin{definition}
  A \textbf{vector field} assigns a vector to each point in 2D or 3D space.
    \[
      \vect{F}=
      \vect{F}(\vect{r})=
      \vect{F}(x,y)=
      \<P(x,y),Q(x,y)\>=
      \<P(\vect{r}),Q(\vect{r})\>=
      \<P,Q\>
    \]
    \[
      \vect{F}=
      \vect{F}(\vect{r})=
      \vect{F}(x,y,z)=
      \<P(x,y,z),Q(x,y,z),R(x,y,z)\>=
      \<P(\vect{r}),Q(\vect{r}),R(\vect{r})\>=
      \<P,Q,R\>
    \]
\end{definition}

          \begin{problem}
            Sketch the vector field $\vect{F}=\<x+y,2y\>$ for
            all $x\in\{0,1,2\}$ and $y\in\{0,1,2\}$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\begin{remark}
  The gradient vector function
    \[
      \nabla f (x,y)
        =
      \<f_x(x,y),f_y(x,y)\>
    \]
    \[
      \nabla f (x,y,z)
        =
      \<f_x(x,y,z),f_y(x,y,z),f_z(x,y,z)\>
    \]
  is a vector field which yields normal vectors
  to the level surfaces of the function $f$.
\end{remark}

          \begin{problem}
            Compute $\nabla f$ for the function
            $f(x,y)=x^2-2xy+y$, and then
            sketch the vector field $\nabla f$
            all $x\in\{0,1,2\}$ and $y\in\{0,1,2\}$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\section{Line Integrals} %16.2

\begin{theorem}
  Some vector functions which parameterize curves follow.
  \begin{itemize}
    \item
    A line segment beginning at $P_0$ and ending at $P_1$:
      \[
        \vect{r}(t) = \vect{P_0} + t\vect{P_0P_1}, 0\leq t\leq 1
      \]
    \item
    A circle centered at the origin with radius $a$:
      \[
        \vect{r}(t) = \<a\cos t,a\sin t\>, 0\leq t\leq 2\pi
        \text{ (full counter-clockwise rotation)}
      \]
      \[
        \vect{r}(t) = \<a\sin t,a\cos t\>, 0\leq t\leq 2\pi
        \text{ (full clockwise rotation)}
      \]
    \item
    A planar curve given by $y=f(x)$ from $(x_0,y_0)$ to $(x_1,y_1)$
      \[
        \vect{r}(t) = \<t,f(t)\>, x_0\leq t\leq x_1
        \text{ (left-to-right)}
      \]
      \[
        \vect{r}(t) = \<-t,f(-t)\>, -x_0\leq t\leq -x_1
        \text{ (right-to-left)}
      \]
    \end{itemize}
\end{theorem}

          \begin{problem}
            Give a vector function which parameterizes the line segment
            from the point $(0,3,-2)$ to the point $(4,-1,0)$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

          \begin{problem}
            Give a vector function which parameterizes the curve
            $y=x^3-2x$ from the point $(1,-1)$ to the point $(-1,1)$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

          \begin{problem}
            Give a vector function which parameterizes the curve
            $x^2+y^2=9$ from the point $(3,0)$ clockwise to the point $(0,-3)$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\begin{definition}
The \textbf{line integral with respect to arclength} of a function of many
variables $f(\vect{r})$ along a curve $C$ is given by
  \[
    \int_C f(\vect{r})\dvar{s} =
    \lim_{n\to\infty}\sum_{i=1}^n f(\vect{r}_{n,i})\Delta s_{n,i}
  \]
where for each positive integer $n$ we've defined a way to partition $C$
into $n$ pieces
  \[
    \Delta C_{n,1},\Delta C_{n,2},\dots,\Delta C_{n,n}
  \]
where $\Delta C_{n,i}$ has length $\Delta s_{n,i}$, contains the position
vector $\vect{r}_{n,i}$, and
  \[
    \lim_{n\to\infty} \max(\Delta s_{n,i}) = 0
  \]
\end{definition}

\begin{theorem}
If $\vect{r}(t)$ is a parametrization of $C$ for $a \leq t \leq b$, then
  \[
    \int_C f(\vect{r})\dvar{s}
    =\int_{t=a}^{t=b} f(\vect{r}(t))\frac{ds}{dt}\dvar{t}
  \]
\end{theorem}

          \begin{problem}
            Evaluate $\int_C z + 2xy\dvar{s}$ where $C$ is the line segment
            from $(0,-1,3)$ to $(2,2,-3)$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

          \begin{problem}
            Prove that $\int_C xy\dvar{s}=\int_0^1 t^3\sqrt{1+4t^2}\dvar{t}$
            where $C$ is the parabolic arc
            on $y=x^2$ from $(0,0)$ to $(1,1)$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\begin{definition}
The \textbf{line integral of a vector field} $\vect F$
over the curve $C$ is given by
  \[
    \int_C \vect F\cdot\dvar{\vect r} =
    \lim_{n\to\infty}\sum_{i=1}^n
    \vect F(\vect{r}_{n,i})\cdot\Delta\vect{C}_{n,i}
  \]
where for each positive integer $n$ we've defined a way to approximate $C$
with $n$ vectors
  \[
    \Delta \vect{C}_{n,1},\Delta \vect{C}_{n,2},\dots,\Delta \vect{C}_{n,n}
  \]
where $\vect{r}_{n,i}+\Delta \vect{C}_{n,i}=\vect{r}_{n,i+1}$
and
  \[
    \lim_{n\to\infty} \max(|\Delta \vect{C}_{n,i}|) = 0
  \]
\end{definition}

\begin{definition}
The line integral of a vector field $\vect F$ over the curve $C$
may be computed by
    \[
      \int_C \vect{F}\cdot\dvar{\vect r}
        =
      \int_C \vect{F}\cdot\vect{T}\dvar{s}
    \]
where $\vect T$ yields the unit tangent vectors to the curve $C$.
\end{definition}

\begin{definition}
If $\vect{r}(t)$ is a parametrization of $C$ for $a \leq t \leq b$, then
    \[
      \int_C \vect{F}\cdot\dvar{\vect r}
        =
      \int_{t=a}^{t=b} \vect{F}\cdot\frac{d\vect{r}}{dt}\dvar{t}
    \]
\end{definition}

          \begin{problem}
            Prove that
            $\int_C \<2x,y-x\>\cdot\dvar{\vect{r}}
              =
            \int_0^1 23t-7 \dvar{t}$
            where $C$ is the line segment given by the vector equation
            $\vect{r}(t)=\<1-2t,3t\>$ for $0\leq t\leq 1$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\begin{remark}
  The work done by a force vector field $\vect{F}$ over the curve $C$
  is given by $\int_C\vect{F}\cdot\dvar{\vect{r}}$.
\end{remark}

          \begin{problem}
            Find the work done by the force vector field
            $\<-3y,3x\>$ moving a particle one rotation counter-clockwise
            around the unit circle $x^2+y^2=1$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\begin{theorem}
  If $C$ may be split into two curves $C_1$ and $C_2$, then
  \[
    \int_C f\dvar{s}
      =
    \int_{C_1} f\dvar{s}
      +
    \int_{C_2} f\dvar{s}
  \]
  and
  \[
    \int_C \vect F\cdot\dvar{\vect r}
      =
    \int_{C_1} \vect F\cdot\dvar{\vect r}
      +
    \int_{C_2} \vect F\cdot\dvar{\vect r}
  \]
\end{theorem}

\begin{theorem}
  If $-C$ is the curve $C$ oriented in the opposite direction, then
  \[
    \int_C f\dvar{s}
      =
    \int_{-C} f\dvar{s}
  \]
  and
  \[
    \int_C \vect F\cdot\dvar{\vect r}
      =
    - \int_{-C} \vect F\cdot\dvar{\vect r}
  \]
\end{theorem}

          \begin{problem}
            Write a paragraph explaining why a negative appears in the
            previous theorem for the
            line integral of a vector field but not for an arclength
            line integral.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\section{The Fundamental Theorem for Line Integrals} %16.3

\begin{definition}
  If $\nabla f=\vect{F}$, then $f$ is a \textbf{potential function}
  for the \textbf{conservative field} $\vect{F}$.
\end{definition}

          \begin{problem} %x^2-3yz
            Prove that $\<2x,-3z,-3y\>$ is a conservative field by
            finding a potential function $f$ for it. Hint: such an $f$
            must satisfy that $f=x^2+\Phi_1(y,z)$, $f=-3yz+\Phi_2(x,z)$,
            and $f=-3yz+\Phi_3(x,y)$ for some functions $\Phi_i$. (Why?)
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\begin{theorem}
  The Fundamental Theorem for Line Integrals:
  If $C$ is any smooth curve beginning at the point $A$ and ending at the
  point $B$, then
  \[
    \int_C \nabla f\cdot \dvar{\vect{r}} = \left[f\right]_A^B = f(B)-f(A)
  \]
\end{theorem}

          \begin{problem}
            Prove that if $C$ is any smooth \textbf{closed curve}
            (beginning and ending at the same point), then
            \[
              \int_C \nabla f\cdot \dvar{\vect{r}} = 0
            \]
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

          \begin{problem}
            Compute $\int_C\<4,z^2,2yz\>\cdot\dvar{\vect r}$ where
            $C$ is the curve given by
            $\vect{r}(t)=\<2^t,\sin (\pi t),4t^2\>$ for $0\leq t\leq 1$.
            Then compute $\int_{C'}\<4,z^2,2yz\>\cdot\dvar{\vect r}$ where
            $C'$ is the line segment starting at $(1,0,0)$ and ending
            at $(2,0,4)$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

          \begin{problem}
            Prove that if $f$ is a potential function for the vector field
            $\<P,Q,R\>$, then
            $P_y=Q_x$, $P_z=R_x$, and $Q_z=R_y$. (Hint: use the mixed derivative
            theorem.)
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\begin{theorem}
  $\vect{F}=\<P,Q,R\>$ is a conservative vector field if and only if
  $P_y=Q_x$, $P_z=R_x$, and $Q_z=R_y$.
\end{theorem}

          \begin{problem}
            Prove that
            $\int_C\<ye^{xy+z},xe^{xy+z},e^{xy+z}\>\cdot\dvar{\vect r}=0$
            where $C$ is the curve given by
            $\vect{r}(t)=\<\frac{1}{1+t^2},\cos t,e^{1-t^2}\>$
            for $-1 \leq t \leq 1$.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\section{Green's Theorem} %16.4

\begin{theorem}
  Let $C$ be the boundary of the region $R$ in the $xy$ plane oriented
  counter-clockwise, and let $\vect{F}$ be a two-dimensional vector field. Then
  \[
    \int_C \vect{F}\cdot\dvar{\vect{r}}
      =
    \iint_R \left(\frac{\p Q}{\p x}-\frac{\p P}{\p y}\right)\dvar{A}
  \]
\end{theorem}

          \begin{problem}
            Evaluate $\int_C\<x^2-y^2,x^3+y-1\>\cdot\dvar{\vect r}$ where
            $C$ is the boundary of the unit square oriented counter-clockwise.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

          \begin{problem}
            Find the work done by a force vector field $\<y,2x\>$ moving an
            object around the
            boundary of the triangle with vertices $(1,2)$, $(-1,-2)$, and
            $(3,-2)$ oriented clockwise.
          \end{problem}

          \begin{solution}

          \end{solution}

          \begin{contributors}

          \end{contributors}

\end{document}


%%%this is taken form loader (1999)

\chapter{Local Regression}
Local regression is used to model a relation between a predictor
variable and response variable. To keep things simple we will consider
the fixed design model. We assume a model of the form
\[
Y_i = f(x_i) + \varepsilon_i
\]
where $f(x)$ is an unknown function and $\varepsilon_i$ is an error term,
representing random errors in the observations or variability from
sources not included in the $x_i$.

We assume the errors $\varepsilon_i$ are IID with mean 0 and finite
variance $\var(\varepsilon_i) = \sigma^2$.

We make no global assumptions about the function $f$ but assume that
locally it can be well approximated with a member of a simple class of
parametric function, e.g. a constant or straight line. Taylor's
theorem says that any continuous function can be approximated with
polynomial.

\section{Taylor's theorem}
We are going to show three forms of Taylor's theorem.
\begin{itemize}

\item This is the original. Suppose $f$ is a real function on $[a,b]$, $f^{(K-1)}$ is continuous on
$[a,b]$,  $f^{(K)}(t)$ is bounded for $t \in (a,b)$ then
for any distinct points $x_0 < x_1$ in $[a,b]$ there exist a point
$x$ between $x_0 < x < x_1$ such that
\[
f(x_1) = f(x_0) + \sum_{k=1}^{K-1} \frac{f^{(k)}(x_0)}{k!}(x_1-x_0)^k +
\frac{f^{(K)}(x)}{K!}(x_1 - x_0)^K.
\]
{\bf Notice:} if we view $f(x_0) + \sum_{k=1}^{K-1}
\frac{f^{(k)}(x_0)}{k!}(x_1-x_0)^k$ as function of $x_1$, it's a
polynomial in the family of polynomials
\[
{\cal  P}_{K+1}= \{f(x) = a_0 + a_1 x + \dots + a_K x^K,
(a_0,\dots,a_K)' \in {\mathbb R}^{K+1}\}.
\]

\item Statistician sometimes use what is called Young's form of Taylor's
Theorem:

Let $f$ be such that $f^{(K)}(x_0)$ is bounded for $x_0$ then
\[
f(x) = f(x_0) +  \sum_{k=1}^{K} \frac{f^{(k)}(x_0)}{k!}(x-x_0)^k +
o(|x-x_0|^K), \mbox{ as } |x-x_0| \rightarrow 0.
\]
{\bf Notice:} again the first two term of the right hand side is in ${\cal P}_{K+1}$.

\item In some of the asymptotic theory presented in this class we are
  going to use another refinement of Taylor's theorem called Jackson's
  Inequality:

  Suppose $f$ is a real function on $[a,b]$ with $K$ is continuous
  derivatives then
  \[
  \min_{g \in {\cal P}_k} \sup_{x \in [a,b]} |g(x) - f(x)| \leq C \left(
    \frac{b-a}{2k}\right)^K
  \]
  with ${\cal P}_k$ the linear space of polynomials of degree $k$.
\end{itemize}

\section{Fitting local polynomials}
We will now define the recipe to obtain a loess smooth for a target
covariate $x_0$.

The first step in loess is to define a weight function (similar to the
kernel $K$ we defined for kernel smoothers). For computational
and theoretical purposes we will define this weight function so that
only values within a {\it smoothing window} $[x_0+h(x_0),x_0-h(x_0)]$ will be
considered in the estimate of $f(x_0)$.

Notice: In local regression $h(x_0)$ is called the span or bandwidth. It
is like the kernel smoother scale parameter $h$. As will be seen a bit
later, in local regression, the span may depend on the target
covariate $x_0$.

This is easily achieved by
considering  weight functions that are $0$ outside of $[-1,1]$. For example
Tukey's tri-weight function
\[
W(u) = \left\{ \begin{array}{cc}
(1 - |u|^3)^3&|u| \leq 1\\
0&|u| > 1.
\end{array}
\right.
\]

The weight sequence is then easily defined by
\[
w_i(x_0) = W \left( \frac{x_i - x_0}{h(x)} \right)
\]

We define a window by a procedure similar to the $k$ nearest
points. We want to include $\alpha\times 100$\% of the data.

Within the smoothing window, $f(x)$ is approximated by a
polynomial. For example, a quadratic approximation
\[
f(x) \approx \beta_0 + \beta_1 (x-x_0) + \frac{1}{2} \beta_2 (x-x_0)^2 \mbox{ for
  } x \in [x_0 - h(x_0), x_0+h(x_0)].
\]
For continuous function, Taylor's theorem tells us something about how
good an approximation this is.

To obtain the local regression estimate $\hat{f}(x_0)$ we simply find
the $\bb = (\beta_0,\beta_1,\beta_2)'$ that minimizes
\[
\hat{\bb} = \arg \min_{\bb \in {\mathbb R}^3} \sum_{i=1}^n w_i(x_0)[ Y_i - \{\beta_0 + \beta_1 (x_i-x_0) + \frac{1}{2} \beta_2 (x_i-x_0)\}]^2
\]
and define $\hat{f}(x_0) = \hat{\beta}_0$.

Notice that the Kernel smoother is a special case of local
regression. Proving this is a Homework problem.

\section{Defining the span}
In practice, it is quite common to have the $x_i$ irregularly
spaced. If we have a fixed span $h$ then one may have local estimates
based on many points and others is very few. For this reason we may
want to consider a nearest neighbor strategy to define a span for
each target covariate $x_0$.

Define $\Delta_i(x_0) = |x_0 -
x_i|$, let $\Delta_{(i)}(x_0)$ be the ordered values of such
distances. One of the arguments in the local regression function {\tt
  loess()} (available in the modreg library) is the {\tt span}. A span
of $\alpha$ means that for each local fit we want to use
 $\alpha \times 100 \%$ of the data.

Let $q$ be equal to $\alpha$n
truncated to an integer. Then we define the span $h(x_0) =
\Delta_{(q)}(x_0)$. As $\alpha$ increases the estimate
becomes smoother.

In Figures \ref{f3.1} -- \ref{f3.3} we see loess smooths for the CD4
cell count data using spans of
0.05, 0.25, 0.75, and 0.95. The smooth presented in the Figures are
fitting a constant, line, and parabola
respectively.

\begin{figure}[htp]
\caption{\label{f3.1} CD4 cell count since seroconversion for HIV infected men.}
\centerline{\epsfig{figure=Plots/plot-03-01.ps,width=\textwidth}}
\end{figure}

\begin{figure}[htp]
\caption{\label{f3.2} CD4 cell count since seroconversion for HIV infected men.}
\centerline{\epsfig{figure=Plots/plot-03-02.ps,width=\textwidth}}
\end{figure}

\begin{figure}[htp]
\caption{\label{f3.3} CD4 cell count since seroconversion for HIV infected men.}
\centerline{\epsfig{figure=Plots/plot-03-03.ps,width=\textwidth}}
\end{figure}

\newpage

\section{Symmetric errors and Robust fitting}
If the errors have a symmetric distribution (with long tails), or if
there appears to be
outliers we can use robust loess.

We begin with the estimate described above $\hat{f}(x)$. The residuals
\[
\hat{\varepsilon}_i = y_i  - \hat{f}(x_i)
\]
are computed.

Let
\[
B(u;b) = \left\{ \begin{array}{cc}
\{1 - (u/b)^2\}^2&|u|<b\\
0& |u|\geq b
\end{array}
\right.
\]
be the bisquare weight function. Let $m$ = median($|\hat{\varepsilon}_i|$).
The robust weights are
\[
r_i = B(\hat{\varepsilon_i}; 6m)
\]
The local regression is repeated but with new weights $r_i w_i(x)$. The
robust estimate is the result of repeating the procedure several times.

If we believe the variance $\var(\varepsilon_i) = a_i \sigma^2$ we could
also use this double-weight procedure with $r_i = 1/a_i$.

\subsection{Example}

Radiolabeling based gene expression measurements are useful for cancer
research because they can be carried out using small amounts of
biological materials.
Statistical issues are different from fluorescence
expression data, because radiolabeling gives absolute intensities that
reflect gene expression and
there is no internal control.

The data-set described here was obtained to identify genes that
may be associated with lung cancer. Lung cancer tissue was obtained
from various subjects. Normal tissues from the same type of cells was
obtained from those same subjects. From each of
these tissues 2 samples were prepared using 2 different isotopic
batches. Each of these 4 samples were hybridized with a filter
spotted with cDNA from many genes in a $48 \times 24$ grid. We refer
to these spotted filters as arrays. Each of these arrays were scanned to
produce an image file
which was then analyzed with
specialized software that produced an intensity level for each grid
point or {\it spot} on the array.

Not all the values read  from the arrays are associated with
genes. There were 207 spots where
no cDNA was spotted. They were left empty. Because there is {\it
  non-specific} binding between the samples and the filters, positive
values are
obtained from these empty spots. The intensities read
from these empty spots provide direct evidence about measurement error
associated with the system. Spots associated with genes that are not
expressed will also have intensities due to non-specific binding.

Can we rank genes by differential expression between
cancer and normal tissues in each subject?

If we denote with $\bx$ and $\by$ the log intensities of each spot we
could say a gene is differentially expressed if $\by - \bx$ is
significantly bigger than 0 for the spot related to that gene.
One problem with this is that there is a filter effect, so $\by$ can
be systematically smaller than $\bx$.

A common procedure in microarray data analysis is to simply normalize the
filters by subtracting the mean of each filter from each value,
i.e. consider $y^{(normalized)}_{i} =  y_{i} -
\bar{y}$ and similarly for the $x$s. The danger with doing
this is that many of the genes spotted on the arrays are usually
selected because researchers consider them likely to be
over-expressed. This means
that the mean of the $y$s should be larger than the $x$s and this
difference in mean is confounded with the difference in filter
effect. By subtracting means we would be subtracting out some of the
differential expression between cancer and normal
tissues.

In Figure \ref{f3.4} we plot the ratio of the intensities vs. the
product of the
intensities in a log scale, i.e. $y-x$ vs. $x + y$, for the two
replicates of subject 1. Notice that the
{\it filter effect} seems to change with the total intensity of a
particular spot. For this reason using medians or trimmed
means to remove the filter effect is not a good solution. If we model $x$
and $y$ as random
variables then we have that the expected filter effect depends on the
total intensity, i.e. $\mbox{E}(y - x | x+y )$ is not constant.
This arises because
specific binding and non-specific binding are two different natural
processes. Because we have no way of knowing which points represent
non-specific binding and which represent specific binding we cannot
normalize by just estimating two means. Rather, we estimate
$\mbox{E}(y-x|y+x)$ using loess. It is critical to use a robust loess,
so that large differences do not affect the fit too much. Notice in
Figure \ref{f3.4} the difference in the robust and non-robust
estimates.

\begin{figure}[htp]
\caption{\label{f3.4} Total intensity plotted against ratio with a
  loess prediction using Gaussian and symmetric kernel.}
\centerline{\epsfig{figure=Plots/plot-03-04.ps,angle=270,width=.8\textwidth}}
\end{figure}

\section{Multivariate Local Regression}
Because Taylor's theorems also applies to multidimensional functions it
is relatively straight forward to extend local regression to cases
where we have more than one covariate. For example if we have a
regression model for two covariates
\[
Y_i = f(x_{i1},x_{i2}) + \varepsilon_i
\]
with $f(x,y)$ unknown. Around a target point $\bx_0 = (x_{01},x_{02})$
a  local quadratic approximation is now
\[
f(x_1,x_2) \approx \beta_0 + \beta_1 (x_1 - x_{01}) + \beta_2 (x_2 - x_{02})
+ \beta_3 (x_1 - x_{01})(x_2 - x_{02}) + \frac{1}{2} \beta_4 (x_1 -
x_{01})^2 +  \frac{1}{2} \beta_5(x_2 -
x_{02})^2
\]

Once we define a distance, between a point $\bx$ and
$\bx_0$, and a span $h$ we can define define waits as in the previous
sections:
\[
w_i(\bx_0) = W\left(\frac{||\bx_i,\bx_0||}{h}\right).
\]
It makes sense to re-scale $x_1$ and $x_2$ so we smooth the same way
in both directions. This can be done through the distance function,
for example by defining a distance for the space ${\mathbb R}^d$ with
\[
||\bx ||^2 = \sum_{j=1}^d (x_j/v_j)^2
\]
with $v_j$ a scale for dimension $j$. A natural choice for these $v_j$
are the standard deviation of the covariates.

Notice: We have not talked about k-nearest neighbors. As we will see in
Chapter VII the {\it curse of dimensionality} will make this hard.

\subsection{Example}
We look at part of the data obtained from a study by Socket
et. al. (1987) on
the factors affecting patterns of insulin-dependent diabetes mellitus
in children. The objective was to investigate the dependence of the
level of serum C-peptide on various other factors in order to
understand the patterns of residual insulin secretion. The response
measurement is the logarithm of C-peptide concentration (pmol/ml) at
diagnosis, and the predictors are age and base deficit, a measure of
acidity. In Figure \ref{f3.5} we show a loess two dimensional
smooth. Notice that the effect of age is clearly non-linear.

\begin{figure}[htp]
\caption{\label{f3.5} Loess fit for predicting C.Peptide from  Base.deficit and Age.}
\centerline{\epsfig{figure=Plots/plot-03-05.ps,width=.7\textwidth}}
\end{figure}

\input{references-03}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\chapter{Classification}
\label{chap:class}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Logistic Regression}
\label{class:logistic}

Logistic regression is a simple method to create a classifier,
typically on two classes $y = 0,1$, though multinomial extensions exist.
Its name comes from the use of the logit, or log-odds, function

\begin{equation}\label{eq:logistic:logic}
l = \text{logit}\left(p\right) = \log\left(\frac{p}{1-p}\right)
\end{equation}

\noindent on the probability $p$ of class $1$.
$l$ is estimated linearly from $n$ input features $x_{j}$ with $n+1$ parameters $\beta_{j}$ as:

\begin{equation}\label{eq:logistic:logicBeta}
l = \beta_{0} + \sum_{j=1}^{n} \, \beta_{j}\,x_{j}\,.
\end{equation}

\noindent The probability $p$ is then

\begin{equation}\label{eq:logistic:p}
p = \frac{e^l}{e^l + 1} = \frac{1}{1+e^{-l}} = \text{logit}^{-1}\left(l\right)
\end{equation}

\noindent which can be turned into a predicted class through the choice of a suitable decision threshold.

The model parameters $\vb*{\beta}$ are chosen by maximizing
the log of the likelihood $L$ \cref{eq:logistic:L} over $m$ known example points $\vb{x}_{i}, y_{i}$.
Note that $P\left(y \mid x\right)$ \cref{eq:logistic:Pr} is simply the Bernoulli distribution.
In practice the log-likelihood $\log\left(L\right)$ is maximized via gradient descent.
An example of logistic regression can be found in \cref{fig:logistic_regression_ex}.

\begin{subequations} \label{eq:logistic:L_Pr}
\begin{align}
L\left(\vb*{\beta} \mid \vb{x}\right) &= \prod_{i=1}^{m} \, P\left(y_{i} \mid \vb{x}_{i};\,\vb*{\beta}\right) \label{eq:logistic:L} \\
P\left(y \mid \vb{x}\right) &= p^y\left(1-p\right)^{1-y}, \quad y \in \{0, 1\} \label{eq:logistic:Pr}
\end{align}
\end{subequations}

\begin{figure}
\centering
% \includegraphics[width=0.8\textwidth]{figures/regression/Exam_pass_logistic_curve.jpeg}
\includegraphics[width=0.7\textwidth]{figures/regression/logistic-regression-probabilities-curve.png}
\caption{
% Example logistic regression curve on one input feature, by \href{https://en.wikipedia.org/wiki/File:Exam_pass_logistic_curve.jpeg}{Michaelg2015}.
Example logistic regression curve on one input feature, by \href{http://www.sthda.com/english/articles/36-classification-methods-essentials/151-logistic-regression-essentials-in-r/}{Kassambara}.
}
\label{fig:logistic_regression_ex}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Assumptions}
\label{class:logistic:assumptions}
% TODO

% TODO any more assumptions?
Some assumptions of the logistic regression approach are:
\begin{enumerate}[noitemsep]
  \item $y$ is either present or absent (dichotomous).
  \item There are minimal correlations between the $x_{j}$ features (no multicollinearity).
  \item There are no major outliers in the data.
\end{enumerate}

% TODO pseudo R2, Wald statistic
% TODO regularized versions?

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Example}
\label{class:logistic:example}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{N{a\"i}ve Bayes Classification}
\label{class:Bayes}
% TODO
% TODO maximum a posteriori (italics) probability (MAP) estimator

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Gaussian N{a\"i}ve Bayes Classification (GNB)}
\label{class:Bayes:GNB}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Support Vector Machines (SVM)}
\label{class:SVM}

Basic support vector machines (SVM) work by finding a hyperplane in the $n$-dimensional
feature space of the training data which best separate the different classes.
This is done by maximizing the margin, $2/\norm{\vb{w}}$,
around the hyperplane defined by $\innerproduct{\vb{w}}{\vb{x}} - b = 0$,
where $\innerproduct{\vb{a}}{\vb{b}}$ is the inner product.
For the separable case, as shown in \cref{fig:svm_sep}, this
can be done by minimizing $\norm{\vb{w}}$, with the hard-margin condition that
$y_{i} \left(\innerproduct{\vb{w}}{\vb{x}_{i}} - b\right)$ for all $1 \leq i \leq m$.

However, in reality the data are frequently inseparable and we must switch
to a soft-margin objective function \cref{eq:svm:soft_margin_obj}.
A hinge loss function is included to penalize points on the ``wrong'' side of the margin
proportionally to their distance from the margin.
Here the $\lambda$ hyperparameter sets the tradeoff between
margin size and ensuring points land on their correct sides.

\begin{equation} \label{eq:svm:soft_margin_obj}
S\left(\vb{w}, b\right) =
\lambda\, \norm{\vb{w}}^{2}
+ \frac{1}{m} \sum_{i=1}^{m} \,
\max{\big(0,\, 1 - y_{i} \left(\innerproduct{\vb{w}}{\vb{x}_{i}} - b\right)\big)}.
\end{equation}

\begin{figure}[H]
\centering
\includegraphics[width=0.42\textwidth]{figures/ml/svm_margin.png}
\vspace{0.2cm}
\caption{
Illustration of the SVM method in the separable case,
by \href{https://en.wikipedia.org/wiki/File:SVM_margin.png}{Larhmam}.
The trained hyperplane in red separates the two classes by the largest margin.
The data points on the margin boundary with black boarders
are known as support vectors, since out of all the data
they are the points really fixing the hyperplane and margin.
}
\label{fig:svm_sep}
\end{figure}

To gain better performance still, we can recast the problem in
a new higher dimensional space where the classes may be easier to separate with a hyperplane.
Fortunately, we don't even need to fully specify the new space,
just a non-linear kernel function\footnote{Common kernel choices include
polynomials of the inner product,
the Gaussian radial basis function,
and the hyperbolic tangent.} $k\left(\vb{a},\vb{b}\right)$
in place of the standard inner product. This is known as the kernel trick.
The classification boundary in the original feature space can then become non-linear,
as can be seen in \cref{fig:svm_kernel_trick}.

\vspace{-0.3cm}% TODo hard coded to fit on one page

\begin{figure}[H]
\centering
\includegraphics[width=0.8\textwidth,trim={4.0cm 0.8cm 4.0cm 1.4cm},clip]{figures/ml/kernel_trick_example.png}% trim={<left> <lower> <right> <upper>}
\caption{
Graphical example of the kernel trick, by \href{https://en.wikipedia.org/wiki/File:Kernel_trick_idea.svg}{Shiyu Ji}.
Here the kernel $k\left(\vb{a},\vb{b}\right) = \innerproduct{\vb{a}}{\vb{b}} + \norm{\vb{a}}^{2} \norm{\vb{b}}^{2}$
transforms the red and purple classes, linearly inseparable in $n=2$ dimensions on the left,
to a separable $3$-dimensional space on the right.
}
\label{fig:svm_kernel_trick}
\end{figure}

In practice minimizing $S\left(\vb{w}, b\right)$ can be
performed more readily by instead solving the Lagrangian dual problem,
which is computationally efficient to solve with quadratic programming algorithms.
Other modern techniques developed to tackle large and sparse data include
sub-gradient methods and coordinate descent\footnote{Sub-gradient methods work better for large $m$,
coordinate descent for large $n$.}.
However, compared to other classifiers SVM training times
tend to slow significantly for large datasets,
in \sklearn\footnote{See the
\href{https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html}{documentation}
for \texttt{sklearn.svm.SVC}.
\texttt{LinearSVC} may be faster.
The best performance I've seen quoted is $\order{n m \log{\left(m\right)}}$.} by
at least $\order{m^{2}}$, limiting $m \sim \num{e4}$.

%\begin{figure}[H]
%  \centering
%  \begin{subfigure}[b]{0.48\textwidth}\centering
%      \includegraphics[width=\textwidth]{figures/ml/svm_separable}
%  \caption{Separable}
%  \label{fig:svm:separable}
%  \end{subfigure}
%  ~
%  \begin{subfigure}[b]{0.48\textwidth}\centering
%      \includegraphics[width=\textwidth]{figures/ml/svm_nonseparable}
%  \caption{Nonseparable}
%  \label{fig:svm:nonseparable}
%  \end{subfigure}
%\caption{
%Illustrations of SVMs in the separable and nonseparable case \cite{HastieTF09}.
%\label{fig:svm}
%}
%\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Decision Trees, \texorpdfstring{\ie}{ie} Classification and Regression Trees (CART)}
\label{class:CART}

A basic classifier can be created from a tree of selections on $\mathbf{X}$ designed to
separate the classes at each branch.
Such a model is known as a classification and regression tree (CART) \cite{Breiman:2253780}
and a simple example can be found in \cref{class:CART:small_example_CART}.
As the splits are just selections on the input variables,
they are --- somewhat --- possible to understand,
and conveniently do not need any kind of feature scaling, unlike other methods.
To make a prediction for an event the tree and its branches are traversed
until the event lands in one of the weighted leaves.
The weight of the leaf $w$ is positive (negative) for signal-like (background-like) events.
A logistic function is used to properly transform $w$ into an output score
$\yhat = 1 /\left(1+e^{-w}\right)$ within $0 < \yhat < 1$.

\begin{figure}[H]
\centering
\includegraphics[width=0.4\textwidth]{figures/ml/tree7_g2000_n1200}
\caption{
Simple classification and regression tree (CART).
Signal-like (background-like) events receive positive (negative) weights in the leaves.
}
\label{class:CART:small_example_CART}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Gini Impurity}
\label{class:CART:gini}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Boosted Decision Trees (BDT)}
\label{class:BDT}

Individual CARTs are rather poor and limited models
in terms of the behaviors they can successfully predict.
However, by taking an ensemble of $K$ complementary trees, \ie boosting \cite{FREUND1997119,friedman2000},
and summing each CART's individual weight $w_{k}$ a much more flexible BDT\footnote{As the leaf weights
are reals rather than integer classes this approach may be better described as a boosted regression tree,
and can indeed handle regression problems without the logistic function.} is formed.
The component trees of a BDT are generated by iteratively adding new trees $f_{k}\left(x_{i}\right)$ to those which came before \cite{XGBoost},

\begin{equation} \label{eq:boosting}
\begin{aligned}
\yhat^{\left(0\right)} &= 0\,, \\
\yhat^{\left(1\right)} &= f_1\left(\mathbf{X}\right) = \yhat^{\left(0\right)} + f_1\left(\mathbf{X}\right), \\
\yhat^{\left(2\right)} &= f_1\left(\mathbf{X}\right) + f_2\left(\mathbf{X}\right)= \yhat^{\left(1\right)} + f_2\left(\mathbf{X}\right), \\
                           &\vdotswithin{\displaystyle =} \\
\yhat^{\left(t\right)} &= \sum_{k=1}^t f_k\left(\mathbf{X}\right)= \yhat^{\left(t-1\right)} + f_t\left(\mathbf{X}\right),
\end{aligned}
\end{equation}

\noindent where each tree $f_{k}$ is grown from zero branches while minimizing $S\left(\beta\right)$.
Through the ingenious use of a second order Taylor expansion this process can
be recast as a form of gradient descent, and thus is known as
stochastic gradient boosting \cite{10.2307/2699986,FRIEDMAN2002367}.
The number of boosting rounds, and thus trees, $K$ can be chosen in advance
but is better optimized during the training process via early stopping.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{\xgboost}% would rather have the \textsc caps than italics
\label{class:BDT:xgboost}
% TODO see https://towardsdatascience.com/boosting-algorithm-xgboost-4d9ec0207d
% TODO how does the Hessian come into play

The \xgboost\footnote{\xgboost: eXtreme Gradient Boosting, \href{https://github.com/dmlc/xgboost}{github.com/dmlc/xgboost}.} library \cite{XGBoost}
is a modern open source implementation of gradient boosted decision tree methods.
Through various algorithmic and memory optimizations \xgboost demonstrates good performance\footnote{\xgboost has lost
its lead in recent years to newer libraries such as LightGBM \cite{LightGBM}
and CatBoost \cite{CatBoost}.}.
L1 and L2 regularization is incorporated via

\begin{equation} \label{eq:bdt_omega_reg}
\Omega\left(f\right) = \alpha T + \frac{1}{2}\lambda \sum_{j=1}^T w_j^2\,,
\end{equation}

\noindent where $T$ is the number of leaves in a tree and $w_{j}$ are the leaf weights;
however, the default hyperparameters $\alpha=0$ and $\lambda=1$ only enable L2 regularization.
Other important hyperparameters in \xgboost include the
learning rate $\eta$, which scales the corrections added by each new tree,
maximum tree depth, which sets a limit on the complexity of any tree via its depth,
and the early stopping validation threshold.
For reference $\eta=0.3$ and a maximum depth of 6 are the default values.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{AdaBoost}
\label{class:BDT:AdaBoost}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Random Forest}
\label{class:RF}
% TODO

% TODO best results occur when you chose $\sqrt{n}$ features randomly to build each tree

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{\texorpdfstring{$k$}{k}-Nearest Neighbors (\texorpdfstring{$k$}{k}-NN)}
\label{class:kNN}
% TODO
% TODO \kNN

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Artificial Neural Networks (NN)}
\label{class:ANN}
% TODO

% TODO add back prop somewhere, here or in grad descent

\begin{figure}[H]
  \centering
  \begin{subfigure}[b]{0.48\textwidth}\centering
      \includegraphics[width=\textwidth]{figures/ml/NN_diagram/NN_diagram}
  \caption{NN Example}
  \label{fig:NN:ex}
  \end{subfigure}
  ~
  \begin{subfigure}[b]{0.48\textwidth}\centering
      \includegraphics[width=\textwidth]{figures/ml/NN_neuron/NN_neuron}
  \caption{Neuron}
  \label{fig:NN:Neuron}
  \end{subfigure}
\caption{
Illustrations of the components of a neural network.
\label{fig:NN}
}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Recursive Neural Networks (RNN)}
\label{class:RNN}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Long Short Term Memory (LSTM)}
\label{class:RNN:LSTM}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Convolutional Neural Networks (CNN)}
\label{class:CNN}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Learning Vector Quantization (LVQ)}
\label{class:kNN:LVQ}
% TODO

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Addressing Class Imbalance}
\label{class:imbalance}
% TODO


\documentclass[letterpaper]{article}

\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{September 24, 2014}
\maketitle
\section*{assignment}
assignment 3.1 no 23, 3.2no 3,6,7

if $(G,\cdot), a\in G$ then the cyclic subgroup generated by a is $<a>=\{\dots,a^{-2},a^{-1},e,a,a^2,a^3,\dots\}\subseteq G$.

$<a>$ is a subgroup of $G$ and if $H$ is a subgroup of $G$ and $a\in H$ then $<a>\subseteq H$, hence $<a>$ is the smallest subgroup of $G$

example: $(\mathbb{Z},+)\to <n>=n\mathbb{Z}$

$<1>=<-1>=\mathbb{Z}$

we say that the group $G$ is cyclic if there exists $a\in G$ such that $<a>=G$. So $(\mathbb{Z},+)$ is cyclic because $<1>=\mathbb{Z}$

another example $(\mathbb{Z}_n,+)$, $<[1]>=\mathbb{Z}_n$.

\section*{definition}
if there exists $n>0, n\in\mathbb{Z}$ such that $a^n=e$ we say that a has finite order and $\text{ord}(a)=\min\{n|n>0,n\in\mathbb{Z},a^n=e\}$. otherwise it has infinite order

\subsection*{proposition}
if $G$ is a finite group, $a\in G$ then $a$ has finite order. $\{e,a,a^2,a^3,\dots\}\in G$. Since $G$ is finite, we have some m,n where $a^m=a^n$ wlog $m>n$, $a^ma^-n=a^na^-n=e=a^{m-n}$
\section*{examples}
$(\mathbb{Q}^*,\cdot)$, $\text{ord}(-1)=2, \text{ord}(1)=1, \text{ord}(2)=\infty$
\subsubsection*{proof for proposition3.2.8ii}
use division algorithm, $k=ord(a)\cdot q+r$ with $0\le r<ord(a)$

then $e=a^k=a^{ord(a)q+r=[a^{ord(a)}]^{q}}a^r=a^r$

excercise, complete this proof
\end{document}


\documentclass{article}

\newcommand{\implies}{\Rightarrow}
\newcommand{\have}{{.~}}
\newcommand{\bstate}{{\mbox{\cal S}}}
\newcommand{\blocks}{{\mbox{\cal B}}}
\newcommand{\tabtop}{{\mbox{\sc t}}}
\newcommand{\tblocks}{{\blocks_\tabtop}}

\begin{document}

\section{Preliminaries}

\begin{itemize}
\item Let
  $$ \blocks = \{ 1 \ldots N \} $$
represent a set of $N$ blocks in a blocks-world problem.
We represent the tabletop with the symbol $\tabtop$, and
let
  $$ \tblocks = \blocks \cup \{ \tabtop \} $$

\item Call any
  $$ \bstate \subseteq \blocks \times \tblocks $$
a {\em state} of the blocks-world problem. The pairs of blocks in
$\bstate$ represent {\em on} relations.

\item A state $\bstate$ is {\em legal} iff
  \begin{itemize}
  \item  Every block is on something: $$
    \forall i \in \blocks \have
      \exists j \in \tblocks \have
	\langle i , j \rangle \in \bstate
  $$
  \item No block is on two things: $$
    \forall i \in \blocks \have
      \langle i , j \rangle \in \bstate \implies
      \not \exists j' \in \tblocks \have
	j \neq j' \wedge \langle i , j' \rangle \in \bstate
  $$
  \item No two blocks are on the same block: $$
    \forall j \in \blocks \have
      \langle i , j \rangle \in \bstate \implies
      \not \exists i' \in \blocks \have
	i \neq i' \wedge \langle i' , j \rangle \in \bstate
  $$
  \end{itemize}

\item A block $j$ is {\em clear} in a legal state $\bstate$ iff no block is above it:
  $$ \not \exists i \in \blocks \have \langle i , j \rangle \in \bstate $$

\item A block $i$ is {\em above} a block $i'$ in a legal
state $\bstate$ if there is a sequence
of blocks whose {\em on} relationships lead from $i$ to $i'$: $$
  \exists \{ j_1 , j_2 , \ldots , j_k \} \subseteq \blocks \have
    \{ \langle i , j_1 \rangle , \langle j_1 , j_2 \rangle ,
       \ldots , \langle j_k , i' \rangle \} \subseteq \bstate
$$

\item A legal state $\bstate$ is {\em realizable} iff no block
is above itself.

\item {\em Moves} are functions $m_{ij}$ which take a realizable state in
which blocks $i$ and $j$ are clear to a
realizable state in which $i$ is on $j$.

Let $i \in \blocks$ and $j \in \tblocks$, and let $C(ij)$ be the set
of realizable states
in which $i$ is clear, and either $j = \tabtop$ or $j$ is clear.
Then the domain of $m_{ij}$ is
$C(ij)$, and the range is the set of realizable states.
Define the functions $\theta_i(\bstate)$ mapping legal states to
blocks to have the value $j$
such that $\langle i, j \rangle \in \bstate$.
By the definition of a legal state, these functions are well-defined.
We then define
$m_{ij}$ by $$
  m_{ij}(\bstate) =
    \left ( \bstate - \{ \langle i, \theta_i(\bstate) \rangle \} \right )
    \cup \{ \langle i, j \rangle \}
$$

\item A blocks world {\em problem} consists of two realizable states: an
initial state $I$ and a goal state $G$.  A {\em solution} consists of
a sequence of pairs $$
  \langle i_1, j_1 \rangle , \langle i_2 , j_2 \rangle , \ldots ,
  \langle i_k , j_k \rangle \in \left ( \blocks \times \tblocks \right ) ^ \ast
$$
such that $$
  \left ( m_{i_k j_k} \circ \cdots \circ m_{i_2 j_2} \circ
  m_{i_1 j_1} \right ) (I) = G
$$
\end{itemize}

\section{Move Restriction}

\end{document}