\documentclass{memoir}
\usepackage{notestemplate}

%\logo{~/School-Work/Auxiliary-Files/resources/png/logo.png}
%\institute{Rice University}
%\faculty{Faculty of Whatever Sciences}
%\department{Department of Mathematics}
%\title{Class Notes}
%\subtitle{Based on MATH xxx}
%\author{\textit{Author}\\Gabriel \textsc{Gress}}
%\supervisor{Linus \textsc{Torvalds}}
%\context{Well, I was bored...}
%\date{\today}

\begin{document}

% \maketitle

% Notes taken on 05/20/21

\section{More Direct Methods}
\label{sec:more_direct_methods}

Let \(u_0 \in C^{1}(\overline{\Omega })\) with \(\Omega \) some open bounded set. We want to minimize the functional:
\begin{align*}
	\inf_{u} \left\{\frac{1}{2} \int_\Omega \left| \nabla u \right|^2 \mid u =u_0 \text{ on }\partial\Omega  \right\}
\end{align*}
Let \(\left\{ u_j \right\}_j\) be a minimizing sequence for \(\inf_{\mathcal{A}} \mathcal{F}\) that satisfies the boundary conditions, and satisfying
\begin{align*}
	\lim_{j \to \infty} \frac{1}{2}\int_\Omega \left| \nabla u_j \right|^2 = \inf_{\mathcal{A}} \mathcal{F}.
\end{align*}
Because the limit supremum of the sequence is finite, we know that there exists a subsequence \(\left\{ \tilde{u}_j \right\}_j\) and \(T \in L^2(\Omega , \R^{n})\) such that
\begin{align*}
	\int_\Omega T \varphi  = \lim_{j \to \infty} \int_\Omega \varphi \nabla \tilde{u}_j \quad \forall \varphi  \in L^2(\Omega )
\end{align*}
Does this subsequence also have a limit of \(u\)? And is this \(T\) simply \(\nabla u\)?

\begin{thm}[Trace Inequality for Minimizers]
	If \(u \in C^{1}(\overline{\Omega })\) and \(\Omega \) is bounded with smooth boundary, then
	\begin{align*}
		\int_\Omega u^2 \leq C(n, \textrm{diam}\Omega ) \left[ \int_{\partial \Omega }u^2 + \int_\Omega \left| \nabla  u\right|^2 \right] .
	\end{align*}
\end{thm}
This quantifies the restrictions of the boundary conditions \(\begin{cases}
	u=0 & \partial\Omega \\
	\nabla u=0 & \Omega
\end{cases} \implies u \cong 0\) on \(\Omega \).

\begin{proof}[Proof of trace inequality]
	Choose \(X(x) = x\) so that \(\textrm{div}(X) = n\). We will pick \(x_0 \in \R^{n}\) later so that the following is satisfied:
	\begin{align*}
		n \int_\Omega u^2 = \int_\Omega u^2 \textrm{div}(x-x_0)\\
		= \int_\Omega \textrm{div}\left( (x-x_0)u^2 \right) - \int_\omega 2u \nabla u \cdot (x-x_0)\\
		\leq  \int_{\partial \Omega }u^2(x-x_0)\cdot \nu_\Omega + \int_\Omega 2 \left| u \right| \left| \nabla u \right| (x-x_0)
	\end{align*}
	Using some standard algebraic inequalities (i.e. \(2ab \leq \varepsilon a^2 + \frac{1}{\varepsilon}b^2\) so that \((\sqrt{\varepsilon} a = \frac{b}{\sqrt{\varepsilon} })^2\geq 0\)), we can put bounds on both terms individually to get
	\begin{align*}
		\int_{\partial \Omega } u^2(x-x_0) \cdot \nu_\Omega + \int_\Omega 2 \left| u \right| \left| \nabla u \right| (x-x_0)\leq 2 \textrm{diam} (\Omega ) \left[ \int_{\partial \Omega }u^2 + \int_\Omega 2\left| u \right| \left| \nabla u \right|  \right]\\
		\leq 2 \textrm{diam}(\Omega ) \int_{\partial \Omega } u^2+ 2 \textrm{diam}(\Omega ) \left[ \varepsilon \int_{\Omega }u^2 + \frac{1}{\varepsilon} \int_{\Omega } \left| \nabla u \right|^2 \right]\\
		\implies \left( n-2\varepsilon\textrm{diam}(\Omega ) \right) \int_{\Omega } u^2 \leq 2 \textrm{diam}(\Omega )\left[ \int_{\partial \Omega }u^2 + \frac{1}{\varepsilon} \int_\Omega  \left| \nabla u \right|^2 \right]
	\end{align*}
	Now we simply choose \(\varepsilon = \dfrac{n}{4 \textrm{diam}(\Omega )}\) and we have the statement
	\begin{align*}
		\int_\Omega u^2 \leq \dfrac{4 \textrm{diam}(\Omega)}{n} \left[ \int_{\partial \Omega }u^2 + \dfrac{4 \textrm{diam}(\Omega )}{n} \int_\Omega  \left| \nabla u \right|^2 \right]
	\end{align*}
\end{proof}

What is the relation between \(T\) and \(u\)?

\begin{defn}
	Let \(u \in L^{1}_{\textrm{loc}}(\Omega )\), and \(T \in L^{1}_{\textrm{loc}}(\Omega , \R^{n})\). We say that \(T\) is the unique vector field called the \textbf{weak} or \textbf{distributional gradient} of \(u\) if the following is satisfied:
	\begin{align*}
		\int_\Omega u \nabla \varphi = - \int_\Omega \varphi T \quad \forall \varphi  \in C^{\infty}_c (\Omega )
	\end{align*}
\end{defn}

Earlier, we asked if the limit of the subsequence was equivalent to the \(u\) earlier. Now we claim that the weak limit \(T\) of \(\left\{ \nabla u_j \right\}_j\) is the weak gradient of
 \begin{align*}
	u = \lim_{n \to \infty}^{w} \left\{ u_j \right\}_j .
\end{align*}
This follows because
\begin{align*}
	\int_\omega u \nabla \varphi = \lim_{j \to \infty} \int_\Omega u_j \nabla \varphi  = - \lim_{j \to \infty} \int_\Omega \varphi \nabla u_j\\
	= - \int_\Omega \varphi T
\end{align*}
In summary, by weak compactness in \(L^2\) and by Trace inequality, we have
\begin{align*}
	\begin{cases}
		u_j \to u & L^2(\Omega )\quad u \in W^{1,2}(\Omega )\\
		\nabla u_j \to \nabla u & L^2(\Omega ,\R^{n})
	\end{cases}
\end{align*}
We need to define a space where these weak objects can live:
\begin{defn}[\(L^2\) Sobolev Space]
	We define the \textbf{Sobolev Space in \(L^2\)} to be
	\begin{align*}
		W^{1,2}(\Omega ) := \left\{u \in L^2(\Omega ) \mid \exists  \text{ weak gradient }\nabla u \in L^2(\Omega ,\R^{n}) \right\}
	\end{align*}
\end{defn}

Now we ask a new question: is it true that
\begin{align*}
	\int_\Omega \left| \nabla u \right|^2 \leq \liminf_{j \to \infty} \int_\Omega  \left| \nabla u_j \right|^2
\end{align*}

Yes. Pick any \(T \in L^2(\Omega ,\R^{n})\). Then
\begin{align*}
	\int_\Omega (\nabla u) \cdot T = \lim_{j \to \infty} \int_\Omega (\nabla u_j) \cdot T\\
	\leq \lim_{j \to \infty} \left( \int_\Omega \left| \nabla u_j \right|^2 \right)^{ \sfrac{1}{2}} \left( \int_{\Omega } \left| T \right|^2 \right)^{ \sfrac{1}{2}}
\end{align*}
Now choose
\begin{align*}
	T = \frac{\nabla u}{\|\nabla u\|_{L^2}}
\end{align*}
to get the inequality we desire.\\

Another question. Is \(u \in \mathcal{A}\), where
\begin{align*}
	\mathcal{A} = \left\{ \mid  \right\}
\end{align*}

\begin{exmp}
	Consider
	\begin{align*}
		u_\alpha (x) = \dfrac{1}{\left| x \right|^{\alpha }}\\
		\nabla u_\alpha (x) = \dfrac{-\alpha }{\left| x \right|^{\alpha +1}} \hat{x}
	\end{align*}
	where \(\hat{x} = \dfrac{x}{\left| x \right| }\). Then
	\begin{align*}
		\int_{B_\varepsilon(0)} \left| \nabla u_\alpha  \right|^2 \approx \int_{0}^{\varepsilon} \dfrac{\rho^{n-1}}{\left( \rho^{\alpha +1} \right)^2} \,d \rho < \infty \\
		\iff n-1 -2(\alpha +1) > -1 \iff\alpha < \frac{n}{2}-1
	\end{align*}
\end{exmp}

	Now we ask a seemingly silly question: is \(\nabla u_{\alpha }\) the weak gradient of \(u_\alpha \)? Yes, but we have to be careful-- if \(\nabla u = 0\) almost everywhere in \(\R^2\), say, \(u\) is a step function, then the weak gradient doesn't exist. One can show that
	\begin{align*}
		\int \varphi \nabla u_{\alpha } = - \int u_\alpha \nabla \varphi \quad \forall \varphi \in C^{\infty}_c(B_1(0))
	\end{align*}
via some standards calculations from distribution theory. Because this holds, we have
\begin{align*}
	u^{(n)} = \sum_{k=1}^{n} \dfrac{2^{-k}}{\left| x-x_k \right|\alpha }
\end{align*}
where \(\left\{ x_k\right\}_{k=1}^{\infty} \) is countably dense in \(B_1(0)\). We have that \(u^{(n)} \in W^{1,2}(\Omega )\) if \(\alpha < \frac{n}{2}-1\).In fact, this is a Cauchy sequence in \(W^{1,2}(\Omega )\) (check this), and so
\begin{align*}
	\exists u \in W^{1,2}(\Omega ) \text{ such that{t} } u = \lim_{N \to \infty} u^{(N)}= \sum_{k=1}^{\infty} \dfrac{2^{-k}}{\left| x-x_k \right|^{\alpha }}
\end{align*}

\begin{exmp}
	Let \(u \not\in L^{\omega }_{\textrm{loc}}(B_{1}(0))\). For all \(B_\varepsilon(x) \subset B_1(0)\), we have
	\begin{align*}
		\textrm{essential sup}_{B_\varepsilon} \left| u \right|  = \infty
	\end{align*}
\end{exmp}

\begin{rmrk}
What is the essential supremum? Let \(f:\Omega \to \R\). Consider the case when \(\tilde{f}:\Omega \to \R\) by
\begin{align*}
	\tilde{f}(x) = \begin{cases}
		f(x) & \Omega \setminus Q^{n}\\
		j & x = x_j \in Q^{n}\cap \Omega
	\end{cases}
\end{align*}
So that \(f = \tilde{f}\) almost everywhere in \(\Omega \) and \(\sup_{j} \tilde{f}= \infty\). A traditional supremum is unbounded and so won't work nicely here. Instead, notice that
\begin{align*}
	\left\{x \in \Omega  \mid f(x) > t \right\} = \left\{f > t \right\} \\
	\mathcal{L}^{n}( \left\{ f > t \right\} ) = \mathcal{L}^{n}\left( \left\{ \tilde{f} > t \right\}  \right)
\end{align*}
With this in mind, we define
\begin{align*}
	\textrm{essential sup}_{\Omega } f := \inf_{t} \left\{ t\mid \mathcal{L}^{n}( \left\{f >t \right\} ) = 0 \right\} .
\end{align*}
\end{rmrk}

\begin{exmp}
	\(u=u_0\) on \(\partial \Omega \)?\\

	\(u_j \to u\) in \(L^2(\Omega )\) but we do \textit{not} have convergence of boundary data. To see this, look at the diagram in \ref{fig:convergence-of-boundary-data}.
\end{exmp}

\begin{figure}[ht]
    \centering
     \def\svgwidth{1\linewidth}
     \input{./figures/convergence-of-boundary-data.pdf_tex}
    \caption{Convergence of Boundary Data}
    \label{fig:convergence-of-boundary-data}
\end{figure}

The moral is that we need more properties to hold: if \(u_j = u_0 \partial \Omega \), \(\sup_{j} \int \left| \nabla u_j \right|^2 < \infty\) and
\begin{align*}
	\begin{cases}
		u_j \to u\\
		\nabla u_j \to \nabla u
	\end{cases}
\end{align*}
then we can say that \(u = u_0\) on \(\partial \Omega \) in the distributional sense.\\

To summarize the work we've done, if we are working with problems of the form
\begin{align*}
	\inf_{u} \left\{\frac{1}{2}\int_\Omega \left| \nabla u \right|^2 \mid u = u_0 \text{ on }\partial \Omega  \text{ in distribution} \right\}
\end{align*}
where \(u \in W^{1,2}(\Omega )\), then the direct method gives you a minimizer. For the restricted problem with \(u \in C^{1}(\overline{\Omega })\) :
\begin{align*}
	\inf_{u} \left\{\frac{1}{2}\int_\Omega \left| \nabla u \right|^2 \mid u = u_0 \text{ on }\partial\Omega  \text{ in classical sense} \right\} \\
	= \frac{1}{2}\int_\Omega  \left| \nabla u \right|^2
\end{align*}
we have the existence of a minimizer \(u \in W^{1,2}(\Omega )\).\\

Some classical results give us some insight:
\begin{thm}[Serrin '61]
	If \(\left\{ u_j \right\}_j \subset L^{1}_{\textrm{loc}}(\Omega )\) and \(\lim_{j \to \infty} u_j = u\) in \(L^{1}_{\textrm{loc}}(\Omega )\) and \(u_j,u\) have weak gradient in \(L^{1}(\Omega ,\R^{n})\), then for all \(f:\R^{n}\to [0,\infty)\) convex, we have
	\begin{align*}
		\int_\Omega f(\nabla u) \leq \liminf_{j \to \infty} \int_\Omega f(\nabla u_j).
	\end{align*}
\end{thm}
We have a converse too. If \(f:\R^{n}\to [0,\infty)\) is continuous and the above result holds, then for all \(u_j,j\) as above, \(f\) is convex.

\end{document}

Topics in Partial Differential Equations/Calculus of Variations/Notes/source/Lecture4.tex


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Copyright (c) 2003-2018 by The University of Queensland
% http://www.uq.edu.au
%
% Primary Business: Queensland, Australia
% Licensed under the Apache License, version 2.0
% http://www.apache.org/licenses/LICENSE-2.0
%
% Development until 2012 by Earth Systems Science Computational Center (ESSCC)
% Development 2012-2013 by School of Earth Sciences
% Development from 2014 by Centre for Geoscience Computing (GeoComp)
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{The \finley Module}\label{chap:finley}
%\declaremodule{extension}{finley}
%\modulesynopsis{Solving linear, steady partial differential equations using finite elements}

The \finley library allows the creation of domains for solving
linear, steady partial differential
equations\index{partial differential equations} (PDEs) or systems
of PDEs using isoparametrical finite elements\index{FEM!isoparametrical}.
It supports unstructured 1D, 2D and 3D meshes.
The PDEs themselves are represented by the \LinearPDE class
of \escript.
\finley is parallelized under both \OPENMP and \MPI.
A more restricted form of this library ({\it dudley}) is described in
Section~\ref{sec:dudley}.

\section{Formulation}
For a single PDE that has a solution with a single component the linear PDE is
defined in the following form:
\begin{equation}\label{FINLEY.SINGLE.1}
\begin{array}{cl} &
\displaystyle{
\int_{\Omega}
A_{jl} \cdot v_{,j}u_{,l}+ B_{j} \cdot v_{,j} u+ C_{l} \cdot v u_{,l}+D \cdot vu \; d\Omega }  \\
+ & \displaystyle{\int_{\Gamma} d \cdot vu \; d{\Gamma} }
+  \displaystyle{\int_{\Gamma^{contact}} d^{contact} \cdot [v][u] \; d{\Gamma} } \\
= & \displaystyle{\int_{\Omega}  X_{j} \cdot v_{,j}+ Y \cdot v \; d\Omega }\\
+ & \displaystyle{\int_{\Gamma} y \cdot v \; d{\Gamma}}  +
\displaystyle{\int_{\Gamma^{contact}} y^{contact}\cdot [v] \; d{\Gamma}} \\
\end{array}
\end{equation}

\section{Meshes}
\label{FINLEY MESHES}

\begin{figure}
\centerline{\includegraphics{FinleyMesh}}
\caption{Subdivision of an Ellipse into triangles order 1 (\finleyelement{Tri3})}
\label{FINLEY FIG 0}
\end{figure}

To understand the usage of \finley one needs to have an understanding of how
the finite element meshes\index{FEM!mesh} are defined.
\fig{FINLEY FIG 0} shows an example of the subdivision of an ellipse into
so-called elements\index{FEM!elements}\index{element}.
In this case, triangles have been used but other forms of subdivisions can be
constructed, e.g. quadrilaterals or, in the three-dimensional case, into
tetrahedra and hexahedra. The idea of the finite element method is to
approximate the solution by a function which is a polynomial of a certain order
and is continuous across its boundary to neighbour elements.
In the example of \fig{FINLEY FIG 0} a linear polynomial is used on each
triangle. As one can see, the triangulation is quite a poor approximation of
the ellipse. It can be improved by introducing a midpoint on each element edge
then positioning those nodes located on an edge expected to describe the
boundary, onto the boundary.
In this case the triangle gets a curved edge which requires a parameterization
of the triangle using a quadratic polynomial.
For this case, the solution is also approximated by a piecewise quadratic
polynomial (which explains the name isoparametrical elements),
see \Ref{Zienc,NumHand} for more details.
\finley also supports macro elements\index{macro elements}.
For these elements a piecewise linear approximation is used on an element which
is further subdivided (in the case of \finley halved).
As such, these elements do not provide more than a further mesh refinement but
should be used in the case of incompressible flows, see \class{StokesProblemCartesian}.
For these problems a linear approximation of the pressure across the element is
used (use the \ReducedSolutionFS) while the refined element is used to
approximate velocity. So a macro element provides a continuous pressure
approximation together with a velocity approximation on a refined mesh.
This approach is necessary to make sure that the incompressible flow has a
unique solution.

The union of all elements defines the domain of the PDE.
Each element is defined by the nodes used to describe its shape.
In \fig{FINLEY FIG 0} the element, which has type \finleyelement{Tri3}, with
element reference number $19$\index{element!reference number} is defined by the
nodes with reference numbers $9$, $11$ and $0$\index{node!reference number}.
Notice that the order is counterclockwise.
The coefficients of the PDE are evaluated at integration nodes with each
individual element.
For quadrilateral elements a Gauss quadrature scheme is used.
In the case of triangular elements a modified form is applied.
The boundary of the domain is also subdivided into elements\index{element!face}.
In \fig{FINLEY FIG 0} line elements with two nodes are used.
The elements are also defined by their describing nodes, e.g. the face element
with reference number $20$, which has type \finleyelement{Line2}, is defined by
the nodes with the reference numbers $11$ and $0$.
Again the order is crucial, if moving from the first to second node the domain
has to lie on the left hand side (in the case of a two-dimensional surface
element the domain has to lie on the left hand side when moving
counterclockwise). If the gradient on the surface of the domain is to be
calculated rich face elements need to be used. Rich elements on a face are
identical to interior elements but with a modified order of nodes such that the
'first' face of the element aligns with the surface of the domain.
In \fig{FINLEY FIG 0} elements of the type \finleyelement{Tri3Face} are used.
The face element reference number $20$ as a rich face element is defined by the
nodes with reference numbers $11$, $0$ and $9$.
Notice that the face element $20$ is identical to the interior element $19$
except that, in this case, the order of the node is different to align the first
edge of the triangle (which is the edge starting with the first node) with the
boundary of the domain.

Be aware that face elements and elements in the interior of the domain must
match, i.e. a face element must be the face of an interior element or, in case
of a rich face element, it must be identical to an interior element.
If no face elements are specified \finley implicitly assumes homogeneous
natural boundary conditions\index{natural boundary conditions!homogeneous},
i.e. \var{d}=$0$ and \var{y}=$0$, on the entire boundary of the domain.
For inhomogeneous natural boundary conditions\index{natural boundary conditions!inhomogeneous},
the boundary must be described by face elements.

\begin{figure}
\centerline{\includegraphics{FinleyContact}}
\caption{Mesh around a contact region (\finleyelement{Rec4})}
\label{FINLEY FIG 01}
\end{figure}

If discontinuities of the PDE solution are considered, contact
elements\index{element!contact}\index{contact conditions} are introduced to
describe the contact region $\Gamma^{contact}$ even if $d^{contact}$ and
$y^{contact}$ are zero.
\fig{FINLEY FIG 01} shows a simple example of a mesh of rectangular elements
around a contact region $\Gamma^{contact}$\index{element!contact}.
The contact region is described by the elements $4$, $3$ and $6$.
Their element type is \finleyelement{Line2_Contact}.
The nodes $9$, $12$, $6$ and $5$ define contact element $4$, where the
coordinates of nodes $12$ and $5$ and nodes $4$ and $6$ are identical, with the
idea that nodes $12$ and $9$ are located above and nodes $5$ and $6$ below the
contact region.
Again, the order of the nodes within an element is crucial.
There is also the option of using rich elements if the gradient is to be
calculated on the contact region. Similarly to the rich face elements these
are constructed from two interior elements by reordering the nodes such that
the 'first' face of the element above and the 'first' face of the element below
the contact regions line up. The rich version of element $4$ is of type
\finleyelement{Rec4Face_Contact} and is defined by the nodes $9$, $12$, $16$,
$18$, $6$, $5$, $0$ and $2$.
\tab{FINLEY TAB 1} shows the interior element types and the corresponding
element types to be used on the face and contacts.
\fig{FINLEY.FIG:1}, \fig{FINLEY.FIG:2} and \fig{FINLEY.FIG:4} show the ordering
of the nodes within an element.

\begin{table}
\centering
\begin{tabular}{l|llll}
\textbf{interior}&\textbf{face}&\textbf{rich face}&\textbf{contact}&\textbf{rich contact}\\
\hline
\finleyelement{Line2} & \finleyelement{Point1} & \finleyelement{Line2Face} & \finleyelement{Point1_Contact} & \finleyelement{Line2Face_Contact}\\
\finleyelement{Line3} & \finleyelement{Point1} & \finleyelement{Line3Face} & \finleyelement{Point1_Contact} & \finleyelement{Line3Face_Contact}\\
\finleyelement{Tri3} & \finleyelement{Line2} & \finleyelement{Tri3Face} & \finleyelement{Line2_Contact} & \finleyelement{Tri3Face_Contact}\\
\finleyelement{Tri6} & \finleyelement{Line3} & \finleyelement{Tri6Face} & \finleyelement{Line3_Contact} & \finleyelement{Tri6Face_Contact}\\
\finleyelement{Rec4} & \finleyelement{Line2} & \finleyelement{Rec4Face} & \finleyelement{Line2_Contact} & \finleyelement{Rec4Face_Contact}\\
\finleyelement{Rec8} & \finleyelement{Line3} & \finleyelement{Rec8Face} & \finleyelement{Line3_Contact} & \finleyelement{Rec8Face_Contact}\\
\finleyelement{Rec9} & \finleyelement{Line3} & \finleyelement{Rec9Face} & \finleyelement{Line3_Contact} & \finleyelement{Rec9Face_Contact}\\
\finleyelement{Tet4} & \finleyelement{Tri6} & \finleyelement{Tet4Face} & \finleyelement{Tri6_Contact} & \finleyelement{Tet4Face_Contact}\\
\finleyelement{Tet10} & \finleyelement{Tri9} & \finleyelement{Tet10Face} & \finleyelement{Tri9_Contact} & \finleyelement{Tet10Face_Contact}\\
\finleyelement{Hex8} & \finleyelement{Rec4} & \finleyelement{Hex8Face} & \finleyelement{Rec4_Contact} & \finleyelement{Hex8Face_Contact}\\
\finleyelement{Hex20} & \finleyelement{Rec8} & \finleyelement{Hex20Face} & \finleyelement{Rec8_Contact} & \finleyelement{Hex20Face_Contact}\\
\finleyelement{Hex27} & \finleyelement{Rec9} & N/A & N/A & N/A\\
\finleyelement{Hex27Macro} & \finleyelement{Rec9Macro} & N/A & N/A & N/A\\
\finleyelement{Tet10Macro} & \finleyelement{Tri6Macro} & N/A & N/A & N/A\\
\finleyelement{Rec9Macro} & \finleyelement{Line3Macro} & N/A & N/A & N/A\\
\finleyelement{Tri6Macro} & \finleyelement{Line3Macro} & N/A & N/A & N/A\\
\end{tabular}
\caption{Finley elements and corresponding elements to be used on domain faces
and contacts.
The rich types have to be used if the gradient of the function is to be
calculated on faces and contacts, respectively.}
\label{FINLEY TAB 1}
\end{table}

The native \finley file format is defined as follows.
Each node \var{i} has \var{dim} spatial coordinates \var{Node[i]}, a reference
number \var{Node_ref[i]}, a degree of freedom \var{Node_DOF[i]} and a tag
\var{Node_tag[i]}.
In most cases \var{Node_DOF[i]}=\var{Node_ref[i]} however, for periodic
boundary conditions, \var{Node_DOF[i]} is chosen differently, see example below.
The tag can be used to mark nodes sharing the same properties.
Element \var{i} is defined by the \var{Element_numNodes} nodes
\var{Element_Nodes[i]} which is a list of node reference numbers.
The order of these is crucial. Each element has a reference number
\var{Element_ref[i]} and a tag \var{Element_tag[i]}.
The tag can be used to mark elements sharing the same properties.
For instance elements above a contact region are marked with tag $2$ and
elements below a contact region are marked with tag $1$.
\var{Element_Type} and \var{Element_Num} give the element type and the number
of elements in the mesh.
Analogue notations are used for face and contact elements.
The following \PYTHON script prints the mesh definition in the \finley file
format:
\begin{python}
  print("%s\n"%mesh_name)
  # node coordinates:
  print("%dD-nodes %d\n"%(dim, numNodes))
  for i in range(numNodes):
     print("%d %d %d"%(Node_ref[i], Node_DOF[i], Node_tag[i]))
     for j in range(dim): print(" %e"%Node[i][j])
     print("\n")
  # interior elements
  print("%s %d\n"%(Element_Type, Element_Num))
  for i in range(Element_Num):
     print("%d %d"%(Element_ref[i], Element_tag[i]))
     for j in range(Element_numNodes): print(" %d"%Element_Nodes[i][j])
     print("\n")
  # face elements
  print("%s %d\n"%(FaceElement_Type, FaceElement_Num))
  for i in range(FaceElement_Num):
     print("%d %d"%(FaceElement_ref[i], FaceElement_tag[i]))
     for j in range(FaceElement_numNodes): print(" %d"%FaceElement_Nodes[i][j])
     print("\n")
  # contact elements
  print("%s %d\n"%(ContactElement_Type, ContactElement_Num))
  for i in range(ContactElement_Num):
     print("%d %d"%(ContactElement_ref[i], ContactElement_tag[i]))
     for j in range(ContactElement_numNodes):
         print(" %d"%ContactElement_Nodes[i][j])
     print("\n")
  # point sources (not supported yet)
  print("Point1 0")
\end{python}

The following example of a mesh file defines the mesh shown in \fig{FINLEY FIG 01}:
\begin{verbatim}
Example 1
2D Nodes 16
0   0 0 0.   0.
2   2 0 0.33 0.
3   3 0 0.66 0.
7   4 0 1.   0.
5   5 0 0.   0.5
6   6 0 0.33 0.5
8   8 0 0.66 0.5
10 10 0 1.0  0.5
12 12 0 0.   0.5
9   9 0 0.33 0.5
13 13 0 0.66 0.5
15 15 0 1.0  0.5
16 16 0 0.   1.0
18 18 0 0.33 1.0
19 19 0 0.66 1.0
20 20 0 1.0  1.0
Rec4 6
 0 1  0  2  6  5
 1 1  2  3  8  6
 2 1  3  7 10  8
 5 2 12  9 18 16
 7 2 13 19 18  9
10 2 20 19 13 15
Line2 0
Line2_Contact 3
 4 0  9 12  6 5
 3 0 13  9  8 6
 6 0 15 13 10 8
Point1 0
\end{verbatim}
Notice that the order in which the nodes and elements are given is arbitrary.
In the case that rich contact elements are used the contact element section
gets the form
\begin{verbatim}
Rec4Face_Contact 3
 4 0  9 12 16 18  6  5  0  2
 3 0 13  9 18 19  8  6  2  3
 6 0 15 13 19 20 10  8  3  7
\end{verbatim}
Periodic boundary conditions\index{boundary conditions!periodic} can be
introduced by altering \var{Node_DOF}.
It allows identification of nodes even if they have different physical locations.
For instance, to enforce periodic boundary conditions at the face $x_0=0$ and
$x_0=1$ one identifies the degrees of freedom for nodes $0$, $5$, $12$ and $16$
with the degrees of freedom for $7$, $10$, $15$ and $20$, respectively.
The node section of the \finley mesh now reads:
\begin{verbatim}
2D Nodes 16
0   0 0 0.   0.
2   2 0 0.33 0.
3   3 0 0.66 0.
7   0 0 1.   0.
5   5 0 0.   0.5
6   6 0 0.33 0.5
8   8 0 0.66 0.5
10  5 0 1.0  0.5
12 12 0 0.   0.5
9   9 0 0.33 0.5
13 13 0 0.66 0.5
15 12 0 1.0  0.5
16 16 0 0.   1.0
18 18 0 0.33 1.0
19 19 0 0.66 1.0
20 16 0 1.0  1.0
\end{verbatim}

\clearpage
\input{finleyelements}
\clearpage

\section{Macro Elements}
\label{SEC FINLEY MACRO}

\begin{figure}[th]
\begin{center}
\includegraphics{FinleyMacroLeg}\\
\subfigure[Triangle]{\label{FINLEY MACRO TRI}\includegraphics{FinleyMacroTri}}\quad
\subfigure[Quadrilateral]{\label{FINLEY MACRO REC}\includegraphics{FinleyMacroRec}}
\end{center}
\caption{Macro elements in \finley}
\end{figure}

\finley supports the usage of macro elements\index{macro elements} which can be
used to achieve LBB compliance when solving incompressible fluid flow problems.
LBB compliance is required to get a problem which has a unique solution for
pressure and velocity. For macro elements the pressure and velocity are
approximated by a polynomial of order 1 but the velocity approximation bases on
a refinement of the elements. The nodes of a triangle and quadrilateral element
are shown in Figures~\ref{FINLEY MACRO TRI} and~\ref{FINLEY MACRO REC},
respectively. In essence, the velocity uses the same nodes like a quadratic
polynomial approximation but replaces the quadratic polynomial by piecewise
linear polynomials. In fact, this is the way \finley defines the macro elements.
In particular \finley uses the same local ordering of the nodes for the macro
element as for the corresponding quadratic element. Another interpretation is
that one uses a linear approximation of the velocity together with a linear
approximation of the pressure but on elements created by combining elements to
macro elements. Notice that the macro elements still use quadratic
interpolation to represent the element and domain boundary.
However, if elements have linear boundaries a macro element approximation for
the velocity is equivalent to using a linear approximation on a mesh which is
created through a one-step global refinement.
Typically macro elements are only required to use when an incompressible fluid
flow problem is solved, e.g. the Stokes problem in \Sec{STOKES PROBLEM}.
Please see \Sec{FINLEY MESHES} for more details on the supported macro elements.

\section{Linear Solvers in \SolverOptions}

Table~\ref{TAB FINLEY SOLVER OPTIONS 1} and
Table~\ref{TAB FINLEY SOLVER OPTIONS 2} show the solvers and preconditioners
supported by \finley through the \PASO library.
Currently direct solvers are not supported under \MPI.
By default, \finley uses the iterative solvers \PCG for symmetric and \BiCGStab
for non-symmetric problems.
If the direct solver is selected, which can be useful when solving very
ill-posed equations, \finley uses the \MKL\footnote{If the stiffness matrix is
non-regular \MKL may return without a proper error code. If you observe
suspicious solutions when using \MKL, this may be caused by a non-invertible
operator.} solver package. If \MKL is not available \UMFPACK is used.
If \UMFPACK is not available a suitable iterative solver from \PASO is used.

\begin{table}
\centering
{\scriptsize
\begin{tabular}{l||c|c|c|c|c|c|c|c}
\member{setSolverMethod} & \member{DIRECT}& \member{PCG} & \member{GMRES} & \member{TFQMR} & \member{MINRES} & \member{PRES20} & \member{BICGSTAB} & lumping \\
\hline
 \hline
 \member{setReordering} & $\checkmark$ & & & & & &\\
 \hline  \member{setRestart} &  & & $\checkmark$ & & & $20$ & \\
 \hline\member{setTruncation} &  & & $\checkmark$ & & & $5$ & \\
   \hline\member{setIterMax} &  & $\checkmark$& $\checkmark$ & $\checkmark$& $\checkmark$& $\checkmark$ & $\checkmark$ \\
 \hline\member{setTolerance} &  & $\checkmark$& $\checkmark$ & $\checkmark$& $\checkmark$& $\checkmark$ & $\checkmark$ \\
 \hline\member{setAbsoluteTolerance} &  & $\checkmark$& $\checkmark$ & $\checkmark$& $\checkmark$& $\checkmark$ & $\checkmark$ \\
\hline\member{setReordering} & $\checkmark$ & & & & & & & \\
\end{tabular}
}
\caption{Solvers available for \finley and the \PASO package and the relevant
options in \class{SolverOptions}.
\MKL supports \member{MINIMUM_FILL_IN}\index{linear solver!minimum fill-in ordering}\index{minimum fill-in ordering}
and \member{NESTED_DISSECTION}\index{linear solver!nested dissection ordering}\index{nested dissection}
reordering.
Currently the \UMFPACK interface does not support any reordering.
\label{TAB FINLEY SOLVER OPTIONS 1}}
\end{table}

\begin{table}
\begin{center}
{\scriptsize
\begin{tabular}{l||c|c|c|c|c|c|c}
\member{NO_PRECONDITIONER}&
\member{AMG}&
\member{JACOBI}&
\member{GAUSS_SEIDEL}&
\member{REC_ILU}&
\member{RILU}&
\member{ILU0}&
\member{DIRECT}\\
\hline
status:& $\checkmark$ &$\checkmark$&$\checkmark$&$\checkmark$&later&$\checkmark$&later\\
\hline
\hline
\member{setLevelMax}&$\checkmark$& & & & & &\\
\hline
\member{setCoarseningThreshold}&$\checkmark$& & & & & &\\
\hline
\member{setMinCoarseMatrixSize}&$\checkmark$& & & & & &\\
\hline
\member{setMinCoarseMatrixSparsity}&$\checkmark$& & & & & &\\
\hline
\member{setNumSweeps}& &$\checkmark$&$\checkmark$& & & &\\
\hline
\member{setNumPreSweeps}&$\checkmark$& & & & & &\\
\hline
\member{setNumPostSweeps}&$\checkmark$& & & & & &\\
\hline
\member{setDiagonalDominanceThreshold}&$\checkmark$& & & & & &\\
\hline
\member{setAMGInterpolation}&$\checkmark$& & & & & &\\
\hline
\member{setRelaxationFactor}& & & & &$\checkmark$& &\\
\end{tabular}
}
\caption{Preconditioners available for \finley and the \PASO package and the
relevant options in \class{SolverOptions}.
\label{TAB FINLEY SOLVER OPTIONS 2}}
\end{center}
\end{table}

\section{Functions}
\begin{funcdesc}{ReadMesh}{fileName \optional{, \optional{integrationOrder=-1}, optimize=True}}
creates a \Domain object from the FEM mesh defined in file \var{fileName}.
The file must be in the \finley file format.
If \var{integrationOrder} is positive, a numerical integration scheme is chosen
which is accurate on each element up to a polynomial of degree
\var{integrationOrder}\index{integration order}.
Otherwise an appropriate integration order is chosen independently.
By default the labeling of mesh nodes and element distribution is optimized.
Set \var{optimize=False} to switch off relabeling and redistribution.
\end{funcdesc}

\begin{funcdesc}{ReadGmsh}{fileName, numDim, \optional{, \optional{integrationOrder=-1}, optimize=True\optional{, useMacroElements=False}}}
creates a \Domain object from the FEM mesh defined in file \var{fileName} for
a domain of dimension \var{numDim}.
The file must be in the \gmshextern file format.
If \var{integrationOrder} is positive, a numerical integration scheme is chosen
which is accurate on each element up to a polynomial of degree
\var{integrationOrder}\index{integration order}.
Otherwise an appropriate integration order is chosen independently.
By default the labeling of mesh nodes and element distribution is optimized.
Set \var{optimize=False} to switch off relabeling and redistribution.
If \var{useMacroElements} is set, second order elements are interpreted as
macro elements\index{macro elements}.
\end{funcdesc}

\begin{funcdesc}{MakeDomain}{design\optional{, integrationOrder=-1\optional{, optimizeLabeling=True\optional{, useMacroElements=False}}}}
creates a \finley \Domain from a \pycad \class{Design} object using \gmshextern.
The \class{Design} \var{design} defines the geometry.
If \var{integrationOrder} is positive, a numerical integration scheme is chosen
which is accurate on each element up to a polynomial of degree
\var{integrationOrder}\index{integration order}.
Otherwise an appropriate integration order is chosen independently.
Set \var{optimizeLabeling=False} to switch off relabeling and redistribution
(not recommended).
If \var{useMacroElements} is set, macro elements\index{macro elements} are used.
Currently \function{MakeDomain} does not support \MPI.
\end{funcdesc}

\begin{funcdesc}{load}{fileName}
recovers a \Domain object from a dump file \var{fileName} created by the
\function{dump} method of a \Domain object.
\end{funcdesc}

\begin{funcdesc}{Rectangle}{n0,n1,order=1,l0=1.,l1=1., integrationOrder=-1, \\
  periodic0=\False, periodic1=\False, useElementsOnFace=\False, optimize=\False}
generates a \Domain object representing a two-dimensional rectangle between
$(0,0)$ and $(l0,l1)$ with orthogonal edges.
The rectangle is filled with \var{n0} elements along the $x_0$-axis and
\var{n1} elements along the $x_1$-axis.
For \var{order}=1 and \var{order}=2, elements of type \finleyelement{Rec4} and
\finleyelement{Rec8} are used, respectively.
In the case of \var{useElementsOnFace}=\False, \finleyelement{Line2} and
\finleyelement{Line3} are used to subdivide the edges of the rectangle, respectively.
If \var{order}=-1, \finleyelement{Rec8Macro} and \finleyelement{Line3Macro}\index{macro elements}
are used. This option should be used when solving incompressible fluid flow
problems, e.g. \class{StokesProblemCartesian}.
In the case of \var{useElementsOnFace}=\True (this option should be used if
gradients are calculated on domain faces), \finleyelement{Rec4Face} and
\finleyelement{Rec8Face} are used on the edges, respectively.
If \var{integrationOrder} is positive, a numerical integration scheme is chosen
which is accurate on each element up to a polynomial of degree
\var{integrationOrder}\index{integration order}.
Otherwise an appropriate integration order is chosen independently.
If \var{periodic0}=\True, periodic boundary conditions\index{periodic boundary conditions}
along the $x_0$-direction are enforced.
That means for any solution of a PDE solved by \finley the values on the line
$x_0=0$ will be identical to the values on $x_0=\var{l0}$.
Correspondingly, \var{periodic1}=\True sets periodic boundary conditions in the
$x_1$-direction.
If \var{optimize}=\True mesh node relabeling will be attempted to reduce the
computation and also ParMETIS will be used to improve the mesh partition if
running on multiple CPUs with \MPI.
\end{funcdesc}

\begin{funcdesc}{Brick}{n0,n1,n2,order=1,l0=1.,l1=1.,l2=1., integrationOrder=-1,
  periodic0=\False, periodic1=\False, \\ periodic2=\False, useElementsOnFace=\False,useFullElementOrder=\False, optimize=\False}
generates a \Domain object representing a three-dimensional brick between
$(0,0,0)$ and $(l0,l1,l2)$ with orthogonal faces. The brick is filled with
\var{n0} elements along the $x_0$-axis,
\var{n1} elements along the $x_1$-axis and
\var{n2} elements along the $x_2$-axis.
For \var{order}=1 and \var{order}=2, elements of type \finleyelement{Hex8} and
\finleyelement{Hex20} are used, respectively.
In the case of \var{useElementsOnFace}=\False, \finleyelement{Rec4} and
\finleyelement{Rec8} are used to subdivide the faces of the brick, respectively.
In the case of \var{useElementsOnFace}=\True (this option should be used if
gradients are calculated on domain faces), \finleyelement{Hex8Face} and
\finleyelement{Hex20Face} are used on the brick faces, respectively.
If \var{order}=-1, \finleyelement{Hex20Macro} and \finleyelement{Rec8Macro}\index{macro elements}
are used. This option should be used when solving incompressible fluid flow
problems, e.g. \class{StokesProblemCartesian}.
If \var{integrationOrder} is positive, a numerical integration scheme is chosen
which is accurate on each element up to a polynomial of degree
\var{integrationOrder}\index{integration order}.
Otherwise an appropriate integration order is chosen independently.
If \var{periodic0}=\True, periodic boundary conditions\index{periodic boundary conditions}
along the $x_0$-direction are enforced.
That means for any solution of a PDE solved by \finley the values on the plane
$x_0=0$ will be identical to the values on $x_0=\var{l0}$.
Correspondingly, \var{periodic1}=\True and \var{periodic2}=\True sets periodic
boundary conditions in the $x_1$-direction and $x_2$-direction, respectively.
If \var{optimize}=\True mesh node relabeling will be attempted to reduce the
computation and also ParMETIS will be used to improve the mesh partition if
running on multiple CPUs with \MPI.
\end{funcdesc}

\begin{funcdesc}{GlueFaces}{meshList, tolerance=1.e-13}
generates a new \Domain object from the list \var{meshList} of \finley meshes.
Nodes in face elements whose difference of coordinates is less than
\var{tolerance} times the diameter of the domain are merged.
The corresponding face elements are removed from the mesh.
\function{GlueFaces} is not supported under \MPI with more than one rank.
\end{funcdesc}

\begin{funcdesc}{JoinFaces}{meshList, tolerance=1.e-13}
generates a new \Domain object from the list \var{meshList} of \finley meshes.
Face elements whose node coordinates differ by less than \var{tolerance} times
the diameter of the domain are combined to form a contact element\index{element!contact}.
The corresponding face elements are removed from the mesh.
\function{JoinFaces} is not supported under \MPI with more than one rank.
\end{funcdesc}

\section{\dudley}
\label{sec:dudley}
The {\it dudley} library is a restricted version of {\it finley}.
So in many ways it can be used as a ``drop-in'' replacement.
Dudley domains are simpler in that only triangular (2D), tetrahedral (3D) and line elements are supported.
Note, this also means that dudley does not support:
\begin{itemize}
\item dirac delta functions
\item contact elements
\item macro elements
\end{itemize}


\documentclass[ 12pt, a4paper]{article}
% Use the option doublespacing or reviewcopy to obtain double line spacing
% \documentclass[doublespacing]{elsart}

\usepackage[utf8]{inputenc}
\usepackage[backend = biber, maxcitenames=2,uniquelist=minyear]{biblatex}

\AtEveryBibitem{\clearfield{number}}
\AtEveryBibitem{\clearfield{doi}}
\AtEveryBibitem{\clearfield{url}}
\AtEveryBibitem{\clearfield{issn}}
\AtEveryBibitem{\clearfield{isbn}}

\addbibresource{references.bib}

\usepackage{color,graphicx,tikz}
\usetikzlibrary{positioning,arrows}
% The amssymb package provides various useful mathematical symbols
\usepackage{mathtools,amssymb,amsmath,mathdots}
\usepackage[mathscr]{eucal} %just for the font \mathscr
\usepackage{setspace}
\usepackage{hyperref}

\usepackage{tikz}

\renewcommand{\vec}[1]{\boldsymbol{#1}}
\renewcommand{\thefootnote}{\fnsymbol{footnote}}

\input{macros}

\begin{document}

\title{Multiple scattering of waves}
\author{
Artur L. Gower$^{a}$,\\
\footnotesize{$^{a}$ School of Mathematics, University of Manchester, Oxford Road, Manchester, M13 9PL,UK}
}
\date{\today}
\maketitle

\begin{abstract}
Here we show and deduce the T-matrix and a general multiple scattering formulation which can be adapted to acoustics, electromagnetism, and elasticity. For details on each specific physical medium see the other documents.
\end{abstract}

\noindent
{\textit{Keywords:} Multiple scattering, T-matrix, Scattering matrix}

\section{Using a T-matrix}
A T-matrix denotes how one single particle scatters waves~\parencite{ganesh_far-field_2010,ganesh_algorithm_2017}.

For convenience and generality we denote:
\begin{equation}
\begin{aligned}
    & \mathrm u_{n}(k\rv) = \text{outgoing spherical waves},
    \label{eqn:outgoing_waves_and_regular_waves}
    \\
    & \mathrm v_{n}(k\rv)= \text{regular spherical waves},
 \end{aligned}
\end{equation}
where $n$ denotes a multi index which depends on the dimension and if the waves are scalar or vector fields.

Any incident wave and scattered wave\footnote{For the scattered wave we need only use outgoing spherical waves when measuring the field outside of a sphere which completely encompasses the particle.}, centred at the same coordinate axis, can be written as
\begin{align}
  & \ui = \sum_{n} g_n \mathrm v_{n}(k\rv),
  \\
  & \us = \sum_{n} f_n \mathrm u_{n}(k\rv).
\end{align}
The T-matrix is an infinite matrix such that
\begin{equation}
  f_n = \sum_{n'} T_{nn'} g_{n'}.
\end{equation}
Such a matrix $T$ exists when scattering is a linear operation (elastic scattering).

We can also estimate the field inside the particle by assuming that the field is smooth and continuous. This approximation is exact for homogeneous spheres and cylinders, but not for a \href{acoustics.pdf}{Circular cylindrical capsule}.

Assume the field inside the particle can be described by a regular spherical series:
\begin{equation}
  \vi = \sum_n b_n \mathrm v_n(k_o \rv),
\end{equation}
where $k_o$ if the particles wavenumber. Now if we assume that the total field is continuous everywhere so that $\ui + \us = \vi$ on the boundary of the particle. If the field was smooth enough, we could analytically extend the field $\vi$ to a spherical boundary, with radius $a$, which contains the particle. Let's take this as an assumption and equate $\ui + \us = \vi$ for $r=a$. Due to orthogonality of the angular components of the basis functions this will result in
\begin{equation}
   g_n \mathrm v_{n}(k\rv) + f_n \mathrm u_{n}(k\rv) = b_n \mathrm v_n(k_o \rv), \quad \text{for} \;\; |\rv| = a
\end{equation}
using the T-matrix we can then write $g_n = T_{nm}^{-1} f_m$, which substituted above leads to
\begin{equation}
    b_n = \frac{1}{\mathrm v_n(k_o \rv)}[ \mathrm v_{n}(k\rv) T_{nm}^{-1} f_m + \mathrm u_{n}(k\rv) f_n], \quad \text{for} \;\; |\rv| = a.
\end{equation}

\section{Multiple scattering in general}

For multiple scattering in higher dimensions and for vector wave equations we use the notation given in \cite{gower2020effective}.

For a point $\rv$, outside of the circumscribed spheres of all particles, we can write the total field $u(\rv)$ as a sum of the incident wave $\ui(\rv)$ and all scattered waves in the form~\cite{Kristensson2015a,Kristensson2016,Linton+Martin2006}
\begin{equation}
    u(\rv) = \ui(\rv) + \us(\rv), \quad \us(\rv) =  \sum_{i=1}^N \sum_n f_n^i \mathrm u_n (k \rv - k \rv_i),
    \label{eqn:total_discrete_wave}
\end{equation}
where we assumed $ |\rv - \rv_i| > a_i $ for $i=1,2,\ldots N$, the $f_n^i$ are coefficients we need to determine, where again:
\begin{equation}
\left\{\begin{aligned}
    & \mathrm u_{n}(k\rv) = \text{outgoing spherical waves},
    \label{eqn:outgoing_waves_and_regular_waves}
    \\
    & \mathrm v_{n}(k\rv)= \text{regular spherical waves},
 \end{aligned}\right.
\end{equation}
where $n$ denotes a multi index which depends on the dimension and if the waves are scalar or vector fields.

In general, we can write the multiple scattering system in the form:
\begin{equation}\label{eqn:multiple-scattering-general}
   \alpha_n^i=g_{n}^i
    +\sum_{\substack{j=1\\j\neq i}}^N \sum_{n' n''}\mathcal{U}_{n''n}(k\rv_i - k\rv_j)T_{n''n'}^j \alpha_{n'}^j,
\end{equation}
for $i=1,2,\ldots,N$, where $f_n^i = \sum_{n'} T^i_{nn'}\alpha_{n'}^i$ and $\mathcal{U}_{nn'}$ is a translation matrix \cite{Bostrom+Kristensson+Strom1991,Friedman+Russek1954}. Let $\rv'=\rv+\dv$, then
the translation matrices for a translation $\dv$ can be defined by the property~\cite{Bostrom+Kristensson+Strom1991}
  \begin{equation}\label{eq:translation_spherical_waves}
 \left\{\begin{aligned}
   &\mathrm{v}_n(k\rv')=\sum_{n'}\mathcal{V}_{nn'}(k\dv)\mathrm{v}_{n'}(k\rv),\quad \text{ for all }\dv
   \\
 &\mathrm{u}_n(k\rv')=\sum_{n'}\mathcal{V}_{nn'}(k\dv)\mathrm{u}_{n'}(k\rv),\quad |\rv|>|\dv|
 \\
   &\mathrm{u}_n(k\rv')=\sum_{n'}\mathcal{U}_{nn'}(k\dv)\mathrm{v}_{n'}(k\rv),\quad |\rv|<|\dv|
   \\
 \end{aligned}\right.
 \end{equation}

\subsection{Turing equations into code}
For easy implementation we need the functions:
\[
\psi_\inc \mapsto g^m_j \quad \text{and} \quad \text{particle} \mapsto T^{nm}_j.
\]

For efficient implementation we rewrite~\eqref{eqn:multiple-scattering-general} as a matrix equation. Let
\begin{align}
  &(\vec \alpha_j)_n =  \alpha_n^j, \quad (\vec g_j)_n =  g_n^j,
  \\
  &(\vec T_j)_{nn'} = T_{nn'}^j, \quad (\vec {\mathcal U}_{j \ell})_{n'n} = \mathcal U_{n'n}(k \rv_j - k\rv_\ell),
  % H_{n'-n}(k R_{\ell j})\ee^{\ii(n'-n)\Theta_{\ell j}},
\end{align}

 Then
\begin{equation}
 \sum_{\ell}(\delta_{j \ell} +  (\delta_{j \ell}-1) \vec {\mathcal U}_{j \ell}^{\mathrm T} \vec T_\ell) \vec \alpha_\ell  =  \vec g_j,
\end{equation}
where $\cdot ^{\mathrm T}$ is the transpose operation. The above then leads to a block matrix equation:
\begin{equation}
  \begin{bmatrix}
    \vec I & - \vec {\mathcal U}_{1 2}^{\mathrm T}\vec T_2 & \cdots & - \vec {\mathcal U}_{1 (N-1)}^{\mathrm T} \vec T_{N-1} & - \vec {\mathcal U}_{1 N}^{\mathrm T} \vec T_N \\
    - \vec {\mathcal U}_{2 1}^{\mathrm T} \vec T_1 & \vec I &  - \vec {\mathcal U}_{2 3}^{\mathrm T} \vec T_3 & \cdots & - \vec {\mathcal U}_{2 N}^{\mathrm T} \vec T_N \\
     & \vdots & & & \vdots \\
     - \vec {\mathcal U}_{N 1}^{\mathrm T} \vec T_1  & \cdots & \cdots & -  \vec {\mathcal U}_{N (N-1)}^{\mathrm T} \vec T_{N-1} & \vec I
  \end{bmatrix}
  \begin{bmatrix}
    \vec \alpha_1 \\
    \vec \alpha_2 \\
    \vdots \\
    \vec \alpha_N
  \end{bmatrix}
   = \begin{bmatrix}
     \vec g_1 \\
     \vdots \\
     \vec g_N
   \end{bmatrix}
\end{equation}

\printbibliography

\end{document}


\chapter{Introduction}  %Title of the First Chapter

Using past experience to update ones behaviour is what differentiates intelligent beings from other creatures in our world. To artificially create said systems, that can learn from data acquired through experience, is the crowning goal of the field of Artificial Intelligence (AI) and Machine Learning (ML) \citep{turing}.
Realising this goal will have a large impact on the world and society we live in, as it will liberate humans from tasks that require cognition, like driving cars, booking appointments, and interpreting and acting on medical records, to give a few examples. Though, arguably, the field has still a long way to go before fulfilling its full potential \citep{grace2018will}.

An elegant approach to formalise the concept of learning through acquired experience is that of \emph{Bayesian learning} \citep{murphy}. In this framework, one specifies beliefs about quantities of interest, and updates them after observing data. The beliefs are represented using probability distributions: the \emph{prior} describes what is known about the quantity before any data is observed, and the so-called \emph{likelihood} describes the relation between the observed data and the quantity of interest. The framework provides a recipe for obtaining an updated belief over the quantity of interest after data has been observed. This distribution is known as the \emph{posterior}. Bayesian learning makes decisions atop these beliefs, and uses the posterior to reason about the optimality of a decision or the cost associated to a certain action.

The performance of a decision-making system will depend on the speed at which it can learn and the optimality of the decisions made---quantified by the \emph{regret}. It can be shown that, under certain assumptions, Bayesian learning gives rise to the optimal regret \citep{Lattimore}. However, such excellence comes at a high computational cost, which in many scenarios renders Bayesian learning---in its purest form---impractical. Fortunately, the literature has proposed many approximate methods which have lightened the computational complexity of the Bayesian paradigm \citep{Neal1993Probabilistic,jordan1999introduction,minka2001expectation}.

Unquestionably, models for decision-making systems need to deal with \emph{uncertainty}. They need to be able to quantify what is known, and what is not known. The importance of quantifying uncertainty for decision-making systems becomes clear from the many sources it can stem from. For example, there can be multiple different settings of a model that explain the data, so one needs to be uncertain about the setting that actually generated it. One also needs to be uncertain about the model itself, as most probably the model at hand is a simplification of the real-world process. Moreover, the environment in which the system operates may also be inherently uncertain, in which case even an infinite amount of data (i.e. experience) would not make the system any smarter (e.g., trying to predict the outcome of rolling a fair dice).

Gaussian Processes (GPs) \citep{rasmussen2006} can be argued to provide the perfect compromise between computational complexity and Bayesian rigour. Their non-parametric nature makes them complex enough to model a wide range of problems, while their kernel formulation makes them applicable to many different domains: graphs, vectors, images, etc. Instead of representing probability distributions on weights, Gaussian processes can be used to represent uncertainty directly on the function that the weights represents. The \emph{function-space} view of Gaussian processes makes them more amenable to mathematical analysis, which in turn leads to strong guarantees on the future performance of the system and overall robustness. This may be necessary for deploying these systems in critical applications. For these reasons studying Gaussian processes is very worthwhile.

This reports presents two pieces of research in the domain of approximate Bayesian inference for GP models conducted during the first year of my PhD degree. Namely, \cref{chapter:vish} introduces an interdomain inducing variable approach that speeds up inference and prediction in GPs by two orders of magnitude by making use of the spectral properties of the kernel. In \cref{chapter:dnn-as-point-estimate-for-dgps} we marry the strengths of deep neural networks and deep GPs by establishing an equivalence between the forward passes of both models. This results in models that can either be seen as neural networks with improved uncertainty prediction or deep GPs with increased prediction accuracy. The final part of this report, \cref{chapter:future-research}, elaborates on a future research agenda. Prior to all of this, we start with covering the theoretical background in \cref{chapter:theoretical-framework}.

The material presented in \cref{chapter:vish,chapter:dnn-as-point-estimate-for-dgps} is either published or is currently under review:
\begin{enumerate}
    \item \fullcite{Dutordoir2020spherical}
    \item \fullcite{dutordoir2021deep}
    \item \fullcite{dutordoir2021gpflux}
\end{enumerate}


% !TEX root = ../main.tex
% chktex-file 21
\section{Introduction}%
\label{sec:intro}

\pagenumbering{arabic}			% arabic page numbering
\setcounter{page}{1}			% set page counter

Over the last few years Big Data processing has become increasingly important in many domains.
This increase in the data volume also poses new challenges for machine learning applications.
The training time of learners is usually polynomially dependent on the size of the training dataset \(\Dtrain\), i.~e. \(\mathcal{O}(|\Dtrain|^\alpha), \alpha \geq 1\).
Since training has to be repeated for every iteration of cross-validation and hyperparameter search, always using the entire dataset quickly becomes infeasible.
This paper gives an overview of approaches to tackle this problem with a focus on the domain of classification problems.

The process of finding a hypothesis can in general be split into three phases:
\begin{enumerate}
	\item \textbf{Model selection:}
		At first a model has to be selected that determines the class of hypothesis spaces out of which the final hypothesis will be selected.
		The choice of model is often implicitly encoded in the class of hypothesis spaces of a learner \(L\).
		Automating this step is non-trivial.
		In practice the model is typically selected by experts with domain specific knowledge about the problem at hand.
	\item \textbf{Hyperparameter search:}
		Optimizing a vector \(\lambda\) in the hyperparameter space \(\Lambda_L\) of the learner \(L\) representing a hypothesis space \(\mathcal{H}_{\lambda}\).
		A na{\"\i}ve approach to do this is to systematically try configurations using a grid search or a random search over \(\Lambda_L\).
		To evaluate the quality of a given \(\lambda\), \(L\) is usually trained on a training dataset \(\Dtrain\) using \(\lambda\). This yields a hypothesis \(\hat{h}_\lambda \in \mathcal{H}_{\lambda}\) that is evaluated using a validation dataset \(\Dvalid\).
		The goal of hyperparameter optimization is to minimize the loss \(l(\lambda)\) of \(\hat{h}_\lambda\) on \(\Dvalid\), i.~e.\@ to find an approximation \(\hat{\lambda}\) of \(\lambda^* := \arg\min_{\lambda}{l(\lambda)}\).
	\item \textbf{Training or parameter search:}
		Let \(w\) be a vector in the parameter space \(W_{\mathcal{H}_\lambda}\), describing a hypothesis \(h_{\lambda, w} \in \mathcal{H}_\lambda\) given a hyperparameter configuration \(\lambda\).
		The goal of parameter search is to find an approximation \(\hat{h}_\lambda\) of the hypothesis \(h^*_\lambda := \arg\min_{h_{\lambda, w}}{\ell(\Dtrain | h_{\lambda, w})}\), with \(\ell(\Dtrain | h_{\lambda, w})\) being the empirical loss of \(h_{\lambda, w}\) on a given training dataset \(\Dtrain\) according to some loss function \(\ell\).
		Depending on the learner \(L\), various kinds of optimization methods are used to find this minimum, e.~g.\@ Bayesian optimization, quadratic programming or, if \(\nabla_w e(\Dtrain | h_{\lambda, w})\) is computable, gradient descent.
		The quality \(l\) of \(\hat{h}_\lambda\) is measured by the loss on a validation or test dataset, i.~e.\@ \(l(\lambda) := \ell(\Dvalid | \hat{h}_\lambda)\).
\end{enumerate}
This paper is structured according to the last two phases, i.~e.\@ we will assume that the learner \(L\) is given.
Section~\ref{sec:hyperparams} describes ways to speed up the hyperparameter search.
Section~\ref{sec:params} then describes how to improve the training methods of existing learners.
Most of the techniques described in this paper improve upon independent components of the hypothesis finding process, allowing them to be combined.


\newcommand\lbxmonad{\texttt{xmonad}\xspace}

\NV{(R3)
 Regarding your XMonad experience, you might want to compare your approach to the work of Wouter Swierstra: "Xmonad in Coq (Experience Report)
}
\section{XMonad: From Testing to Static Verification}\label{sec:xmonad}

In this section we discuss our experience in the on-going endeavour of
static verification of \lbxmonad.
%
We start by summarizing the data structures used in \lbxmonad, as described in the documentation.
%
We continue by specifying and proving \lbxmonad's invariant, \ie, that
virtual workspaces are unique
%
Finally, we comment on proving \lbxmonad properties to conclude
that static verification can be not much harder than testing.

\subsection{The structure}

\lbxmonad is a dynamically tiling \texttt{X11}
window manager that is written and configured in Haskell.

The window (@a@) is a set of virtual workspaces.
On each workspace is a stack of windows.
A given workspace is always current, and a given
window on each workspace has focus.
The focused window on the current
workspace is the one which will take user input.
%
\lbxmonad uses a Zipper~\citep{zipper} data structure
to directly track focus on the virtual workspaces,
and calls this structure @Stack a@.

A workspace is just a @Stack@ of virtual workspaces
tagged with a tag @i@ and its layout @l@
@Workspace i l a@.

To view a workspace on a physical screen one needs to
associate the workspace with physical screen's id @sid@
and details @sd@,
forming the new data structure @Screen i l a sid sd@.

Xinerama in X11 allows viewing multiple virtual workspaces
simultaneously.
%
While only one the current one will ever be in focus (i.e. will
receive keyboard events), other workspaces may be passively
viewable.
%
We thus need to track which virtual workspaces are
associated (viewed) on which physical screens.
%
To keep track of
this, \lbxmonad's main data structure  @StackSet i l a sid sd@
keeps, apart from the current screen,
separate lists of visible but non-focused
workspaces (@Screen@) , and non-visible workspaces (@Workspace@).

\subsection{Uniqueness of windows}
We proved that each window @a@ appears once in the @StackSet i l a sid sd@
bottom-up:
We proved that each @Stack@ has unique elements
and that all the stacks that appear in @StackSet@ are disjoint.

\mypara{Unique Stack}
Assume the existence of a predicate
@ULNeq (X::a) (XS::[a])@ that ensures that
(1)~the list @XS@ has no duplicates and
(2)~@X@ is not an element of @XS@.
%
With that magical predicate we define an \emph{almost unique}
stack:
\begin{code}
data Stack a = Stack { focus :: a
                     , up    :: ULNEq a focus
                     , down  :: ULNEq a focus }
\end{code}
The stack has a @focus@ element @a@ and
and two unique lists of @a@'s @up@ and @down@ of the focus.

The above Stack is almost unique, as an element
may appear both in the @up@ and the @down@ lists.
%
We define a type alias for @U@nique@Stack@
that rejects with possibility:

\begin{code}
type UStack a = {v:(Stack a) |
  (LDisjoint (up v) (down v))}

predicate LDisjoint X Y =
  (Set_emp (Set_cap (listElts X) (listElts Y)))
\end{code}

where @listElts@ is a recursively defined measure that
returns the Set of elements of the input list,
that is build in \toolname.
\begin{code}
measure listElts  :: [a] -> (Set a)
   listElts([])   = {v | (Set_emp v) }
   listElts(x:xs) = {v | v =
     (Set_cup (Set_sng x) (listElts xs))}
\end{code}

Note that the above definitions crucially depend on set theoretic properties.
Thus, verification of \lbxmonad is achieved using an SMT back-end
that supports set theory (like Z3~\citep{z3}).

We slightly modify the above measure definition
to define @listDup@, a measure that returns the duplicate
elements of a list.
%
\begin{code}
measure listDup :: [a] -> (Set a)
 listDup([])   = {v | (Set_emp v)}
 listDup(x:xs) = {v | v =
   if (Set_mem x (listElts xs))
    then (Set_cup (Set_sng x) (listDup xs))
    else (listDup xs))                      }
\end{code}

Using @listDup@ we define the magical list type @ULNEq a N@
as a list @v@ that has no duplicates, \ie the set @listDup v@
is empty, and @N@ does not belong to the set @listElts v@.

\begin{code}
type ULNEq a N = {v:[a] | ( (UL v) && (not (LElt N v))}

predicate LElt N LS  = (Set_mem N (listElts LS))
predicate UL     LS  = (Set_emp (listDup LS)   )
\end{code}

Throughout verification
we need to establish and use the invariant
that each Stack is Unique.
%
This is achieved by the following annotation
\begin{code}
using (Stack a) as (UStack a)
\end{code}
that allows \toolname to
(1)~ use the invariant:
each time a Stack value is retrieved from the environment
\toolname strengthens its type with the disjointness information;
(2)~ prove the invariant:
each time @Stack@ data constructor is used,
an disjoint constraint should be proved.
Failure to prove this constraint will raise an
``Invariant Check'' error.

\mypara{Unique StackSet}
Establishing Uniqueness on StackSets is a generalization of the above procedure.

The definition of a @StackSet@ includes
the @current@ Screen,
the list of @visible@ screens,
and the list of @hidden@ workspaces.
\begin{code}
data StackSet i l a sid sd = StackSet
   { lcurrent  ::  Screen i l a sid sd
   , lvisible  :: [Screen i l a sid sd]
   , lhidden   :: [Workspace i l a]
   , lfloating :: M.Map a RationalRect
   }
\end{code}
%
\toolname automatically turns the record selectors of refined data types
to measures that return the appropriate fields.
%
Thus we infix the refined selectors with an @l@
to distinguish between the haskell (\eg, @current@)
and the logical (\eg, @lcurrent@) selectors.

To prove absence of duplicates we need to @use@
only @StackSet@s that have no duplicates:
\begin{code}
using (StackSet i l a sid sd)
 as  {v:StackSet i l a sid sd|(NoDuplicates v)}
\end{code}

@NoDuplicates@ ensures that the elements of
@hidden@, @current@, and @visible@
workspaces are mutually disjoint
and that the @visible@ and @hidden@ workspaces
have no duplicates.
%
\begin{code}
predicate NoDuplicates SS =
    (Disjoint3  (workspacesElts (lhidden  SS))
                (screenElts     (lcurrent SS))
                (screensElts    (lvisible SS)))
  &&
    (Set_emp (screensDups    (lvisible SS)))
  &&
    (Set_emp (workspacesDups (lhidden  SS)))
\end{code}
%
Again we used recursively defined measures to grap
the elements and the duplicates of the structures.
For example, @screenElts@ returns
the elements of the stack of the workspace of the screen,
and is used by @screensDup@ to grap the duplicates of
a list of Screens.

\mypara{Verification Procedure}
Using the above unique types make \toolname verify the absence of duplicates.
%
It was not surprising that when we first run \toolname
against these stronger types many type errors where created.
%
The
can be summarized as follows:
\begin{itemize}
	\item\emph{Strengthening library functions.}
      \lbxmonad repeatedly concatenates the list fields of a Stack.
      %
      To prove that for some `s::UStack a`, `(up s ++ down s)` is a unique list,
      the type of `(++)`
      needs to capture that concatenation of two unique and disjoint lists is a unique list.
      %
      For verification, we assumed that Prelude's `(++)` satisfies this property.
      %
      But, not all arguments of `(++)` are unique disjoint lists:
      @"StackSet" ++ "error"@ is a trivial example that does not satisfy
      the assumed preconditions of `(++)` thus creating a type error.
      %
      Sadly, \toolname does not currently support sum types,
      thus we used an unrefined `(++.)` variant of `(++)` for such cases.
%%
%%	Thus function @mapLayout@ seemed en auto-provable one,
%%	as it only modify the layout elements @l@
%%	which cannot affect the uniqueness of @a@ elements.
%%	%
%%\begin{code}
%%mapLayout :: (l -> l')
%%          -> StackSet i l a s sd
%%          -> StackSet i l' a s sd
%%\end{code}

	\item\emph{Restrict the functions' domain.}
	@modify@ is a @maybe@ like function
	that given a default value @x@,
	a function @f@ and a StackSet @s@,
	applies @f@ on the @Maybe (Stack a)@ values inside @s@.
	%
\begin{code}
modify ::
    x:{v:Maybe (UStack a) | (isNothing v)}
 -> f:(y:(UStack a)
     -> Maybe {v:UStack a) | (SubElts v y)})
 -> s:StackSet i l a s sd
 -> StackSet i l a s sd
\end{code}
	%
	Since inside the StackSet each @y:Maybe (Stack a)@
	could be replaced with either the default value @x@
	or @f y@ we need to ensure that both there alternatives
	will not insert duplicates.
	%
	This imposes the interesting precondition that the default
	value should be @Nothing@, which dramatically restricts
	@modify@'s domain.
	%
	Given this restriction, \toolname should verify
	that all user functions satisfy it.

	\item\emph{Code inlining}
    %
    Given a tag @i@ and a StackSet @s@,
    @view i s@ will set the current Screen
    to the screen with tag @i@,
    if such screen exists in @s@.
    %
    Below is the original definition for @view@
    in case when a screen with tag @i@ exists in
    visible screens
    %
\begin{code}
view :: (Eq s, Eq i) => i
    -> StackSet i l a s sd -> StackSet i l a s sd
view i s
  | Just x <- L.find ((i==).tag.workspace)
                     (visible s)
  = s { current = x
      , visible = current s :
          L.deleteBy (equating screen) x
                     (visible s)}
\end{code}
    %
    Verification of this code is difficult
    as the properties of all intermediate values
    are too complicated to be expressed.
    %
    Instead we replaces this code with a call to a
    recursive function @raiseIfVisible i s@
    that in-place replaces @x@ with the current screen.

\end{itemize}

\subsection{QuickCheck Properties}

\lbxmonad is tested against $113$ \texttt{quickcheck} properties.
%
Of those $15$ check the uniqueness invariant
and the rest $113$ check various functional properties.
%
We started the endeavour of verifying these properties with \toolname.
%
We looked at a sample of $15$ properties to conclude that
which we categorized as follows:
\begin{itemize}
\item\emph{Easy to be proved.}
Consider the \texttt{quickcheck} property that checks that @view@ing
a @StackSet a@ is idempotent:
\begin{code}
prop_view_idem (x :: T) (i :: NonNegative Int)
  = i `tagMember` x ==> view i (view i x) == (view i x)
\end{code}
%
The above property directly translated to a haskell function
\begin{code}
type Valid     = {v:Bool | (Prop v) }

prop_view_idem :: StackSet i l a sid sd -> i -> Valid
prop_view_idem x i
  | i `tagMember` x = view i (view i) == v
  | otherwise       = True
\end{code}
%
By the above type signature,
\ie by the result type @Valid@,
we specify that the function should always returns True.
%
When typechecking the above function,
\toolname proves that the property holds.
%
\toolname is able to verify this property as the result type of @view@
is strengthens with a refinement
(in this case @EqTag x i => x = v@)
that directly implies this property.

The above is generalizing to (10/17) properties that we checked:
strengthening the function types by refinements that \toolname can prove
is sufficient to verify these properties.

\item\emph{Can be estimated.}
In some other properties (like checking that @view@ing is reversible),
two StackSets (@s1@ and @s2@) were normalized before being compared,
that is their elements were first sorted.
%
In our logic we do not support any operation that can normalize structures in such a way.
%
Thus we cannot prove this exact property.
%
Instead we approximated it, by proving that proving that @s1@ and @s2@
have the same sets of elements.
%
We approximated (3/17) of the properties.

\item\emph{Their proof cannot be supported, currently.} [1]
One \texttt{quickcheck} property checks
that @i@ cannot belong to an empty stackset.
%
We used abstract refinements to encode empty stacksets.
%
Proving the above property would be easy
if we could mix abstract and concrete refinements in logical formulas
or if \toolname supported sum types.
%
Both the alternatives constitutes features that
we would like to extend \toolname with in the near future.
%
Still, currently we are not able to prove such kind of properties.
\item\emph{Cannot be expressed}
Other properties %, like @prop_focus_left_master@
check that the order of the windows is not affected by certain operations.
%
Though not infeasible we acknowledge that \toolname
is not appropriate for reasoning about order preserving
and verification of such properties
would require many code modifications.
%
(3/17) are order preserving properties.
\end{itemize}