\documentclass{article}
\author{a1trl9}
\title{Probability Review}
\date{}

\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{environ}

\newcommand{\V}[1]{\boldsymbol{#1}}
\newcommand{\TM}{\bigtriangledown}

\NewEnviron{mulequation}{%
\begin{equation}\begin{split}
  \BODY
\end{split}\end{equation}
}

\setlength\parindent{0pt}
\counterwithin*{equation}{section}

\begin{document}
\maketitle

\section{Random Variables}

\subsection{Characteristics of Random Variables}

\subsubsection{Expected Value}

For discrete random variable \(X\), expected value is denfined as:

\begin{equation}
\sum_{i=1}^{\infty}x_ip_i
\end{equation}

Where \(p_i\) is the probability for \(X=x_i\).

For continuous random variable \(X\), expected value is defined as:

\begin{equation}
    \int_{\mathbf{R}}f(x)xdx
\end{equation}

Where \(f(x)\) is the probability density function for \(X=x\).

\vspace{2mm}
\textbf{Proposition 1:} For random variables \(X_1, X_2\), no matter if
they are independent, \(E(X_1+X_2)=E(X_1)+E(X_2)\).

\textbf{Proof}

If \(X_1, X_2\) are continuous variables. Then:

\begin{equation}
\begin{split}
    E(X_1 + X_2)&=\int_{-\infty}^{\infty}
    \int_{-\infty}^{\infty}f(x_1, x_2)(x_1+x_2)dx_1dx_2\\
    &=\int_{-\infty}^{\infty}x_1dx_1\int_{-\infty}^{\infty}f(x_1,x_2)dx_2\\
    &+\int_{-\infty}^{\infty}x_2dx_2\int_{-\infty}^{\infty}f(x_1,x_2)dx_1\\
    &=\int_{-\infty}^{\infty}x_1f(x_1)dx_1+\int_{-\infty}^{\infty}x_2f(x_2)dx_2\\
    &=E(X_1)+E(X_2)
\end{split}
\end{equation}

Noting it actually requires rigorous proof to show why the order of
integrations on \(x_1, x_2\) could be interchanged.

\vspace{2mm}
\textbf{Proposition 2:} If \(X_1, X_2\) are two independent random variables,
\(E(X_1X_2)=E(X_1)E(X_2)\).

\vspace{2mm}
\textbf{Proof}

\textit{Only prove when \(X_1, X_2\) are continuous variables.}

Since \(X_1, X_2\) are independent:
\begin{equation}
    f(x_1, x_2)=f(x_1)f(x_2)
\end{equation}

Therefore:

\begin{equation}
\begin{split}
    E(X_1X_2)&=\int_{-\infty}^{\infty}\int_{-\infty}^{\infty}
    x_1x_2f(x_1, x_2)dx_1dx_2\\
    &=\int_{-\infty}^{\infty}f(x_1)dx_1\int_{-\infty}^{\infty}f(x_2)dx_2\\
    &=E(X_1)E(X_2)
\end{split}
\end{equation}

\subsubsection{Variance}

\vspace{2mm}
\textbf{Proposition 3:} If \(X_1, X_2\) are two independent random variables,
\(Var(X_1+X_2)=Var(X_1)+Var(X_2)\).

\vspace{2mm}
\textbf{Proof}
\begin{equation}
\begin{split}
    &Var(X_1+X_2)=E\{[(X_1+X_2)-E(X_1+X_2)]^2\}\\
    &=E[(X_1+X_2)^2]-E^2(X_1+X_2)\\
    &=E(X_1^2)+E(X_2^2)+2E(X_1X_2)-E^2(X_1)-E^2(X_2)-2E(X_1)E(X_2)\\
    &=E(X_1^2)-E^2(X_1)+E(X_2^2)-E^2(X_2)\\
    &=Var(X_1)+Var(X_2)
\end{split}
\end{equation}

\section{Distributions}
\subsection{Discrete Distributions}
\subsubsection{Poisson Distribution}
\textbf{PDF}

\begin{equation}
p(k)=\frac{e^{-\lambda}\lambda^k}{k!}
\end{equation}

Where \(k \in \{0\} \cup \mathbb{N}\).

According to Taylor series (at 0, Maclaurin series):
\begin{equation}
\begin{split}
f(\lambda)&=e^{\lambda}=f(0)+\frac{f'(0)\lambda}{1!}+\frac{f''(0)\lambda^2}{2!}+\cdots\\
&=\frac{\lambda^0}{0!}+\frac{\lambda^1}{1!}+\frac{\lambda^2}{2!}+\cdots\\
&=\sum_{k=0}^\infty\frac{\lambda^k}{k!}
\end{split}
\end{equation}

Therefore:
\begin{equation}
\begin{split}
\sum_{k=0}^{\infty}\frac{e^{-\lambda}\lambda^k}{k!}&=e^{-\lambda}\sum_{k=0}^{\infty}\frac{\lambda^k}{k!}\\
&=e^{-\lambda}e^{\lambda}\\
&=1
\end{split}
\end{equation}

\subsection{Continuous Distributions}
\subsubsection{Exponential Distribution}
\textbf{PDF}

\begin{equation}
p(x)=\lambda e^{-\lambda x}
\end{equation}

It is obvious:
\begin{equation}
\begin{split}
\int_{0}^{\infty}\lambda e^{-\lambda x}&=-e^{-\lambda x}|^{\infty}_0\\
&=-e^{\infty}+e^0=0+1=1
\end{split}
\end{equation}

\subsubsection{Gamma Distribution}
\vspace{2mm}
\textbf{PDF}
\begin{equation}
\int_0^1x^{a-1}(1-x)^{b-1}dx
\end{equation}

\subsubsection{Beta Distribution}
Beta distribution is used as conjugate prior distibution for Bernoulli Distribution.

\vspace{2mm}
\textbf{PDF}
\begin{equation}
p(x)=x^{a-1}(1-x)^{b-1}dx
\end{equation}

\vspace{2mm}
\textbf{Further: Beta function}

Beta function holds an interesting property that:

\begin{equation}
\mathbf{B}(a, b) = \frac{\Gamma{(a)}\Gamma{(b)}}{\Gamma{(a, b)}}
\end{equation}

\vspace{2mm}
\textbf{Proof}

Let \(\Gamma(a)=\int_0^{\infty}e^{-x}x^{a-1}dx\) and \(\Gamma(b)=\int_0^{\infty}e^{-y}y^{b-1}dy\), then:
\begin{equation}
\begin{split}
\Gamma(a)\Gamma(b)&=\int_0^{\infty}e^{-x}x^{a-1}dx \cdot \int_0^{\infty}e^{-y}y^{b-1}dy\\
&=\int_0^{\infty}\int_0^{\infty}e^{-(x+y)}x^{a-1}y^{b-1}dxdy
\end{split}
\end{equation}

Let \(f(u, v): (u, v) \rightarrow (x, y), x=uv, y=u(1-v)\), then:
\begin{equation}
\Gamma{(a)}\Gamma{(b)}=\int_0^{\infty}\int_0^1e^{-u}(uv)^{a-1}(u(1-v))^{b-1}
|D(u, v)|dudv
\end{equation}

Where \(D(\mathbf{J}(u, v))\) is the determinant of Jacobian matrix \(\mathbf{J}\). As:
\begin{equation}
\mathbf{J}=
\begin{bmatrix}
v & 1 - v \\
u & -u
\end{bmatrix}
\end{equation}

\(|D(\mathbf{J}(u, v))|=|-uv-u(1-v)=-u|=u\). So:
\begin{mulequation}
\Gamma{(a)}\Gamma{(b)}=
&\int_0^{\infty}\int_0^1e^{-u}u^{a+b-2}v^{a-1}(1-v)^{b-1}ududv\\
&=\int_0^{\infty}\int_0^1e^{-u}u^{a+b-1}v^{a-1}(1-v)^{b-1}dudv\\
&=\int_0^{\infty}u^{a+b-1}du\int_0^1v^{a-1}(1-v)^{b-1}dv\\
&=\Gamma{(a+b)}\mathbf{B}(a, b)
\end{mulequation}

Here one fact is important:
\(\Gamma(a)=\int_0^{\infty}e^{-x}x^{a-1}dx\) is convergent, which can be proved
as:
\begin{equation}
\begin{split}
\Gamma{(a)}&=\int_0^{k}e^{-x}x^{a-1}dx+\int_k^{\infty}e^{-x}x^{a-1}dx\\
\end{split}
\end{equation}

And it can be shown \(\forall a, \exists k\):
\begin{equation}
\begin{split}
\int_k^{\infty}e^{-x}x^{a-1}dx
<
\int_k^{\infty}e^{\frac{-x}{2}}dx=-2e^{\frac{-x}{2}}|^{\infty}_1=2e^{\frac{-k}{2}}
\end{split}
\end{equation}

and:

\begin{equation}
\int_0^{k}e^{-x}x^{a-1}dx
\end{equation}

is constant.

\section{Linear Regression}
\textbf{Least Square Approach in the form of matrix}
As for:

\begin{equation}
\mathbf{y}=X\beta + \epsilon
\end{equation}

The least square approach tries to minimize:

\begin{equation}
f=||y-X\beta||^2
\end{equation}

Let each partial derivative of \(f\) on \(\beta\) is equal to zero. That is,
for \(\beta_j\):
\begin{equation}
\sum_{i=1}^nx_{ij}(y_i-\mathbf{x}_i\beta)=0
\end{equation}

It could be transformed to:
\begin{equation}
\begin{split}
&\sum_{i=1}^nx_{ij}y_i=\sum_{i=1}^nx_{ij}\mathbf{x}_i\beta\\
&\mathbf{x}'_j\mathbf{y}=\mathbf{x}'_jX\beta
\end{split}
\end{equation}

Now for all partial derivatives:
\begin{equation}
X'\mathbf{y}=X'X\beta
\end{equation}

That is:
\begin{equation}
\beta=(X'X)^{-1}X'\mathbf{y}
\end{equation}

\end{document}


\section{Mathematical Properties}
Consider $\mathcal{D}$ to be a set of observations, such that $\mathcal{D} = X \biguplus Y$, $\mathcal{D}$ is the set of two sets $X$ and $Y$ with no shared elements between them; $X$ is the set of normal observations, while $Y$ is the set of outliers; Denote $\epsilon$ as threshold; $Fe$ is the feature of a function $F$ of the an observation; Any function $F$ uses feature $Fe$ must satisfy the following properties:

\begin{def}\textbf{(Outlier Existence)} If there are at least two clusters then we should check for outliers:
\begin{align}
degree(D) \geq 2
\end{align}
\end{def}

\begin{def}\textbf{(Normality)} To check whether a certain observation is normal:
\begin{align}
x \in D \text{ is normal observation if } x \in X | d(x) \geq \epsilon
\end{align}
\end{def}

\begin{def}\textbf{(Outlier)} To check whether a certain observation is an outlier:
\begin{align}
y \in D \text{ is an outlier if } y \in Y | d(y) < \epsilon
\end{align}
\end{def}

\section{Motivation}
The motivation of this research was to compare two algorithms of outlier detection. Precisely, One Class Support Vector Machine and Local Outlier Factor, bringing together two advantages:
\begin{itemize}
\item Ability to know the algorithm with the higher efficiency.
\item Ability to know the algorithm with the higher effectiveness.
\end{itemize}

\section{Objective}
In order to achieve the above goals, the following objectives must be satisfied:
\begin{itemize}
\item Simplicity of structure.
\item Functional Completeness.
\end{itemize}


\documentclass[10pt]{article}

\usepackage{amsmath}
\usepackage{algpseudocode}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{enumerate}

\title{Loop Diagonalization}
\author{Vedant Kumar \\ \texttt{vsk@berkeley.edu}}

\begin{document}
\maketitle

\section{Abstract}

The eigenvalue equation $Mv = \lambda v$ is a powerful statement about
matrix and scalar multiplications. In the context of compiler optimization,
we can use it to transform linearizable loops which run in $O(n)$ steps into
matrix operations which run in $O(\log n)$ steps.  This paper defines loop
diagonalization (the process of rewriting linearizable loops as two
matrix multiplies), discusses the implementation of this optimization
with LLVM, and discusses the advantages and limitations of this method.

\section{Overview}

Roughly speaking, a linearizable loop may equally well be represented as a
matrix $M$. Let the loop's state variables reside in a vector $v$. Each
iteration of the loop effects the update $v' = Mv$.

It follows inductively that if one iteration of a linearizable loop gives
$Mv$, $n$ iterations of the loop can be simulated by computing $M^n v$.
Performing this matrix exponentiation naively requires up to $O(nm^3)$
scalar multiplictaions.

Linear algebra comes to the rescue. Recall the equation $M = PDP^{-1}$.  The
column vectors of $P$ are the eigenvectors of $M$, and the diagonal matrix
$D$ contains the corresponding eigenvalues. We can find this
eigendecomposition for any square $M$ so long as $P$ is invertible.

It is known that $M^n = PD^nP^{-1}$, and that $D^n$ can be computed in
$O(m\log n)$ steps using the repeated squaring algorithm. With these facts
we can replace linearizable loops with fast matrix operations.

\section{Diagonalization}

The following definitions are useful: \\

\noindent
\textbf{Def. Linearizable Loop}: A tuple $(L, M, v)$ where $L$ is a list of
basic blocks with a preheader and one backedge, $M$ is a $m$ by $m$ square
matrix with $m$ linearly independent eigenvectors, and $v$ is a vector of
state variables. Let $v' = Mv$: the \emph{only} permissible instructions in
$L$ are the arithmetic operations which contribute to the goal of updating
$v$ to $v'$ with no other side effects, excepting the branch and jump
instructions required to construct a loop. \\

\noindent
\textbf{Def. Loop Diagonalization}: Given a program containing a linearizable
loop $(L, M, v)$, replace $L$ with a single basic block that computes $M^n
v$ using $PD^nP^{-1}v$. \\

Loop diagonalization transforms functions in $O(nm^3)$ into two matrix
multiplications and a diagonal matrix exponentiation, which is in $O(m^3 +
m\log n)$. This can result in an appreciable increase in program efficiency,
as will be shown.

\section{The Fibonacci Example}

Consider the iterative procedure for computing the $n$-th Fibonacci number:

\begin{center}
    \parbox{4cm}{
        \begin{algorithmic}
            \Function{fib}{n}
                \State $a\gets 1$
                \State $b\gets 1$
                \For{$i \in [2 ... n]$}
                    \State $tmp\gets a$
                    \State $a\gets b$
                    \State $b\gets tmp + b$
                \EndFor
                \State \Return b
            \EndFunction
        \end{algorithmic}
    }
\end{center}

The state vector is $v = (a, b)^T$. Initially, $v_0 = (1, 1)^T$. After each
iteration of the loop, $a' = b$ and $b' = a + b$. These linear combinations
are encoded by:

\begin{displaymath}
M = \begin{bmatrix}
        0 & 1 \\
        1 & 1
    \end{bmatrix}
\end{displaymath}

Let $\phi = \frac{1 + \sqrt{5}}{2}$. The eigenvalues of $M$ are $\phi$ and
$1-\phi$. The corresponding eigenvectors are $(1, \phi)^T$ and $(\phi,
-1)^T$. $M$ is diagonalizable because its eigenvalues are distinct and its
eigenvectors are linearly independent:

\begin{displaymath}
M = \begin{bmatrix}
    1 & \phi \\
    \phi & -1
    \end{bmatrix}
    \begin{bmatrix}
    \phi & 0 \\
    0 & 1-\phi
    \end{bmatrix}
    \begin{bmatrix}
    1 & \phi \\
    \phi & -1
    \end{bmatrix}^{-1}
\end{displaymath}

Given $M = PDP^{-1}$, $M^n = PD^nP^{-1}$:

\begin{align*}
M^n &= \begin{bmatrix}
    1 & \phi \\
    \phi & -1
    \end{bmatrix}
    \begin{bmatrix}
    \phi & 0 \\
    0 & 1-\phi
    \end{bmatrix}^n
    \begin{bmatrix}
    1 & \phi \\
    \phi & -1
    \end{bmatrix}^{-1} \\
    &= \begin{bmatrix}
    1 & \phi \\
    \phi & -1
    \end{bmatrix}
    \begin{bmatrix}
    \phi^n & 0 \\
    0 & (1-\phi)^n
    \end{bmatrix}
    \begin{bmatrix}
    1 & \phi \\
    \phi & -1
    \end{bmatrix}^{-1}
\end{align*}

Now the Fibonacci loop has been diagonalized:

\begin{center}
    \parbox{6cm}{
        \begin{algorithmic}
            \Function{fib}{n}
                \State $(a, b)^T \gets PD^{n-1}P^{-1}(1, 1)^T$
                \State \Return b
            \EndFunction
        \end{algorithmic}
    }
\end{center}

\section{Implementation}

I implemented automatic loop diagonalization using the LLVM compiler
infrastructure. This involved creating an instance of
\texttt{llvm::LoopPass} (\texttt{ADPass}) which can be loaded from a dynamic
library. \texttt{ADPass} determines if a loop is linearizable by filtering
out unsupported instructions. It then builds the loop matrix $M$ by
performing a depth-first search on the phi-nodes in the loop. This
search allows the pass to determine the coefficients of every linear
combination of $v$ in the loop, which are exactly the entries of $M$. Next,
the pass uses the \texttt{Eigen} library to find the eigendecomposition of
$M$. Finally the pass deletes the original loop, inserts newly generated
bitcode corresponding to the decomposition into the program, and rewires the
values flowing into the exit phi-nodes. The source code is available under a
non-restrictive free software license
\href{https://github.com/vedantk/auto-diagonalize}{here}.

\section{Testing}

To test the loop diagonalization algorithm, I created a test suite of
iterative processes written in C++ and compiled them at the highest
optimization levels available in LLVM 3.2 and clang++ on Linux x86-64. I
then created alternate versions of the binaries which were post-processed
with \texttt{ADPass}. \texttt{ADPass} was able to diagonalize basic loops
but depended upon pre-processing by the compiler to eliminate stack-spills
and canonicalize loops. It produced good results for all tested programs. I
did not attempt to analyze loops which I knew the pass cannot optimize.

Consider the iterative Fibonacci procedure discussed in a previous section.
I measured the time it took to call $fib(0)$, ..., $fib(200)$ 50,000 times
each. I first tried this with the normal binary and then compared its
performance to the auto-diagonalized binary. As expected, the normal program
(left) scales linearly with the size of the input, whereas the diagonalized
program (right) exhibits $O(\log n)$ performance.

\begin{center}
\includegraphics[scale=0.42]{"fib-O3"}
\includegraphics[scale=0.42]{"fib-O3-diag"}
\end{center}

\section{Discussion}

Matrix decompositions have applications in many fields, but their use in
compiler loop optimizations appears to be novel \footnote{This might have
been true when it was written in 2012, but papers on abstract acceleration
of general linear loops have been published since then.}. It isn't clear if
this optimization pays for its implementation complexity.

Leaving the issue of practicality aside, there are important technical
limitations to \texttt{ADPass}. The most severe restriction is that function
calls, branches, and other instructions with unpredictable side effects
cannot occur within a linearizable loop. In general it is not possible to
lift this draconian restriction without solving undecidable problems such as
`when exactly is this branch taken?' and `does this function call return?'.
Another problem is that there is inherent numerical instability in IEEE 754
floating point numbers, which means that the effects of auto-diagonalization
are not always completely transparent.

With that said, we're left with a somewhat academic optimization that
produces some interesting results. It allows linearizable $O(nm^3)$
processes to run in $O(m^3 + m\log n)$, using the simple magic trick $Mv =
\lambda v$.

\section{References}

\begin{enumerate}[1.]
    \item LLVM, \url{http://llvm.org/}.
    \item Eigen, v3. Ga\"{e}l Guennebaud and Beno\^{i}t Jacob and others.
        \url{http://eigen.tuxfamily.org}.
\end{enumerate}

\end{document}


\newpage
\section{Jinchao: XN}
XN is independent with the SGD training process.

After an update of the neural network, XN can be applied to each hidden layer successively, and won't change the output of it. In this example, we have only one hidden layer. The XN algorithm can be written as follows.
\begin{enumerate}
\item Denote
$$\Theta=(W_1\ b_1)\in\mathbb{R}^{n_1\times(n+1)}$$
and
$$
\hat X=(X;\mathbf{1})\in\mathbb{R}^{(n+1)\times m},
Y=\text{ReLU1}(W_1X+b_1).
$$
	Split the columns of $\hat X$ to $X_I$, $X_J$, $X_K$, s.t. for every row $\theta_i$ of $\Theta$, $i=1,...,n_1$,
	\begin{equation}
	\begin{aligned}
	\theta_i X_I\leq 0,\\
	\theta_i X_J\geq 1,\\
	\theta_i X_K\in[0,1].
	\end{aligned}
	\end{equation}
	Denote IJ as the union of I and J. For every $\theta_i$, the optimization problem can be rewritten as
	\begin{equation}\label{standardv2opt}
	\begin{aligned}
	\min_\theta&\quad\|\theta  X_{IJ}-Y_{IJ}\|_2\\
	\text{subject to}&\quad\theta X_{I}\leq 0,\\
	&\quad\theta X_{J}\geq 1,\\
	&\quad\theta X_{K}=Y_{K}.
	\end{aligned}
	\end{equation}
\end{enumerate}


\section{Linear smoothers}
Most of the smoother presented here are linear smoothers which means
that the fit at any point $x_0$ can be written as
\[
s(x) = \sum_{j=1}^n S_{j}(x) y_j.
\]

In practice we usually have the model
\[
Y_i = f(X_i) + \epsilon_i
\]
and we have observations $\{(x_i, y_i)\}$. Many times it is the vector
${\mathbf f} = \{f(x_1),\dots,f(x_n)\}'$ we are after. In this case
the vector of estimates $\hat{\mathbf f} =
\{\hat{f}(x_1),\dots,\hat{f}(x_n)\}'$ can be written as
\[
\hat{\mathbf f} = \bS \by
\]
with $\bS$ a matrix with the i,j-th entry $S_{j}(x_i)$. We will call
$\hat{\mathbf f}$ the {\it smooth}.

This makes it easy to figure out things like the variance of
$\hat{\f}$ since
\[
\var[\bS \by] = \bS \var[\by] \bS'
\]
which in the case of IID data is $\sigma^2 \bS\bS'$.


\section{Graphs}

\begin{definition}[Arborescence]
    an arborescence is a directed graph in which, for a vertex u called the
    root and any other vertex v, there is exactly one directed path from u to v.
 \end{definition}

\begin{definition}[Clique]
    A subset of vertices $C \subset V$, such that in this subgraph all nodes
    are connected, i.e.\ there is an edge from every pair of nodes.
\end{definition}

\begin{definition}[Connected graph (component)]\label{connectedcomp}
    A graph in which, from any $v \in V$, you can reach any other $u \neq v$
\end{definition}

\begin{definition}[Directed graph]
    Properties:
    \begin{itemize}
        \item $\sum\limits_{v \in V}{d(v)} = 2|E|$
        \item $\sum\limits_{v \in V}{indegree(v)} =
            \sum\limits_{v \in V}{outdegree(v)}$
    \end{itemize}
\end{definition}

\begin{definition}[Eularian]\label{eularian}
    Visit each \textit{edge} once.
\end{definition}

\begin{definition}[Hamiltonian]
    Visit each \textit{vertex} once.
\end{definition}

\begin{definition}[Metric spaces]\label{metric}
    See also~\nameref{semimetric}
    Properties:
    \begin{itemize}
        \item $d_{u,v} = 0 \iff u = v$
        \item $d_{u,v} = d_{v,u}$
        \item $\forall k, d_{u,v} \leq d_{u,k} + d_{k,v}$
    \end{itemize}
\end{definition}

\begin{definition}[Minimum mean cost]
    minimize ratio of cost of arcs (directed edges) to number of arcs
\end{definition}

\begin{definition}[MST]
    A mininum~\nameref{spantree}, such the weight of this tree is less than
    or equal to all other possible~\nameref{spantree}.
\end{definition}

\begin{definition}[Offline Graph Algorithm]
    Algorithms on a graph that requires knowing the entire structure
    For example PageRank, Clustering, so on.
\end{definition}

\begin{definition}[Online Graph Algorithm]
    Process the graph in a serial fashion, e.g. operate on the graph without knowing
    the entire structure. For example shortest path, pattern matching, etc.
\end{definition}

\begin{definition}[Planar graph]
    No edges need to cross each other.
\end{definition}

\begin{definition}[Semi-metric]\label{semimetric}
    Like a~\nameref{metric}, without the property that
    $d_{u,v} = 0 \iff u = v$
\end{definition}

\begin{definition}[Spanning tree]\label{spantree}
    Connected, undriected graph that has all vertices and some subset
    of edges to form a tree.
\end{definition}

\begin{definition}[Strongly connected component]
    Similar to~\nameref{connectedcomp}, but in a directed graph.
\end{definition}

\begin{definition}[Topological sort]\label{topsort}
    Runs in O(V + E), same as DFS, although it can be supered to $O(\log_{2} n)$
\end{definition}

\begin{definition}[Tree]\label{tree}
    Properties of a tree:
    \begin{itemize}
        \item in a tree, there will always be an even amount of nodes that has
            an uneven degree.
        \item Degree of nodes in a tree is at most twice the number of nodes
    \end{itemize}
\end{definition}

\begin{definition}[Vertex-cover]
    A selection of vertices such that each edge is incident to at least one
    of them.
\end{definition}


\subsection{The Ramsey RESET test}

The Ramsey Regression Equation Specification Error Test (RESET)

Once we have done our OLS we have \(\hat y\).

The Ramsey RESET test is an additional stage, which takes these predictions and estimates:

\(y=\theta x+\sum_{i=1}^3\alpha_i \hat {y^i}\)

We then run an F-test on \(\alpha\), with the null that \(\alpha = 0\).


\chapter{The Definition of a Graph}
\label{chapter:graph-theory-basics}
In this chapter we start a very important topic in discrete mathematics, which
became even more important with the rise of computers, we start the discussion
of graph theory. Graphs are used in mathematics and computer science to describe
networks, maps, and dependencies of objects.

\begin{definition}
  A \emph{graph} $G$ is a pair $(V, E)$ such that $E \subseteq V^2$ is a multiset.

  We say that $G$ is \emph{unoriented} iff $(u, v) \in E$ iff $(v, u) \in E$ for
  any $u, v \in V$. Otherwise the graph is \emph{oriented}.
  We say that a graph does not have \emph{loops} iff $(u, u) \notin E$ for any
  $u \in V$. Finally, we say that the graph has \emph{parallel edges} if
  $E$ is not a set.
\end{definition}
A graph is \emph{simple} iff it has no loops, it has not parallel edges, and
it is unoriented.

From now on we will follow a standard convention and think about the set of
edges of unoriented graphs as sets of \emph{unordered} pairs.

It is very convenient to draw graphs using pictures like this.
\begin{center}
  \begin{tikzpicture}
    \node[shape=circle,draw=black] (v1) at (0,0) {\tiny A};
    \node[shape=circle,draw=black] (v2) at (0,1) {\tiny B};
    \node[shape=circle,draw=black] (v3) at (1,1) {\tiny C};
    \node[shape=circle,draw=black] (v4) at (-1,2) {\tiny D};

    \draw (v1) -- (v2);
    \draw (v1) -- (v3);
    \draw (v2) -- (v4);
    \draw (v2) -- (v3);

  \end{tikzpicture}
\end{center}
In this picture, each circle corresponds to a vertice and each line corresponds
to a an edge; i.e. this diagram describes the graph
\[
  (\underbrace{\set{A, B, C, D}}_V,
  \underbrace{\set{(A, B), (A, C), (B, C), (B, D)}}_E).
\]
Note that we already use the convention that in an unoriented graph the pairs
are unordered and we have not listed $(B, A)$, $(C, A)$ etc.

To talk about graphs we need to fix the vocubulary.
An edge is said to \emph{connect} its endpoints; two vertices that are
connected by an edge are called \emph{adjacent}; and a vertex that is an
endpoint of a loop is said to be \emph{adjacent to itself}. An edge is said to
be \emph{incident} on each of its endpoints, and two edges incident
on the same end point are called \emph{adjacent}. A vertex on which no edges
are incident is called \emph{isolated}.

One of the most important examples of graphs are complete graphs defined as
follows.
\begin{definition}
  Let $n$ be a natural number. A complete graph on $n$ vertices, denoted
  $K_n$,\footnote[][-2cm]{%
    Some sources claim that the letter K in this notation stands for the German
    word komplett, but the German name for a complete graph, vollst\"{a}ndiger
    Graph, does not contain the letter K, and other sources state that the
    notation honors the contributions of Kazimierz Kuratowski to graph theory.
  }
  is a simple graph with $n$ vertices and exactly one edge connecting
  each pair of distinct vertices.
\end{definition}
\nomenclature[G]{$K_n$}{denotes the complete graph on $n$ vertices}

\begin{exercise}
\label{exercise:maximal-number-of-edges}
  Show that for all natural numbers $n$, the number of edges of $K_n$
  is $\frac{n(n - 1)}{2}$.
\end{exercise}

\section{Operations on Graphs}
Quite often in order to prove a theorem we need to modify a graph. The most
often operations are the following four. Let $G = (V, E)$ be a graph,
$F \subseteq E$ be a set of edges, $U \subseteq V$ be a set of edges,
$e \in E$ be an edge, and $v \in V$ be a vertex.
\begin{enumerate}
  \item $G[U]$ denotes the graph $(U, \set[e \in U^2]{e \in E})$,
    $G[U]$ is called the induced subgraph of $G$ on the vertices $U$;
    \nomenclature[G]{$G[U]$}{denotes the induced subgraph of $G$ on the vertices $U$
    (i.e. $(U, \set[e \in U^2]{e \in E})$)}
  \item $G[F]$ denotes the graph $(V, F)$, $G[F]$ is called the induced
    subgraph of $G$ on the edges $F$;
    \nomenclature[G]{$G[F]$}{denotes the induced subgraph of $G$ on the edges $F$
    (i.e. $(V, F)$)}
  \item $G - e$ denotes the graph $(V, E \setminus \set{e})$, i.e., the graph $G$
    without the edge $e$.
    \nomenclature[G]{$G - e$}{denotes the graph $(V, E \setminus \set{e})$}
  \item $G - v$ denotes the graph
    $(V \setminus \set{v}, E \cap (V \setminus \set{v})^2)$, i.e., the graph $G$
    without the vertex $v$.
    \nomenclature[G]{$G - v$}{denotes the graph
      $(V \setminus \set{v}, E \cap (V \setminus \set{v})^2)$}
\end{enumerate}

Note that we used the word ``subgraph'', in fact we can define formally the
meaning of this word.
\begin{definition}
  We say that a graph $H = (U, F)$ is a \emph{subgraph} of $G = (V, E)$ iff
  $U \subseteq V$ and $F \subseteq E$.
\end{definition}

\section{Degrees of Vertices}
The degree of a vertex is the number of endsegments of edges that ``stick out
of'' the vertex.
\begin{definition}
  Let $G = (V, E)$ be a graph, and $v$ be a vertex. Then
  $\deg_G(v) = |\set[{e \text{is connected to } v}]{e \in E}|$.
\end{definition}

\begin{exercise}
  Let $G = (V, E)$ be a graph and $v \in V$ be a vertex.
  What are the possible values of $\deg_G(v)$?
\end{exercise}

Note that Lemma~\ref{lemma:handshaking} shows that in any simple graph the
number of vertices with an odd degree is even.
The essence of the proof of this lemma is the following statement.
\begin{theorem}
\label{theorem:handshaking-graphs}
  Let $G = (V, E)$ be a simple graph. Then $\sum_{v \in V} \deg_G(v) = 2|E|$.
\end{theorem}

It is intresting to note that this result allows to give another proof
\Cref{corollary:binommials-equality}. Let us consider a graph $H_n =
(\set{0, 1}^n, E_n)$ such that $(x, y) \in E_n$ iff $x$ and $y$ are
different in exactly one position. Note that $\deg_{H_n} x = n$ for all
$x \in \set{0, 1}^n$; hence, $|E_n| = n 2^n / 2 = n 2^{n - 1}$. In addition,
note that there are $\binom{n}{k}$ vertices with $k$ many 1's and there are
$k$ edges that turn $1$ into $0$; i.e., there are
$\sum_{k = 0}^n k \binom{n}{k}$ edges in $H_n$. Therefore,
$\sum_{k = 0}^n k \binom{n}{k} = n 2^{n - 1}$.

\begin{chapterendexercises}
  \exercise Either draw a graph with the specified properties or explain why
    no such graph exists:
    \begin{enumerate}[nolistsep]
      \item simple graph with five vertices of degrees $1$, $2$, $3$, $3$,
        and $5$;
      \item simple graph with four vertices of degrees $1$, $2$, $3$, and $3$;
      \item simple graph with four vertices of degrees $1$, $1$, $1$, and $5$;
      \item simple graph with four vertices of degrees $1$, $2$, $3$, and $4$;
      \item simple graph with four vertices of degrees $1$, $2$, $3$, and $5$.
    \end{enumerate}
  \exercise Let $G$ be a \emph{simple} graph. Show that there are two different
    vertices $u$ and $v$ such that they have the same degree.
    \begin{solution}
      Assume $G$ is a simple graph with $n$ vertices. Suppose all vertices in
      $G$ have different degrees. The set of degrees of the vertices is then
      $\set{0, 1, \dots, n - 1}$. However this means that one vertex is
      connected to all other vertices and another vertex is connected to no
      other vertices, it is a contradiction. Therefore there are at least two
      different vertices $u$ and $v$ such that they have the same degree.
    \end{solution}
  \exercise Let $G = ([5] \times [5], E)$ be a graph such that
    $((x_1, y_1), (x_2, y_2)) \in E$ iff
    \begin{itemize}
      \item either $x_1 = x_2$ and $|y_1 - y_2| = 1$, or
      \item $|x_1 - x_2| = 1$ and $y_1 = y_2$.
    \end{itemize}

    \begin{enumerate}
      \item Is it possible to cover the graph $G$ with $8$ paths of length $5$
        (these paths should not visit any vertex twice)? Note that by covering we
        mean that every edge of $G$ is an edge of one of the paths.
      \item Is it possible to cover the graph $G$ with $5$ paths of length $8$
        (these paths should not visit any vertex twice)? Note that by covering
        we mean that every edge of $G$ is an edge of one of the paths.
    \end{enumerate}
  \exercise Let $G$ be a graph on $n$ vertices such that every vertex has odd
    degree. Show that $n$ is even.
    \begin{solution}
      One may notice that for any odd and even numbers the following identities
      hold:
      $\mathrm{odd} + \mathrm{odd} = \mathrm{even}$ and $\mathrm{even} +
      \mathrm{odd} = \mathrm{odd}$.

      Assume there are $k$ edges in $G$ where $k \in \Z$ and let $v_i$ denote
      the $i$th vertex of $G$ and $d_i$ denote degree of $v_i$, where $i \in
      \range{n}$. Then the total degree of $G$ (i.e. $\sum_{i = 1}^n d_i$)
      is $2k$ by \Cref{theorem:handshaking-graphs}. Since every vertex has odd
      degree $2k = \sum_{i = 1}^{n} d_i = \mathrm{odd} + \mathrm{odd} + \dots +
      \mathrm{odd}$.

      Notice if $n$ is odd then there is an odd number of vertices and
      $\sum_{i = 1}^n d(v_i) = \mathrm{odd} \neq 2k$ as $2k$ is even. Thus $n$
      must be even.
    \end{solution}
  \exercise In a group of $25$ people, is it possible for each to shake hands
    with exactly $3$ other people?
  \exercise Suppose that $G$ is a graph with $v$ vertices and $e$ edges and
    that the degree of each vertex is at least $d_\mathrm{min}$ and at most
    $d_\mathrm{max}$. Show that
    \[
      \frac{1}{2} v d_\mathrm{min} \le e \le \frac{1}{2} v d_\mathrm{max}.
    \]
\end{chapterendexercises}


\subsection{\texorpdfstring{$\lambda$-c}{Lambda C}alculus}\label{ssec:lambda-calculus}

    Developed by Alonzo Church in the 1930s, the $\lambda$-calculus was the first such computational calculus and describes a mathematical representation of a computable function.
    When it was first designed, it was not expected to be as relevant to the newly-emerging field of theoretical Computer Science but instead Discrete Mathematics and Category Theory, the $\lambda$-calculus in fact forms a universal model of computation and can contain an encoding of any single-taped Turing machine.
    It has since become a popular formal system within which to study properties of computation.

    \begin{definition}{Syntax\\}
        \label{lambda-calculus-syntax}
        The $\lambda$-calculus, as defined by Church but here explained by~\cite{lambda-calculus}\footnotemark consists of an expression $M$ built from the following terms:
        \footnotetext{While Church's original paper is still available, the source cited is found to be more relevant due to research in the subject area since the original paper's publication in the 1930s as well as a narrower scope than Church initially provides.}
        \begin{center}
            \begin{tabular}{ l l l }
                $M  \quad \defeq$       & $a$               & (variable) \\
                                        & $\lambda x . M$   & (abstraction) \\
                                        & $M N$             & (application)
            \end{tabular}
        \end{center}
    \end{definition}
    From this, any computable function can be constructed and computation is achieved through a series of operations on the expression.

    \begin{definition}{$\alpha$-substitution\\}
        \label{lambda-alpha-relation}
        Unbound variables within an expression may be substituted for any given value.
        This is formally expressed as:
        \begin{align*}
            x[y \defeq P] \defeq &
                \begin{cases}
                    P & \text{if } x = y \\
                    x & \text{otherwise}
                \end{cases} \\
            (\lambda x . M)[y \defeq P] \defeq &
                \begin{cases}
                    \lambda x . M & \text{if } x = y \\
                    \lambda x . (M[y \defeq P]) & \text{if } x \neq y \text{ and } x \notin FV(P)~\footnotemark
                \end{cases}
        \end{align*}\footnotetext{Here, $FV(P)$ is the set of all variables $x$ such that $x$ is free (unbound) in $P$.}
    \end{definition}
    This operation may be thought of as variable renaming as long as both old and new names are free in the expression in which they were substituted.

    \begin{corollary*}{$\alpha$-equivalence\\}
        The above definition of $\alpha$-substitution may be extended to give an equivalence relation on expressions, $\alpha$-equivalence, defined as:
        \begin{align*}
            y \notin FV(M) & \implies \lambda x . M \equiv_\alpha \lambda y \, (M[x \defeq y]) \\
            M \equiv_\alpha M' & \implies
                \begin{cases}
                    M \, P \equiv_\alpha M' \, P \\
                    P \, M \equiv_\alpha P \, M' \\
                    \lambda x . M \equiv_\alpha \lambda x. M'
                \end{cases}
        \end{align*}
    \end{corollary*}

    \begin{definition}{$\beta$-reduction\\}
        An expression may be simplified by applying one term to another through substitution of a term for a bound variable.
        This is formally expressed as:
        \begin{align*}
            (\lambda x . P) \, Q   & \rightarrow_\beta P[x \defeq Q] \\
            M \rightarrow_\beta M' & \implies
                \begin{cases}
                    P \, M \rightarrow_\beta P M' \\
                    M \, P \rightarrow_\beta M' P \\
                    \lambda x . M \rightarrow_\beta \lambda x . M'
                \end{cases}
        \end{align*}
    \end{definition}
    Often $\beta$-reduction requires several steps at once and as such these multiple $\beta$-reduction steps are abbreviated to $\rightarrow_\beta^*$.

    \begin{corollary*}{$\beta$-equivalence\\}
        The above definition of $\beta$-reduction may be extended to give an equivalence relation on expressions, $beta$-equivalence, defined as:
        \begin{equation*}
            M \rightarrow_\beta^* P \text{ and } N \rightarrow_\beta^* P \implies M \equiv_\beta N
        \end{equation*}
    \end{corollary*}

    \begin{example*}
        The above corollaries can be seen to have desirable properties when examining whether two expressions describe equivalent computation.
        \begin{align*}
            \lambda a . x \, a [x \defeq y] & \equiv \lambda a . y \, a \\
            \\
            \lambda x . x \, y & \equiv_\alpha \lambda z . z \, y \\
            \lambda x . x \, y & \not\equiv_\alpha \lambda y . y \, y \\
            \\
            (\lambda a . x \, a) \, y & \rightarrow_\beta x \, a [a \defeq y] \equiv x \, y
        \end{align*}
    \end{example*}

    As computational calculus shares many parallels with modern functional programming, the following are encodings of some common functional concepts within the $\lambda$-calculus.

    \begin{definition}{List\\}
        Within the $\lambda$-calculus, lists may be encoded through the use of an arbitrary ${cons}$ function that takes a head element and a tail list and of a ${null}$ function that signifies the end of a list.
        The list is then constructed as a singly-linked list might be constructed in other languages:
        \begin{equation*}
            [x_1, \ldots, x_n] \defeq \lambda c . \lambda n . (c \, x_1 \, (\ldots (c \, x_n \, n) \ldots) )
        \end{equation*}
    \end{definition}

    \begin{definition}{Map\\}
        The $map$ function takes two arguments --- a function $F$ that itself takes one argument and a list of suitable arguments $[x_1, \ldots, x_n]$ to this function.
        The output is then a list of the output of F when applied to each $x_1 \ldots x_n$.
        \begin{equation*}
            {map} \defeq \lambda f . \lambda l . (\lambda c . (l \, (\lambda x . c \, (f \, x))))
        \end{equation*}
    \end{definition}

    \begin{example*}
        As follows is an example of the reductions on the ${map}$ function for a list of length $n \defeq 3$:
        \begin{align*}
            {map} \, F \, [x_1, x_2, x_3]
                  & \equiv_{\alpha} \lambda f . \lambda l . (\lambda c . (l \, (\lambda x . c \, (f \, x)))) \, F \, \lambda c . \lambda n . (c \, x_1 \, (c \, x_2 \, (c \, x_3 \, n))) \\
                  & \rightarrow_\beta^* \lambda c . (\lambda c . \lambda n . (c \, x_1 \, (c \, x_2 \, (c \, x_3 \, n))) \, (\lambda x . c \, (F \, x))) \\
                  & \rightarrow_\beta^* \lambda c . (\lambda n . ((\lambda x . c \, (F \, x)) \, x_1 \, ((\lambda x . c \, (F \, x)) \, x_2 \, ((\lambda x . c \, (F \, x)) \, x_3 \, n)))) \\
                  & \rightarrow_\beta^* \lambda c . (\lambda n . (c \, (F \, x_1) \, (c \, (F \, x_2) \, (c \, (F \, x_3) \, n)))) \\
                  & \equiv_{\alpha} [F \, x_1, F \, x_2, F \, x_3]
        \end{align*}
    \end{example*}

    \begin{remarks}
        While the $\lambda$-calculus has been successful and been studied by various areas of academia outside Computer Science, it is limited in modern-day application by its fundamentally `single-process' model and struggles to describe multiple systems working and communicating together.
        While certain additional properties --- specifically the $Y$-combinator and simply-typed $\lambda$-calculus --- are not mentioned here, the calculus is defined from a few simple rules.
        This simplicity allows implementations of $\lambda$-calculus interpreters to be relatively painless.\footnote{There exists an example of such an interpreter, available online at the time of writing, at \url{http://www.cburch.com/lambda/}}
        \textit{In my opinion, the simplicity and expressiveness of the $\lambda$-calculus should be the standard to which other computational calculi are held.}\\

        The ${map}$ example is particularly relevant to study within concurrent calculi as multiple large-scale systems follow a MapReduce programming model, as described by~\cite{mapreduce}, which utilises massive parallelism of large detacenters.
        The model requires a ${map}$ function that applies a function to a key and set of values, similar to that described above, and a ${reduce}$ function that collects all values with matching keys.
        This has found to be useful for modelling many real-world tasks for performance reasons, but concurrent calculi may provide a simple case for study and understanding for any of these tasks which scales as necessary.
    \end{remarks}


\subsection{Axiom of regularity}

The axiom of regularity states that:

\(\forall x[x\ne \varnothing \rightarrow \exists y \in x (y\land x )=\varnothing]\)

That is, for all non-empty sets, there is an element of the set which is disjoint from the set itself.

This means that no set can be a member of itself.


\lab{Anisotropic Diffusion}{Anisotropic Diffusion}
\label{lab:AnisotropicDiffusion}

\objective{Demonstrate the use of finite difference schemes in image analysis.}

A common task in image processing is to remove extra static from an image.
This is most easily done by simply blurring the image, which can be accomplished by  treating the image as a rectangular domain and applying the diffusion (heat) equation:
\[u_t = c \Delta u\]
where $c$ is some diffusion constant and $\Delta$ is the Laplace operator.
Unfortunately, this also blurs the boundary lines between distinct elements of the image.

A more general form of the diffusion equation in two dimensions is:
\[u_t = \nabla\cdot \left( c(x,y,t) \nabla u \right)\]
where $c$ is a function representing the diffusion coefficient at each given point and time.
In this case, $\nabla \cdot$ is the divergence operator and $\nabla$ is the gradient.

To blur a picture uniformly, choose $c$ to be a constant function.
% , but we are often interested in \textit{preserving the edges} between features of the image.
Since $c$ controls how much diffusion is allowed at each point, it can be modified so that diffusion is minimized across edges in the image.
In this way we attempt to limit diffusion near the boundaries between different features of the image, and allow smaller details of the image (such as static) to blur away.
This method for image denoising is especially useful for denoising low quality images, and was first introduced by Pietro Perona and Jitendra Malik in 1987.
It is known as Anisotropic Diffusion or Perona-Malik Diffusion.

\section*{A Finite Difference Scheme}
Suppose we have some estimate $E$ of the rate of change at a given point in an image.
$E$ will be largest at the boundaries in the image.
We will then let $c(x,y,t) = g(E(x,y,t))$ where $g$ is some function such that $g(0)=1$ and $\displaystyle{\lim_{x \to \infty} g(x) = 0}$.
Thus $c$ will be small where $E$ is large, so that little diffusion occurs near the boundaries of different portions of the image.

We will model this system using a finite differencing scheme with an array of values at a 2D grid of points, and iterate through time.
Let $U_{l,m}^n$ be the discretized approximation of the function $u$, $n$ be the index in time, $l$ be the index along the $x$-axis, and $m$ be the index along the $y$-axis.

The Laplace operator can be approximated with the finite difference scheme
\[\Delta u = u_{xx}+u_{yy} \approx \frac{U_{l-1,m}^n - 2 U_{l,m}^n + U_{l+1,m}^n}{(\Delta x)^2} + \frac{U_{l,m-1}^n-2 U_{l,m}^n + U_{l,m+1}^n}{(\Delta y)^2}.\]
A good metric to use with images is to let the distance between each pixel be equal to one, so $\Delta x = \Delta y = 1$.
Rearranging terms, we obtain
\[\Delta u \approx (U_{l-1,m}^n - U_{l,m}^n) + (U_{l+1,m}^n - U_{l,m}^n) + (U_{l,m-1}^n - U_{l,m}^n) + (U_{l,m+1}^n - U_{l,m}^n).\]
Again, since we are working with images and not some time based problem, we can without loss of generality let $\Delta t = 1$, so we obtain the finite difference scheme
\[U_{l,m}^{n+1} = U_{l,m}^n + (U_{l-1,m}^n - U_{l,m}^n) + (U_{l+1,m}^n - U_{l,m}^n) + (U_{l,m-1}^n - U_{l,m}^n) + (U_{l,m+1}^n - U_{l,m}^n).\]
We will now limit the diffusion near the edges of objects by making the modification
\begin{equation}\label{eq:anisotropic}
\begin{aligned}
U_{l,m}^{n+1} = U_{l,m}^n + & \lambda \Big(g(|U_{l-1,m}^n - U_{l,m}^n|)(U_{l-1,m}^n - U_{l,m}^n) \\
			&+ g(|U_{l+1,m}^n - U_{l,m}^n|)(U_{l+1,m}^n - U_{l,m}^n) \\
			&+ g(|U_{l,m-1}^n - U_{l,m}^n|)(U_{l,m-1}^n - U_{l,m}^n) \\
			&+ g(|U_{l,m+1}^n - U_{l,m}^n|)(U_{l,m+1}^n - U_{l,m}^n)\Big),
\end{aligned}
\end{equation}
where $\lambda \leq \frac{1}{4}$ is the stability condition.

In this difference scheme, each term is affected most by nearby terms that are most similar to it, so less diffusion will happen anywhere there is a sharp difference between pixels.
This scheme also has the useful property that it does not increase or decrease the total brightness of the image.
Intuitively, this is because the effect of each point on its neighbors is exactly the opposite effect its neighbors have on it.

Two commonly used functions for $g$ are $g(x) = e^{-\left(\frac{x}{\sigma}\right)^2}$ and $g(x) = \frac{1}{1+\left(\frac{x}{\sigma}\right)^2}$.
The parameter $\sigma$ allows us to control how much diffusion decreases across boundaries, with larger $\sigma$ values allowing more diffusion.
Note that $g(0) = 1$ and $\displaystyle{\lim_{x\to \infty} g(x) = 0}$ for both functions.
In this lab we use $g(x)=e^{-\left(\frac{x}{\sigma}\right)^2}$.

It is worth noting that this particular difference scheme is \textit{not} an accurate finite difference scheme for the version of the diffusion equation we discussed before, but it \textit{does} accomplish the same thing in the same way.
As it turns out, this particular scheme is the solution to a slightly different diffusion PDE, but can still be used the same way.

For this lab's examples we read in the image using the \li{imageio.imread} function, and normalized it so that the colors are represented as floating point values between 0 and 1.
An image can converted to black and white when it is read by including the argument \li{as_gray=True}.

\begin{lstlisting}
from matplotlib import cm, pyplot as plt
from imageio import imread

# To read in an image, convert it to grayscale, and rescale it.
picture = imread('balloon.png', as_gray=True) * 1./255

# To display the picture as grayscale
plt.imshow(picture, cmap=cm.gray)
plt.show()
\end{lstlisting}

\subsection*{Simplifying Calculations}\label{sec:simp}

You will notice that the algorithm given in \ref{eq:anisotropic} does not describe what to do for the edges and corners of $U^{n+1}$.
In these cases we will simply eliminate the undefined terms in the algorithm.
For example, the top edge equation becomes
\begin{align*}
U_{l,m}^{n+1} = U_{l,m}^n + & \lambda (g(|U_{l+1,m}^n - U_{l,m}^n|)(U_{l+1,m}^n - U_{l,m}^n) \\
                    & + g(|U_{l,m+1}^n - U_{l,m}^n|)(U_{l,m+1}^n - U_{l,m}^n)) \\
					& + g(|U_{l,m-1}^n - U_{l,m}^n|)(U_{l,m-1}^n - U_{l,m}^n)),
\end{align*}
and top left corner equation becomes
\begin{align*}
U_{l,m}^{n+1} = U_{l,m}^n + & \lambda (g(|U_{l+1,m}^n - U_{l,m}^n|)(U_{l+1,m}^n - U_{l,m}^n) \\
					& + g(|U_{l,m+1}^n - U_{l,m}^n|)(U_{l,m+1}^n - U_{l,m}^n)).
\end{align*}
Essentially we are only using the terms of the difference scheme that are actually defined.

To help facilitate this we can create a larger "padded" matrix that will make these calculations easy to do.
This padded matrix will have an extra row on the top and bottom, and an extra column on either side of the original matrix.
These extra rows and columns will duplicate the outer edge of the original matrix.

So if our original array \li{X} has shape \li{m,n}, then our padded array \li{Y} has shape \li{m+2,n+2}.
The top edge of \li{Y} will be defined so that \li{Y[0,1:-1] == X[0,:]} is true, and the rest of the edges of \li{Y} follow the same pattern.

Notice that this allows us to simply implement the algorithm found in \ref{eq:anisotropic} without having to make special cases for the edges and corners, since those previously undefined terms become zero when using the padded matrix.

\begin{problem}
\label{prob:anisdiff_bw}
Complete the following function, by implementing the anisotropic diffusion algorithm found in \ref{eq:anisotropic} for black and white images.
Use the padded array technique found in the Simplifying Calculations section.

In your function, use
\[g(x) = e^{-\left(\frac{x}{\sigma}\right)^2}\]

\begin{lstlisting}
def anisdiff_bw(U, N, lambda_, g):
    """ Run the Anisotropic Diffusion differencing scheme
    on the array U of grayscale values for an image.
    Perform N iterations, use the function g
    to limit diffusion across boundaries in the image.
    Operate on U inplace to optimize performance. """
    pass
\end{lstlisting}
Run the function on \li{balloon.jpg}.
Show the original image and the diffused image for $\sigma = .1$, $\lambda = .25$, $N = 5, 20, 100$.
\end{problem}

\newpage
\vfill
\begin{figure}[ht]
\begin{minipage}[b]{0.45\linewidth}
\centering
\includegraphics[width=\textwidth]{balloon_grayscale.jpg}
\caption*{original image}
\end{minipage}
\hspace{0.5cm}
\begin{minipage}[b]{0.45\linewidth}
\centering
\includegraphics[width=\textwidth]{balloon_grayscale_5.jpg}
\caption*{5 iterations with $\sigma = .1$ and $\lambda = .25$}
\end{minipage}
\begin{minipage}[b]{0.45\linewidth}
\centering
\includegraphics[width=\textwidth]{balloon_grayscale_20.jpg}
\caption*{20 iterations}
\end{minipage}
\hspace{0.5cm}
\begin{minipage}[b]{0.45\linewidth}
\centering
\includegraphics[width=\textwidth]{balloon_grayscale_100.jpg}
\caption*{100 iterations}
\end{minipage}
\end{figure}
\vfill
\clearpage

\section*{Color Schemes}

Colored images can be processed in a similar manner.
Instead of being represented as a two-dimensional array, colored images are represented as three dimensional arrays.
The third dimension is used to store the intensities of each of the standard 3 colors.
This diffusion process can be carried out in the exact same way, on each of the arrays of intensities for each color, but instead of detecting edges just in one color, we need to detect edges in any color, so instead of using something of the form $g(|U_{l+1,m}^n - U_{l,m}^n|)$ as before, we will now use something of the form $g(||U_{l+1,m}^n - U_{l,m}^n||)$, where $U_{l+1,m}^n$ and $U_{l,m}^n$ are vectors now instead of scalars.
The difference scheme can be treated as an equation on vectors in 3-space and now reads:
\begin{align*}
U_{l,m}^{n+1} = U_{l,m}^n + & \lambda (g(||U_{l-1,m}^n - U_{l,m}^n||)(U_{l-1,m}^n - U_{l,m}^n) \\
					& + g(||U_{l+1,m}^n - U_{l,m}^n||)(U_{l+1,m}^n - U_{l,m}^n) \\
					& + g(||U_{l,m-1}^n - U_{l,m}^n||)(U_{l,m-1}^n - U_{l,m}^n) \\
					& + g(||U_{l,m+1}^n - U_{l,m}^n||)(U_{l,m+1}^n - U_{l,m}^n))
\end{align*}

When implementing this scheme for colored images, use the $2$-norm on 3-space, i.e $||x||=\sqrt{x_1^2+x_2^2+x_3^2}$ where $x_1$, $x_2$, and $x_3$ are the different coordinates of $x$.

\begin{problem}
Complete the following function to process a colored image.
You may modify your code from the previous problem.
Measure the difference between pixels using the $2$-norm.
Use the corresponding vector versions of the boundary conditions given in Problem \ref{prob:anisdiff_bw}.

\begin{lstlisting}
def anisdiff_color(U, N, lambda_, sigma):
    """ Run the Anisotropic Diffusion differencing scheme
    on the array U of grayscale values for an image.
    Perform N iterations, use the function g = e^{-x^2/sigma^2}
    to limit diffusion across boundaries in the image.
    Operate on U inplace to optimize performance. """
    pass
\end{lstlisting}
Run the function on \li{balloons_color.jpg}.
Show the original image and the diffused image for $\sigma = .1$, $\lambda = .25$, $N = 5, 20, 100$.

Hint: If you have an $m \times n \times 3$ matrix representing the RGB differences of each pixel, then to find a matrix representing the norm of the differences, you can use the following code.
This code squares each value and sums along the last axis, and takes the square root.
In order to keep the dimension size of the matrix and aid in broadcasting, you must use \li{keepdims=True}.

\begin{lstlisting}
# x is mxnx3 matrix of pixel color values
norm = np.sqrt(np.sum(x**2, axis=2, keepdims=True))
\end{lstlisting}

\end{problem}

\newpage
\section*{Noisy Images}

\begin{problem}
Use the following code to add noise to your grayscale image.

\begin{lstlisting}
from numpy.random import randint

image = imread('balloon.jpg', as_gray=True)
x, y = image.shape
for i in xrange(x*y//100):
	image[randint(x),randint(y)] = 127 + randint(127)
\end{lstlisting}

Run \li{anisdiff_bw()} on the noisy image with $\sigma=.1$, $\lambda=.25$, $N=20$.
Display the original image and the noisy image.
Explain why anisotropic diffusion does not smooth out the noise.

Hint: Don't forget to rescale.
\end{problem}

\section*{Minimum Bias (Optional)}

This sort of anisotropic diffusion can be very effective, but, depending on the image, it may also smear out edges that do not have large differences between them.
An example of this limitation can be seen in Figure \ref{fig:anisdif_smearing}

\begin{figure}
\begin{minipage}[b]{.45\linewidth}
\centering
\includegraphics[width=\textwidth]{baloons_resized_bw.jpg}
\caption*{original image}
\end{minipage}
\hspace{0.5cm}
\begin{minipage}[b]{0.45\linewidth}
\centering
\includegraphics[width=\textwidth]{baloons_resized_bw_50.jpg}
\caption*{after 50 iterations}
\end{minipage}
\caption{Smearing of similar colors when using an anisotropic diffusion filter.}
\label{fig:anisdif_smearing}
\end{figure}

As we can see, after 100 iterations, some of the boundaries between similar shades of grey have smeared unevenly. You may still have to look closely to see it.
This can be counteracted somewhat by further decreasing the $\sigma$ value, but if we have random noise throughout the image, this will not remove it.
If we have random static in the image, we can remove this using a modified version of the filter.
Instead of measuring the rate of change in the picture in each direction, we change each point according to whether or not any of its adjacent points have roughly the same value it has.
This is called a minimum-biased filter.
This sort of trick is especially good for removing isolated pixels that are different from those around them.
A very simple way to do this is by taking the average of the two smallest differences between each pixel and its eight neighbors and using that in place of $g$ in the difference scheme above.
Along the boundaries, we do not have 8 neighbors for each pixel, but we can get by by just using the pixels we have and eliminating the other terms in the difference scheme, just as we did before.
This will make it so that points that neighbor points of similar value will not be changed, while points that do not match their surroundings will be faded to become more like the points surrounding them.
This does not have the same symmetrical diffusion as the other scheme, i.e. if one pixel changes, it does not necessarily change its neighboring pixels by the same amount.
As long as you leave $\lambda \leq \frac{1}{4}$ and you have scaled the pixels to have floating point values between 0 and 1, the scheme will still remain within its minimum and maximum bounds, since the tendency is always to move points closer to the values of their neighbors.
To demonstrate the action of such a filter, we make changes to random pixels in the color version of the same photo and use both filters to remove the noise we have added.
Below, we include an example where we have added noise to the color version of that same picture, then used a minimum-biased filter to diminish the noise and the original filter to smooth what remains.

\begin{problem*}
(Optional)

Implement the minimum-biased finite difference scheme described above.
Add noise to \li{balloons_color.jpg} using the provided code below, and clean it using your implementation.
Show the original image, the noised image, and the cleaned image.

\begin{lstlisting}
image = imread('balloons_color.jpg')
x,y,z = image.shape
for dim in xrange(z):
    for i in xrange(x*y//100):
        # Assign a random value to a random place
        image[randint(x),randint(y),dim] = 127 + randint(127)
\end{lstlisting}
Hint: Don't forget to rescale.

\end{problem*}

\newpage
\vfill
\begin{figure}[ht]
\begin{minipage}[b]{0.45\linewidth}
\centering
\includegraphics[width=\textwidth]{baloons_resized_color.jpg}
\caption*{original image}
\end{minipage}
\hspace{0.5cm}
\begin{minipage}[b]{0.45\linewidth}
\centering
\includegraphics[width=\textwidth]{baloons_resized_noisy.png}
\caption*{randomly changed 100000 color values}
\end{minipage}
\begin{minipage}[b]{0.45\linewidth}
\centering
\includegraphics[width=\textwidth]{baloons_resized_minbias.jpg}
\caption*{300 iterations of a min-biased scheme}
\end{minipage}
\hspace{0.5cm}
\begin{minipage}[b]{0.45\linewidth}
\centering
\includegraphics[width=\textwidth]{baloons_resized_both.jpg}
\caption*{after 8 additional iterations of the first filter with $\lambda=.25$ and $\sigma=.04$.}
\end{minipage}
\end{figure}
\vfill
\clearpage

\nocite{Perona1988,Kim2009}


\subsection{Real numbers aren't closed}

Define as an ordered pair of reals

We have a complete set of real numbers. Do we need any more?

For the real numbers, we showed there were functions on the rational numbers which did not have rational solutions. We can similarly show that there are functions on real numbers which do not have real solutions.

Consider:

\(f(x)=\sqrt x\)

This has no real solution for \(x<0\).

We define:

\(i:=\sqrt {-1}\)

\(i\) and \(-i\) can be used interchangeably.

\((-i)^2=(-1)^2i^2=i^2=-1\)

Complex numbers can be shown more generally as:

\(a+bi\)

We define the complex conjugate of

\(x=a+bi\)

As

\(\bar x=a-bi\)

Note that

\(x\bar x=(a+bi)(a-bi)=a^2-b^2\)

We can take exponents of imaginary numbers

\(c^{i\theta}=a+bi\)

We know the opposite is true.

\(c^{-i\theta}=a-bi\)

So

\(c^{i\theta}c^{-i\theta}=(a+bi)(a-bi)\)

\(1=a^2-b^2\)

The case where \(c=e\) is of particular note. We explore this later.


\subsection{The score}

The score is defined as the differential of the log-likelihood function with respect to \(\theta \).

\(V(\theta, X)=\dfrac{\delta }{\delta \theta }l(\theta ; X) \)

\(V(\theta, X)=\dfrac{1 }{\prod_{i=1}^nP(X_i|\theta )}\dfrac{\delta }{\delta \theta}L(\theta; X) \)

\subsection{Expectation of the score}

The expectation of the score, given the true value of \(\theta \) is:

\(E[V(X|\theta)]=\int V(X|\theta) dX\)

\(E[V(X|\theta)]=E[\dfrac{1 }{\prod_{i=1}^nP(X_i|\theta )}\dfrac{\delta }{\delta \theta}L(\theta; X) ]\)

\(E[V(X|\theta)]=\int \dfrac{1 }{\prod_{i=1}^nP(X_i|\theta )}\dfrac{\delta }{\delta \theta}L(\theta; X) \)

\(E[\dfrac{1 }{\prod_{i=1}^nP(X_i|\theta )}]\)

\(\int \dfrac{1 }{\prod_{i=1}^nP(X_i|\theta )}P(\theta )d\theta \)

We can show that the expected value of this is \(0\).

\subsection{Variance of the score}

The variance of the score is:

\(var [\dfrac{\delta }{\delta \theta }l(\theta ; X) ]\)

\(var [\dfrac{1 }{\prod_{i=1}^nP(X_i|\theta )}]\)


\section*{Dimension Reduction}
\subsection*{Principal component analysis (PCA)}
Given: $D={x_1,...,x_n} \subset \mathbb{R}^d$, $1\leq k \leq d$\\
$\Sigma_{d \times d} = \frac{1}{n}\sum_{i=1}^n x_i x_i^T$, $\mu =\frac{1}{n}\sum_{i = 1}^n x_i = 0$\\
Sol.:
$(W,z_1,...,z_n) = argmin \sum_{i=1}^n||W z_i - x_i||_2^2$,\\
where $W \in \mathbb{R}^{d \times k}$ is orthogonal, $z_1,...,z_n\in\mathbb{R}^k$ is given by $W = (v_1|...|v_k)$ and $z_i = W^T x_i$ where $\Sigma = \sum_{i=1}^d \lambda_i v_i v_i^T$, $\lambda_1 \geq ... \geq \lambda_d \geq 0$

\subsection*{Kernel PCA}
For general $k\geq1$, the Kernel PC are given by $\alpha^{(1)},...,\alpha^{(k)}\in \mathbb{R}^n$, where $\alpha^{(i)} = \frac{1}{\sqrt{\lambda_i}}v_i$ is obtained from: $K = \sum_{i=1}^n \lambda_i v_i v_i^T$, $\lambda_1 \geq ... \geq \lambda_d \geq 0$\\
Given this, a new point $x$ is projected as $z \in \mathbb{R}^k$:\\
$z_i = \sum_{j=1}^n\alpha_j^{(i)}k(x,x_j)$

\subsection*{Autoencoders}
Try to learn identity function: $x \approx f(x;\theta)$\\
$f(x;\theta) = f_2(f_1(x;\theta_1);\theta_2)$; $f_1:$ en-, $f_2:$ decoder\\
Training: $ \underset{w}{\operatorname{min}}\sum_{i=1}^n||x_i-f(x_i;W)||_2^2$


\subsubsection{Simulations for Fourier-Collocation Method}

	Following the same ideas from the previous section for Fourier-Collocation, we will seek solutions in the space given by $S_N = \widetilde{B}_N \cap H^2_p [0, 2 \pi]$, and using the discrete expansion to the problem (\ref{IVP_Burgers}) as follows
	\begin{align*}
		\mathcal{J}_N u (x, t) =  \displaystyle \sum_{|n| \leq \frac{N}{2}} \widetilde{u}_n (t) e^{inx}, \hspace{2mm}
		\widetilde{u}_n (t) =  \displaystyle \sum_{j=0}^{2N} u (x_j, t)  e^{-in x_j}
	\end{align*}
	or equivalently
	\begin{align*}
		\mathcal{J}_N u (x, t) =  \displaystyle \sum_{j=0}^{2N} u (x_j, t) \psi_j (x)
	\end{align*}
	where $x_j$ are given by
	\begin{align*}
		xj = \frac{2 \pi j}{2N + 1}, \hspace{2mm} j = 0, 1, \dots, 2N.
	\end{align*}

	For this case, the Fourier-Collocation method will be given by the following problem
	\begin{align}
	\label{Collocation_Nonlinear}
		R_{N} (x_j, t) = \frac{\partial u_N}{\partial t} (x_j, t) - \frac{\partial^2 u_N }{\partial x^2} (x_j, t) + \frac{1}{2} \frac{\partial \left[ u_N \right]^2}{\partial x} (x_j, t) = 0
	\end{align}
	which is a system of $2N + 1$ ordinary differential equations that can be solved using the initial condition given by
	\begin{align*}
		\mathcal{J}_N u (x_j, t) =  u_0 (x_j), , \hspace{2mm} j = 0, 1 \dots, 2N
	\end{align*}

	By Setting a vector as follows
	\begin{align*}
		u_N (t) &= (u_N (x_0 , t), u_N (x_1 , t), \dots , u_N (x_{2N} , t))^T,
	\end{align*}
	and the above system of ordinary differential equations can be written as follows
	\begin{align*}
		\frac{d u_N (t)}{dt} =  \alpha D_N^2 u_N (t) - \frac{1}{2} D_N u^2_N (t)
	\end{align*}
	where $D_N$ is the matrix given by (\ref{matrix_DN_odd}), that represents discrete Fourier differentiation, which can be rewrite as
	\begin{align}
		D_N = \displaystyle C^{-1} \Lambda^2_N C
	\end{align}
	where $\Lambda_N = diag \{ ik \}_{|k|\leq N}$, $C$ represents the discrete Fourier transform and $C^{-1}$ the inverse. \\

	Note that the situation here is very different compared to the systems obtained with Fourier-Galerkin. The representation of the non-linear term is more practical to handle with implicit methods, for example, an implicit approach is as follows
	\begin{align*}
		u_N (t_{i + 1} ) =  C^{-1} \Lambda_N C u_N (t_{i}) - \frac{\Delta t}{2} D_N u^2_N (t_{i+1})
	\end{align*}

	It is possible to formulate the above problem as
	\begin{align*}
		W(u_N (t_{i + 1})) =  C^{-1} \Lambda_N C u_N (t_{i}) - u_N (t_{i + 1} ) - \frac{\Delta t}{2} D_N u^2_N (t_{i+1}),
	\end{align*}
	and then we must find $u_N(t_{i + 1})$ that satisfies $W(u_N(t_{i + 1})) = 0$, for example, using the Newton-Raphson method. However, this formulation may require many calculations because the differentiation matrix increases as a function of $N$, and therefore the number of operations. Here we are going to develop a numerical solution that is more practical to implement, implicitly approaching as we did with Fourier-Galerkin on the linear term as follows
	\begin{align*}
		\left[I_N + \frac{\Delta t}{2} u_{0} \Lambda_N \right] u_N (t_{i + 1} ) =  \displaystyle e^{-\Delta t \Lambda^0_N} \left[ u_N (t_{i}) - \frac{\Delta t}{2} D_N u^2_N (t_{i}) \right]
	\end{align*}
	where $\Lambda^0_N = diag \{ \alpha k^2 + \frac{ik}{2} u_0 \}_{|k|\leq N}$, and solving for $u_N (t_{i + 1})$ gives us
	\begin{align}
	\label{Collocation_Euler}
		u_N (t_{i + 1} ) =  \displaystyle \left[I_N + \frac{\Delta t}{2} u_{0} \Lambda_N \right]^{-1} e^{-\Delta t \Lambda^0_N} \left[ u_N (t_{i})  - \frac{\Delta t}{2} D_N u^2_N (t_{i}) \right]
	\end{align}
	which is very similar to \ref{Galerkin_Euler}, except for the nonlinear term. \\

	This formulation will be used for its implementation, and we will present some numerical results that were obtained using the same information that was used with Fourier-Galerkin, the difference is that the calculations are performed in real space using the differentiation matrix $D_N$, and in this case, we have the advantage of approximating the second derivative with $D^2_N = D_N \cdot D_N$. \\

	The description of the following results is as follows. In the figure \ref{Collocation_alphas} shows the maximum distance over every $t \in [0, 100]$ between the exact solution and its approximations given by (\ref{Collocation_Euler}) for $N = 2^m$, $m = 4, \dots, 12$, $\Delta t = 1.0 \times 10^{-5}$, and different values of $\alpha$. Furthermore, in Tables \ref{Collocation_tabla_L2_alpha=1} and \ref{Collocation_tabla_max_alpha=1}, we can see the numerical values of these distances for different configurations of $N$ and $\Delta t$. Similarly, in Tables \ref{Collocation_tabla_L2_alpha=005} and \ref{Collocation_tabla_max_alpha=005} but for $\alpha = 0.005$.

	\newpage
	\begin{figure}[H]
		\centering
		\includegraphics[width=12cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/collocation/alphas_Error_N.png}
		\caption{(a) $L^2$-norm between the exact solution and its approximations using Collocation method. (b) Max norm between the exact solution and its approximations.}
		\label{Collocation_alphas}
	\end{figure}

	\newpage
	\begin{figure}[H]
		\centering
		\caption{Numerical solution for (\ref{IVP_Burgers}) using (\ref{Collocation_Euler}) with $\alpha = 1.0$, $N=2048$, and $\Delta t = 1.0 \times 10^{-5}$.}
		\includegraphics[width=12cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/collocation/Numerical_Solution_alpha=1.png}
		\label{Collocation_alpha=1}
		\caption{Numerical solution for (\ref{IVP_Burgers}) using (\ref{Collocation_Euler}) at the time $T = 100$ with $\alpha = 1.0$, and $\Delta t = 1.0 \times 10^{-5}$. (b) Point-wise error of approximation}
		\includegraphics[width=12.5cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/collocation/Numerical_Solution_alpha=1_T=100.png}
		\label{Collocation_alpha=1_T}
	\end{figure}
	\begin{table}[H]
		\begin{tabular}{lcccc}
			\toprule
			\multicolumn{1}{c}{\textbf{Expansion}} & \multicolumn{4}{c}{\textbf{Error}} \\
			$\hspace{9mm}N$ & $\Delta t=1\times 10^{-2}$ & $\Delta t=1\times 10^{-3}$ & $\Delta t=1\times 10^{-4}$ & $\Delta t=1\times 10^{-5}$ \\
			\midrule
			\hspace{7mm} 16 & 0.721112    & 0.721112    & 0.721112    & 0.721112    \\
			\midrule
			\hspace{7mm} 32 & 4.71797 $\times 10^{-2}$   & 4.72892 $\times 10^{-2}$   & 4.73004 $\times 10^{-2}$   & 4.73015 $\times 10^{-2}$   \\
			\midrule
			\hspace{7mm} 64 & 1.17954 $\times 10^{-3}$  & 7.35344 $\times 10^{-4}$ & 7.27561 $\times 10^{-4}$ & 7.27283 $\times 10^{-4}$  \\
			\midrule
			\hspace{7mm} 128 & 9.43454 $\times 10^{-4}$ & 1.75152 $\times 10^{-4}$ & 1.74583 $\times 10^{-4}$ & 1.74574 $\times 10^{-4}$ \\
			\midrule
			\hspace{7mm} 256 & 9.43454 $\times 10^{-4}$ & 1.15509 $\times 10^{-4}$ & 1.14669 $\times 10^{-4}$ & 1.14659 $\times 10^{-4}$ \\
			\midrule
			\hspace{7mm} 512 & 9.43454 $\times 10^{-4}$ & 9.41793 $\times 10^{-5}$ & 7.78847 $\times 10^{-5}$ & 7.78707 $\times 10^{-5}$ \\
			\midrule
			\hspace{7mm} 1024 & $\ast$         & 9.41793 $\times 10^{-5}$ & 5.32213 $\times 10^{-5}$ & 5.32019 $\times 10^{-5}$ \\
			\midrule
			\hspace{7mm} 2048 & $\ast$           & $\ast$           & 3.56779 $\times 10^{-5}$ & 3.56498 $\times 10^{-5}$ \\
			\\
			\bottomrule
		\end{tabular}
		\caption{Error using $L^2$-norm with $\alpha=1.0$}
		\label{Collocation_tabla_L2_alpha=1}
		\vspace{1cm}
		\begin{tabular}{lcccc}
			\toprule
			\multicolumn{1}{c}{\textbf{Expansion}} & \multicolumn{4}{c}{\textbf{Error}} \\
			$\hspace{9mm}N$ & $\Delta t=1\times 10^{-2}$ & $\Delta t=1\times 10^{-3}$ & $\Delta t=1\times 10^{-4}$ & $\Delta t=1\times 10^{-5}$ \\
			\midrule
			\hspace{7mm} 16 & 0.317617    & 0.317617    & 0.317617    & 0.317617    \\
			\midrule
			\hspace{7mm} 32 & 1.95279 $\times 10 ^{-2}$  & 1.96812 $\times 10 ^{-2}$   & 1.96965 $\times 10 ^{-2}$   & 1.96981 $\times 10 ^{-2}$   \\
			\midrule
			\hspace{7mm} 64 & 6.21793 $\times 10 ^{-4}$ & 2.9813 $\times 10 ^{-4}$  & 2.80086 $\times 10 ^{-4}$ & 2.78934 $\times 10 ^{-4}$ \\
			\midrule
			\hspace{7mm} 128 & 4.74952 $\times 10 ^{-4}$ & 1.64746 $\times 10 ^{-4}$ & 1.6473 $\times 10 ^{-4}$  & 1.64728 $\times 10 ^{-4}$ \\
			\midrule
			\hspace{7mm} 256 & 4.74936 $\times 10 ^{-4}$ & 1.52482 $\times 10 ^{-4}$ & 1.52467 $\times 10 ^{-4}$ & 1.52465 $\times 10 ^{-4}$ \\
			\midrule
			\hspace{7mm} 512 & 4.74936 $\times 10 ^{-4}$ & 1.47249 $\times 10 ^{-4}$ & 1.47234 $\times 10 ^{-4}$ & 1.47232 $\times 10 ^{-4}$ \\
			\midrule
			\hspace{7mm} 1024 & $\ast$           & 1.45032 $\times 10 ^{-4}$ & 1.45017 $\times 10 ^{-4}$ & 1.45016 $\times 10 ^{-4}$ \\
			\midrule
			\hspace{7mm} 2048 & $\ast$           & $\ast$           & 1.43941 $\times 10 ^{-4}$ & 1.4394 $\times 10 ^{-4}$ \\
			\\
			\bottomrule
		\end{tabular}
		\caption{Error using Max norm with $\alpha=1.0$}
		\label{Collocation_tabla_max_alpha=1}
	\end{table}

	\begin{figure}[H]
		\centering
		\caption{Numerical solution for (\ref{IVP_Burgers}) using (\ref{Collocation_Euler}) with $\alpha = 0.005$, $N=2048$, and $\Delta t = 1.0 \times 10^{-5}$.}
		\includegraphics[width=12cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/collocation/Numerical_Solution_alpha=0005.png}
		\label{Collocation_alpha=005}
		\caption{Numerical solution for (\ref{IVP_Burgers}) using (\ref{Collocation_Euler}) at the time $T = 100$ with $\alpha = 0.005$, and $\Delta t = 1.0 \times 10^{-5}$. (b) Point-wise error of approximation.}
		\includegraphics[width=12.5cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/collocation/Numerical_Solution_alpha=0005_T=100.png}
		\label{Collocation_alpha=005_T}
	\end{figure}
	%L2

	\begin{table}[H]
	\begin{tabular}{lcccc}
		\toprule
		\multicolumn{1}{c}{\textbf{Expansion}} & \multicolumn{4}{c}{\textbf{Error}} \\
		$\hspace{9mm}N$ & $\Delta t=1\times 10^{-2}$ & $\Delta t=1\times 10^{-3}$ & $\Delta t=1\times 10^{-4}$ & $\Delta t=1\times 10^{-5}$ \\
		\midrule
		\hspace{7mm} 16 & 1.36189   & 1.35883    & 1.35852   & 1.35849   \\
		\midrule
		\hspace{7mm} 32 & 2.67506   & 2.65305    & 2.65078   & 2.65055   \\
		\midrule
		\hspace{7mm} 64 & 2.50365   & 2.45855    & 2.45432   & 2.45387   \\
		\midrule
		\hspace{7mm} 128 & 2.15795   & 2.0632     & 2.05589   & 2.05497   \\
		\midrule
		\hspace{7mm} 256 & 1.362     & 1.18393    & 1.16697   & 1.16532   \\
		\midrule
		\hspace{7mm} 512 & 0.350775  & 0.304595   & 0.300865  & 0.300499  \\
		\midrule
		\hspace{7mm} 1024 & 0.168462  & 0.140332   & 0.13803   & 0.137804  \\
		\midrule
		\hspace{7mm} 2048 & 6.56161 $\times 10^{-2}$ & 4.63808 $\times 10^{-2}$  & 4.49226 $\times 10^{-2}$ & 4.47813 $\times 10^{-2}$ \\
		\\
		\bottomrule
	\end{tabular}
	\caption{Error using $L^2$-norm with $\alpha=0.005$}
	\label{Collocation_tabla_L2_alpha=005}
	\vspace{1cm}
	\begin{tabular}{lcccc}
		\toprule
		\multicolumn{1}{c}{\textbf{Expansion}} & \multicolumn{4}{c}{\textbf{Error}} \\
		$\hspace{9mm}N$ & $\Delta t=1\times 10^{-2}$ & $\Delta t=1\times 10^{-3}$ & $\Delta t=1\times 10^{-4}$ & $\Delta t=1\times 10^{-5}$ \\
		\midrule
		\hspace{7mm} 16 & 0.695784 & 0.695659  & 0.695646  & 0.695645 \\
		\midrule
		\hspace{7mm} 32 & 1.20278  & 1.19418   & 1.19329   & 1.1932   \\
		\midrule
		\hspace{7mm} 64 & 1.22454  & 1.18903   & 1.18507   & 1.18467  \\
		\midrule
		\hspace{7mm} 128 & 1.11999  & 1.0238    & 1.01754   & 1.01701  \\
		\midrule
		\hspace{7mm} 256 & 0.927954 & 0.877058  & 0.872508  & 0.87201  \\
		\midrule
		\hspace{7mm} 512 & 0.664133 & 0.415288  & 0.39714   & 0.395563 \\
		\midrule
		\hspace{7mm} 1024 & 0.247742 & 0.259451  & 0.260605  & 0.26072  \\
		\midrule
		\hspace{7mm} 2048 & 0.126824 & 0.103297  & 0.107102  & 0.10748  \\
		\\
		\bottomrule
	\end{tabular}
	\caption{Error using Max norm with $\alpha=0.005$}
	\label{Collocation_tabla_max_alpha=005}
	\end{table}


In \texttt{MoveUrBody} code there are various implementations: some of them simulate colliding bodies.
In this section we try to explain what kind of collisions are used if the implementation considers colliding bodies.

\subsection{Collisions detection}
We choose \textit{a posteriori} method in order to detect collisions: this is a discrete and easy to implement method.
We detect the collisions after they append.
This is less expensive in term of computations than \textit{a priori} method (continuous).
In \texttt{MoveUrBody} we consider that \textbf{the shape of all the bodies is a sphere}, so two bodies $i$ and $j$ are colliding if:
\begin{equation}
\label{eq:collsDetection}
	||\vec{r_{ij}}|| - (r_i + r_j) \leq 0,
\end{equation}
where $r_i$ and $r_j$ are respectively the radii of body $i$ and body $j$.

\subsection{Elastic collisions}
Notation: Throughout this section, $m$ is the mass and $v$ is the velocity.
Be aware that $v = ||\vec{v}||$.
Subscripts $i$ and $j$ distinguish between the two colliding bodies.
An apostrophe after a variable means that the value is taken after the collision (called prime; i.e., $v$' is "$v$ prime").\\

In \texttt{MoveUrBody} we consider that \textbf{all collisions are perfectly elastic}.
An elastic collision is a collision in which kinetic energy is conserved.
That means no energy is lost as heat or sound during the collision.
In the real world, there are no perfectly elastic collisions on an everyday scale of size.
But you can get the sense of an elastic collision by imagining a perfect pool ball which doesn't waste any energy when it collides.
In an elastic collision, both kinetic energy and momentum are conserved (the total before and after the collision remains the same).
Momentum is the product of mass and velocity:
\begin{equation}
\label{eq:momentum}
	\vec{p} = m . \vec{v}.
\end{equation}
The kinetic energy of an object is one-half times its mass times the square of its velocity:
\begin{equation}
\label{eq:kinetic}
	E_k = \frac{1}{2} . m . v^2.
\end{equation}
Now it is easy to write the conservation of momentum and kinetic energy as two equations:
\begin{enumerate}
	\item Conservation of momentum
		\begin{equation}
		\label{eq:momentumCons}
			m_i . \vec{v_i} + m_j . \vec{v_j} = m_i . \vec{v'_i} + m_j . \vec{v'_j}
		\end{equation}

	\item Conservation of kinetic energy
		\begin{equation}
		\label{eq:kineticCons}
			\frac{1}{2} . m_i . (v_i)^2 + \frac{1}{2} . m_j . (v_j)^2 = \frac{1}{2} . m_i . (v'_i)^2 + \frac{1}{2} . m_j . (v'_j)^2
		\end{equation}
\end{enumerate}

\subsubsection{1-Dimensional elastic collisions}
Combining the two previous equations (Eq.~\ref{eq:momentumCons} and \ref{eq:kineticCons}) and doing a lot of algebra gives the final (after collision) velocities of body $i$ and $j$:
\begin{equation}
\label{eq:1DelasticSols}
	v'_i = \frac{v_i(m_i-m_j) + 2.m_j.v_j}{m_i + m_j},~~v'_j = \frac{v_j(m_j-m_i) + 2.m_i.v_i}{m_j + m_i}.
\end{equation}
This result allows us to find the velocity of two objects after undergoing a one-dimensional elastic collision.
We will use this result later in the 3-dimensional case.

\subsubsection{3-Dimensional elastic collisions}
In previous sub-section (1D elastic collisions) vectors representation was not very important.
Now in 3D, we will use the component representation of a vector: $\vec{v} = \left\{ v_x, v_y, v_z \right\}$.\\

We will follow a 7-step process to find the new velocities of two objects after a collision.
The basic goal of the process is to project the velocity vectors of the two objects onto the vectors which are normal (perpendicular) and tangent to the plan of the collision.
This gives us a normal component and two tangential components (defining a plan) for each velocity.
The tangential components of the velocities are not changed by the collision because there is no force along the tangent plan to the collision surface.
The normal components of the velocities undergo a one-dimensional collision, which can be computed using the one-dimensional collision formulas presented above.
Next the unit normal vector is multiplied by the scalar (plain number, not a vector) normal velocity after the collision to get a vector which has a direction normal to the collision surface and a magnitude which is the normal component of the velocity after the collision.
The same is done with the unit tangent vectors and the tangential velocity components.
Finally the new velocity vectors are found by adding the normal velocity and the two tangential velocity vectors for each object.

\paragraph{Step 1}
~\\
Find the unit normal and the two unit tangent vectors.
The unit normal vector is a vector which has a magnitude of 1 and a direction that is normal (perpendicular) to the surfaces of the objects at the point of collision.
The two unit tangent vectors are vectors with a magnitude of 1 which are forming a tangent plan to the circle's surfaces at the point of collision.

First find a normal vector.
This is done by taking a vector whose components are the difference between the coordinates of the centers of the spheres.
Let $q_i = \left\{q_{ix}, q_{iy}, q_{iz}\right\}$, $q_j = \left\{q_{jx}, q_{jy}, q_{jz}\right\}$ coordinates of the centers of the spheres (it does not matter which spheres is labelled $i$ or $j$; the end result will be the same).
Then the normal vector $\vec{n}$ is:
\begin{equation*}
	\vec{n} = \left\{ q_{jx} - q_{ix},~q_{jy} - q_{iy},~q_{jz} - q_{iz}  \right\}.
\end{equation*}
Next, we have to find the unit vector of $\vec{n}$, which we will call $\vec{un}$. This is done by dividing by the magnitude of $\vec{n}$:
\begin{equation*}
	\vec{un} = \frac{\vec{n}}{||\vec{n}||} = \frac{\vec{n}}{\sqrt{n_x^2 + n_y^2 + n_z^2}}.
\end{equation*}
Once it's done we need to find the two unit tangent vectors which are forming the collision tangent plan.
Those two unit tangent vectors ($\vec{ut_1}$ and $\vec{ut_2}$) are perpendicular to the unit normal vector $\vec{un}$ and there are also perpendicular between themselves (in order to form a Cartesian coordinate system).
Before trying to determine two unit tangent vectors $\vec{ut_1}$ and $\vec{ut_2}$ we will try to find a tangent vector $\vec{t_1}$.
If $\vec{t_1}$ is perpendicular to $\vec{un}$ then the scalar product (alias dot product) of the two should be null:
\begin{equation*}
	\vec{t_1} . \vec{un} = 0 \Leftrightarrow	(t_{1x} . un_x) + (t_{1y} . un_y) + (t_{1z} . un_z) = 0.
\end{equation*}
The first and easy solution to this previous equation is the null vector $\vec{0}$ but we obviously want to avoid it.
An idea is to fix one of the three components to 0 and an other to 1 in order to determine a perpendicular vector $\vec{t_1}$ (be aware that there is an infinity of perpendicular vectors to $\vec{un}$ and we just need to find one of them).\\
If $un_x \neq 0$:
\begin{equation*}
	\vec{t_1} = \left\{ -\frac{un_y}{un_x}, 1, 0 \right\}.
\end{equation*}
If $un_y \neq 0$:
\begin{equation*}
	\vec{t_1} = \left\{ 1, -\frac{un_x}{un_y}, 0 \right\}.
\end{equation*}
If $un_z \neq 0$:
\begin{equation*}
	\vec{t_1} = \left\{ 1, 0, -\frac{un_x}{un_z} \right\}.
\end{equation*}
We have to choose one of the three previous propositions and be sure to avoid to divide by 0.
Now we have to normalize $\vec{t_1}$ in order to find the unitary $\vec{ut_1}$ vector:
\begin{equation*}
	\vec{ut_1} = \frac{\vec{t_1}}{||\vec{t_1}||} = \frac{\vec{t_1}}{\sqrt{t_{1x}^2 + t_{1y}^2 + t_{1z}^2}}.
\end{equation*}
It remains to determine the $\vec{ut_2}$ vector, this can be done by computing the cross product (alias vector product) between $\vec{un}$ and $\vec{ut_1}$:
\begin{equation*}
	\vec{ut_2} = \vec{un} \wedge \vec{ut_1} = \left\{ (un_y . ut_{1z} - un_z . ut_{1y}), (un_z . ut_{1x} - un_x . ut_{1z}), (un_x . ut_{1y} - un_y . ut_{1x})\right\}.
\end{equation*}

\paragraph{Step 2}
~\\
Create the initial (before the collision) velocity vectors, $\vec{v_i}$ and $\vec{v_j}$.
These are just the $x$, $y$ and $z$ components of the velocities put into vectors: $\vec{v_i} = \left\{ v_{ix}, v_{iy} \right\}$ (and similarly for $\vec{v_j}$).
Note that this step really isn't necessary if the velocities are already represented as vectors.
This step is needed only if the velocities are initially represented as separate $x$, $y$ and $z$ values.

\paragraph{Step 3}
~\\
Keep in mind that after the collision the tangential components of the velocities are unchanged and the normal component of the velocities can be found using the one-dimensional collision formulas presented earlier.
So we need to resolve the velocity vectors, $\vec{v_i}$ and $\vec{v_j}$, into normal and tangential components.
To do this, project the velocity vectors onto the unit normal and unit tangent vectors by computing the dot product.
Let $v_{in}$ be the scalar (plain number, not a vector) velocity of body $i$ in the normal direction.
Let $v_{it1}$ and $v_{it2}$ be the scalar velocity of body $i$ in the tangential directions.
Similarly, let $v_{jn}$, $v_{jt1}$ and $v_{jt2}$ be for body $j$.
These values are found by projecting the velocity vectors onto the unit normal and unit tangent vectors, which is done by taking the dot (alias scalar) product:
\begin{equation*}
	v_{in} = \vec{v_i}.\vec{un},~~v_{it1} = \vec{v_i}.\vec{ut_1},~~v_{it2} = \vec{v_i}.\vec{ut_2},
\end{equation*}
\begin{equation*}
	v_{jn} = \vec{v_j}.\vec{un},~~v_{jt1} = \vec{v_j}.\vec{ut_1},~~v_{jt2} = \vec{v_j}.\vec{ut_2}.
\end{equation*}

\paragraph{Step 4}
~\\
Find the new tangential velocities (after the collision).
This is the simplest step of all.
The tangential components of the velocity do not change after the collision because there is no force between the spheres in the tangential direction during the collision.
So, the new tangential velocities are simply equal to the old ones:
\begin{equation*}
	v'_{it1} = v_{it1},~~v'_{it2} = v_{it2},
\end{equation*}
\begin{equation*}
	v'_{jt1} = v_{jt1},~~v'_{jt2} = v_{jt2}.
\end{equation*}

\paragraph{Step 5}
~\\
Find the new normal velocities.
This is where we use the one-dimensional collision formulas (Eq.~\ref{eq:1DelasticSols}).
The velocities of the two spheres along the normal direction are perpendicular to the surfaces
of the spheres at the point of collision, so this really is a one-dimensional collision:
\begin{equation*}
	v'_{in} = \frac{v_{in}(m_i-m_j) + 2.m_j.v_{jn}}{m_i + m_j},
\end{equation*}
\begin{equation*}
	v'_{jn} = \frac{v_{jn}(m_j-m_i) + 2.m_i.v_{in}}{m_j + m_i}.
\end{equation*}

\paragraph{Step 6}
~\\
Convert the scalar normal and tangential velocities into vectors.
This is easy just multiply the unit normal vector by the scalar normal velocity and you get a vector which has a direction that is normal to the surfaces at the point of collision and which has a magnitude equal to the normal component of the velocity.
It is similar for the tangential components:
\begin{equation*}
	\vec{v'_{in}} = v'_{in} . \vec{un},~~\vec{v'_{it1}} = v'_{it1} . \vec{ut_1},~~\vec{v'_{it2}} = v'_{it2} . \vec{ut_2},
\end{equation*}
\begin{equation*}
	\vec{v'_{jn}} = v'_{jn} . \vec{un},~~\vec{v'_{jt1}} = v'_{jt1} . \vec{ut_1},~~\vec{v'_{jt2}} = v'_{jt2} . \vec{ut_2}.
\end{equation*}

\paragraph{Step 7}
~\\
Find the final velocity vectors by adding the normal and tangential components for each body:
\begin{equation*}
	\vec{v'_i} = \vec{v'_{in}} + \vec{v'_{it1}} + \vec{v'_{it2}},
\end{equation*}
\begin{equation*}
	\vec{v'_j} = \vec{v'_{jn}} + \vec{v'_{jt1}} + \vec{v'_{jt2}}.
\end{equation*}
Now we have the final (after collision) velocity of each body as a vector.


\appendix
\chapter{Gaussian Integrals}
\section{Basic Gaussians}
Lots of Gaussian integrals happen in this subject. In the interests of not getting stuck, here are the results.

The basic Gaussian integral for \( a \in \mathbb{R}^+\) is
\[ \int_\mathbb{R} e^{-ax^2} \dd{x} = \sqrt\frac{\pi}{a} \] which we can then use Feynman's trick of differentiation with respect to the parameter and symmetry arguments to find
\begin{align*}
  \int_\mathbb{R} x^n e^{-ax^2} \dd{x} &= \begin{cases}
    \pdv[\nicefrac{n}{2}]{a} \sqrt\frac{\pi}{a} & \text{if \(n\) even}\\
    0 & \text{otherwise}
  \end{cases}.
\end{align*}

If there is an \(e^{bx} \) power, this may be taken care of using completing the square, given
\begin{align*}
  \int_\mathbb{R} e^{-ax^2+bx} \dd{x} &= \int_\mathbb{R} e^{-a(x - \nicefrac{b}{2a})^2+\nicefrac{b^2}{4a}} \dd{x}\\
  &= \sqrt\frac{\pi}{a} e^{\nicefrac{b^2}{4a}}
\end{align*}
and if we make \(b = ik\) this still works, which shows that the fourier transform of a Gaussian is still a Gaussian.
With these, Feynman's trick still works, and works even better than before as we now have single \(x\) powers to play with --- just keep differentiating.

By closing one's contour cleverly, one can also solve Fresnel (and similar) integrals with a complex integrand. This gives
\begin{align*}
  \int_\mathbb{R} e^{-iax^2+ikx} \dd{x}
  &= \sqrt\frac{i\pi}{a} e^{\nicefrac{-k^2}{4a}}
\end{align*}
which is the same as what you get if you just substitute into the previous result, but it's not obvious that that should be the case.

\section{Nested Gaussians}\label{sec:nested-gaussians}
When it is said that there are a lot of Gaussians, what makes it all the harder is that they all happen \emph{all at once}.

Firstly, we look at a double Gaussian, in the form
\begin{align*}
  I &= \int_{-\infty}^\infty e^{\nicefrac{i}{a} {(x-u)^2}}e^{ \nicefrac{i}{b}(u-y)^2}\dd{u}
\end{align*}
which is a bit tricky because they're coupled.
We use a clever trick, by which we use the translational invariance of these integrals, making a mapping \( u \mapsto u + y \) which makes the whole thing a function of \( z = x - y \).
Utilising this gives us
\begin{align*}
  \frac{i}{a}(z-u)^2 + \frac{i}{b}u^2 &= \frac{i}{a}(z^2 - 2zu + u^2) + \frac{i}{b}u^2\\
  &= i\qty(\frac{1}{a} + \frac{1}{b})\qty(u - \frac{z}{a \qty(\frac{1}{a} + \frac{1}{b})})^2 + \frac{i}{a}z^2 + i \frac{z^2}{a^2 \qty(\frac{1}{a}  + \frac{1}{b})}\\
  &= i\qty(\frac{1}{a} + \frac{1}{b})\qty(u - \frac{z}{ia \qty(\frac{1}{a} + \frac{1}{b})})^2 + iz^2\frac{1}{a + b}
\end{align*}
which with another translation gives us the really nice answer that \[ I = e^{i\frac{z^2}{a + b}} \sqrt{\frac{iab\pi}{a+b}}.\]

The $N$-th nested Gaussian, \( I_N \), is the logical extension of this, and telescopes rather nicely.
We set
\begin{align*}
  I_N &= \idotsint \dd{x_1} \cdots \dd{x_N} e^{i \qty[ (x_1 - x_0 )^2 + (x_2 - x_1)^2 + \ldots + (x_{N+1} - x_{N})^2]}\\
      &= \sqrt{\frac{i\pi}{2}}\idotsint \dd{x_2} \cdots \dd{x_N} \exp[i \qty[ \nicefrac{1}{2}(x_2 - x_0 )^2 + (x_2 - x_1)^2 + \ldots + (x_{N+1} - x_{N})^2]]\\
      &= \sqrt{\frac{i\pi}{N+1}}\exp[i \nicefrac{(x_{N+1} - x_0)^2}{N+1}]
\end{align*}
which is a nice enough statement.

\chapter{Functional Derivatives}
A functional derivative generalises the derivative from
\begin{align*}
  \pdv{J_i}\sum_j J_j x_j &= x_i & \pdv{J_i} &= \delta_{ij}
\end{align*}
to cover a smooth function on a continuous domain such that
\begin{align*}
  \pdv{J(t)}J(t^\prime) &= \delta(t-t^\prime) & \fdv{J(t)}  \int_{t_a}^{t_b} J(t^\prime) x(t^\prime) \dd{t^\prime} &= \int_{t_a}^{t_b} \delta(t - t^\prime) x(t^\prime) \dd{t^\prime}\\
  &&&= \begin{cases}
    x(t) & t_a < t^\prime < t_b\\
    0 & \text{otherwise}.
  \end{cases}
\end{align*}

Functional derivatives have the following properties:
\begin{description}
  \item [Linearity] \[ \fdv{(\lambda F + \mu G)[\rho]}{\rho(x)} = \lambda \fdv{F[\rho]}{\rho(x)} + \mu \fdv{G[\rho]}{\rho(x)}\]
  for \(\lambda, \mu\) constant,
  \item [Chain Rule] \[ \fdv{(FG)[\rho]}{\rho} = \fdv{F[\rho]}{\rho(x)}G[\rho] + F[\rho]\fdv{F[\rho]}{\rho(x)} \]
  \item [Product Rules] These are logical. Look them up if you need them.
\end{description}

We can use an equivalent of the Euler-Lagrange equation to find the functional derivative, \[ \fdv{F}{\rho(r)} = \pdv{f}{\rho} - \partial^\mu \pdv{f}{\qty(\partial^\mu \rho)}.\]


% arara: pdflatex
% arara: pdflatex

\documentclass[letterpaper,10pt]{article}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{siunitx}
\usepackage{physics}
\usepackage[left=1in,right=1in,top=1in,bottom=1in]{geometry}

\begin{document}

\section{Introduction}

Simpson's rule is a method for performing numerical integration. While there are several different approximations under the name ``Simpson's rule'', the most
common version, Simpson's 1/3 rule, is equivalent fitting a 2'nd degree polynomial to three points so that the integration can be carried out analytically.
It is usually written as:
\begin{equation}
  \label{eq:simpsons}
  \int_a^b f(x) \dd x \approx \frac{b-a}{6}\left[ f(a) + 4f\qty(\frac{a+b}{2}) + f(b)\right]
\end{equation}
This can be extended to larger domains with more function evaluations by breaking the domain into sub-domains and applying
\ref{eq:simpsons} to each.
However, equation \ref{eq:simpsons} assumes that the function $f(x)$ is evaluated at equally spaced points. If we have a discretized
function that is \emph{not} evaluated at equally spaced points, equation \ref{eq:simpsons} cannot be used.

\section{Generalizing to Non-Uniform Spacing}

To generalize equation \ref{eq:simpsons} to non-equally spaced points, we just need to perform the polynomial fit using non-equally spaced points an
perform the integration on the new fit. Using Lagrange polynomial interpolation, we have
\begin{equation}
  f(x) \approx f(x_1) \frac{ (x-x_2)(x-x_3) }{ (x_1 - x_2)(x_1 - x_3) }
             + f(x_2) \frac{ (x-x_1)(x-x_3) }{ (x_2 - x_1)(x_2 - x_3) }
             + f(x_3) \frac{ (x-x_1)(x-x_2) }{ (x_3 - x_1)(x_3 - x_2) }
\end{equation}
where $x_1$, $x_2$, and $x_3$ are the points at which the function is evaluated.

The Lagrange basis polynomials can be written as
\begin{equation}
  \label{eq:lagrange}
  l(x,A,B,C) = \frac{(x-A)(x-B)}{(C-A)(C-B)}
\end{equation}
and we can rewrite the interpolation as
\begin{equation}
  f(x) \approx
    f(x_1) l(x,x_2,x_3,x_1)
  + f(x_2) l(x,x_1,x_3,x_2)
  + f(x_3) l(x,x_1,x_2,x_3)
\end{equation}

\subsection{Method 1: Integrating the Lagrange polynomials directly}
Integrating the Lagrrange basis polynomials from $a$ to $b$ gives
\begin{equation}
  L(A,B,C,a,b) = \int_a^b l(x,A,B,C) \dd x = \frac{1}{(C-A)(C-B)} \left[ \frac{1}{3} \qty(b^3 - a^3) + \frac{A+B}{2}\qty(b^2 - a^2) + AB(b-a)  \right]
\end{equation}
and the integral of $f(x)$ is then approximated as
\begin{equation}
  \int_a^b f(x) \dd x \approx
    f(x_1) L(x_2,x_3,x_1,a,b)
  + f(x_2) L(x_1,x_3,x_2,a,b)
  + f(x_3) L(x_1,x_2,x_3,a,b)
\end{equation}

To reproduce equation \ref{eq:simpsons}, we let $x_1 = a$, $x_2 = \frac{a+b}{2}$, and $x_3 = b$. For example,
\begin{align}
  L(x_2,x_3,x_1,a,b) = \frac{1}{(a-\frac{a+b}{2})(a-b)} \qty[\frac{1}{3} \qty(b^3 - a^3) + \frac{\frac{a+b}{2} + b}{2}\qty(b^2 - a^2) + \frac{a+b}{2}b\qty(b-a) ]
\end{align}
apparently reduces to $\frac{b-a}{6}$...

\subsection{Method 2: Integrating the Lagrange polynomials by parts}

Integrating the Lagrange polynomials by parts gives
\begin{align}
  L(A,B,C,a,b) &= \int_a^b l(x,A,B,C) \dd x = \frac{1}{(C-A)(C-B)} \int_a^b (x - A)(x - B) \dd x  \\
  \dv{x} \qty[\frac{1}{2}(x-A)(x-B)^2] &= \frac{1}{2}(x - B)^2 + (x-A)(x-B) \\
  \int (x-A)(x-B) \dd x &= \frac{1}{2}(x-A)(x-B)^2 - \frac{1}{2} \int (x-B)^2 \dd x \\
  L(A,B,C,a,b) &= \left. \frac{1}{(C-A)(C-B)} \qty[ \frac{1}{2}(x-A)(x-B)^2 - \frac{1}{6} (x-B)^3] \right|_a^b
\end{align}

\subsection{Interpolating to uniform spacing}

Since Simpon's rule is derived by interpolating a function with a polynomial and integrating, we can use the polynomial to interpolate to the
medpoint of $[a,b]$ and just use the usual formula. Given the function at points $x_1$, $x_2$, and $x_3$, the interpolated function value at $\frac{x_1 + x_2}{2}$ is
\begin{equation}
  f\qty(\frac{x_1 + x_3}{2}) \approx
  f(x_1) l( \frac{x_1+x_2}{2}, x_2,x_3,x_1)
  + f(x_2) l( \frac{x_1+x_2}{2}, x_1,x_3,x_2)
  + f(x_3) l( \frac{x_1+x_2}{2}, x_1,x_2,x_3)
\end{equation}
Once we interpolate to the midpoint, we can use equation \ref{eq:simpsons}.

\end{document}