\documentclass[12pt,a4paper]{article}
\usepackage[latin1]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{enumitem}
\usepackage{graphicx}
\usepackage{xcolor}
\usepackage{float}
\usepackage[ruled,vlined,linesnumbered]{algorithm2e}
\usepackage[left=1in, right=1in, top=1in, bottom=1in]{geometry}
\author{Dan Johnson\\dansj}
\title{Problem Set 7\\CME 305}
\begin{document}
    \DeclareFontFamily{U}{comicdans}{}
    \DeclareFontShape{U}{comicdans}{m}{n}{ <-> comicdans }{}

    \usefont{U}{comicdans}{m}{n}

	\maketitle

	\section*{Problem 1}
	First, we show that $\vec{p}$ is unique. We write the PageRank expression in vector form.
	\begin{gather*}
		\vec{p} = \frac{1 - \alpha}{|V|} \vec{1} + \alpha W \vec{p}\\
		(I - \alpha A D^{-1}) D D^{-1} \vec{p} = \frac{1 - \alpha}{|V|} \vec{1}\\
		(D - \alpha A) D^{-1} \vec{p} = \frac{1 - \alpha}{|V|} \vec{1}
	\end{gather*}
	If we can show that $(D - \alpha A)$ is invertible, then we can multiply both sides by $D(D - \alpha A)^{-1}$, which will show that $\vec{p}$ has one solution and is therefore unique. $(D - \alpha A)$ is nonsingular and invertible if it is strictly diagonally dominant. A matrix $B$ is strictly diagonally dominant if $|B_{ii}| > \sum_{j \neq i} |B_{ij}|, \forall i$. In our case, we need to show that $|D_{ii}| > \sum_{j \neq i} |-\alpha A_{ij}|, \forall i$. By definition, each $D_{ii} = \sum_{j \neq i} A_{ij}$.
	\begin{gather*}
		|D_{ii}| > \sum_{j \neq i} |-\alpha A_{ij}|\\
		\left|\sum_{j \neq i} A_{ij}\right| > \sum_{j \neq i} |-\alpha A_{ij}|\\
		\sum_{j \neq i} A_{ij} > \sum_{j \neq i} \alpha A_{ij}
	\end{gather*}
	$\alpha \in (0, 1)$, so this is clearly true. Therefore, $(D - \alpha A)$ is invertible which means that $\vec{p}$ has one solution and is therefore unique.
	$\blacksquare$

	Now we look to compute $\vec{p}$. We construct a new graph $G' = (V', E')$ so that we can solve a Laplacian system. Let $V' = V \cup \{ v \}$.
	Let $E' = \{ e \text{ with weight } \alpha w_e | \forall e \in E \} \cup \{ \{u,v \} \text{ with weight } (1 - \alpha) \deg_G(u) | \forall u \in V \}$. We now construct $D'$, $A'$, and $\mathcal{L}'$.
	\begin{gather*}
		D' =
		\begin{pmatrix}
		\alpha D + (1 - \alpha) D & \vec{0}\\
		\vec{0}^T & (1-\alpha) \vec{1}^T D \vec{1}
		\end{pmatrix}\\
		A' =
		\begin{pmatrix}
		\alpha A & (1-\alpha) D \vec{1}\\
		(1-\alpha) (D \vec{1})^T & 0
		\end{pmatrix}\\
		\mathcal{L}' =
		\begin{pmatrix}
		D - \alpha A & -(1-\alpha) D \vec{1}\\
		-(1-\alpha) (D \vec{1})^T & (1-\alpha) \vec{1}^T D \vec{1}
		\end{pmatrix}
	\end{gather*}
	We now want to solve a Laplacian system that computes $\vec{p}$. We note that $G'$ is fully connected by construction, so $\mathcal{L}' \vec{1} = 0 \vec{1}$. We construct $\vec{x}'$ and $\vec{b}'$ so that solving the Laplacian system $\mathcal{L}' \vec{x}' = \vec{b}'$ solves for $\vec{p}$.
	\begin{gather*}
		\vec{x}' =
		\begin{pmatrix}
		D^{-1}\vec{p}\\
		1
		\end{pmatrix}\\
		\vec{b}' =
		\begin{pmatrix}
		\frac{1 - \alpha}{|V|} \vec{1} - (1-\alpha) D\vec{1}\\
		-(1-\alpha) (D \vec{1})^T D^{-1}\vec{p} + (1-\alpha) \vec{1}^T D \vec{1}
		\end{pmatrix}\\
		\vec{b}' =
		\begin{pmatrix}
		\frac{1 - \alpha}{|V|} \vec{1} + (\alpha - 1) D\vec{1}\\
		(\alpha-1) \vec{1}^T \vec{p} + (1-\alpha) \vec{1}^T D \vec{1}
		\end{pmatrix}\\
		\vec{b}' =
		\begin{pmatrix}
		\frac{1 - \alpha}{|V|} \vec{1} + (\alpha - 1) D\vec{1}\\
		\alpha-1 + (1-\alpha) \vec{1}^T D \vec{1}
		\end{pmatrix}
	\end{gather*}
	So, now we have our Laplacian system $\mathcal{L}' \vec{x}' = \vec{b}'$. The system is solved in $O(\mathcal{T})$ time and gives us back some $\vec{x}''$. Since $\mathcal{L}' \vec{1} = 0 \vec{1}$, we notice that our target solution $\vec{x}' = \vec{x}'' + c\vec{1}$. So, we solve for $c$ knowing that the last component of $\vec{x}'$ is 1 and using the last component of the solution to the Laplacian $\vec{x}''$.
	\begin{gather*}
		1 = [\vec{x}'']_{n+1} + c
	\end{gather*}
	We then get $\vec{x}'$ easily. The first $n$ components of this vector are $D^{-1}\vec{p}$. So, we get $\vec{p} = D \vec{x}'$. So, the algorithm constructs $G'$ and then $\mathcal{L}$ and $\vec{b}'$ all in $O(|V| + |E|)$ time. It then solves this Laplacian system in $O(\mathcal{T})$ time and gets $\vec{p}$ in $O(|V| + |E|)$ time. Therefore, the total run time of this algorithm is $O(|V| + |E| + \mathcal{T})$.
	$\blacksquare$

	\section*{Problem 2}
	\begin{align*}
		M
		&= D^{-1/2} \tilde{W} D^{1/2}\\
		&= D^{-1/2} \frac{1}{2} (I + A D^{-1}) D^{1/2}\\
		&= \frac{1}{2} (D^{-1/2} D^{1/2} + D^{-1/2} A D^{-1/2})\\
		&= \frac{1}{2} (I + D^{-1/2} A D^{-1/2})
	\end{align*}
	This matrix is clearly symmetric because $I$, $D^{-1/2}$, and $A$ are symmetric. We now use the normalized Laplacian.
	\begin{align*}
		M
		&= \frac{1}{2} (I + D^{-1/2} A D^{-1/2})\\
		&= \frac{1}{2} (I + I - N(G))\\
		&= I - \frac{1}{2} N(G)
	\end{align*}
	We find the relationship between eigenvalues of $N$ and $M$.
	\begin{gather*}
		N(G) \vec{v}_i = \lambda_i(N(G)) \vec{v}_i\\
		\frac{1}{2} N(G) \vec{v}_i = \frac{1}{2} \lambda_i(N(G)) \vec{v}_i\\
		\vec{v}_i - \frac{1}{2} N(G) \vec{v}_i = \vec{v}_i - \frac{1}{2} \lambda_i(N(G)) \vec{v}_i\\
		\left( I - \frac{1}{2} N(G) \right) \vec{v}_i = \left(1 - \frac{1}{2} \lambda_i(N(G)) \right) \vec{v}_i\\
		M \vec{v}_i = \left(1 - \frac{1}{2} \lambda_i(N(G)) \right) \vec{v}_i\\
		\lambda_i(M) = 1 - \frac{1}{2} \lambda_i(N)
	\end{gather*}
	The largest eigenvalue for $M$ is the smallest for $N$. From the notes, we know that $\lambda_1(N) = 0$, so $\lambda_{\max}(M) = 1$. The second smallest eigenvalue of $N$ is $\lambda_2(N)$, so the second largest eigenvalue of $M$ is $\lambda_{n-1}(M) = 1 - \frac{1}{2} \lambda_2(N)$.
	$\blacksquare$

	\section*{Problem 3}
	First, we want to get $\tilde{W}^k$ into an easier form. Let $M = D^{-1/2} \tilde{W} D^{1/2}$ like in problem 2.
	\begin{align*}
		\tilde{W} = D^{1/2} M D^{-1/2}\\
		\tilde{W}^k = D^{1/2} M^k D^{-1/2}
	\end{align*}
	Let $x = p - \frac{1}{||d||_1}d$.
	\begin{align*}
		\left|\left|\tilde{W}^k p - \frac{1}{||d||_1}d\right|\right|_1
		&= \left|\left|\tilde{W}^k p - \frac{1}{||d||_1} \tilde{W}^k d\right|\right|_1\\
		&= \left|\left|\tilde{W}^k x\right|\right|_1
	\end{align*}
	Why does $\tilde{W}^k d = d$?
	\begin{align*}
		\tilde{W} d
		&= \frac{1}{2} (I + A D^{-1}) d\\
		&= \frac{1}{2} (d + A \vec{1})\\
		&= \frac{1}{2} (d + d)\\
		&= d\\
		&\downarrow\\
		\tilde{W}^k d &= d
	\end{align*}
	We now look back at the LHS.
	\begin{gather*}
		\left|\left|\tilde{W}^k x\right|\right|_1
		\leq \sqrt{n} \left|\left|\tilde{W}^k x\right|\right|_2
		= \sqrt{n} \left|\left|D^{1/2} M^k D^{-1/2} x\right|\right|_2
		\leq \sqrt{n} \left|\left|D^{1/2}\right|\right|_2 \left|\left|M^k D^{-1/2} x\right|\right|_2\\
		\left|\left|\tilde{W}^k p - \frac{1}{||d||_1}d\right|\right|_1
		\leq \sqrt{n} \sqrt{d_{\max}} \left|\left|M^k D^{-1/2} x\right|\right|_2\\
	\end{gather*}
	From problem 2, we know the largest eigenvalue of $M$ is 1.
	\begin{align*}
		M \sqrt{d}
		&= \frac{1}{2} (I + D^{-1/2} A D^{-1/2}) \sqrt{d}\\
		&= \frac{1}{2} (\sqrt{d} + D^{-1/2} A \vec{1})\\
		&= \frac{1}{2} (\sqrt{d} + D^{-1/2} d)\\
		&= \frac{1}{2} (\sqrt{d} + \sqrt{d})\\
		&= \sqrt{d}\\
	\end{align*}
	So, we know that $\sqrt{d}$ is the eigenvector of the largest eigenvalue. If $\sqrt{d} \perp D^{-1/2} x$, then we can bound the 2 norm with the second largest eigenvalue.
	\begin{align*}
		\sqrt{d}^T (D^{-1/2} x)
		&= \vec{1}^T \left( p - \frac{1}{||d||_1}d \right)\\
		&= 1 - 1\\
		&= 0
	\end{align*}
	The first term is zero because the sum of probabilities is 1 and the second term is 1 by definition. So, we can bound $\left|\left|M^k D^{-1/2} x\right|\right|_2$ using the second largest eigenvalue: $1 - \frac{1}{2} \lambda_2(N)$.
	\begin{gather*}
	\left|\left|\tilde{W}^k p - \frac{1}{||d||_1}d\right|\right|_1
	\leq \sqrt{n} \sqrt{d_{\max}} \left|\left|M^k D^{-1/2} x\right|\right|_2
	\leq \sqrt{n} \sqrt{d_{\max}} \left|\left|(1 - \frac{1}{2} \lambda_2(N))^k D^{-1/2} x\right|\right|_2\\
	\left|\left|\tilde{W}^k p - \frac{1}{||d||_1}d\right|\right|_1
	\leq \sqrt{n} \sqrt{d_{\max}} (1 - \frac{1}{2} \lambda_2(N))^k \left|\left|D^{-1/2}\right|\right|_2  \left|\left|x\right|\right|_2
	\end{gather*}
	Bound $\left|\left|D^{-1/2}\right|\right|_2$, $\left|\left|x\right|\right|_2$, and $(1 - \frac{1}{2} \lambda_2(N))^k$.
	\begin{gather*}
		\left|\left|D^{-1/2}\right|\right|_2 \leq \frac{1}{\sqrt{d_{\min}}}\\
		\left|\left|x\right|\right|_2 = \left|\left|p - \frac{1}{||d||_1}d\right|\right|_2 \leq 1 + 1 = 2\\
		(1 - \frac{1}{2} \lambda_2(N))^k \leq \exp \left( -\frac{k}{2} \lambda_2(N) \right)
	\end{gather*}
	Now we can finally bound it.
	\begin{gather*}
		\left|\left|\tilde{W}^k p - \frac{1}{||d||_1}d\right|\right|_1
		\leq \sqrt{n} \sqrt{d_{\max}} \exp \left( -\frac{k}{2} \lambda_2(N) \right) \frac{1}{\sqrt{d_{\min}}}  2
		\leq \epsilon\\
		\left|\left|\tilde{W}^k p - \frac{1}{||d||_1}d\right|\right|_1
		\leq \sqrt{n} \sqrt{d_{\max}} \exp \left( -\frac{c}{2} \log(\frac{n d_{\max}}{d_{\min}}) \right) \frac{1}{\sqrt{d_{\min}}}  2
		\leq \epsilon\\
		\left|\left|\tilde{W}^k p - \frac{1}{||d||_1}d\right|\right|_1
		\leq \sqrt{n} \sqrt{d_{\max}} \exp (c) \frac{\sqrt{d_{\min}}}{\sqrt{n} \sqrt{d_{\max}}} \frac{1}{\sqrt{d_{\min}}}  2
		\leq \epsilon\\
		\left|\left|\tilde{W}^k p - \frac{1}{||d||_1}d\right|\right|_1
		\leq  \exp (c)   2
		\leq \epsilon\\
		\text{Suppose } c = \log (1/4)\\
		\exp (\log (1/4))   2 = 1/2 \leq \epsilon
	\end{gather*}
	So, we can clearly pick a $c$ such that the LHS is less than or equal to $\epsilon$. We can always pick a $c$ such that this is true given a $\epsilon$.
	\begin{gather*}
		c \leq \log \left( \frac{\epsilon}{2}\right)\\
		\text{Example: } c = \log \left( \frac{\epsilon}{2}\right) - 1
	\end{gather*}
	Therefore, we have shown that $k$ is of the given form.
	$\blacksquare$

	\section*{Problem 4}
	\begin{enumerate}[label=\alph*.]
		\item We think about this problem using effective resistance. We know what the commute time from $a$ to $b$ is from lemma 3 in lecture 14.
		\begin{gather*}
			C_{ab} = ||\vec{d}||_1 (\vec{1}_a - \vec{1}_b)^T  \mathcal{L}^\dagger (\vec{1}_a - \vec{1}_b)
		\end{gather*}
		We also know the effective resistance between $a$ and $b$ from the same lecture.
		\begin{gather*}
			\tilde{R}_{ab} = \sum_{e \in E} r_e f_e^2 = (\vec{1}_a - \vec{1}_b)^T  \mathcal{L}^\dagger (\vec{1}_a - \vec{1}_b)
		\end{gather*}
		So, we know that the commute time between $a$ and $b$ in terms of the effective resistance between $a$ and $b$.
		\begin{gather*}
			C_{ab} = \left( \sum_{v \in V} \deg(v) \right) \tilde{R}_{ab}
		\end{gather*}
		We know that the size of the min cut separating $a$ and $b$ is 1, so the only path between $a$ and $b$ is the edge $\{a, b\}$. The effective resistance $\tilde{R}_{ab}$ is therefore just the resistance between $a$ and $b$ which is 1. We also know that $\sum_{v \in V} \deg(v) =2m$ because every edge is counted twice, once at each endpoint.
		\begin{gather*}
		C_{ab} = (2m) (1) = 2m
		\end{gather*}

		\item Again, we have the commute time in terms of effective resistance.
		\begin{gather*}
		C_{ab} = \left( \sum_{v \in V} \deg(v) \right) \tilde{R}_{ab} = 2 m \tilde{R}_{ab}
		\end{gather*}
		We want to show that this is upper-bounded by $2m d(a, b)$.
		\begin{align*}
			C_{ab} &\leq 2m d(a, b)\\
			2 m \tilde{R}_{ab} &\leq 2m d(a, b)\\
			\tilde{R}_{ab} &\leq d(a, b)
		\end{align*}
		When we find the effective resistance, we have to consider all paths between $a$ and $b$. If all paths use the same edge, the edges are in series and the resistance is added normally. If the edges are used in some of the paths, the edges are in parallel, so the effective resistance is defined as $\frac{1}{\tilde{R}_{ab}} = \frac{1}{R_1} + \frac{1}{R_2}$. In our case, $R = 1$ for all edges. This means that the largest effective resistance is if all the edges between $a$ and $b$ are in series or, equivalently, there is just one path between the two nodes. In this case, every edge in the path would sum to $d(a, b)$, which means $C_{ab} = 2m d(a, b)$. Now if we have any other path between the two nodes, they will be in parallel.
		\begin{gather*}
			\frac{1}{\tilde{R}_{ab}} = \frac{1}{R_1} + \frac{1}{R_2}
		\end{gather*}
		Clearly, $\tilde{R}_{ab}$ is largest when the effective resistance of the second path is infinitely large which can be interpreted as no path existing. Since adding any path lowers the effective resistance, the commute time must be upper-bounded by the case of a single shortest path.
		\begin{align*}
		C_{ab} &\leq 2m d(a, b)
		\end{align*}
	\end{enumerate}

	\section*{Problem 5}
	\begin{enumerate}[label=\alph*.]
		\item If $G$ is a tree, there is only one path between $a$ and $b$. Otherwise, this induces a cycle and $G$ is not a tree. As argued in problem 4 part b, in the case where there is only one path between $a$ and $b$, the effective resistance is the sum of all the weights of this path. With each weight being one, the effective resistance $\tilde{R}_{ab}$ becomes $d(a, b)$.
		\begin{gather*}
		C_{ab} = \left( \sum_{v \in V} \deg(v) \right) \tilde{R}_{ab} = 2 m d(a, b)
		\end{gather*}

		\item
		First, we note the relationship between hit time and commute time.
		\begin{gather*}
			C_{ab} = H_{ab} + H_{ba}
		\end{gather*}
		I state that the max commute time in a graph is larger than the max hit time. We easily prove this by taking the nodes from the max hit time and see that the commute time for these nodes is the max hit time plus the time it takes to get back to the first node. In the lecture notes, we found an upper bound on the cover time.
		\begin{align*}
			\text{Cover}(G)
			&\leq \left( \max_{a, b \in V} H_{ab} \right) O(\log n)\\
			&\leq \left( \max_{a, b \in V} C_{ab} \right) O(\log n)\\
			&\leq 2m d(a, b) O(\log n)\\
			&\leq 2m n O(\log n)\\
			&\leq O( mn \log n)
		\end{align*}
		In the third line, we use the upper bound found in problem 4 part b. In the forth line we note that $d(a, b) < n$.
	\end{enumerate}
\end{document}


\chapter{Constructing an formula}
\label{chapter:constructingaformula}
The class \formulaClass represents SMT formulas, which are
defined according to the following abstract grammar

\[
\begin{array}{rccccccccccccc}
  p &\quad ::=\quad & a & | & b & | & x & | & (p + p) & | & (p \cdot p) & | & (p^e) \\
  v &\quad ::=\quad & u & | & x \\
  s &\quad ::=\quad & f(v,\ldots,v) & | & u & | & x \\
  e &\quad ::=\quad & s = s \\
  c &\quad ::=\quad & p = 0 & | & p < 0 & | & p \leq 0 & | & p > 0 & | & p \geq 0 & | & p \neq 0 \\
 \varphi &\quad ::=\quad & c & | & (\neg \varphi) & | &
 (\varphi\land\varphi) & | &
 (\varphi\lor\varphi) & | &
 (\varphi\rightarrow\varphi) & | \\ &&
 (\varphi\leftrightarrow\varphi) & | &
 (\varphi\oplus\varphi)
\end{array}
\]

where $a$ is a rational number, $e$ is a natural number greater one, $b$ is a \emph{Boolean variable} and the \emph{arithmetic variable} $x$ is an inherently existential quantified and either real- or integer-valued. We call $p$ a \emph{polynomial} and use a \carl multivariate polynomial with \cln rationals as coefficients to represent it. The \emph{uninterpreted function} $f$ is of a certain \emph{order} $o(f)$ and each of its $o(f)$ arguments are either an arithmetic variable or an \emph{uninterpreted variable} $u$, which is also inherently existential quantified, but has no domain specified. Than an \emph{uninterpreted equation} $e$ has either an uninterpreted function, an uninterpreted variable or an arithmetic variable as left-hand respectively right-hand side. A \emph{constraint} $c$ compares a polynomial to zero, using a \emph{relation symbol}. Furthermore, we keep constraints in a normalized representation to be able to differ them better.

\section{Normalized constraints}
A normalized constraint has the form
\[a_1\overbrace{x_{1,1}^{e_{1,1}}\cdot\ldots\cdot x_{1,k_1}^{e_{1,k_1}}}^{m_1}+\ldots+a_n\overbrace{x_{n,1}^{e_{n,1}}\cdot\ldots\cdot x_{n,k_n}^{e_{n,k_n}}}^{m_n}\ + \ d\ \sim \ 0\]
with $n\geq0$, the \emph{$i$th coefficient} $a_i$ being an integral number ($\neq 0$), $d$ being a integral number, $x_{i,j_i}$ being a real- or integer-valued variable and $e_{i,j_i}$ being a natural number greater zero (for all $1\leq i\leq n$ and $1\leq j_i\leq k_i$). Furthermore, it holds that
$x_{i,j_i}\neq x_{i,l_i}$ if $j_i\neq l_i$ (for all $1\leq i\leq n$ and $1\leq j_i, l_i\leq k_i$) and $m_{i_1}\neq m_{i_2}$ if $i_1\neq i_2$ (for all $1\leq i_1,i_2\leq n$). If $n$ is $0$ then $d$ is $0$ and $\sim$ is either $=$ or $<$. In the former case we have the normalized representation of any variable-free consistent constraint, which semantically equals \true, and in the latter case we have the normalized representation of any variable-free inconsistent constraint, which semantically equals \false. Note that the monomials and the variables in them are ordered according the \polynomialOrder of \carl.
Moreover, the first coefficient of a normalized constraint (with respect to this order) is always positive and the greatest common divisor of $a_1,\ldots,\ a_n,\ d$ is $1$. If all variable are integer valued the constraint is further simplified to
\[\frac{a_1}{g}\cdot m_1\ +\ \ldots\ +\ \frac{a_n}{g}\cdot m_n\ + \ d'\  \sim' \ 0,\]
where $g$ is the greatest common divisor of $a_1,\ldots,\ a_n$,
\[\sim'=\left\{
\begin{array}{ll}
\leq, &\text{ if }\sim\text{ is }< \\
\geq, &\text{ if }\sim\text{ is }> \\
\sim, &\text{ otherwise }
\end{array}
\right.\]
and
\[
d' = \left\{
\begin{array}{ll}
\lceil\frac{d}{g}\rceil &\text{ if }\sim'\text{ is }\leq \\[1.5ex]
\lfloor\frac{d}{g}\rfloor &\text{ if }\sim'\text{ is }\geq \\[1.5ex]
\frac{d}{g} &\text{ otherwise }
\end{array}
\right.\]
If additionally $\frac{d}{g}$ is not integral and $\sim'$ is $=$, the constraint is simplified $0<0$, or if $\sim'$ is $\neq$,
the constraint is simplified $0=0$.

We do some further simplifactions, such as the elimination of multiple roots of the left-hand sides in equations and inequalities with the relation symbol $\neq$, e.g., $x^3=0$ is simplified to $x=0$. We also simplify constraints whose left-hand sides are obviously positive (semi)/negative (semi) definite, e.g., $x^2\leq 0$ is simplified to $x^2=0$, which again can be simplified to $x=0$ according to the first simplification rule.

\section{Boolean combinations of constraints and Boolean variables}
A formula is stored as a directed acyclic graph, where the intermediate nodes represent the Boolean operations on the sub-formulas represented by the successors of this node. The leaves (nodes without successor) contain either a Boolean variable, a constraint or an uninterpreted equality. Equal formulas, that is formulas being leaves and containing the same element or formulas representing the same operation on the same sub-formulas, are stored only once.

The construction of formulas, which are represented by the \formulaClass, is mainly based on the presented abstract grammar. A formula being a leaf wraps the corresponding objects representing a Boolean variable, a constraint or an uninterpreted equality. A Boolean combination of Boolean variables, constraints and uninterpreted equalities consists of a Boolean operator and the sub-formulas it interconnects. For this purpose we either firstly create a set of formulas containing all sub-formulas and then construct the Formula or (if the formula shall not have more than three sub-formulas) construct the formula directly passing the operator and sub-formulas. Formulas, constraints and uninterpreted equalities are non-mutable, once they are constructed. %TODO: explain mutable member of formulas for information storage

We give a small example constructing the formula \[(\neg b\ \land\ x^2-y<0\ \land\ 4x+y-8y^7=0 )\ \rightarrow\ (\neg(x^2-y<0)\ \lor\ b ),\] with the Boolean variable $b$ and the real-valued variables $x$ and $y$, for demonstration. Furthermore, we construct the UF formula
\[v = f(u,u)\ \oplus\ w \neq u\]
with $u$, $v$ and $w$ being uninterpreted variables of not specified domains $S$ and $T$, respectively,
and $f$ is an uninterpreted function with not specified domain $T^{S\times S}$.

Firstly, we show how to create real valued (integer valued analogously with \texttt{VT\_INT}), Boolean and uninterpreted variables:
\scriptsize
\begin{verbatim}
carl::Variable x = smtrat::newVariable( "x", carl::VariableType::VT_REAL );
carl::Variable y = smtrat::newVariable( "y", carl::VariableType::VT_REAL );
carl::Variable b = smtrat::newVariable( "b", carl::VariableType::VT_BOOL );
carl::Variable u = smtrat::newVariable( "u", carl::VariableType::VT_UNINTERPRETED );
carl::Variable v = smtrat::newVariable( "v", carl::VariableType::VT_UNINTERPRETED );
carl::Variable w = smtrat::newVariable( "w", carl::VariableType::VT_UNINTERPRETED );
\end{verbatim}
\normalsize
Uninterpreted variables, functions and function instances combined in equations or inequalities comparing them are constructed the following way.
\scriptsize
\begin{verbatim}
carl::Sort sortS = smtrat::newSort( "S" );
carl::Sort sortT = smtrat::newSort( "T" );
carl::UVariable uu( u, sortS );
carl::UVariable uv( v, sortT );
carl::UVariable uw( w, sortS );
carl::UninterpretedFunction f = smtrat::newUF( "f", sortS, sortS, sortT );
carl::UFInstance f1 = smtrat::newUFInstance( f, uu, uw );
carl::UEquality ueqA( uv, f1, false );
carl::UEquality ueqB( uw, uu, true );
\end{verbatim}
\normalsize
Next we see an example how to create polynomials, which form the left-hand sides of the constraints:
\scriptsize
\begin{verbatim}
smtrat::Poly px( x );
smtrat::Poly py( y );
smtrat::Poly lhsA = px.pow(2) - py;
smtrat::Poly lhsB = smtrat::Rational(4) * px + py - smtrat::Rational(8) * py.pow(7);
\end{verbatim}
\normalsize
Constraints can then be constructed as follows:
\scriptsize
\begin{verbatim}
smtrat::ConstraintT constraintA( lhsA, carl::Relation::LESS );
smtrat::ConstraintT constraintB( lhsB, carl::Relation::EQ );
\end{verbatim}
\normalsize
Now, we can construct the atoms of the Boolean formula
\scriptsize
\begin{verbatim}
smtrat::FormulaT atomA( constraintA );
smtrat::FormulaT atomB( constraintB );
smtrat::FormulaT atomC( b );
smtrat::FormulaT atomD( ueqA );
smtrat::FormulaT atomE( ueqB );
\end{verbatim}
\normalsize
and the formulas itself (either with a set of arguments or directly):
\scriptsize
\begin{verbatim}
smtrat::FormulasT subformulasA;
subformulasA.insert( smtrat::FormulaT( carl::FormulaType::NOT, atomC ) );
subformulasA.insert( atomA );
subformulasA.insert( atomB );
smtrat::FormulaT phiA( carl::FormulaType::AND, std::move(subformulasA) );
smtrat::FormulaT phiB( carl::FormulaType::NOT, atomA )
smtrat::FormulaT phiC( carl::FormulaType::OR, phiB, atomC );
smtrat::FormulaT phiD( carl::FormulaType::IMPLIES, phiA, phiC );
smtrat::FormulaT phiE( carl::FormulaType::XOR, atomD, atomE );
\end{verbatim}
\normalsize
Note, that $\land$ and $\lor$ are $n$-ary constructors, $\neg$ is a unary constructor and all the other Boolean operators are binary.


\documentclass{acmsiggraph}

\usepackage{parskip}
\usepackage{graphicx}
\usepackage{footmisc}
\usepackage{amsmath}
\usepackage{url}
\onlineid{0}

\title{\Large Plotting the Single-Electron Solution to Schr\"{o}dinger's Equation with OpenCL}

\author{Tim Horton\thanks{e-mail: hortot2@rpi.edu}\\Rensselaer Polytechnic Institute}

\begin{document}

\maketitle

\begin{figure}
    \includegraphics[width=84.5mm]{320.png}
    \caption{The 3-2-0 orbital of the hydrogen atom}
\end{figure}

\section{Abstract}

The realm of physics provides many readily parallelizable algorithms --- often involving the simple evaluation of a function at an enormous number of points in space --- which also happen to produce attractive visualizations. One such visualization is that of the probability distribution of the single electron within a hydrogen atom (or He$^+$, Li$^{2+}$, and so on). By evaluating Schr\"{o}dinger's equation at millions of points, one can construct a representation of the likely location of the electron --- the orbital cloud, if you will. Luckily, the evaluation of this function at a particular point is entirely independent of nearby points, so it is effectively perfect for parallelization. We will use OpenCL to develop and benchmark (on a varied array of hardware, from dated CPUs to very modern GPUs) an implementation of this algorithm which will produce a simple --- but attractive --- image of the atomic orbital of the single-electron hydrogen atom.

\section{Physics}

Quantum mechanics --- since Louis de Broglie's paper {\it Research on Quantum Theory} --- tells us that matter behaves sometimes as a wave, and other times as a particle. The electron in a hydrogen atom is no different: when it isn't being observed, it doesn't exist at a particular point in space, but instead expands as a wave. The atomic orbital cloud which we're plotting here is actually simply a plot of the density of that wave at all points surrounding the atom's nucleus.

All of the equations below are borrowed from \cite{quantumBook}, but have been slightly modified to fit our purposes, as discussed in section \ref{simpSection}.

\subsection{Schr\"{o}dinger's Equation}

From \cite{quantumBook}, we find the single-electron solution to Schr\"{o}dinger's equation:

\begin{equation}\label{psi}
\psi\left(n, l, m\right)=\psi_c
\left(\mathit{e}^{-r/n}\right)
\left(\frac{2r}{n}\right)^l
\left[L_{n-l-1}^{2l+1}
    \left(\frac{2r}{n}\right)\right]
Y_l^m\left(\theta,\phi\right),
\end{equation}

\begin{equation}\label{psiConstant}
\psi_c=\sqrt{\left(\frac{2}{n}\right)^3
    \frac{\left(n-l-1\right)!}{2n\left[\left(n+l\right)!\right]^3}}.
\end{equation}

This is the fundamental formula involved in our computation --- it is this equation which we evaluate at each sample. The result of this equation gives the probability that the single electron of our hydrogen atom is at the particular chosen location in the spherical coordinate system, $\left(\theta, \phi, r\right)$, which is the information that we eventually aim to plot.

One will notice that $\psi_c$ is independent of the coordinates of evaluation, $\left(\theta, \phi, r\right)$, so it --- as an obvious optimization --- can be computed once and cached for the entire image.

Indeed, while all of the benchmarks in this paper were done with $\psi_c$ included, one might notice that --- since it's a constant term across the entire image, and we're scaling the image by {\it another} constant scale factor --- that it doesn't actually affect the visualization, and can be entirely discarded. It appears that discarding the overhead required to pass this value in as an argument to each of the kernel instances and to multiply it into the value of $\psi$ gives a performance increase of between 6 and 10 percent: quite a significant improvement for no change in output!

Also, $\psi$ depends on two external functions: $Y$, the spherical harmonic equation (equation \ref{yFunc}), and $L$, the Laguerre polynomial (equation \ref{laguerre}), which will be detailed in sections \ref{sphericalHarmonics} and \ref{laguerreLegendre}, respectively.

\subsection{Spherical Harmonics}

\label{sphericalHarmonics}

$Y$, a solution to Laplace's spherical harmonic function, provides the angular component of $\psi$:

\begin{equation}\label{yFunc}
Y_l^m\left(\theta,\phi\right)=\epsilon
\sqrt{\frac{\left(2l+1\right)}{4\pi}
    \frac{\left(l-\left|m\right|\right)!}{\left(l+\left|m\right|\right)!}}
\mathit{e}^{{\rm i}m\phi}
P^m_l\left(\cos\theta\right),
\end{equation}

\begin{equation}\label{yEpsilon}
\epsilon=\begin{cases}
\left(-1\right)^m & \text{$m\ge0$} \\
1 & \text{$m<0$}
\end{cases}.
\end{equation}

$Y$ depends on an additional external function, $P$, the Legendre polynomial (equation \ref{legendre}).

\subsection{Laguerre and Legendre}

\label{laguerreLegendre}

Key to the evaluation of Schr\"{o}dinger's equation are the Laguerre and Legendre polynomials. Unfortunately, the generation of both requires symbolically solving differential equations. \cite{legendreCite} Since OpenCL (rightfully) lacks a symbolic differential equation solver (and the implementation of such a program is far outside of the scope of this project), we have added to our kernel a simple table of the first few polynomials, computed with Mathematica. This limits the range of input $n, l, m$ parameters which can be used with this program, but extending the tables is very simple and requires only patience.

The generating equations for these polynomials are as follows:

\begin{equation}\label{laguerre}
L^\alpha_n\left(x\right)=\frac{x^{-\alpha}e^x}{n!}\frac{d^n}{dx^n}\left(e^{-x}x^{n+\alpha}\right),
\end{equation}

\begin{equation}\label{legendre}
P^u_v\left(z\right)=\frac{\left(1+z\right)^{\mu/2}}{\left(1-z\right)^{\mu/2}}
\tilde{F}_{2,1}
\left(-v,v+1;1-\mu;\frac{1-z}{2}\right),
\end{equation}

where $\tilde{F}$ is Gauss' hypergeometric function, which we don't need to bother ourselves with since we're simply allowing Mathematica to evaluate it and using the comparatively simple symbolic results in our table.

\section{Implementation}

\subsection{Overview}

Our implementation is twofold: an OpenCL kernel (written in the C-like OpenCL kernel language) which performs the evaluation of all of the Schr\"{o}dinger's equation samples required for a single pixel of the output image, and a Python script which parses command line options, uses PyOpenCL to load and run the kernel and PIL to create the output image, and performs timed benchmarks.

\subsection{Simplifications}

\label{simpSection}

The decision was made during the implementation phase to redefine the Bohr radius so that $a=1$; this was done in order to keep intermediate numbers within the range of single-precision floating point values, as OpenCL doesn't currently support native double-precision math on any of the GPUs available for benchmarking. Since we're only using this software to generate visualizations, this doesn't affect the output; it simply scales the distances away from the atomic level. The equations above have $a=1$ already substituted in, for brevity.

To keep the implementation of this algorithm within the scope of this project, we also decided to use a fixed-camera orthogonal projection. This worked to significantly simplify the process of iteration over all of the samples, as with this restriction, the program simply has to iterate over all of the pixels in the image, evaluating many points in the $z$ dimension for each pixel, without worrying about complex transformations between coordinate systems.

\subsection{Evaluation}

After arguments have been parsed and an OpenCL context has been created, we create a buffer large enough to store the entirety of the resultant image, and also compute the constant term of $\psi$, both of which are passed to each instance of the kernel.

For each pixel in the image, we spawn an instance of our OpenCL kernel which evaluates 2000 samples in the $z$ dimension. Each sample represents the value of the density function, $\psi$, at that point in space (the cartesian coordinates of the image have to be converted into spherical coordinates in order to evaluate $\psi$). All 2000 samples are summed and stored into their respective pixel in the image buffer.

Once computation has completed, the buffer is copied back from working memory (either video memory or system memory) to the final image buffer in system memory. Once there, pixel values are scaled linearly so that the brightest pixel is 100\% white, and the image is compressed to PNG and saved to the filesystem.

\subsection{Complex Math}

The OpenCL specification unfortunately currently does not include complex math primitives, which are necessary for the evaluation of $\psi$. It does reserve the {\bf complex} keyword, which suggests that perhaps support for something similar to GCC's complex type is coming in a future version of OpenCL, which would put it on ground more similar to CUDA.

In order to solve this problem, we implemented a small library of complex math functions which make use of the OpenCL {\bf float2} type to store complex numbers. This library includes various complex number constructors, as well as exponentiation, multiplication, square root, the exponential function, and conjugation. These functions are used extensively within the evaluation of $\psi$.

\section{Hardware}

Benchmarks will be performed on a number of different computation devices across a few different computers:

\subsection{GPU}

\begin{itemize}

\item ATI Radeon 4890, 800$\times$850MHz, 1GB, 250\$\footnote{All hardware prices listed are approximate launch prices. Prices of older hardware, especially the Core 2 Quad, have dropped significantly since introduction.\label{fn:prices}}\footnote{Tested on Windows, with ATI Stream SDK\label{fn:windows}}

\end{itemize}

\subsection{CPUs}

\begin{itemize}

\item Intel Core i7 620M, 2$\times$3333MHz, 4GB, 332\$\footref{fn:prices}\footnote{Tested on Mac OS X, with Apple OpenCL\label{fn:osx}}

\item Intel Core 2 Quad Q6600, 4$\times$3000MHz, 4GB, 851\$\footref{fn:prices}\footnote{Tested on Linux, with ATI Stream SDK\label{fn:linux}}

\item Intel Core 2 Duo E7200, 2$\times$2530MHz, 8GB, 133\$\footref{fn:prices}\footref{fn:osx}

\end{itemize}

These machines run a variety of different operating systems (including Mac OS X, Windows, and Linux). Comparisons made later in this paper assume that each OS and the drivers and OpenCL implementation used within them are created equally; this is only somewhat reasonable, and should be kept in mind when interpreting results.

Also, it should be noted that while each core on a given CPU could potentially be evaluating samples in parallel, a GPU's cores are much more restricted (less general-purpose) and must work in small groups to accomplish their work. Therefore, given the kernel used for this project, the 4890 listed above only has 50 compute units (16 stream processors work together to compute a single sample).

\section{Results}

Three computers have spent countless hours performing calculations to bring you the following results, as each benchmark point is the minimum of five individual trials.

\subsection{Communication Overhead}

\begin{figure}
    \includegraphics[width=84.5mm]{overheadPlot.pdf}
    \caption{Overhead from image buffer copy as a percentage of the total runtime; smaller values are better}
    \label{fig:overheadPlot}
\end{figure}

The algorithm we implemented for this project required no communication between  cores whatsoever during the course of the computation. The primary overhead involved in the entire process is the single copy of the $n\times n$ output buffer from working memory into system memory, where it is then normalized, compressed, and output (we don't consider these parts of the process when measuring performance or overhead, as they're written in Python, a language not known for its performance characteristics).

The overhead incurred during this copy can be seen in the chart in figure \ref{fig:overheadPlot} to be negligible --- in the range of a tenth of a percent of the total runtime when working on the GPU, and a one-hundred-thousandth of a percent when working on the CPU, where the data has a shorter distance to travel.

\subsection{Strong Scaling}

\label{strongScaling}

\begin{figure}
    \includegraphics[width=84.5mm]{strongPlotOne.pdf}
    \caption{Runtime as core count increases on the 4890 with a fixed resolution ($400\times400$); smaller values are better}
    \label{fig:strongPlotOne}
\end{figure}

\begin{figure}
    \includegraphics[width=84.5mm]{strongPlotTwo.pdf}
    \caption{Speedup as core count increases (measured against the single-core run) on the 4890 with a fixed resolution ($400\times400$); the orange line is the theoretical maximum speedup, given linear scaling and 50 execution units (800 cores / 16 cores per unit); the red line is a linear speedup curve; larger values are better}
    \label{fig:strongPlotTwo}
\end{figure}

In order to measure the strong scalability of our implementation, we fix the problem size and vary the number of processing units used to compute it. For this benchmark, we will fix the image at $400\times400$ pixels, requiring $320,000,000$ total samples, and we will vary the number of cores of the 4890 that we use.

One can see in figure \ref{fig:strongPlotTwo} that this implementation is strongly scalable; the speedup is almost perfectly linear as the core count increases. It should be noted that since --- for our kernel --- the 4890 only has 50 compute units, the linear increase is only maintained until we reach 50 cores; after that, the performance gains plateau as they should.

\subsection{Large- vs. Small-scale Parallelism}

\begin{figure}
    \includegraphics[width=84.5mm]{runtimePlot.pdf}
    \caption{Runtime vs. Resolution on all hardware; smaller values are better}
    \label{fig:runtimePlot}
\end{figure}

As one can see in figure \ref{fig:runtimePlot}, the GPU significantly outperforms all of the CPUs. This is not surprising, as it can perform more than an order of magnitude more parallel computations, but it does validate the premise of this experiment.

\begin{figure}
    \includegraphics[width=84.5mm]{speedupPlot.pdf}
    \caption{Speedup of fastest GPU (4890) over fastest CPU (Core 2 Quad) as resolution increases; larger values are better}
    \label{fig:speedupPlot}
\end{figure}

Figure \ref{fig:speedupPlot} shows that our implementation converges to an approximately 14.7x speedup (between the fastest of each class of hardware, namely, the 4890 and Core 2 Quad) as the resolution of the output image increases. Noise in the speedup values with smaller images is likely due to measurement error and the overhead and unpredictability involved when copying data to/from video memory.

It should be noted that these speedup values are across different pieces of hardware at different core clock speeds, and, as such, do not represent the actual parallel speedup gained (for that, look at section \ref{strongScaling}). This is, instead, a representation of the performance potentially gained by moving computation to the GPU.

\subsection{Cost Effectiveness of Hardware}

\begin{figure}
    \includegraphics[width=84.5mm]{dollarPlot.pdf}
    \caption{Samples per second per dollar with a fixed resolution ($500\times500$); larger values are better}
    \label{fig:costEffectiveness}
\end{figure}

\begin{figure}
    \includegraphics[width=84.5mm]{wattPlot.pdf}
    \caption{Samples per second per heavy-load watt with a fixed resolution ($500\times500$); larger values are better}
    \label{fig:wattPlot}
\end{figure}

One oft-cited measure of the performance of a computational device is its {\it performance-per-dollar}; this is especially of interest for consumers with budgets or people building large computation clusters. In order to measure performance-per-dollar, we have to define {\it performance} in the context of our problem. In all of the plots in this section, performance is measured as the number of samples per second. Figures \ref{fig:costEffectiveness} and \ref{fig:wattPlot} all use a 500$\times$500 image, with 2000 samples per pixel, for $500,000,000$ total samples.

Figure \ref{fig:costEffectiveness} illustrates the performance-per-dollar of our four devices. As is quite evident already, the 4890 --- though in the same price range as most of the other devices, and significantly cheaper than the Core 2 Quad --- manages to dominate the chart. It's safe to say that if an institution is looking into buying a large number of a particular device for scientific computation, they should seriously consider looking into high-end graphics cards. Indeed; each dollar spent on a 4890 goes 18 times farther, performance-wise, than it would if spent on the next-most-cost-effective chip, the Core 2 Duo E7200.

Another important point when considering the cost of various pieces of hardware is how much it will cost to run. One way to measure this is to consider the power efficiency of the chip, in terms of performance-per-watt. Again using the same measure of performance from our performance-per-dollar comparison, we can see in figure \ref{fig:wattPlot} that the GPU continues to dominate, even though it requires three times more energy to run than its closest competitor, the Core 2 Quad. As one might expect, the older --- thus, less power efficient --- Core 2 Duo falls behind the other CPUs on this scale: though it was the performance-per-dollar winner out of the three CPUs, it would cost significantly more than the other two to run for any period of time.

\section{Future Work}

There are quite a few things which have come up while developing this project that would be interesting to implement if we were to continue work on the codebase. We will discuss a few of these below:

\subsection{Camera Transformation}

\label{cameraTransformation}

One key feature that is missing from the current implementation is the ability to manipulate the "camera" location, changing your viewport on the orbital cloud. Implementing this would add a bit of complexity, but would allow for interesting video rendering --- for example, video of the camera spinning around the atom, providing a better idea of the actual shape of the cloud. It can be hard --- without prior knowledge --- to comprehend the shape of the cloud from our current visualization.

In addition, if the number of samples was sufficiently reduced, or a fast enough video card was obtained, one could implement {\it live} manipulation of the camera's view, which could assist in inspection of the orbital cloud to an even greater extent.

\subsection{Slicing}

\begin{figure}
    \includegraphics[width=84.5mm]{320-slice.png}
    \caption{A slice through the 3-2-0 orbital of the hydrogen atom}
    \label{fig:slice}
\end{figure}

Section \ref{cameraTransformation} covered the ideal solution to the problem of comprehending the shape of the orbital from our 2D representation --- however, if that solution proved to be too challenging, one could instead generate an animation of slices through the cloud, each slice being similar to figure \ref{fig:slice}. This would provide another form of interesting visualization of the resulting data, rather similar to the data collected by an MRI device.

\subsection{Automatic Scaling}

Our OpenCL kernel currently contains an arbitrary zoom factor which needs to be adjusted based on the orbital configuration --- higher energy electron states lead to larger probability clouds. Automatically generating this zoom factor would lead to significantly improved ease of use when changing between different configurations. This could most effectively be implemented by finding a formula for a reasonable scale factor given a set of $n, l, m$ values.

\subsection{Coloring}

A third way to make the visualization more appealing (or even more informative) would be to provide some manner of colorization. One potential algorithm would be to colorize each sample based on its depth in $z$; the resulting image would then have more depth information, which might increase the ease with which it is interpreted. A variety of other coloring functions are possible, but none were pursued during this project, for time's sake.

\subsection{Generalization of Parameters}

Equations \ref{laguerre} and \ref{legendre} provide a generalized way to construct Laguerre and Legendre polynomials, respectively. However, our implementation does not make use of these functions, instead using a table of simplified solutions, generated with Mathematica, which is limited in scope.

The length of the table determines the range of $n, l, m$ values which can be passed into our program, and is the only restriction on generality within our implementation. If one were to construct these functions on the fly from their generating equations, the algorithm would be completely generalized in terms of $n, l, m$, and would be significantly more useful from an exploratory standpoint.

However, as discussed in section \ref{laguerreLegendre}, this would require (at least) the implementation of a symbolic differential equation solver in OpenCL, which could potentially be a very time-consuming task (it could also increase the number of GPU stream processors per kernel required, significantly decreasing the speedup gained).

Alternatively, one could write a translation program to automatically generate the table from Mathematica, removing the slow, error-prone human translation step, and very quickly expand the size of the table to something more useful.

\subsection{Multiple Electrons}

Another feature that would be incredibly cool to implement would be simulation of atoms with more than one electron. In this case, interference between the electrons changes the pattern to be significantly more complex (and much more interesting, as well). It's also a good bit more complex to compute: there is no exact solution of the Schr\"{o}dinger's equation for multiple electrons. Instead, an implementation would depend on a numerical differential equation solver, which would significantly affect performance.

\section{Conclusion}

After consuming many computer-hours computing the 3-2-0 orbital repeatedly, we've come to a few clear conclusions. Firstly, scientific computation --- the calculation of atomic orbitals, at the very least --- should clearly be a prime target for porting to parallel computation on the GPU, as the field takes off. Projects like Folding@Home are already taking advantage of this (besides being massively parallel, as a web-distributed project) by providing ATI Stream SDK and CUDA ports of their client, and the last few years have seen an explosion in other such projects.

We've also found that --- for problems which are embarrassingly easy to parallelize, like this one --- it's very easy to implement your problem using OpenCL. We've used MPI and pthreads in the past, and it seems that OpenCL (perhaps because it's a more modern API) is a bit easier to use in terms of implementation. The added benefit of kernels running on a wider array of hardware (not just the CPU, like MPI, but on the GPU as well) means that OpenCL is a no-brainer --- at least for this sort of simple project. It's quite likely that a problem requiring a large amount of inter-kernel communication would be better suited by something with more communications primitives, like MPI. Indeed, besides {\it barrier}, OpenCL doesn't seem to have any manner of inter-kernel communications functions.

\section{Code}

All of the code developed for this project is available under the two-clause BSD license, and is hosted on GitHub:

\url{http://github.com/hortont424/orbitals}

\url{git://github.com/hortont424/orbitals.git}

The code has only been tested with the Apple and ATI OpenCL compilers, but should work with few to no changes on NVIDIA's SDK.

\bibliographystyle{acmsiggraph}
\nocite{*}
\bibliography{paper}

\end{document}


% This LaTeX was auto-generated from MATLAB code.
% To make changes, update the MATLAB code and republish this document.

\documentclass{article}
\usepackage{graphicx}
\usepackage{color}

\sloppy
\definecolor{lightgray}{gray}{0.5}
\setlength{\parindent}{0pt}

\begin{document}

\section*{ECE302 Project 2: Estimation Techniques}

\subsection*{Contents}

\begin{itemize}
\setlength{\itemsep}{-1ex}
   \item Scenario 1
   \item Bayes MMSE Estimator
\end{itemize}

\subsection*{Scenario 1}

\begin{par}
An implementation of a Bayes MMSE and Linear MMSE estimators of the random variable Y from the random variable X where X = Y + W. Here Y \ensuremath{\tilde{\;}} U(-1,1) and W \ensuremath{\tilde{\;}} U(-2,2). Note that in this project, N represents the number of samples taken from the respective distributions.
\end{par} \vspace{1em}
\begin{verbatim}
clear;
clc;
close all;

N = 10000;

Y1 = random('Uniform',-1,1,N,1);
W = random('Uniform',-2,2,N,1);
X1 = Y1 + W;
\end{verbatim}

\subsection*{Bayes MMSE Estimator}

\begin{par}
Here we take $\hat{y} = E[Y | X = x]$. Calculating this condition expectation leads to: \begin{verbatim}latex\end{verbatim} \ensuremath{\backslash}usepackage\{mathworks\} \ensuremath{\backslash}[ \ensuremath{\backslash}hat\{y\} =       \ensuremath{\backslash}begin\{cases\}       0 \& x\ensuremath{\backslash}leq 0 \ensuremath{\backslash}\ensuremath{\backslash}       \ensuremath{\backslash}frac\{100-x\}\{100\} \& 0\ensuremath{\backslash}leq x\ensuremath{\backslash}leq 100 \ensuremath{\backslash}\ensuremath{\backslash}       0 \& 100\ensuremath{\backslash}leq x    \ensuremath{\backslash}end\{cases\} \ensuremath{\backslash}] \begin{verbatim}/latex\end{verbatim}
\end{par} \vspace{1em}
\begin{verbatim}
Y1est = zeros(N,1);
for i = 1:N
    if X1(i) < -1
        Y1est(i) = .5 + .5*X1(i);
    elseif X1(i) < 1
        Y1est(i) = 0;
    else
        Y1est(i) = -.5 + .5*X1(i);
    end
end

empmmse1 = mean((Y1 - Y1est).^2);

actmmse1 = .25;
\end{verbatim}
\begin{verbatim}
Y1est = (1/5)*X1;

emplmmse1 = mean((Y1 - Y1est).^2);

actlmmse1 = 4/15;
\end{verbatim}
\begin{verbatim}
table([empmmse1; emplmmsel],[actmmse1; actlmmse1],'VariableNames', ...
    ["Empirical MSE","Theorhetical MSE"],'RowNames', ...
    ["MMSE","LMMSE"])
\end{verbatim}

        \color{lightgray} \begin{verbatim}Unrecognized function or variable 'emplmmsel'.

Error in proj2 (line 55)
table([empmmse1; emplmmsel],[actmmse1; actlmmse1],'VariableNames', ...
\end{verbatim} \color{black}
    \begin{verbatim}
m = 5;
n = 5;

Y2est = zeros(N,m);
emplmmse2 = zeros(1,m);
actlmmse2 = zeros(1,m);
leg = strings(1,m);

for j = 1:m
    muY2 = 1;
    varY2 = j;
    varR = j;

    Y2 = random('Normal',muY2,sqrt(varY2),N,1);
    R = random('Normal',muR,sqrt(varR),N,n);
    X2 = Y2 + R;

    CXX = varY2*ones(n) + diag(varR*ones(1,n));
    CXY = varY2*ones(n,1);

    a = CXX\CXY;

    a0 = muY2 - dot(a,muY2*ones(n,1));

    Y2est(:,j) = a0 + dot(repmat(a',N,1),X2,2);

    emplmmse2(j) = mean((Y2 - Y2est(:,j)).^2);

    actlmmse2(j) = varY2 - CXY'*a;

    leg(j) = "\sigma_Y^2 = " + j + ", \sigma_R^2 = " + j;
end

sz = 25;

figure;
for j = 1:m
    scatter(emplmmse2(j),actlmmse2(j),sz,j,'filled','DisplayName',leg(j));
    hold on;
end
xlabel("Empirical LMMSE");
ylabel("Theorhetical LMMSE");
legend('location','northwest');
grid on;
title("LMMSE with " + n + " observations");
\end{verbatim}

\end{document}


\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,graphicx}
\usepackage{../setspace}
\addtolength{\textwidth}{1.5in}
\addtolength{\hoffset}{-1in}
\addtolength{\textheight}{1.5in}
\addtolength{\voffset}{-1in}

\title{STAT3401: Lab exercises concerning principal components}
\author{Paul Hewson}
\date{11th January 2007}
\usepackage{/usr/share/R/share/texmf/Sweave}
\begin{document}
\setlength{\parindent}{0pt}
\setlength{\parskip}{12pt}
\sffamily
\maketitle

Do note that the Heptathalon data, and code for two functions are available in the portal!

First, the data needs to be removed from the portal, and loaded into \textbf{R}.   Then, as explained in the lecture, some rescaling is necessary:

\begin{Schunk}
\begin{Sinput}
> hept.df <- read.csv("Heptathalon.csv", row.names = 1)
> hept.df$X100mHurdles.S. <- hept.df$X100mHurdles.S. * -1
> hept.df$X200m.sec. <- hept.df$X200m.sec. * -1
> hept.df$r800m.s. <- hept.df$r800m.s. * -1
\end{Sinput}
\end{Schunk}

You may like to consider whether this particular rescaling is a good one.   Perhaps it would have been better to take reciprocals?

\section{A principal components analysis}

Essentially, all we need to carry out a principal component analysis are the eigenvalues and eigenvectors of the covariance or correlation matrix.   This is as simple as:

\begin{Schunk}
\begin{Sinput}
> hept.cormat <- cor(hept.df[,-1])
> hept.covmat <- cov(hept.df[,-1])
> hep1.ev <- eigen(hept.cormat)
> hep1.ev
> hep2.ev <- eigen(hept.covmat)
> hep2.ev
\end{Sinput}
\end{Schunk}

And a scree plot can be quite easily obtained using \verb+plot(hep1.ev$values, type = "b")+

\section{Using the in-built functions}

In practice, we will use the inbuilt functions.   There is an older function, \texttt{princomp()}, which is retained partly for compatibility with S-Plus and partly because it allows us to provide a covariance matrix to the function (such as a robust estimate rather than the usual estimator).   The advantage of using the inbuilt functions are that the objects created have various methods associated.

\begin{Schunk}
\begin{Sinput}
> hept.princomp <- princomp(hept.df[,-1], scale = TRUE)
> summary(hept.princomp)
> plot(hept.princomp) ## produces scree plot
> biplot(hept.princomp) ## produces biplot
> loadings(hept.princomp) ## pretty printed
> predict(hept.princomp) ## scores
> par(mfrow = c(3,3))
> apply(predict(hept.princomp), 2, qqnorm)
\end{Sinput}
\end{Schunk}

The latter two lines are a reminder that if the original data were multivariate normal, then all the infinite linear combinations (including the principal components) will be univariate normal.

\begin{itemize}
\item Why did we add the instruction \verb+scale = TRUE+ in the call to \verb+princomp()+.   What is the analagous instruction to the command \verb+prcomp()+
\item How many components do you wish to retain in the heptathalon analysis?
\item Can you interpret the principal components (by altering the arguments to \verb+choices+ you can examine different variables, e.g.  \verb+biplot(hept.princomp,choices = c(1,3))+)?
\item Do the principal component scores suggest a different method for awarding medals (details on the actual winner can be found on the web)?
\end{itemize}

\begin{itemize}
\item Before moving on, it is a very good idea to conduct a principal component analysis of the US Arrests data: contrast the scree plot from an analysis based on the covariance matrix and an analysis based on the correlation matrix?   What information does the former analysis give you?
\end{itemize}

\section{Using \texttt{prcomp()}}

\texttt{prcomp()} uses the original data, and do note that it returns the square roots of the eigenvalues (in principle at least).   We use this routine to examine the turtles data:

\begin{Schunk}
\begin{Sinput}
> library(Flury)
> data(turtles)
\end{Sinput}
\end{Schunk}

The helpfile gives you one suggestion for eda.   We follow Flury (1997) below, and consider only the males.   We take natural logarithms and multiply the values by 10.

\begin{Schunk}
\begin{Sinput}
> data(turtles)
>   turtles.m <- subset(turtles, turtles$Gender == "Male")
>   turtles.m <- 10 * log(turtles.m[,-1])
>   turtles.m.prcomp <- prcomp(turtles.m)
>   summary(turtles.m.prcomp)
> plot(turtles.m.prcomp)
> turtles.m.prcomp$sdev^2 ## extract eigenvalues
> par(xpd = NA)
> biplot(turtles.m.prcomp)
\end{Sinput}
\end{Schunk}

\begin{itemize}
\item How many principal components do you need to retain here?
\item How do you interpret the principal components?
\end{itemize}

\section{Aside: the value of reducing dimensions}

Here's a lot of code we've seen before, it produces a three cluster solution from five variables collected on mammalian milk.   We tried plotting the cluster solutions against the original data - but perhaps we don't need a five dimensional representation.   The last five lines try plotting the cluster solution against the first two principal components.   Some of the inbuilt visulisation functions in the \texttt{cluster} library do this for you!

\begin{Schunk}
\begin{Sinput}
> library(cluster)
> library(flexclust)
> data(milk)
> milk.dist <- dist(milk)
> milk.hclust <- hclust(milk.dist)
> plot(milk.hclust)
> milk.cut <- cutree(milk.hclust, 3)
> z <- predict(prcomp(milk, scale = TRUE)) ##a
> plot(z, col = milk.cut, pch = milk.cut, main = "a")
> ## run the command windows() to compare these side by side
> z <- predict(prcomp(milk, scale = FALSE)) ##b
> plot(z, col = milk.cut, pch = milk.cut, main = "b")
\end{Sinput}
\end{Schunk}

\begin{itemize}
\item What's the difference between graph $a$ and $b$.
\end{itemize}

\section{Further routines for considering the number of dimensions needed}

(R code for the two functions, \texttt{Horn()} and \texttt{stickometer()} is available from the portal - improvements and suggestions are most welcome)

Here we consider Horn's method for simulating from the sample covariance matrix:

\begin{Schunk}
\begin{Sinput}
> require(MASS)
> Horn <- function(data, reps){
+   p <- dim(data)[2]
+   n <- dim(data)[1]
+   Varmat <- matrix(0,p,p)
+   Mean <- mean(data)
+   diag(Varmat) <- diag(var(data))
+     Evals <- princomp(data, cor = TRUE)$sdev^2
+     idx <- barplot(Evals, names.arg = paste("PC", c(1:7)),
+     xlab = "Component", ylab = "Proportion of trace",
+     main = "Proportion of trace explained")
+       results <- matrix(0,reps,p)
+       for (i in 1:reps){
+       SimData <- mvrnorm(n, Mean, Varmat)
+       ExpEvalsH <- princomp(SimData, cor = TRUE)$sdev^2
+       results[i,] <- ExpEvalsH
+       lines(idx, ExpEvalsH, type = "b", pch = 16)
+       }
+     lines(idx, apply(results, 2, mean), type = "b", col = "red")
+   legend("topright", lty = 1, pch = 16, legend = "Expected values")
+   Results <- data.frame(Evals = Evals, ExpEvalsH = ExpEvalsH)
+ }
\end{Sinput}
\end{Schunk}

Having entered this function (it is available in the portal), it can be used on the heptathalon data simply by entering:

\begin{Schunk}
\begin{Sinput}
> Horn(hept.df[-1], 10)
\end{Sinput}
\end{Schunk}
\includegraphics{STAT3401Week5PCAlab-hornhept}
to get ten replicates.  You may wish to consider more than 10.

In a similar way, we can get Joliffe's stick estimates as follows:

\begin{Schunk}
\begin{Sinput}
>  stickometer <- function(p){
+   vec <- 1 / (1:p)
+   stick <- vector("numeric", p)
+   stick[1] <- sum(vec)
+      for (i in 2:p){
+      stick[i] <- sum(vec[-(1:(i-1))])}
+   stick <- 1/p * stick
+   names(stick) <- paste("Comp.", c(1:p), sep = "")
+   return(stick)
+ }
\end{Sinput}
\end{Schunk}

And so, for the heptathalon data (we created the \texttt{hept.princomp} object earlier), we can create a barplot of the proportion of variance explained and superimpose a line from the expected values as follows:

\begin{Schunk}
\begin{Sinput}
>  stick <- stickometer(7)
>  proptrace <- hept.princomp$sdev^2 / sum(hept.princomp$sdev^2)
>  stick ## checking the values
>  proptrace ## checking the values
>  idx <- barplot(proptrace, names.arg = paste("PC", c(1:7)),
+  xlab = "Component", ylab = "Proportion of trace",
+  main = "Proportion of trace explained")
>  lines(idx, stick, type = "b", pch = 16)
>  legend("topright", lty = 1, pch = 16, legend = "Expected values")
\end{Sinput}
\end{Schunk}
\includegraphics{STAT3401Week5PCAlab-stickhept}

It would be nice to see this laid out properly!

\begin{itemize}
\item Does either method alter your recommendation as to how many principal components should be retained?
\item What alterations need to be be made to the line:
\begin{verbatim}
proptrace <- hept.princomp$sdev^2 / sum(hept.princomp$sdev^2)
\end{verbatim}
in order to work with \texttt{prcomp()} objects?
\end{itemize}

\section{Examining the Mahalanobis distance}

The following function will partition the Mahalanobis distance between the retained and non-retained components:

\begin{Schunk}
\begin{Sinput}
> princomp2dist <- function(obj.princomp, retain){
+  scores <- t(t(obj.princomp$scores^2) / obj.princomp$sdev)
+  dtot <- apply(scores, 1, sum)
+  d1 <- apply(scores[,c(1:retain)], 1, sum)
+  d2 <- apply(scores[,-c(1:retain)], 1, sum)
+  dists <- data.frame(dtot = dtot, d1 = d1, d2 = d2)
+  return(dists)
+ }
\end{Sinput}
\end{Schunk}

So for example, if we wanted to retain three components from the heptathalon data we could use the following:

\begin{Schunk}
\begin{Sinput}
> hept.princomp <- princomp(hept.df[-1], scores = TRUE, scale = TRUE)
> ## form a princomp object
> hept.m <- princomp2dist(hept.princomp, 3)
\end{Sinput}
\end{Schunk}

All that remains to be done is to plot the retained distances as a suitable qq plot.   One could also consider a scatter plot of retained versus non-retained distances to see if there are any individuals who are not well explained by the 3 dimensional representation using something like \verb+plot(hept.m$d1, hept.m$d2)+

\end{document}


\section{Simplicial Homology}
\subsection{Oriented Simplices and Boundary Homomorphism}
We slightly modify our definition of simplex by associating each simplex with an orientation.
For a simplex $\langle v_0,\ldots,v_n\rangle$, we can shuffle the vertices around which defines an action of $S_{n+1}$ on it.
Write $A_{n+1}\unlhd S_{n+1}$ as the alternating group, then it also acts on the simplex by restriction.
An orientation on $\sigma=\langle v_0,\ldots,v_n\rangle$ is a choice of ordering defined up to the action of $A_{n+1}$.
So we can now view $\langle v_0,\ldots,v_n\rangle$ as a simplex with an orientation.
\begin{example}
    $0$-simplex does not have a notion of orientation.\\
    There are two possible orderings of $1$-simplex, namely $\langle v_0,v_1\rangle$ and $\langle v_1,v_0\rangle$ which have different orientation.\\
    For $2$-simplices, again there are two orientations and they are $\langle v_0,v_1,v_2\rangle=\langle v_1,v_2,v_0\rangle=\langle v_2,v_0,v_1\rangle$ and $\langle v_1,v_0,v_2\rangle=\langle v_0,v_2,v_1\rangle=\langle v_2,v_1,v_0\rangle$.
\end{example}
So afterwards when we mention simplex we mean oriented simplex.
\begin{definition}
    Let $K$ be a simplicial complex, we define the group $C_n(K)$ of $n$-chains to be the free abelian group generated by the simplices of dimension $n$ in $K$, i.e.
    $$C_n(K)=\bigoplus_{\sigma\in K,\dim\sigma=n}\langle\sigma\rangle$$
    where $\langle\sigma\rangle$ is the free group generated by $\sigma$.
\end{definition}
We will convention that we have made our choice of some orientation.
For a simplex $\sigma$ in this orientation, the same simplex in the other orientation is denoted $\bar\sigma$.
In $\langle\sigma\rangle$, we identify $\bar\sigma$ by $-\sigma$.
\begin{definition}
    The $n^{th}$ boundary homomorphism is a homomorphism $\partial=\partial_n:C_n(K)\to C_{n-1}(K)$ induced by
    $$\partial_n\langle v_0,\ldots,v_n\rangle = \sum_{i=0}^n(-1)^i\langle v_1,\ldots,\hat{v}_i,\ldots,v_n\rangle$$
    where $\langle v_1,\ldots,\hat{v}_i,\ldots,v_n\rangle=\langle v_0,\ldots,v_{i-1},v_{i+1},\ldots,v_n\rangle$.
\end{definition}
\begin{example}
    $\partial\langle v_0,v_1\rangle=\langle v_1\rangle-\langle v_0\rangle$.
    $\partial\langle v_0,v_1,v_2\rangle=\langle v_1,v_2\rangle-\langle v_0,v_2\rangle+\langle v_0,v_1\rangle=\langle v_1,v_2\rangle+\langle v_2,v_0\rangle+\langle v_0,v_1\rangle$ which is indeed the (oriented) topological boundary of the simplex $\langle v_0,v_1,v_2\rangle$.
\end{example}
\begin{remark}
    We have $\partial\bar\sigma=-\partial\sigma$.
\end{remark}
\subsection{The Homology Groups of Simplicial Complexes}
\begin{definition}
    Let $K$ be a simplicial complex and $n\in\mathbb Z$, the group of $n$-cycles of $K$ is $Z_n(K)=\ker\partial_n$.
    The group of $n$-boundaries is $B_n(K)=\operatorname{Im}\partial_{n+1}$.
\end{definition}
\begin{lemma}
    $B_n(K)\subset Z_n(K)$, or in other words $\partial_{n-1}\circ\partial_n=0$.
\end{lemma}
\begin{proof}
    Pick an $n$-simplex $\langle v_0,\ldots,v_n\rangle$, then
    \begin{align*}
        \partial_{n-1}\circ\partial_n(\langle v_0,\ldots,v_n\rangle)&=\partial_{n-1}\left( \sum_{i=0}^n(-1)^i\langle v_0,\ldots,\hat{v}_i,\ldots,v_n\rangle \right)\\
        &=\sum_{i=0}^n(-1)^i\partial_{n-1}(\langle v_0,\ldots,\hat{v}_i,\ldots,v_n\rangle)\\
        &=\sum_{j<i}(-1)^j(-1)^i\langle v_0,\ldots,\hat{v}_j,\ldots,\hat{v}_i,\ldots,v_n\rangle\\
        &\quad+\sum_{j>i}(-1)^{j-1}(-1)^i\langle v_0,\ldots,\hat{v}_i,\ldots,\hat{v}_j,\ldots,v_n\rangle\\
        &=0
    \end{align*}
    as desired.
\end{proof}
\begin{definition}
    The $n^{th}$ (simplicial) homology group of the simplicial complex $K$ is defined as $H_n(K)=Z_n(K)/B_n(K)$.
\end{definition}
\begin{example}
    Take the simplicial complex $K$ generated by boundary of a $2$-simplex (which can be taken as a triangulation of $S^1$).
    Then $C_0(K)\cong C_1(K)\cong\mathbb Z^3$ and $C_n(K)=0$ for all $n>1$.
    So we only need to understand $\partial_1$.
    Say its $0$-simplices are $\langle v_0\rangle,\langle v_1\rangle,\langle v_2\rangle$ and $1$-simplices are $\langle v_0,v_1\rangle,\langle v_1,v_2\rangle,\langle v_2,v_0\rangle$, then $\partial_1:C_1(K)\to C_0(K)$ maps
    \begin{align*}
        \langle v_0,v_1\rangle &\mapsto \langle v_1\rangle-\langle v_0\rangle\\
        \langle v_1,v_2\rangle &\mapsto \langle v_2\rangle-\langle v_1\rangle\\
        \langle v_2,v_0\rangle &\mapsto \langle v_0\rangle-\langle v_2\rangle
    \end{align*}
    So if we take the free abelian groups as free $\mathbb Z$-modules generated by the simplices, then $\partial_1$ has the matrix
    $$\begin{pmatrix}
        -1&0&1\\
        1&-1&0\\
        0&1&-1
    \end{pmatrix}$$
    So $Z_1(K)=\ker\partial_1=\langle(1,1,1)\rangle=\langle \langle v_0,v_1\rangle +\langle v_1,v_2\rangle + \langle v_2,v_0\rangle\rangle$ and $B_1(K)=\ker\partial_2=0$, therefore $H_1(K)\cong\mathbb Z$.\\
    As for the $H_0(K)$, we have $Z_0(K)=\ker\partial_0=C_0(K)\cong\mathbb Z^3$ and $B_0(K)=\operatorname{Im}\partial_1\cong\langle (-1,1,0),(0,-1,1)\rangle$, hence $H_0(K)\cong\mathbb Z^3/\langle (-1,1,0),(0,-1,1)\rangle\cong\mathbb Z$.
    And $H_n(K)=0$ for $n>1$.
\end{example}
\begin{example}
    Take $L$ the simplicial complex generated by the $2$-simplex (so it is a solid triangle which is a triangulation of the closed unit disk).
    Then $C_0(L)=C_0(K),C_1(L)=C_1(K)$ but $C_2(L)=\langle\langle v_0,v_1,v_2\rangle\rangle$ where $K$ is as in the previous example.
    Now $\partial_2(\langle v_0,v_1,v_2\rangle)=\langle v_0,v_1\rangle+\langle v_1,v_2\rangle+\langle v_2,v_0\rangle$ which is the generator of $\ker\partial_1$, therefore $Z_1(L)=B_1(L)$ and hence $H_1(L)=0$.
    Note that $L,K$ coincides on $0$ and $1$-simplices they contain, so $H_0(L)\cong H_0(K)\cong\mathbb Z$.
    Now easily $Z_2(L)=\ker\partial_2=0$ and therefore $H_2(L)=0$, therefore $H_0(L)\cong\mathbb Z$ and $H_n(L)=0$ for any $n\neq 0$.
\end{example}
\begin{lemma}
    Let $K$ be a simplicial complex.
    If $d$ is the number of path components of $|K|$, then $H_0(K)\cong\mathbb Z^d$.
\end{lemma}
\begin{proof}
    Denote by $\pi_0(K)$ the set of path-connected components of $|K|$.
    Write $\mathbb Z[A]$ as the free abelian group generated by a set $A$, then $\mathbb Z[\pi_0(K)]\cong\mathbb Z^d$.
    Consider a $q:C_0(K)\to\mathbb Z[\pi_0(K)]$ sending a vertex $\langle v\rangle$ to the path component containing $v$ and extend it to a homomorphism.
    Then $q$ is surjective.
    Note that $\partial_0=0$, so $Z_0(K)=C_0(K)$, therefore $H_0(K)\cong C_0(K)/B_0(K)$, therefore it suffices to show that $\ker q=B_0(K)$.
    If $\langle v_0,v_1\rangle\in C_1(K)$, then $q\circ\partial_1(\langle v_0,v_1\rangle)=q(\langle v_1\rangle-\langle v_1\rangle)=0$ since $v_0$ and $v_1$ are joined by a path.
    This means that $B_0(K)\subset\ker q$.
    Conversely, $\ker q$ is generated by elements of $C_0(K)$ of the form $\langle w\rangle-\langle v\rangle$ where $v,w$ are in the same path components of $|K|$.
    But then there is a sequence of vertices $v=v_1,v_2,\ldots,v_n=w$ in $K$ with $\langle v_i,v_{i+1}\rangle\in K$.
    Then
    $$\langle w\rangle-\langle v\rangle=(\langle v_n\rangle-\langle v_{n-1}\rangle)+(\langle v_{n-1}\rangle-\langle v_{n-2}\rangle)\cdots +(\langle v_2\rangle-\langle v_1\rangle)\in\operatorname{Im}\partial_1=B_0(K)$$
    Hence $\ker q\subset B_0(K)$.
    This completes the proof.
\end{proof}
\begin{remark}
    There is a very rough analogy between $\pi_1(|K|)$ and $H_1(K)$ as $B_1(K)$ is kind of homotopies between the loops in $Z_1(K)$.
    They are of course not the same as $H_1(K)$ is always abelian, but in fact, there does exist a certain connection as one can show that $H_1(K)\cong\pi_1(|K|)^{\operatorname{ab}}$.
\end{remark}
\subsection{Chain Maps and Homotopies}
We want to understand the maps on homology that is induced by maps of simplicial complexes.
\begin{definition}
    A chain complex $C_\bullet$ is a sequence of abelian groups $C_n,n\in\mathbb Z$ with homomorphisms $\partial_n:C_n\to C_{n-1}$ such that $\partial_{n-1}\circ\partial_n=0$ for all $n$.\\
    A chain map $f_\bullet:C_\bullet\to D_\bullet$ between chain complexes is a collection of homomorphisms $f_n:C_n\to D_n$ indexed by $n\in\mathbb Z$ such that
    \[
        \begin{tikzcd}
            C_n\arrow{r}{\partial_n}\arrow[swap]{d}{f_n}&C_{n-1}\arrow{d}{f_{n-1}}\\
            D_n\arrow[swap]{r}{\partial_n}&D_{n-1}
        \end{tikzcd}
    \]
    commutes for any $n$.
\end{definition}
Homology usually deals with the cases where nontrivial groups only occur at nonnegative $n$.
In these situations, we can just define $C_n$ for $n\ge 0$ and $\partial_n$ for $n\ge 1$ -- because those are what we care about -- and leave the rest of the groups and boundary maps to be zero.
This will be the case for the simplicial complexes.
\begin{definition}
    Given a chain complex $C_\bullet$, we define the group of $n$-cycles to be $Z_n(C_\bullet)=\ker\partial_n$ and the group of $n$-boundaries to be $B_n(C_\bullet)=\operatorname{Im}\partial_n$.
    Then $B_n(C_\bullet)\unlhd Z_n(C_\bullet)$, so we define the $n^{th}$ homology group is then $H_n(C_\bullet)=Z_n(C_\bullet)/B_n(C_\bullet)$.
\end{definition}
\begin{lemma}
    If $f_\bullet:C_\bullet\to D_\bullet$ is a chain map, then for any $n\in\mathbb Z$, we have a well-defined homomorphism $f_\ast:H_n(C_\bullet)\to H_n(D_\bullet)$ via $[c]\mapsto [f_n(c)]$ for $c\in Z_n(C_\bullet)$.
\end{lemma}
\begin{proof}
    Suffices to show that the map is well-defined.
    For $c\in Z_n(C_\bullet)$, we have $\partial_n\circ f_n(c)=f_{n-1}\circ\partial_n(c)=0$, so indeed $f_n(c)\in Z_n(D_\bullet)$.
    Also, if $c\in B_n(C_\bullet)$, then there is some $c'\in C_{n-1}$ such that $c=\partial_{n+1}(c')$, so $f_n(c)=f_n\circ\partial_{n+1}(c')=\partial_{n+1}\circ f_{n+1}(c')$, therefore $f_n(c)\in B_n(D_\bullet)$.
    Hence $f_\ast$ is well-defined.
\end{proof}
\begin{lemma}
    A simplicial map $f:K\to L$ induces a chain map $f_\bullet:C_\bullet(K)\to C_\bullet(L)$ via
    $$f_n:\sigma\mapsto\begin{cases}
        f(\sigma)\text{, if $\dim f(\sigma)=n$}\\
        0\text{, otherwise}
    \end{cases}$$
    for $\sigma\in K,\dim\sigma=n$.
    Hence for each $n\in\mathbb N$, $f_\bullet$ induces a homomorphism $f_\ast:H_n(K)\to H_n(L)$.
\end{lemma}
\begin{proof}
    We need to show that $\partial_n\circ f_n=f_{n-1}\circ\partial_n$.
    Suffices to demonstrate this on generators.
    Let $\sigma=\langle v_0,\ldots,v_n\rangle$.
    If $\dim f(\sigma)=n$, then $f(\sigma)=\langle f(v_0),\ldots,f(v_n)\rangle$, then there is a one-to-one correspondence between faces of $\sigma$ and fases of $f(\sigma)$, hence necessarily $f_{n-1}\circ\partial_n(\sigma)=\partial_n\circ f_n(\sigma)$.
    If $\dim f(\sigma)\le n-2$, then $f_{n-1}\circ\partial_n(\sigma)=0=\partial_n\circ f_n(\sigma)$.
    We are left with the case $\dim f(\sigma)=n-1$.
    Assume $f(v_0)=f(v_1)$ and $f(v_1),\ldots,f(v_n)$ are all distinct.
    In that case $f(\langle v_0,\ldots,v_n\rangle)=f(\langle v_1,\ldots,v_n\rangle)$.
    We know that $f_n(\sigma)=0$, so $\partial_n\circ f_n(\sigma)=0$.
    Now
    \begin{align*}
        f_{n-1}\circ\partial_n(\sigma)&=f_{n-1}\left(\sum_{i=0}^n(-1)^i\langle v_0,\ldots,\hat{v}_i,\ldots,v_n\rangle\right)\\
        &=\sum_{i=0}^n(-1)^if_{n-1}(\langle v_0,\ldots,\hat{v}_i,\ldots,v_n\rangle)\\
        &=f_{n-1}(\langle v_1,\ldots,v_n\rangle)-f_{n-1}(\langle v_0,v_2,\ldots,v_n\rangle)\\
        &=0
    \end{align*}
    Therefore $\partial_n\circ f_n(\sigma)=0=f_{n-1}\circ\partial_n(\sigma)$ too, which means $f_\bullet$ is indeed a chain map.
\end{proof}
\begin{remark}
    If $f:K\to L$ and $g:L\to M$ are simplicial maps, then $(g\circ f)_\ast=g_\ast\circ f_\ast$.
    Also, if $K$ is a simplicial complex, then $(\operatorname{id}_K)_\ast=\operatorname{id}_{H_n(K)}$.
\end{remark}
A natural question is that when do chain maps induce the same maps on homology.
\begin{definition}
    Let $f_\bullet,g_\bullet:C_\bullet\to D_\bullet$ be chain maps.
    A chain homotopy $h_\bullet$ between $f_\bullet$ and $g_\bullet$ is a collection of homomorphisms $h_n:C_n\to D_{n+1}$ such that $g_n(c)-f_n(c)=\partial_{n+1}\circ h_n(c)+h_{n-1}\circ\partial_n(c)$.
    We say $f_\bullet$ and $g_\bullet$ are chain homotopic, written as $f_\bullet\simeq g_\bullet$ if such $h_\bullet$ exists.
\end{definition}
\begin{lemma}
    If $f_\bullet\simeq g_\bullet:C_\bullet\to D_\bullet$, then $f_\ast=g_\ast$.
\end{lemma}
\begin{proof}
    Let $c\in Z_n(C_\bullet)$, then $g_n(c)-f_n(c)=\partial_{n+1}\circ h_n(c)+h_{n-1}\circ\partial_n(c)=\partial_{n+1}\circ h_n(c)\in B_n(D_\bullet)$, therefore $[g_n(c)]=[f_n(c)]$.
\end{proof}
\begin{example}
    Consider the triangle $K$ and the line segment $L$, both as simplicial complexes in the obvious way.
    Say the vertices in $K$ are $e_0,e_1,e_2$ and those in $L$ are $e_0,e_1$ and let $i:L\to K$ be the natural inclusion $e_0\mapsto e_0,e_1\mapsto e_1$, $r$ be the simplicial retraction $e_0\mapsto e_0,e_1\mapsto e_1,e_2\mapsto e_0$, both as simplicial maps.
    Now $r\circ i=\operatorname{id}_L$, but $i\circ r\neq \operatorname{id}_K$.
    However, we can define a chain homotopy between $(i\circ r)_\bullet$ and $\operatorname{id}_{C_\bullet(K)}$.
    This would be given by $h_\bullet$ which is everywhere zero except $h_0(\langle e_2\rangle)=\langle e_2,e_0\rangle$ and $h_1(\langle e_1,e_2\rangle)=-\langle e_0,e_1,e_2\rangle$ which works since
    \begin{align*}
        (\partial_1\circ h_0+h_{-1}\circ\partial_0)(\langle e_2\rangle)&=\partial_1(\langle e_2,e_0\rangle)\\
        &=\langle e_0\rangle-\langle e_2\rangle\\
        &=(i_0\circ r_0-\operatorname{id}_{C_0(K)})(\langle e_2\rangle)\\
        (\partial_2\circ h_1+h_0\circ\partial_1)(\langle e_1,e_2\rangle)&=\partial_2(-\langle e_0,e_1,e_2\rangle)+h_0(\langle e_2\rangle-\langle e_1\rangle)\\
        &=-\langle e_0,e_1\rangle-\langle e_1,e_2\rangle-\langle e_2,e_0\rangle+\langle e_2,e_0\rangle\\
        &=(i_1\circ r_1-\operatorname{id}_{C_1(K)})(\langle e_1,e_2\rangle)\\
        (\partial_3\circ h_2+h_1\circ\partial_2)(\langle e_0,e_1,e_2\rangle)&=h_1(\langle e_0,e_1\rangle+\langle e_1,e_2\rangle+\langle e_2,e_0\rangle)\\
        &=-\langle e_0,e_1,e_2\rangle\\
        &=(i_2\circ r_2-\operatorname{id}_{C_2(K)})(\langle e_0,e_1,e_2\rangle)
    \end{align*}
    Therefore $(i\circ r)_\ast=i_\ast\circ r_\ast$ would just be the identity on $H_n(K)$.
    In particular, $r_\ast$ is an isomorphism on the homology groups.
\end{example}
\begin{definition}
    A simplicial complex $K$ is a cone if there is a vertex $x_0$ such that for all other simplices $\tau\in K$, there exists $\sigma\in K$ such that $x_0\in\sigma$ and $\tau\le\sigma$.
\end{definition}
Perhaps nonsurprisingly,
\begin{lemma}
    If $K$ is a cone, then $H_0(K)\cong\mathbb Z$ and $H_n(K)=0$ if $n\neq 0$.
\end{lemma}
\begin{proof}
    Let $i:\{\langle x_0\rangle\}\to K$ be the obvious inclusion and $r:K\to\{\langle x_0\rangle\}$ be constant.
    Then $r\circ i=\operatorname{id}_{\{\langle x_0\rangle\}}$, therefore $r_\ast\circ i_\ast=\operatorname{id}_{H_n(\{\langle x_0\rangle\})}$.
    We shall show that $i_\ast\circ r_\ast=\operatorname{id}_{H_n(K)}$ which implies $H_n(K)\cong H_n(\{\langle x_0\rangle\})$ from where the result follows.\\
    We will build a chain homotopy between $\operatorname{id}_{C_\bullet(K)}$ and $i_\bullet\circ r_\bullet$ where $i_\bullet$ and $r_\bullet$ are the induced chain maps.
    Let $\sigma=\langle v_0,\ldots,v_n\rangle\in K$, then we define
    $$h_n(\sigma)=\begin{cases}
        0\text{, if $x_0\in\sigma$}\\
        \langle x_0,v_0,\ldots,v_n\rangle\text{, otherwise}
    \end{cases}$$
    which is well-defined as $K$ is a cone with vertex $x_0$.
    We want to show that $\partial_{n+1}\circ h_n+h_{n-1}\circ\partial_n=\operatorname{id}_{C_n(K)}-i_n\circ r_n$.
    Suppose $n>0$ and $x_0\notin\sigma$, then
    \begin{align*}
        &\quad(\partial_{n+1}\circ h_n+h_{n-1}\circ\partial_n)(\sigma)\\
        &=\partial_{n+1}(\langle x_0,v_0,\ldots,v_n\rangle)+h_{n-1}\left( \sum_{i=0}^n(-1)^i\langle v_0,\ldots,\hat{v}_i,\ldots,v_n \rangle\right)\\
        &=\langle v_0,\ldots,v_n\rangle+\sum_{i=0}^n(-1)^{i+1}\langle x_0,v_0,\ldots,\hat{v}_i,\ldots,v_n \rangle\\
        &\quad+\sum_{i=0}^n(-1)^i\langle x_0,v_0,\ldots,\hat{v}_i,\ldots,v_n \rangle\\
        &=\langle v_0,\ldots,v_n\rangle=\sigma\\
        &=(\operatorname{id}_{C_n(K)}-i_n\circ r_n)(\sigma)
    \end{align*}
    which works.
    If $n>0$ but $x_0\in\sigma$, then $x_0=v_j$ for some $j$.
    Consequently,
    \begin{align*}
        &\quad(\partial_{n+1}\circ h_n+h_{n-1}\circ\partial_n)(\sigma)\\
        &=0+h_{n-1}\left( \sum_{i=0}^n(-1)^i\langle v_0,\ldots,\hat{v}_i,\ldots,v_n \rangle\right)\\
        &=(-1)^j\langle x_0=v_j,v_0,\ldots,\hat{v}_j,\ldots,v_n\rangle\\
        &=\langle v_0,\ldots,v_n\rangle=\sigma\\
        &=(\operatorname{id}_{C_n(K)}-i_n\circ r_n)(\sigma)
    \end{align*}
    The case $n=0$ is trivial.
    Therefore $h_\bullet$ is indeed a chain homotopy as desired.
\end{proof}
\begin{example}
    Any $n$-simplex $K$ is a cone, so by the lemma
    $$H_n(K)\cong\begin{cases}
        \mathbb Z\text{, if $n=0$}\\
        0\text{, otherwise}
    \end{cases}$$
    Take $L=\partial\sigma_n$ be the boundary of the $n$-simplex, then $|L|\cong S^{n-1}$ for $n\ge 2$.
    The obvious inclusion $L\hookrightarrow K$ of simplicial complexes induces a chain map
    \[
        \begin{tikzcd}
            0\arrow{r}&0\arrow{r}\arrow{d}&C_{n-1}(L)\arrow{r}{\partial_{n-1}}\arrow[equal]{d}&\cdots\arrow{r}{\partial_1}&C_0(L)\arrow{r}\arrow[equal]{d}&0\\
            0\arrow{r}&C_n(K)\arrow[swap]{r}{\partial_n}&C_{n-1}(K)\arrow[swap]{r}{\partial_{n-1}}&\cdots\arrow[swap]{r}{\partial_1}&C_0(K)\arrow{r}&0
        \end{tikzcd}
    \]
    So evidently $H_d(L)\cong H_d(K)$ for $d\le n-2$.
    Since $H_{n-1}(K)=0$, we have $Z_{n-1}(L)=Z_{n-1}(K)=B_{n-1}(K)$.
    Also $B_{n-1}(L)=0$, therefore $H_{n-1}(L)\cong Z_{n-1}(L)=B_{n-1}(K)$.
    But this is easy enough to calculate:
    $K$ only has one $n$-simplex $\sigma$, so $C_n(K)=\mathbb Z\sigma$ and hence $B_{n-1}\cong\mathbb Z$ as $\partial_n$ has to be injective.
    Thus
    $$H_d(L)=\begin{cases}
        \mathbb Z\text{, if $d=0$ or $d=n-1$}\\
        0\text{, otherwise}
    \end{cases}$$
    This means homology can actually detect ``higher dimensional holes''.
\end{example}
\subsection{Continuous Maps and Homotopies}
For a map $\phi:|K|\to|L|$, we want to associate to it a homomorphism $\phi_\ast:H_n(K)\to H_n(L)$.
Note that a chief difficulty in this is that $\phi$ may not contain much information about the structures of $K,L$ as simplicial complexes, since it is just a continuous map between topological spaces.
The idea is to use a simplicial approximation.
That is, instead of looking for $\phi_\ast:H_n(K)\to H_n(L)$ directly, we seek $\phi_\ast:H_n(K^{(r)})\to H_n(L^{(r)})$ for sufficiently large $r$ and show that $H_n(K^{(r)})\cong H_n(K)$ for any simplicial complex $K$ and $r\in\mathbb N$.
Eventually, we will show that $H_n(K)$ only depends on $|K|$.\\
First step on that journey is the notion of homotopy on simplicial maps.
\begin{definition}
    Two simplicial maps $f,g:K\to L$ are contiguous if, for every $\sigma\in K$, there exists some $\tau\in L$ such that $f(\sigma)$ and $g(\sigma)$ are both faces of $\tau$.
\end{definition}
\begin{remark}
    Suppose given $\phi:|K|\to |L|$ with $f,g:K\to L$ different simplicial approximations to $\phi$.
    Then choose any $x\in\sigma^\circ,\phi(x)\in\tau^\circ$, we know that $f(\sigma)\le\tau$ and $g(\sigma)\le\tau$ and hence $f,g$ are contiguous.
\end{remark}
\begin{lemma}
    If $f,g:K\to L$ are contiguous, then $f_\ast=g_\ast:H_n(K)\to H_n(L)$ for all $n$.
\end{lemma}
\begin{proof}
    We shall construct a chain homotopy.
    Fix a total order $<$ on vertices of $K$ and use the convention that $\sigma=\langle v_0,\ldots,v_n\rangle$ is oriented in such a way that $v_0<\ldots<v_n$.
    We write $\langle v_0,\ldots,v_n\rangle=0$ if $v_0,\ldots,v_n$ are not in general position.
    Easy to see this is compatible with everything.
    Define $h_n:C_n(K)\to C_{n+1}(L)$ by
    $$h_n(\langle v_0,\ldots,v_n\rangle)=\sum_{i=0}^n(-1)^i\langle f(v_0),\ldots,f(v_i),g(v_i),\ldots,g(v_n)\rangle$$
    Note that whenever the summand is nonzero, it would be an $(n+1)$-simplex of $L$ since $f,g$ are contiguous.
    We have some calculations to do.
    \begin{align*}
        (\partial\circ h+h\circ\partial)(\sigma)&=\partial\left( \sum_{i=0}^n(-1)^i\langle f(v_0),\ldots,f(v_i),g(v_i),\ldots,g(v_n)\rangle \right)\\
        &\quad+h\left( \sum_{i=0}^n(-1)^i\langle v_0,\ldots,\hat{v}_i,\ldots,v_n\rangle \right)\\
        &=\sum_{i\le j}(-1)^{i+j}\langle f(v_0),\ldots,\widehat{f(v_i)},\ldots,f(v_j),g(v_j),\ldots,g(v_n)\rangle\\
        &\quad-\sum_{i\ge j}(-1)^{i+j}\langle f(v_0),\ldots, f(v_j),g(v_j),\ldots,\widehat{g(v_i)},\ldots,g(v_n)\rangle\\
        &\quad+\sum_{j<i}(-1)^{i+j}\langle f(v_0),\ldots, f(v_j),g(v_j),\ldots,\widehat{g(v_i)},\ldots,g(v_n)\rangle\\
        &\quad-\sum_{j>i}(-1)^{i+j}\langle f(v_0),\ldots,\widehat{f(v_i)},\ldots,f(v_j),g(v_j),\ldots,g(v_n)\rangle\\
        &=\sum_{i=0}^n\langle f(v_0),\ldots,f(v_{i-1}),g(v_i),\ldots,g(v_n)\rangle\\
        &\quad-\sum_{i=0}^n\langle f(v_0),\ldots,f(v_i),g(v_{i+1}),\ldots,g(v_n)\rangle\\
        &=\langle g(v_0),\ldots,g(v_n)\rangle-\langle f(v_0),\ldots,f(v_n)\rangle\\
        &=g(\sigma)-f(\sigma)
    \end{align*}
    as desired.
\end{proof}
\begin{lemma}
    Let $K$ be a simplicial complex and $K'$ be its barycentric subdivision.
    A simplicial map $s:K'\to K$ is a simplicial approximation to $\operatorname{id}_{|K|}$ iff for every $\sigma\in K$, $s(\hat\sigma)$ is a vertex of $\sigma$.
    Also, such $s$ always exists.
\end{lemma}
\begin{proof}
    Let $s:K'\to K$ be a simplicial approximation to $\operatorname{id}_{|K|}$, which just means $\operatorname{id}_{|K|}(\operatorname{St}_{K'}(\hat\sigma))\subset\operatorname{St}_K(s(\hat\sigma))$.
    In particular, $\sigma\circ\subset\operatorname{St}_K(s(\hat\sigma))$, therefore $s(\hat\sigma)$ is a vertex of $\sigma$.\\
    Conversely, suppose $s(\hat\sigma)$ be a vertex of $\sigma$ for any $\sigma\in K$.
    Choose any $\tau'\in K'$ with $\tau'^\circ\subset\operatorname{St}_{K'}(\hat\sigma)$ (i.e. $\hat\sigma$ is a vertex of $\tau'$).
    Then $\tau'^\circ$ is contained in the interior of a simplex $\tau\in K$ such that $\sigma\le\tau$.
    Thus $s(\hat\sigma)$ is also a vertex of $\tau$.
    But $\tau'^\circ\subset\tau^\circ\subset\operatorname{St}_K(s(\hat\sigma))$.
    But such $\tau'^\circ$ necessarily cover $\operatorname{St}_{K'}(\hat\sigma)$, therefore $\operatorname{id}_{|K|}(\operatorname{St}_{K'}(\hat\sigma))\subset\operatorname{St}_K(s(\hat\sigma))$ as desired.\\
    To see such an $s$ exists, we simply just need to send $\hat\sigma$ to an arbitrarily chosen vertex of $\sigma$ which, as one can verify, works.
\end{proof}
\begin{proposition}\label{barycentric_iso_homol}
    Let $s:K'\to K$ be the simplicial approximaion to the identity obtained like in the proof above, then $s_\ast:H_n(K')\to H_n(K)$ is an isomorphism for all $n$.
\end{proposition}
We will postpone this proof until more machinery is developed.
But let us see some implications first.
\begin{corollary}
    Let $K$ be a simplicial complex, then for all $r$, there is a canonical isomorphism $H_n(K)\cong H_n(K^{(r)})$.
\end{corollary}
\begin{proof}
    Suffices to make the choice for $r=1$.
    Choose a simplicial approximation $s:K'\to K$ to the identity on $|K|$ which induces an ismorphism $s_\ast:H_n(K')\to H_n(K)$ by the preceding proposition.
    To see this is canonical, we shall show that this isomorphism is independent of the choice of $s$.
    But this is obvious since any other choice $s'$ is contiguous with $s$.
\end{proof}
We write $\nu_{K,r,s}:H_n(K^{(r)})\to H_n(K^{(s)})$ as the canonical ismorphism for $r\ge s$ and write $\nu_{K,r}=\nu_{K,r,0}$.
Then $\nu_{K,r_2,r_3}\circ \nu_{K,r_1,r_2}=\nu_{K,r_1,r_3}$.
\begin{proposition}
    To each continuous map $f:|K|\to|L|$, there is an associated homomorphism $f_\ast:H_n(K)\to H_n(L)$ given by $f_\ast=s_\ast\circ\nu_{K,r}^{-1}$ where $s:K^{(r)}\to L$ is a simplicial approximation to $f$.
    This homomorphism does not depend on the choice of $r$ or $s$.
    Furthermore, if $g:|M|\to |K|$ is continuous for some other simplicial complex $M$, then $(f\circ g)_\ast=f_\ast\circ g_\ast$.
\end{proposition}
\begin{proof}
    We already know that the homomorphism does not depend on the choice of $s$.
    If $s:K^{(r)}\to L,t:K^{(q)}\to L$ are both simplicial approximations to $f$ where WLOG $r\ge q$, then let $a:K^{(r)}\to K^{(q)}$ be a simplicial approximation to the identity on $|K|=|K^{(q)}|$.
    Now $s,t\circ a:K^{(r)}\to L$ are both simplicial approximations to $f$, hence are contiguous and induces the same homomorphism $s_\ast=(t\circ a)_\ast=t_\ast\circ a_\ast=t_\ast\circ\nu_{K,r,q}$, so $s_\ast\circ\nu_{K,r}^{-1}=t_\ast\circ\nu_{K,r,q}\circ\nu_{K,r}^{-1}=t_\ast\circ\nu_{K,q}$ as desired.\\
    Now let $s:K^{r}\to L$ and $t:M^{q}\to K^{r}$ be simplicial approximations to $f,g$ respectively (here we used $|K|=|K^{(r)}|$).
    Then $s\circ t$ is a simplicial approximation of $f\circ g$, so
    $$(f\circ g)_\ast=(s\circ t)_\ast\circ\nu_{M,q}^{-1}=s_\ast\circ t_\ast\circ \nu_{M,q}^{-1}=(s_\ast\circ\nu_{K,r}^{-1})\circ(\nu_{K,r}\circ t_\ast\circ\nu_{M,q}^{-1})=f_\ast\circ g_\ast$$
    as desired.
\end{proof}
And now we finally arrive at:
\begin{corollary}
    If $|K|\cong|L|$, then $H_n(K)\cong H_n(L)$.
\end{corollary}
\begin{proof}
    Immediate.
\end{proof}
We can do even better.
\begin{lemma}
    If $L$ is a simplicial complex residing in $\mathbb R^m$, then there exists $\epsilon=\epsilon(L)>0$ such that if $f,g:|K|\to|L|$ satisfies $|f(x)-g(x)|<\epsilon$ for any $x\in |K|$, then $f_\ast=g_\ast$.
\end{lemma}
\begin{proof}
    The set $\{\operatorname{St}_L(w):w\in L\}$ forms an open cover of $|L|$.
    So by Lebesgue number lemma, there exists $\epsilon>0$ such that each ball of radius $2\epsilon$ in $L$ lies in some $\operatorname{St}_L(w)$.
    We take $\epsilon(L)=\epsilon$.
    Let $f,g:|K|\to|L|$ as in the statement and consider the open cover of $|K|$ given by $\{f^{-1}(B_\epsilon(y)):y\in L\}$ which admits $\delta>0$ such that each $B_\delta(x)$ is contained in some member of this cover again by Lebesgue number lemma.
    This would mean that $f(B_\delta(x))\subset B_\epsilon(x)$, so $g(B_\delta(x))\subset B_{2\epsilon}(y)$.
    Choose some large $r$ such that $\operatorname{mesh}(K^{(r)})<\delta/2$, then for each vertex $v\in K^{(r)}$, the diameter of $\operatorname{St}_{K^{(r)}}(v)$ is strictly less than $\delta$, so both $f(\operatorname{St}_{K^{(r)}}(v))$ and $g(\operatorname{St}_{K^{(r)}}(v))$ are contained in some $\operatorname{St}_L(w)$.
    Set $s(v)=w$, then $s$ is a simplicial approximation to both $f$ and $g$, hence $f_\ast=s_\ast\circ \nu_{K,r}^{-1}=g_\ast$.
\end{proof}
\begin{theorem}
    If two maps $f,g:|K|\to|L|$ are homotopic, then $f_\ast=g_\ast$.
\end{theorem}
\begin{proof}
    Let $H:|K|\times I\to |L|$ be the homotopu between $f,g$, so $H(\cdot,0)=f,H(\cdot,1)=g$.
    As $|K|\times I$ is compact, $H$ is uniformly continuous.
    Thus for $\epsilon=\epsilon(L)$ as in the preceding lemma, there is some $\delta>0$ such that $|H(x,s)-H(x,t)|<\epsilon$ whenever $|s-t|<\delta$.
    Now choose $0=t_0<t_1<\ldots<t_k=1$ such that $t_i-t_{i-1}<\delta$ for any $i$ and let $f_i(x)=H(x,t_i)$.
    By construction $|f_i(x)-f_{i-1}(x)|<\epsilon$ for any $x\in |K|$, therefore $(f_i)_\ast=(f_{i-1})_\ast$ for all $i$.
    In particular, $f_\ast=(f_0)_\ast=\cdots=(f_k)_\ast=g_\ast$.
\end{proof}
\begin{corollary}
    If $|K|,|L|$ are homotopy equivalent, then $H_n(K)\cong H_n(L)$ for all $n$.
\end{corollary}
\begin{proof}
    Follows directly.
\end{proof}
\begin{definition}
    We write $H_n(X)=H_n(K)$ if $X=|K|$.
\end{definition}


\label{sec:Averaging}

A {\it mesh catalog} is a set $\{ \M_0 \ldots \M_{n-1} \}$
of $n$ meshes which are all embeddings
of the same simplicial complex
(that is, they have the same vertices, edges, and faces,
but the vertex positions may be different).

\subsection{Linear weights}
\label{sec:Linear-weights}

We can compute a {\it weighted average mesh} by averaging the vertex positions:
\begin{equation}
\p_i(\w) = \sum_{j=0}^{n-1} w_j \p_{ij}
\end{equation}
where $\p_{i}; (i=0 \ldots m-1)$ is the position of the $i$th vertex of the average mesh,
and $\p_{ij}$ is the position of the $i$th vertex of the $j$th catalog element.

In {\it catalog fitting} we fit a registered weighted average mesh,
to a set of data, $\{ \d_k \in \Reals^3; k=0 \ldots p-1 \}$
by minimizing
\begin{equation}
f(\M(\Tr,\w)) = \sum_{k=0}^{p-1} \| \d_k - \Pr_{\M(\Tr,\w)} (\d_k) \|^2 ,
\end{equation}
over a family of registration transforms $\{\Tr\}$
and vectors of weights $\w \in \Reals^n$.
The vertex positions $\p(\Tr,\w) = (\p_0(\Tr,\w) \ldots  \p_{n-1}(\Tr,\w))$
of the registered average mesh, $\M(\Tr,\w)$, are given by:
\begin{equation}
\p_i(\Tr,\w) = \Tr ( \sum_{j=0}^{n-1} w_j \p_{ij} )
\end{equation}

The partial derivative with respect to $\Tr$
is:
\begin{eqnarray}
\De{\Tr}{f(\M(\Tr,\w))}{\Tr^0,\w^0}
& = &
\De{\p}{f(\M))}{\M(\Tr^0,\w^0)}
\circ
\De{\Tr}{\p(\Tr,\w)}{\Tr^0,\w^0}
\\
& = &
\De{\p}{f(\M))}{\M(\Tr^0,\w^0)}
\circ
\De{\Tr}{\Tr\p(\w^0)}{\Tr^0}
\nonumber
\end{eqnarray}

$\De{\p}{f(\M))}{\M(\Tr^0,\w^0)}$ is the derivative of $f$ with respect to
the vertex positions of of the registered average mesh.
It's value for data fitting distance functions
is given in \autoref{sec:data-fitting}.

$\Df{\Tr}{\Tr\p}$ is given in \autoref{sec:Transforms}
for various families of transforms $\{\Tr\}$: affine, euclidean, and rigid.

Using the chain rule, the partial derivative with respect to $\w$ is:
\begin{eqnarray}
\De{\w}{f(\M(\Tr,\w))}{\Tr^0,\w^0}
& = &
\De{\w}{f(\Tr(\M(\w)))}{\Tr^0,\w^0}
\\
& = &
\De{\p}{f(\M))}{\M(\Tr^0,\w^0)}
\circ
\De{\Tr}{\Tr\p(\w^0)}{\Tr^0}
\circ
\De{\w}{\p(\w)}{\w^0}
\nonumber
\end{eqnarray}

$\Df{\p}{f}$ and $\Df{\Tr}{\Tr\p}$ are already known
so we need only determine $\Df{\w}{\p(\w)}$.
Using reasoning similar
to equation \ref{eq:total-registration-transform-derivative},
we have $\p(\w) = \bigoplus_{j=0}^{m-1} \p_i(\w)$,
and
$\Df{\w}{\p(\w)}
=
\Df{\w}{\bigoplus_{j=0}^{m-1} \p_i(\w)}
=
\bigoplus_{j=0}^{m-1} \Df{\w}{\p_i(\w)}$,
so we can restrict ourselves to
$\Df{\w}{\p(\w)}$, where $\p(\w) = \sum_{i=0}^{n-1} w_i \p_i$,
and $\p, \p_i \in \Reals^3$.

Note that $\p(\w)$ is a linear transform from $\Reals^n \mapsto \Reals^3$,
which can be expressed as $\P = \sum_{i=0}^{n-1} \p_i \otimes \e_i$,
where $\e_i$ are the canonical basis vectors of $\Reals^n$.
($\P$ can be written as a matrix whose $i$th column is the vector $\p_i$.)
Thus it follows that
\begin{equation}
\Df{\w}{\p(\w)} = \Df{\w}{\P\w} = \P
\end{equation}

\subsection{Convex weights}
\label{sec:Convex-weights}

It may be desireable to restrict the weights to be convex,
that is, $0 \leq w_i \leq 1; \sum w_i = 1$.
To use unconstrained optimization methods with convex weights,
we re-parameterize:
\begin{equation}
u_i(\w) = {{w_i^2} \over {\| \w \|^2}}
\end{equation}
where, as usual, $\| \w \|^2 = \sum_{j=0}^{n-1} w_j^2$.

Then we need to compute
\begin{eqnarray}
\De{\w}{\p(\u(\w))}{\w^0}
& = &
\De{\u}{\p(\u)}{\u(\w^0)}
\circ
\De{\w}{\u(\w)}{\w^0}
\\
& = &
\P
\circ
\De{\w}{\u(\w)}{\w^0}
\nonumber
\end{eqnarray}

The partial derivatives are
\begin{eqnarray}
\Df{w_j}{u_i(\w))}
& = &
\Df{w_j}{{w_i^2} \over {\| \w \|^2}}
\\
& = &
{{2 w_j} \over {\| \w \|^4}} \left( \delta_{ij} \| \w \|^2 - w_i^2 \right)
\nonumber
\end{eqnarray}


\section{Hyperparameter Tuning}\label{section:background-hyperparameter}
Hyperparameter tuning is the process of optimizing a model to find the hyperparameters that will perform optimally given a specified metric. This metric is usually either the validation loss or the validation accuracy.
Hyperparameters are generally chosen by hand or by some automated process, but are not learned as a part of the model itself as opposed to the model weights.

Methods to find the optimal hyperparameters include manual tuning, grid search, random search, genetic algorithms, bayesian optimization, and hyperband, among others.
Random and grid searches are very time-consuming and computationally expensive, as they potentially run less than optimal parameters and do not use knowledge of previous runs to select the next set of parameters.
Bayesian optimization attempts to curtail this problem by taking previous runs into account, greatly speeding up the learning process and not experimenting with less than optimal hyperparameters~\cite{bayesian}.
Hyperband is a bandit-based approach to hyperparameter optimizations that can abandon or reduce the budget of an experimental run if the network performance is below with a chosen parameter set is below a certain threshold~\cite{hyperband}.

\subsection{Bayesian Optimization}\label{subsection:background-bayesian}
The idea behind Bayesian optimization is that the hyperparameters of a network follow a probabilistic distribution $\text{P}(y | x)$ where $y$ is the score determined by some objective function and $x$ is a set of hyperparameters.
Based on this probabilistic model, an optimal set of hyperparameters can then be calculated.
Applying these hyperparameters to the model, the objective function can then be used to calculate the scores.
The probabilistic distribution is then updated and the process is repeated until the maximum number of iterations is reached.

Bayesian optimization relies on the following~\cite{bayesiantds}:
\begin{enumerate}
	\itemsep-1em
	\item A clearly defined hyperparameter domain of possible configurations $\chi$ over which to search,
	\item An objective function which calculates a score that we wish to minimize using a set of hyperparameters,
	\item A surrogate model that represents the probabilistic distribution of the hyperparameters,
	\item A selection function to evalute which hyperparameters should be evaluated next, and
	\item A history of score-hyperparameter pairs used to update the surrogate model.
\end{enumerate}
The domain must be chosen by the researcher and is usually defined with respect to previous knowledge or based on related works.
The objective function is evaluated by running the model and calculating its error using a predetermined loss function.

The surrogate model used is essentially a mapping of hyperparameters to scores from the objective function.
An example for the surrogate model is the Tree-structured Parzen Estimator (TPE)~\cite{tpe}.
TPE takes advantage of Bayes' rule and represents the probability function $\text{P}(y~|~x)$ as
\begin{equation}\label{eq:bohb-1}
	\begin{split}
		\text{P}(y~|~x) &= \frac{\text{P}(x~|~y) \cdot \text{P}(x)}{\text{P}(y)}\\
		\text{with } \text{P}(x~|~y) &=
		\begin{cases}
		l(x) &\text{when } y < y^* \\
		g(x) &\text{when } y \geq y^*
		\end{cases}
	\end{split}
\end{equation}
where $y^*$ is a threshold value. $l(x)$ is thus a probability density function formed from the set of observations using hyperparameters $x^i \in \chi$ which have been performed such that the loss of the network is less than $y^*$.
Conversely, $g(x)$ is the density function formed from the remaining observations.
$y^*$ is chosen by the algorithm such that $\text{P}(y<y^*) = \gamma$ with $\gamma$ being a value chosen by the researcher.

The selection function chooses which hyperparameters $x \in \chi$ should be chosen for each successive experiment and is commonly based on the expected improvement~\cite{bayesiantds}:
\begin{align}
	EI_{y^*}(x) &= \int_{-\infty}^{y^*} (y^* - y) p(y~|~x)dy
\end{align}
Here, $y^*$ is again a threshold value, $x$ is the proposed set of hyperparameters, $y$ is the actual value of the objective function using hyperparameters $x$ and $p(y~|~x)$ is the surrogate probability model expressing the probability of $y$ given $x$.
Substituting the values of $p(y~|~x)$ from the surrogate function we get
\begin{equation}
	\begin{split}
		EI_{y^*}(x) &= \frac{\gamma y^* l(x) - l(x) \int_{-\infty}^{y^*}p(y)dy}{\gamma l(x) + (1 - \gamma)g(x)}\\
		&\propto \left(\gamma + \frac{g(x)}{l(x)} (1 - \gamma)\right)^{-1}
	\end{split}
\end{equation}
We can see from this function that the expected improvement is proportional to $\frac{l(x)}{g(x)}$ and thus, to maximize the expected improvement, samples should be drawn from the maximum value in $l(x)$.
The selected hyperparameters are then evaluated with regards to the objective function and the results used to update the surrogate model.

\subsection{Hyperband}\label{section:background-hyperband}
Hyperband is a learning strategy that "relies on a principled early-stopping strategy to allocate resources"~\cite{hyperband}.
It is a variation of successive halving, and in fact uses successive halving in its inner loop~\cite{successivehalving}.

Successive halving randomly samples $n$ hyperparameter sets in the search domain $\chi$.
It then evaluates all of these sets for $B$ iterations to calculate the validation loss.
The lowest performing half is then discarded and the remaining evaluated for a further $B$ iterations.
This is repeated until only one hyperparameter set remains.

Successive halving suffers from the $n \text{ vs } \frac{B}{n}$ problem, which is whether to train more configurations $n$ or to explore fewer, but with more resources $B$.
If a larger $n$ is chosen, configurations which are slower to converge might be killed off too quickly.
If a larger $\frac{B}{n}$ is chosen, then lower performing configurations may be given more resources, thus wasting resources that could otherwise be spent on higher performing configurations~\cite{hyperband}.

Hyperband attempts to solve this issue by considering several possible values of $n$ for a fixed $B$.
The pseudocode in Algorithm \ref{algorithm:hyperband} shows how Hyperband uses an outer loop, performing essentially a grid search with multiple values of $n$ and an inner loop utilizing a method similar to successive halving but with the top $\left\lfloor n_i/\eta\right\rfloor$ instead of the top half.

\input{algorithms/hyperband}

\subsection{Bayesian Optimization with Hyperband}\label{section:background-bohb}
"BOHB: Robust and Efficient Hyperparameter Optimization at Scale" by Stefan Falkner, Aaron Klein, and Frank Hutter proposes a hyperparameter optimization approach which seeks to combine the benefits of both Bayesian optimization and Hyperband~\cite{bohb}.
This approach seeks to have strong anytime performance, strong final performance, effectively use parallel resources, be easily scalable, and be robust and flexible.

BOHB works by using Hyperband to determine the number of configurations to evaluate with a given budget, but replaces the random selection of configurations with a Bayesian model based search.
The process of using Bayesian optimization for configuration sampling can be seen in Algorithm \ref{algorithm:bohb}. In this algorithm, $l'(x)$ is the $l(x)$ component of the newly updated probability density function.

Even though Hyperband also provides strong anytime performance compared to random search and Bayesian optimization, Fakner et al. observed an improvement of over 55x against a random search at larger budgets, converging to the global optimum much faster than either Hyperband or Bayesian optimization alone~\cite{bohb}.

\input{algorithms/bohb}