\chapter{Limits and series}
\label{ch:calc_limits}
Now that we have developed the theory
of metric (and topological) spaces well,
we give a three-chapter sequence which
briskly covers the theory of single-variable calculus.

Much of the work has secretly already been done,
For example, if $x_n$ and $y_n$ are real sequences
with $\lim_n x_n = x$ and $\lim_n y_n = y$,
then in fact $\lim_n (x_n + y_n) = x+y$
or $\lim_n (x_n y_n) = xy$,
because we showed in \Cref{prop:arithmetic_continuous}
that arithmetic was continuous.
We will also see that completeness plays a crucial role.

\section{Completeness and inf/sup}
\prototype{$\sup [0,1] = \sup (0,1) =  1$.}
As $\RR$ is a metric space,
we may discuss continuity and convergence.
There are two important facts about $\RR$
which will make most of the following sections tick.

The first fact you have already seen before:
\begin{theorem}
	[$\RR$ is complete]
	\label{thm:complete_R}
	As a metric space, $\RR$ is complete:
	sequences converge if and only if they are Cauchy.
\end{theorem}

The second one we have not seen before --- it is the
existence of $\inf$ and $\sup$.
Your intuition should be:
\begin{moral}
	$\sup$ is $\max$ adjusted slightly for infinite sets.
	(And $\inf$ is adjusted $\min$.)
\end{moral}
Why the ``adjustment''?
\begin{example}
	[Why is max not good enough?]
	Let's say we have the open interval $S = (0,1)$.
	The elements can get arbitrarily close to $1$,
	so we would like to think ``$1$ is the max of $S$'';
	except the issue is that $1 \notin S$.
	In general, infinite sets don't necessarily \emph{have}
	a maximum, and we have to talk about bounds instead.

	So we will define $\sup S$ in such a way that $\sup S = 1$.
	The definition is that ``$1$ is the smallest number
	which is at least every element of $S$''.
\end{example}
To write it out:
\begin{definition}
	If $S$ is a set of real numbers:
	\begin{itemize}
		\ii An \emph{upper bound} for $S$ is a real number $M$
		such that $x \le M$ for all $x \in S$.
		If one exists, we say $S$ is \vocab{bounded above};
		\ii A \emph{lower bound} for $S$ is a real number $m$
		such that $m \le x$ for all $x \in S$.
		If one exists, we say $S$ is \vocab{bounded below}.
		\ii If both upper and lower bounds exist,
		we say $S$ is \vocab{bounded}.
	\end{itemize}
\end{definition}
\begin{theorem}
	[$\RR$ has inf's and sup's]
	\label{thm:inf_sup}
	Let $S$ be a nonempty set of real numbers.
	\begin{itemize}
		\ii If $S$ is bounded above
		then it has a \emph{least} upper bound,
		which we denote by $\sup S$
		and refer to as the \vocab{supremum} of $S$.
		\ii If $S$ is bounded below
		then it has a \emph{greatest} lower bound,
		which we denote by $\inf S$
		and refer to as the \vocab{infimum} of $S$.
	\end{itemize}
\end{theorem}

\begin{definition}
	For convenience, if $S$ has not bounded above, we write $\sup S = +\infty$.
	Similarly, if $S$ has not bounded below, we write $\inf S = -\infty$.
\end{definition}

\begin{example}
	[Supremums]
	Since the examples for infimums are basically the same,
	we stick with supremums for now.
	\begin{enumerate}[(a)]
		\ii If $S = \left\{ 1, 2, 3, \dots \right\}$
		then $S$ is not bounded above, so we have $\sup S = +\infty$.
		\ii If $S = \left\{ \dots, -2, -1 \right\}$
		denotes the set of negative integers, then $\sup S = -1$.
		\ii Let $S = [0,1]$ be a closed interval.
		Then $\sup S = 1$.
		\ii Let $S = (0,1)$ be an open interval.
		Then $\sup S = 1$ as well, even though $1$ itself
		is not an element of $S$.
		\ii Let $S = \QQ \cap (0,1)$ denote the set of rational
		numbers between $0$ and $1$.
		Then $\sup S = 1$ still.
		\ii If $S$ is a finite nonempty set,
		then $\sup S = \max S$.
	\end{enumerate}
\end{example}

\begin{definition}
	[Porting definitions to sequences]
	If $a_1$, \dots is a sequence we will often write
	\begin{align*}
		\sup_n a_n &\defeq \sup \left\{ a_n \mid n \in \NN \right\} \\
		\inf_n a_n &\defeq \inf \left\{ a_n \mid n \in \NN \right\}
	\end{align*}
	for the supremum and infimum of the set of elements of the sequence.
	We also use the words ``bounded above/below'' for sequences
	in the same way.
\end{definition}
\begin{example}
	[Infimum of a sequence]
	The sequence $a_n = \frac 1n$ has infimum $\inf a_n = 0$.
\end{example}

\section{Proofs of the two key completeness properties of $\RR$}
Careful readers will note that we have not actually
proven either \Cref{thm:inf_sup} or \Cref{thm:complete_R}.
We will do so here.

First, we show that the ability
to take infimums and supremums lets you prove completeness of $\RR$.
\begin{proof}
	[Proof that \Cref{thm:inf_sup} implies \Cref{thm:complete_R}]
	Let $a_1$, $a_2$, \dots be a Cauchy sequence.
	By discarding finitely many leading terms,
	we may as well assume that $|a_i - a_j| \le 100$ for all $i$ and $j$.
	In particular, the sequence is now bounded;
	it lies between $[a_1-100, a_1+100]$ for example.

	We want to show this sequence converges,
	so we have to first describe what the limit is.
	We know that to do this we are really going
	to have to use the fact that we live in $\RR$.
	(For example we know in $\QQ$ the limit of
	$1$, $1.4$, $1.41$, $1.414$, \dots is nonexistent.)

	We propose the following: let
	\[ S = \left\{ x \in \RR \mid a_n \ge x \text {
		for infinitely many $n$ } \right\}.  \]
	We claim that the sequence converges to $M = \sup S$.
	\begin{exercise}
		Show that this supremum makes sense by proving
		that $a_1 - 100 \in S$ (so $S$ is nonempty)
		while all elements of $S$ are at most $a_1 + 100$
		(so $S$ is bounded above).
		Thus we are allowed to actually take the supremum.
	\end{exercise}

	You can think of this set $S$ with the following picture.
	We have a Cauchy sequence drawn in the real line which we think converges,
	which we can visualize as a bunch of dots on the real line,
	with some order on them.
	We wish to cut the line with a knife such that
	only finitely many dots are do to the left of the knife.
	(For example, placing the knife all the way to the left always works.)
	The set $S$ represents the places where we could put the knife,
	and $M$ is ``as far right'' as we could go.
	Because of the way supremums work,
	$M$ might not \emph{itself} be a valid knife location,
	but certainly anything to its left is.
	\begin{center}
	\begin{asy}
		size(10cm);
		draw( (-8,0)--(8,0), Arrows );
		label("$\mathbb R$", (8,0), dir(-90));
		draw( (3.1,1)--(3.1,-1), red );
		draw( (2.7,1.5)--(2.7,-1.5), deepgreen );
		label("$M$", (3.1,-1), dir(-45), red);
		label("$M-\frac12\varepsilon$", (2.7,1.5), dir(90), deepgreen);
		dot("$a_1$", (-5, 0), dir(-90), blue);
		dot("$a_2$", (-1, 0), dir(-90), blue);
		dot("$a_3$", ( 6, 0), dir(-90), blue);
		dot("$a_4$", ( 0, 0), dir(-90), blue);
		dot("$a_5$", (4.8, 0), dir(-90), blue);
		dot("$a_6$", (4.1, 0), dir(-90), blue);
		dot("$a_8$", (3.5, 0), dir(-90), blue);
		dot("$a_7$", (1.5, 0), dir(-90), blue);
		dot((3.3, 0), blue);
		dot((3.2, 0), blue);
		dot((3.05, 0), blue);
	\end{asy}
	\end{center}
	Let $\eps > 0$ be given;
	we want to show eventually all terms are within $\eps$ of $M$.
	Because the sequence is Cauchy,
	there is an $N$ such that
	eventually $\left\lvert a_m - a_n \right\rvert < \half\eps$
	for $m \ge n \ge N$.

	Now suppose we fix $n$ and vary $m$.
	By the definition of $M$,
	it should be possible to pick the index $m$
	such that $a_m \ge M - \half \eps$
	(there are infinitely many to choose from
	since $M - \half\eps$ is a valid knife location,
	and we only need $m \ge n$).
	In that case we have
	\[ \left\lvert a_n - M \right\rvert
		\le \left\lvert a_n - a_m \right\rvert
		+ \left\lvert a_m - M \right\rvert <
		\half\eps + \half\eps = \eps \]
	by the triangle inequality.
	This completes the proof.
\end{proof}

Therefore it is enough to prove the latter \Cref{thm:inf_sup}.
To do this though, we would need to actually give a rigorous definition of
the real numbers $\RR$, since we have not done so yet!

One approach that makes this easy to use the so-called
\vocab{Dedekind cut} construction.
Suppose we take the rational numbers $\QQ$.
Then one \emph{defines} a real number to be a ``cut''
$A \mid B$ of the set of rational numbers:
a pair of subsets of $\QQ$ such that
\begin{itemize}
	\ii $\QQ = A \sqcup B$ is a disjoint union;
	\ii $A$ and $B$ are nonempty;
	\ii we have $a < b$ for every $a \in A$ and $b \in B$, and
	\ii $A$ has no largest element (i.e.\ $\sup A \notin A$).
\end{itemize}
This can again be visualized by taking what you think of
as the real line, and slicing at some real number.
The subset $\QQ \subset \RR$ gets cut into two halves $A$ and $B$.
If the knife happens to land exactly at a rational number,
by convention we consider that number to be in the right half
(which explains the last fourth condition that $\sup A \notin A$).

With this definition \Cref{thm:inf_sup} is easy:
to take the supremum of a set of real numbers,
we take the union of all the left halves.
The hard part is then figuring out how to define $+$, $-$, $\times$, $\div$
and so on with this rather awkward construction.
If you want to read more about this construction in detail,
my favorite reference is \cite{ref:pugh}$,$
in which all of this is done carefully in Chapter 1.

\section{Monotonic sequences}
Here is a great exercise.
\begin{exercise}
	[Mandatory]
	\label{exer:inf_exists}
	Prove that if $a_1 \ge a_2 \ge \dots \ge 0$
	then the limit
	\[ \lim_{n \to \infty} a_n \]
	exists.
	Hint: the idea in the proof of the previous section helps;
	you can also try to use completeness of $\RR$.
	Second hint: if you are really stuck,
	wait until after \Cref{thm:nonneg_bounded},
	at which point you can use essentially copy its proof.
\end{exercise}

The proof here readily adapts by shifting.
\begin{definition}
	A sequence $a_n$ is \vocab{monotonic}
	if either $a_1 \ge a_2 \ge \dots$
	or $a_1 \le a_2 \le \dots$.
\end{definition}

\begin{theorem}
	[Monotonic bounded sequences converge]
	Let $a_1$, $a_2$, \dots be a monotonic bounded sequence.
	Then $\lim_{n \to \infty} a_n$ exists.
	\label{thm:monotonic_bounded}
\end{theorem}

\begin{example}
	[Silly example of monotonicity]
	Consider the sequence defined by
	\begin{align*}
		a_1 &= 1.2 \\
		a_2 &= 1.24 \\
		a_3 &= 1.248 \\
		a_4 &= 1.24816 \\
		a_5 &= 1.2481632 \\
		&\vdotswithin=
	\end{align*}
	and so on, where in general we stuck
	on the decimal representation of the next power of $2$.
	This will converge to \emph{some} real number,
	although of course this number
	is quite unnatural and there is probably no good description for it.
\end{example}
In general, ``infinite decimals''
can now be defined as the limit of the truncated finite ones.

\begin{example}
	[$0.9999\dots = 1$]
	In particular, I can finally make precise the notion
	you argued about in elementary school that
	\[ 0.9999\dots = 1. \]
	We simply \emph{define} a repeating decimal
	to be the limit of the sequence $0.9$, $0.99$, $0.999\dots$.
	And it is obvious that the limit of this sequence is $1$.
\end{example}

Some of you might be a little surprised since
it seems like we really should have
$0.9999 = 9 \cdot 10^{-1} + 9 \cdot 10^{-2} + \dots$ ---
the limit of ``partial sums''.
Don't worry, we're about to define those in just a moment.

\medskip

Here is one other great use of monotonic sequences.
\begin{definition}
	Let $a_1$, $a_2$, \dots be a sequence
	(not necessarily monotonic) which is bounded below.
	We define
	\[
		\limsup_{n \to \infty} a_n
		\defeq \lim_{N \to \infty} \sup_{n \ge N} a_n
		= \lim_{N \to \infty} \sup \left\{ a_N, a_{N+1}, \dots \right\}.
	\]
	This is called the \vocab{limit supremum} of $(a_n)$.
	We set $\limsup_{n \to \infty} a_n$ to be $+\infty$
	if $a_n$ is not bounded above.

	If $a_n$ is bounded above,
	the \vocab{limit infimum} $\liminf_{n \to \infty} a_n$
	is defined similarly.
	In particular, $\liminf_{n \to \infty} a_n = -\infty$
	if $a_n$ is not bounded below.
\end{definition}
\begin{exercise}
	Show that these definitions make sense,
	by checking that the supremums are non-increasing,
	and bounded below.
\end{exercise}
We can think of $\limsup_n a_n$ as
``supremum, but allowing finitely many terms to be discarded''.

\section{Infinite series}
\prototype{$\sum_{k \ge 1}^\infty \frac{1}{k(k+1)}
= \lim_{n \to \infty} \left( 1 - \frac{1}{n+1} \right) = 1$.}

We will actually begin by working with infinite series,
since in the previous chapters we defined limits of sequences,
and so this is actually the next closest thing to work
with.\footnote{Conceptually: discrete things are easier
	to be rigorous about than continuous things,
	so series are actually ``easier'' than derivatives!
	I suspect the reason that most schools teach series last in calculus
	is that most calculus courses do not have proofs.}

This will give you a rigorous way to think about
statements like
\[ \sum_{n = 1}^{\infty} \frac{1}{n^2} = \frac{\pi^2}{6} \]
and help answer questions like
``how can you add rational numbers and get an irrational one?''.

\begin{definition}
	Consider a sequence $a_1$, \dots{} of real numbers.
	The series $\sum_k a_k$ \vocab{converges} to a limit $L$ if
	the sequence of ``partial sums''
	\begin{align*}
		s_1 &= a_1 \\
		s_2 &= a_1 + a_2 \\
		s_3 &= a_1 + a_2 + a_3 \\
		&\vdotswithin= \\
		s_n &= a_1 + \dots + a_n
	\end{align*}
	converges to the limit $L$.
	Otherwise it \vocab{diverges}.
\end{definition}
\begin{abuse}
	[Writing divergence as $+\infty$]
	It is customary, if all the $a_k$ are nonnegative,
	to write $\sum_k a_k = \infty$ to denote that the series diverges.
\end{abuse}

You will notice that by using the definition of sequences,
we have masterfully sidestepped the issue of
``adding infinitely many numbers''
which would otherwise cause all sorts of problems.
\begin{moral}
	An ``infinite sum'' is actually the \emph{limit} of its partial sums.
	There is no infinite addition involved.
\end{moral}

That's why it's for example okay to have
$\sum_{n \ge 1} \frac{1}{n^2} = \frac{\pi^2}{6}$ be irrational;
we have already seen many times that sequences of rational
numbers can converge to irrational numbers.
It also means we can gladly ignore all the
irritating posts by middle schoolers about
$1+2+3+\dots = -\frac{1}{12}$;
the partial sums explode to $+\infty$, end of story,
and if you want to assign a value to that sum
it had better be a definition.

\begin{example}
	[The classical telescoping series]
	We can now prove the classic telescoping series
	\[ \sum_{k = 1}^\infty \frac{1}{k(k+1)} \]
	in a way that doesn't just hand-wave the ending.
	Note that the $k$th partial sum is
	\begin{align*}
		\sum_{k=1}^n \frac{1}{k(k+1)}
		&= \frac{1}{1 \cdot 2}
			+ \frac{1}{2 \cdot 3}
			+ \dots + \frac{1}{n(n+1)} \\
		&= \left( \frac11 - \frac12 \right)
			+ \dots + \left( \frac 1n - \frac{1}{n+1} \right) \\
		&= 1 - \frac{1}{n+1}.
	\end{align*}
	The limit of this partial sum as $n \to \infty$ is $1$.
\end{example}

\begin{example}
	[Harmonic series diverges]
	We can also make sense of the statement
	that $\sum_{k=1}^\infty \frac 1k = \infty$
	(i.e.\ it diverges).
	We may bound the $2^n$th partial sums from below:
	\begin{align*}
		\sum_{k=1}^{2^n} \frac1k &= \frac11 + \frac12 + \dots + \frac1{2^n} \\
		&\ge \frac11 + \frac12 + \left( \frac14+\frac14 \right)
		+ \left( \frac18+\frac18+\frac18+\frac18 \right) \\
		&+ \dots  +
		\underbrace{\left( \frac{1}{2^n} + \dots + \frac{1}{2^n} \right)}_{2^{n-1} \text{ terms}} \\
		&= 1 + \half + \half + \dots + \half = 1 + \frac{n-1}{2}.
	\end{align*}
	A sequence satisfying $s_{2^n} \ge 1 + \half(n-1)$
	will never converge to a finite number!
\end{example}

I had better also mention that for nonnegative sums,
convergence is just the same as having ``finite sum''
in the following sense.

\begin{proposition}
	[Partial sums of nonnegatives bounded implies convergent]
	\label{thm:nonneg_bounded}
	Let $\sum_k a_k$ be a series of \emph{nonnegative} real numbers.
	Then $\sum_k a_k$ converges to some limit
	if and only if there is a constant $M$ such that
	\[ a_1 + \dots + a_n < M \]
	for every positive integer $n$.
\end{proposition}
\begin{proof}
	This is actually just \Cref{thm:monotonic_bounded} in disguise,
	but since we left the proof as an exercise back then,
	we'll write it out this time.

	Obviously if no such $M$ exists then convergence will not happen,
	since this means the sequence $s_n$ of partial sums is unbounded.

	Conversely, if such $M$ exists then we have
	$s_1 \le s_2 \le \dots < M$.
	Then we contend the sequence $s_n$ converges to
	$L \defeq \sup_n s_n < \infty$.
	(If you read the proof that completeness implies Cauchy,
	the picture is nearly the same here, but simpler.)
	\begin{center}
	\begin{asy}
		size(10cm);
		draw( (-8,0)--(8,0), Arrows );
		label("$\mathbb R$", (8,0), dir(-90));
		draw( (3.1,1)--(3.1,-1), red );
		draw( (2.4,1.5)--(2.4,-1.5), deepgreen );
		label("$L$", (3.1,-1), dir(-45), red);
		label("$L-\varepsilon$", (2.4,1.5), dir(90), deepgreen);
		dot("$s_1$", (-5, 0), dir(-90), blue);
		dot("$s_2$", (-4, 0), dir(-90), blue);
		dot("$s_3$", (-1.7, 0), dir(-90), blue);
		dot("$s_4$", (-0.2, 0), dir(-90), blue);
		dot("$s_5$", (0.3, 0), dir(-90), blue);
		dot("$s_6$", (1.3, 0), dir(-90), blue);
		dot("$s_7$", (2.2, 0), dir(-90), blue);
		dot("$s_8$", (2.7, 0), dir(-90), blue);
		dot((2.8, 0), blue);
		dot((2.88, 0), blue);
		dot((2.97, 0), blue);
	\end{asy}
	\end{center}

	Indeed, this means for any $\eps$ there
	are infinitely many terms of the sequence exceeding $L-\eps$;
	but since the sequence is monotonic, once $s_n \ge L-\eps$
	then $s_{n'} \ge L-\eps$ for all $n' \ge n$.
	This implies convergence.
\end{proof}

\begin{abuse}
	[Writing $\sum < \infty$]
	For this reason, if $a_k$ are nonnegative real numbers,
	it is customary to write \[ \sum_k a_k < \infty \]
	as a shorthand for ``$\sum_k a_k$ converges to a finite limit'',
	(or perhaps shorthand for ``$\sum_k a_k$ is bounded'' --- as
	we have just proved these are equivalent).
	We will use this notation too.
\end{abuse}

\section{Series addition is not commutative: a horror story}
One unfortunate property of the above definition
is that it actually depends on the order of the elements.
In fact, it turns out that there is an explicit way
to describe when rearrangement is okay.

\begin{definition}
	A series $\sum_k a_k$ of real numbers
	is said to \vocab{converge absolutely} if
	\[ \sum_k \left\lvert a_k \right\rvert < \infty \]
	i.e.\ the series of absolute values converges to some limit.
	If the series converges, but not absolutely,
	we say it \vocab{converges conditionally}.
\end{definition}

\begin{proposition}
	[Absolute convergence $\implies$ convergence]
	If a series $\sum_k a_k$ of real numbers
	converges absolutely, then it converges in the usual sense.
\end{proposition}
\begin{exercise}
	[Great exercise]
	Prove this by using the Cauchy criteria:
	show that if the partial sums of $\sum_k |a_k|$ are Cauchy,
	then so are the partial sums of $\sum_k a_k$.
\end{exercise}

Then, rearrangement works great.
\begin{theorem}
	[Permutation of terms okay for absolute convergence]
	Consider a series $\sum_k a_k$ which is absolutely convergent
	and has limit $L$.
	Then any permutation of the terms will also converge to $L$.
\end{theorem}
\begin{proof}
	Suppose $\sum_k a_k$ converges to $L$,
	and $b_n$ is a rearrangement.
	Let $\eps > 0$.
	We will show that the partial sums of $b_n$
	are eventually within $\eps$ of $L$.

	The hypothesis means that there is a large $N$ in terms of $\eps$
	such that
	\[ \left\lvert \sum_{k=1}^N a_k - L \right\rvert < \half\eps
		\quad\text{and}\quad
		\sum_{k = N+1}^{n} \left\lvert a_k \right\rvert < \half\eps
	\]
	for every $n \ge N$ (the former from vanilla convergence of $a_k$
	and the latter from the fact that $a_k$ converges absolutely,
	hence its partial sums are Cauchy).

	Now suppose $M$ is large enough that $a_1$, \dots, $a_N$
	are contained within the terms $\{b_1, \dots, b_M\}$.
	Then
	\begin{align*}
		b_1 + \dots + b_M
		&= (a_1 + \dots + a_N) \\
		&+ \underbrace{a_{i_1} + a_{i_2} + \dots + a_{i_{M-N}}}%
		_{\text{$M-N$ terms with indices $\ge N$}}
	\end{align*}
	The terms in the first line sum up to within $\half\eps$ of $L$,
	and the terms in the second line have sum at most $\half\eps$
	in absolute value, so the total $b_1 + \dots + b_M$
	is within $\half\eps + \half\eps = \eps$ of $L$.
\end{proof}
In particular, when you have nonnegative terms, the world is great:
\begin{moral}
	Nonnegative series can be rearranged at will.
\end{moral}
And the good news is that actually,
in practice, most of your sums will be nonnegative.

The converse is not true,
and in fact, it is almost the worst possible converse you can imagine.
\begin{theorem}
	[Permutation of terms meaningless for conditional convergence]
	Consider a series $\sum_k a_k$ which converges \emph{conditionally}
	to some real number.
	Then, there exists a permutation of the series
	which converges conditionally to $1337$.

	(Or any constant.  You can also get it to diverge, too.)
\end{theorem}
So, permutation is as bad as possible for conditionally convergent
series, and hence don't even bother to try.

\section{Limits of functions at points}
\prototype{$\lim_{x \to \infty} 1/x = 0$.}
We had also better define the notion
of a limit of a real function,
which (surprisingly) we haven't actually defined yet.
The definition will look like what we have seen before with continuity.

\begin{definition}
	Let $f \colon \RR \to \RR$ be a function\footnote{Or
		$f \colon (a,b) \to \RR$, or variants.
		We just need $f$ to be defined on an open neighborhood of $p$.
	} and let $p \in \RR$ be a point in the domain.
	Suppose there exists a real number $L$ such that:
	\begin{quote}
		For every $\eps > 0$, there exists $\delta > 0$
		such that if $\left\lvert x - p \right\rvert < \delta$
		and $x \neq p$ then $\left\lvert f(x) - L \right\rvert < \eps$.
	\end{quote}
	Then we say $L$ is the \vocab{limit} of $f$ as $x \to p$, and write
	\[ \lim_{x \to p} f(x) = L. \]
\end{definition}
There is an important point here: in this definition
we \emph{deliberately} require that $x \ne p$.
\begin{moral}
	The value $\lim_{x \to p} f(x)$ does not depend on $f(p)$,
	and accordingly we often do not even bother to define $f(p)$.
\end{moral}
\begin{example}
	[Function with a hole]
	Define the function $f \colon \RR \to \RR$ by
	\[ f(x) = \begin{cases}
			3x & \text{if } x \ne 0 \\
			2019 & \text{otherwise}.
		\end{cases} \]
	Then $\lim_{x \to 0} f(x) = 0$.
	The value $f(0) = 2019$ does not affect the limit.
	Obviously, because $f(0)$ was made up to be some artificial
	value that did not agree with the limit,
	this function is discontinuous at $x = 0$.
\end{example}

\begin{ques}
	[Mandatory]
	Show that a function $f$ is continuous at $p$
	if and only if $\lim_{x \to p} f(x)$ exists and equals $f(p)$.
\end{ques}

\begin{example}
	[Less trivial example: a rational piecewise function]
	\label{ex:rational_piecewise}
	Define the function $f \colon \RR \to \RR$ as follows:
	\[ f(x) = \begin{cases}
			1 & \text{if } x = 0 \\
			\frac1q & \text{if } x = \frac pq \text{ where } q > 0
				\text{ and } \gcd(p,q) = 1 \\
			0 & \text{if } x \notin \QQ.
		\end{cases} \]
	For example, $f(\pi) = 0$, $f(2/3) = \frac13$, $f(0.17) = \frac{1}{100}$.
	Then \[ \lim_{x \to 0} f(x) = 0. \]
	For example, if $|x| < 1/100$ and $x \neq 0$
	then $f(x)$ is either zero (for $x$ irrational)
	or else is at most $\frac{1}{101}$ (if $x$ is rational).

	As $f(0) = 1$, this function is also discontinuous at $x = 0$.
	However, if we change the definition so that $f(0) = 0$ instead,
	then $f$ becomes continuous at $0$.
\end{example}

\begin{example}
	[Famous example]
	Let $f(x) = \frac{\sin x}{x}$, $f \colon \RR \to \RR$,
	where $f(0)$ is assigned any value.
	Then
	\[ \lim_{x \to 0} f(x) = 1. \]
\end{example}
We will not prove this here,
since I don't want to get into trig yet.
In general, I will basically only use trig functions
for examples and not for any theory,
so most properties of the trig functions will just be quoted.
\begin{abuse}
	[The usual notation]
	\label{abuse:limit}
	From now on, the above example
	will usually be abbreviated to just
	\[ \lim_{x \to 0} \frac{\sin x}{x} = 1. \]
	The reason there is a slight abuse here
	is that I'm supposed to feed a function $f$ into the limit,
	and instead I've written down an expression
	which is defined everywhere --- except at $x=0$.
	But that $f(0)$ value doesn't change anything.
	So the above means: ``the limit of the function described
	by $f(x) = \frac{\sin x}{x}$,
	except $f(0)$ can be whatever it wants because it doesn't matter''.
\end{abuse}

\begin{remark}
	[For metric spaces]
	You might be surprised that I didn't define
	the notion of $\lim_{x \to p} f(x)$ earlier
	for $f \colon M \to N$ a function on metric spaces.
	We can actually do so as above, but there is one nuance:
	what if our metric space $M$ is discrete,
	so $p$ has no points nearby it?
	(Or even more simply, what if $M$ is a one-point space?)
	We then cannot define $\lim_{x \to p} f(x)$ at all.

	Thus if $f \colon M \to N$ and we want to define
	$\lim_{x \to p} f(x)$, we have the requirement that $p$
	should have a point within $\eps$ of it, for any $\eps > 0$.
	In other words, $p$ should not be an isolated point.
\end{remark}

As usual, there are no surprises with arithmetic,
we have $\lim_{x \to p} (f(x) \pm g(x))
	= \lim_{x \to p} f(x) \pm \lim_{x \to p} g(x)$,
and so on and so forth.
We have effectively done this proof before
so we won't repeat it again.

\section{Limits of functions at infinity}
Annoyingly, we actually have to make this
definition separately,
even though it will not feel any different
from earlier examples.
\begin{definition}
	Let $f \colon \RR \to \RR$.
	Suppose there exists a real number $L$ such that:
	\begin{quote}
		For every $\eps > 0$, there exists a constant $M$
		such that if $x > M$, then $\left\lvert f(x)-L \right\rvert < \eps$.
	\end{quote}
	Then we say $L$ is the \vocab{limit} of $f$ as
	$x$ approaches $\infty$ and write
	\[ \lim_{x \to \infty} f(x) = L. \]
	The limit $\lim_{x \to -\infty} f(x)$ is defined similarly,
	with $x > M$ replaced by $x < M$.
\end{definition}
Fortunately, as $\infty$ is not an element of $\RR$,
we don't have to do the same antics about $f(\infty)$
like we had to do with ``$f(p)$ set arbitrarily''.
So these examples can be more easily written down.
\begin{example}
	[Limit at infinity]
	The usual:
	\[ \lim_{x \to \infty} \frac 1x = 0. \]
	I'll even write out the proof:
	for any $\eps > 0$, if $x > 1/\eps$
	then $\left\lvert \frac 1x - 0 \right\rvert < \eps$.
\end{example}

There are no surprises with arithmetic:
we have $\lim_{x \to \infty} (f(x) \pm g(x))
	= \lim_{x \to \infty} f(x) \pm \lim_{x \to p} g(x)$,
and so on and so forth.
This is about the fourth time
I've mentioned this, so I will not say more.

\section{\problemhead}

\begin{problem}
	Define the sequence
	\[ a_n = (-1)^n + \frac{n^3}{2^n} \]
	for every positive integer $n$.
	Compute the limit infimum and the limit supremum.
\end{problem}

\begin{problem}
	For which bounded sequences $a_n$
	does $\liminf_n a_n = \limsup_n a_n$?
	\begin{hint}
		Iff the sequence is convergent!
	\end{hint}
\end{problem}

\begin{dproblem}
	[Comparison test]
	Let $\sum a_n$ and $\sum b_n$ be two series.
	Assume $\sum b_n$ is absolutely convergent,
	and $|a_n| \le |b_n|$ for all integers $n$.
	Prove that $\sum_n a_n$ is absolutely convergent.
\end{dproblem}

\begin{problem}
	[Geometric series]
	\label{prob:geometric}
	Let $-1 < r < 1$ be a real number.
	Show that the series
	\[ 1 + r + r^2 + r^3 + \dots \]
	converges absolutely and determine what it converges to.
	\begin{hint}
		The $n$th partial sum is $\frac{1}{1-r} (1-r^{n+1})$.
	\end{hint}
\end{problem}

%\begin{problem}
%	[An order-free definition for infinite nonnegative sums]
%	Let $a_1$, \dots{} be a sequence of \emph{nonnegative} real numbers.
%	Prove that
%	\[ \sum_{k=1}^\infty a_k = \sup \left\{ \sum_{k \in K} a_k
%			\mid K \subseteq \NN \right\} \]
%	where if the right-hand side is $+\infty$,
%	we mean that the left-hand side is a divergent series.
%\end{problem}

\begin{problem}
	[Alternating series test]
	Let $a_0 \ge a_1 \ge a_2 \ge a_3 \ge \dots$ be a weakly decreasing sequence
	of nonnegative real numbers,
	and assume that $\lim_{n \to \infty} a_n = 0$.
	Show that the series $\sum_n (-1)^n a_n$ is convergent
	(it need not be absolutely convergent).
	\begin{sol}
		This is an application of Cauchy convergence, since
		one can show that
		\[ \left\lvert \sum_{n=M}^N (-1)^n a_n \right\rvert \le a_{\min\{M,N\}}. \]
		Indeed, if $M$ and $N$ are even
		(for simplicity; other cases identical) then
		\begin{align*}
			a_M - a_{M+1} + a_{M+2} - \dots
			&= a_M - (a_{M+1}-a_{M+2}) - (a_{M+3}-a_{M+4}) \\
			&\quad- \dots - (a_{N-1} - a_N) \\
			&\le a_M \\
			a_M - a_{M+1} + a_{M+2} - \dots
			&= a_M - a_{M+1} + (a_{M+2} - a_{M+3}) + (a_{M+4} - a_{M+5}) \\
			&\quad+ \dots + (a_{N-2} - a_{N+1}) + a_N \\
			&\ge -a_{M+1}.
		\end{align*}
		In this way we see that the sequence of partial sums is Cauchy,
		hence converges to some limit.
	\end{sol}
\end{problem}

\begin{problem}
	[{\cite[Chapter 3, Exercise 55]{ref:pugh}}]
	\gim
	Let $(a_n)_{n \ge 1}$ and $(b_n)_{n \ge 1}$ be sequences of real numbers.
	Assume $a_1 \le a_2 \le \dots \le 1000$
	and moreover that $\sum_n b_n$ converges.
	Prove that $\sum_n a_n b_n$ converges.
	(Note that in both the hypothesis and statement,
	we do not have absolute convergence.)
	\begin{hint}
		This is a very tricky algebraic manipulation.
		Try setting $a_n = x_1 + \dots + x_n$ for $x_i \ge 0$.
	\end{hint}
	\begin{sol}
		To capture $(a_n)_n$ monotonic and bounded,
		write $a_n = x_1 + \dots + x_n$ for some $x_i$.
		Then $x_2$, \dots are all the same sign and so $\sum |x_i| = A < \infty$
		for some constant $A$.

		We now prove that the partial sums of $\sum a_n b_n$ are a Cauchy sequence.
		Consider any $\varepsilon > 0$.
		Let $K$ be such that the tails of $b_n$
		starting after $K$ have absolute value less than $\frac{\varepsilon}{A}$.
		Then for any $N > M \ge K$ we have
		\begin{align*}
			\left\lvert \sum_{k=M}^N a_k b_k \right\rvert
			&= \left\lvert \sum_{k=M}^N \sum_{j=1}^k b_k x_j \right\rvert \\
			&= \left\lvert \sum_{j=1}^N \sum_{k=\max\{j,M\}}^N b_k x_j \right\rvert \\
			&= \left\lvert \sum_{j=1}^N x_j \cdot \sum_{k=\max\{j,M\}}^N b_k\right\rvert \\
			&\le \sum_{j=1}^N |x_j| \left\lvert \sum_{k=\max\{j,M\}}^N b_k \right\rvert \\
			&< \sum_{j=1}^N |x_j| \cdot \frac{\varepsilon}{A} \\
			&< \varepsilon
		\end{align*}
		as desired.
	\end{sol}
\end{problem}

\begin{problem}
	[Putnam 2016 B1]
	\gim
	Let $x_0, x_1, x_2, \dots$ be the sequence
	such that $x_0=1$ and for $n\ge 0$,
	\[ x_{n+1} = \log(e^{x_n}-x_n) \]
	(as usual, $\log$ is the natural logarithm).
	Prove that the infinite series $x_0 + x_1 + \dots$
	converges and determine its value.
	\begin{hint}
		This is trickier than it looks.
		We have $x_n = e^{x_n} - e^{x_{n+1}}$
		but it requires some care to prove convergences.
		Helpful hint: $e^t \ge t+1$ for all real numbers $t$,
		therefore all $x_n$'s are nonnegative.
	\end{hint}
	\begin{sol}
		The answer is $e-1$.

		We begin by noting $x_{n+1} = \log(e^{x_n} - x_n) \ge \log 1 = 0$,
		owing to $e^t \ge 1+t$.
		So $x_n \ge 0$ for all $n$.

		Next notice that
		\[ x_{n+1} = \log\left( e^{x_n} - x_n \right) < \log e^{x_n} = x_n. \]
		So $(x_n)_n$ is strictly decreasing in addition to nonnegative.
		Thus it must converge to some limit $L$.

		Third, observe that
		\[ x_n = e^{x_n} - e^{x_{n+1}}
			\implies x_0 + x_1 + \dots + x_n
			= e^{x_0} - e^{x_n} = e - e^{x_n} < e. \]
		Since the partial sums are bounded by $e$,
		and $x_i \ge 0$, we conclude $L = 0$.

		Finally, the limit of the partial sums is then
		\[ \lim_{n \to \infty} e - e^{x_n} = e - e^0 = e - 1. \]
	\end{sol}
\end{problem}

\begin{problem}
	Consider again the function $f \colon \RR \to \RR$
	in \Cref{ex:rational_piecewise} defined by
	\[ f(x) = \begin{cases}
			1 & \text{if } x = 0 \\
			\frac1q & \text{if } x = \frac pq
			\text{ where } q > 0 \text{ and } \gcd(p,q) = 1 \\
			0 & \text{if } x \notin \QQ.
		\end{cases} \]
	For every real number $p$,
	compute $\lim_{x \to p} f(x)$, if it exists.
	At which points is $f$ continuous?
	\begin{hint}
		The limit always exists and equals zero.
		Consequently, $f$ is continuous exactly at irrational points.
	\end{hint}
\end{problem}


\subsection{Recursive calculation of PMW}\label{sec:paritydelay}

%The Partitioned-Growth data structure allows for prioritizing the growth of \emph{nodes} on an odd-parity node-tree $\dot{\nset}$ with low PWM, instead of \emph{vertices}.
The PMW can be used to prioritize the growth of vertices with low PMW. However, in order to implement this heuristic, the relevant figure of merit is not the PMW itself but the difference in PMW between the vertices. In the following we explain how to track the differences in PMW efficiently.

%%%%%%%%%%%%%%%%%%%
%We define the delay of a node $N$ which we denote by $\delta_N$, as the difference between the PMW of $N$ and the minimum PMW in the node-tree multiplied by two.
%\begin{equation}\label{eq:delayequation}
%    \delta_N = 2\left(\text{PMW}(N) - \min_{X\in\nset}{\text{PMW}(X)}\right).
%\end{equation}
%The delay of a node can be interpreted operationally as the number of times the node with minimum PMW should grow to have the same PMW as node $N$. %rowth iterations for a node to wait for all nodes in the node-tree to have equal PWM:

%To grow a cluster, we loop over all nodes in $\nset$ and grow all boundary vertices of $N$ if $\delta_N=0$.
%However, to find $\delta_N$, the PMW of all nodes in $\dot{\nset}$ must be known, which is no trivial task as the entire tree must be considered for the calculation in every node. Furthermore, the PWM of a node changes as it and its cluster grow. Instead, we will compute for the difference in node delay between child node and its parent.
%%%%%%%%%%%%%%%%%%%%%

Let a node $R\in \nset$ be the root node of the node-tree. As $\nset$ is an acyclic tree, $R$ defines implicitly the parent-child relations for all the nodes in $\nset$, where a node has one parent, but potentially multiple children. We let $P\geq Q$ denote that $P$ is the parent of $Q$. Abusing notation, we represent any odd-parity node-tree of $\abs{\dot\nset}\geq 3$ by the tuple $\{\nset_P,P-Q,\nset_Q\}$. In this notation, $\nset_P$ denotes the sub-tree consisting of all ancestors of $P$, \textcolor{cyan}{including the edge from $P$ to its parent}, and all descendant branches of $P$ excluding the branch starting from $(P,Q)$.  $\nset_Q$ denotes the sub-tree consisting of all descendant branches of $Q$, \textcolor{cyan}{including the edges from $Q$ to its children}. With this notation and based on the node types of $P,Q$ and the parity of $\nset_Q$, we can deduce the parity of $\nset_P$, since $\dot{\nset}$ is odd.

In the following, we derive the difference in PMW between a parent node $P$ and its child $Q$. For this, let us consider that a syndrome-node is connected to either $P$ or $Q$.

We first argue that any matching is equivalent to a matching where all nodes in the even note-tree are matched internally. If the sub-tree $\nset_Q$ is of even parity, denoted by $\overbar{\nset_Q}$, it must consist of a combination of even branches and an even number odd branches, where a branch is the subtree hanging from a child node. %, where the parity of the branch refers to the parity of the number of syndrome-nodes.
For any even branch $\bar{b}$, the full matching must be within $\bar{b}$: since there is a single edge connecting $\bar b$ with $Q$ and the branch is even, it would not be possible to support two matches out of $\bar{b}$.
If an odd branch $\dot{b}_1$ matches outside $\overbar{\nset_Q}$, on edge $(Q, \dot{b}_1)$, there must exist another odd branch $\dot{b}_2$ that also matches outside $\overbar{\nset_Q}$, in this case via edge $(Q, \dot{b}_1)$. Since the matchings meet at node $Q$ there is an equivalent matching that matches $\dot{b}_1$ and $\dot{b}_2$.
%If the node-tree merges with another odd tree either at $Q$ or $P$, the minimum weight matching will be equivalent to a matching where all nodes in $\overbar{\nset_Q}$ match internally.
The same argument holds for an even $\nset_P$.

If the subtree $\nset_Q$ is of odd parity, denoted by $\dot{\nset_Q}$, it must consist of a combination of even branches and an odd number of odd branches. By the same argument as before, for any matching, there exists an equivalent matching where the even branches and all odd branches except for one (which is an even number) match internally.
The remaining odd branch in $\dot{\nset_Q}$ must be connected with either $Q$, $P$, or the new syndrome-node connected. The same argument holds for an odd $\nset_P$.

Using this argument, we can deduce the required edges for a matching during a merger on $Q$ and $P$ and find the difference in the PMW of $Q$ and $P$. For example, if both $P,Q$ are syndrome-nodes and $\nset_Q$ is even, $\nset_P$ must be odd. The potential matching weights on $\dot{P},\dot{Q}$ are thus
\begin{align*}
  \text{PMW}(\dot{P}) &= \rho_{\dot{P}} + \abs{(\dot{\nset_P},\dot{P})} + \abs{(\dot{P}, \dot{Q})} + C \\
  \text{PMW}(\dot{Q}) &= \rho_{\dot{Q}} + \abs{(\dot{\nset_P},\dot{P})} + C,
\end{align*}
where $C$ is a constant equal to the weight of the matching \textcolor{cyan}{entirely within} the subtrees $\dot{\nset_P}$ and $\overbar{\nset_Q}$ and \textcolor{cyan}{$\abs{(\dot{\nset_P},\dot{P})}$ denotes the distance between the $P$ and its pair in $\nset_P$}. In consequence, we find that their difference is
\begin{equation*}
  \text{PMW}(\dot{Q}) - \text{PMW}(\dot{P}) = \rho_{\dot{Q}} - \rho_{\dot{P}} - \abs{(\dot{P}, \dot{Q})}.
\end{equation*}
Similarly, the PMW difference can be deduced for any combination of node types of $P,Q$ and parity of $\nset_Q$ (see \Cref{fig:nmcombi}):
\begin{align*}
  \text{PMW}({Q}) - \text{PMW}({P}) &= \rho_Q - \rho_P - \abs{(P,Q)}  &\dot{P}, \dot{Q}, \overbar{\nset_Q} \\
  &= \rho_Q - \rho_P + \abs{(P,Q)}  &\dot{P}, \dot{Q}, \dot{\nset_Q} \\
  &= \rho_Q - \rho_P + \abs{(P,Q)}  &\dot{P}, \bar{Q}, \overbar{\nset_Q} \\
  &= \rho_Q - \rho_P - \abs{(P,Q)}  &\dot{P}, \bar{Q}, \dot{\nset_Q} \\
  &= \rho_Q - \rho_P - \abs{(P,Q)}  &\bar{P}, \dot{Q}, \overbar{\nset_Q} \\
  &= \rho_Q - \rho_P + \abs{(P,Q)}  &\bar{P}, \dot{Q}, \dot{\nset_Q} \\
  &= \rho_Q - \rho_P + \abs{(P,Q)}  &\bar{P}, \bar{Q}, \overbar{\nset_Q} \\
  &= \rho_Q - \rho_P - \abs{(P,Q)}  &\bar{P}, \bar{Q}, \dot{\nset_Q} \\
\end{align*}
which can be simplified to
    %n_d = m_d + \Big\lfloor 2C\big(n_r-\rho_P - (-1)^{n_p}\abs{(n,m)}\big )\Big\rfloor
\begin{equation}\label{eq:pmwdif}
    \text{PMW}({Q}) =  \text{PMW}({P}) + \rho_Q-\rho_P + (-1)^{\pi_Q}\abs{(P,Q)},
\end{equation}
where $\pi_N$ is the \textbf{node parity} of $N$, i.e. the number of syndrome-nodes modulo 2 \textcolor{cyan}{in the sub-tree containing $N$ and all its descendants}. The node parity of a node can also be calculated recursively from the node parities of its children nodes, \textcolor{cyan}{based on the chosen reference root node $R$}. In particular, they are given by the following expressions for syndrome-nodes and junction-nodes:
\begin{align}
    \pi_{\dot{P}} = \hspace{.6cm} \Bigg(\hspace{.2cm} \sum_{\mathclap{\forall Q \leq \dot{P}}} (1-\pi_Q) \Bigg ) \bmod 2 \label{eq:snodeparity} \\
    \pi_{\bar{P}} = \Bigg(1 + \hspace{.2cm}  \sum_{\mathclap{\forall Q \leq \bar{P}}} (1-\pi_Q) \Bigg) \bmod 2.  \label{eq:jnodeparity}
\end{align}

\Figure[tb](topskip=0pt, botskip=0pt, midskip=0pt){figures/tikz/build/main-figure3.pdf}{
  Any odd-parity node-tree of $\abs{\dot{\nset}}\geq 3$ can be simplified to $\{\nset_P,P,Q,\nset_Q\}$, where $\nset_P$ is the subtree consisting of all ancestors of $P$ and all descendant branches excluding the branch starting from $(P,Q)$, $\nset_Q$ is the subtree consisting of all descendant branches of $Q$, and $(\nset_P, P), (Q, \nset_Q)$ can represent any number of edges on $\dot{\nset}$. The figure shows which edges are part of the matching for a hypothetical merger on either $P$ (cyan edges) or $Q$ (magenta edges), for any combination of node types of $P,Q$ and the parity of $\nset_Q$, similarly to \Cref{eq:pmwdif}.
  \label{fig:nmcombi}}

The previous relations allow to compute the difference in PMW between nodes directly connected in the node-tree. These relations can be used to obtain the difference with respect to a reference node, for instance with respect to $R$ the root of the node-tree, which can be calculated by a depth-first search from root $R$ to all descendant nodes by adding the differences between each parent and its child.

% \begin{equation}\label{eq:pseudodelay}
%     {\delta}_Q = {\delta}_P + 2\left(\text{PMW}(Q) - \text{PMW}(P)\right), \hspace{.3cm} Q\leq P.
% \end{equation}
% where ${\delta}_N = \text{PMW}(N) - \text{PMW}(R)$.

% \Cref{eq:pmwdif} can be easily explained through an example of the cluster in \Cref{fig:pmw}. The PMW's of $A, B, C$ are respectively
% \begin{align}
%     \nonumber \text{PMW}(A) &= \rho_A + \abs{(B, C)}, \\
%     \nonumber \text{PMW}(B) &= \rho_B + \abs{(A, B)} + \abs{(B, C)}, \\
%     \nonumber \text{PMW}(C) &= \rho_C + \abs{(A, B)}.
% \end{align}
% Take $A\geq B\geq C$. The differences in node delay are thus
% \begin{align}
%     \nonumber \Delta_B &= \delta_B - \delta_A =& 2\left(\rho_B - \rho_A + \abs{(A, B)}\right), \\
%     \nonumber \Delta_C &= \delta_C - \delta_B =& 2\left(\rho_C - \rho_B - \abs{(B, C)}\right).
% \end{align}


\documentclass[12pt]{article}
\usepackage[usenames]{color} %used for font color
\usepackage{amsmath, amssymb, amsthm}
\usepackage{wasysym}
\usepackage[utf8]{inputenc} %useful to type directly diacritic characters
\usepackage{graphicx}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{float}
\usepackage{mathtools}
\usepackage [english]{babel}
\usepackage [autostyle, english = american]{csquotes}
\MakeOuterQuote{"}
\graphicspath{ {./} }
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\prob}{\mathbb{P}}
\newcommand{\degrees}{^{\circ}}
\DeclarePairedDelimiter\ceil{\lceil}{\rceil}
\DeclarePairedDelimiter\floor{\lfloor}{\rfloor}

\author{Tianshuang (Ethan) Qiu}
\begin{document}
\title{Math 104, HW8}
\maketitle
\newpage

\section{Q1}
For $x \not = 1$, $\frac{1}{x}$ is differentiable, and according to the inverse theorem, it is equal to $-\frac{1}{x^2}$
\newline
Since $\sin'(x) = \cos(x)$, and $\sin$ is well defined on all of $\R$ (the codomain of $\frac{1}{x}: \R \setminus 0 \to \R$), we can apply the chain rule to the second factor: $(\sin(\frac{1}{x}))' = \cos(\frac{1}{x})(-\frac{1}{x^2})$
\newline
Now since $x^2: \R \to \R$ is continuous in all its domain, we attempt to differentiate at an arbitrary point $x_0$:
$$\lim_{x \to x_0}\frac{(x^2)-(x_0^2)}{x-x_0} = \lim_{x \to x_0}\frac{(x+x_0)(x-x_0)}{x-x_0} = \lim_{x \to x_0}(x+x_0) = 2x_0$$
Therefore $(x^2)'=2x$
\newline
Finally we use the product rule and the derivative is $$2x\sin(\frac{1}{x}) + x^2\cos(\frac{1}{x})(-\frac{1}{x^2})= 2x\sin(\frac{1}{x})-\cos(\frac{1}{x})$$
\newpage

\section{Q2}
We claim that $f'(0)$ exists and is equal to $0$.
\newline
Consider the definition of the derivative: $$\lim_{x \to 0}\frac{x^2\sin(\frac{1}{x})-0}{x-0}$$
Since this function is defined on every point but 0, we have $f': \R \setminus 0 \to \R$
Now we can simplify to:
$$\lim_{x \to 0}x \sin(\frac{1}{x}) $$
Now we apply the squeeze theorem with $-1 \leq \sin(s) \leq 1$, and
$$-x \leq x \sin(\frac{1}{x}) \leq x$$
Since $-x, x$ both converge to 0, our derivative also converges to 0.
\newpage

\section{Q3}
We will use the fact that the derivative at $\R \setminus 0$ contains $\cos(\frac{1}{x})$, which fluctuates rapidly when $x$ is close to 0 to show a contradiction.
\newline
Assume that the derivative function $f'$ is continuous on $\R$. So let $\epsilon = 0.1, x_0=0$, then by our assumption there exists $\delta$ such that $|x-x_0|<\delta$ implies $|f'(x)-f'(0)|<\epsilon$
\newline
We have shown above that $f'(0)=0$, so we expand $|f'(x)-f'(0)|$:
$$=|2x\sin(\frac{1}{x})-\cos(\frac{1}{x})-0| \leq |2x\sin(\frac{1}{x})|+|\cos(\frac{1}{x})| \leq |\cos(\frac{1}{x})|$$
\newline
By the Archimedean Principle we know that there exists $n$ such that $\frac{1}{n}<\delta$. Set $x = \frac{1}{2\pi n} <\frac{1}{n}<\delta$. Consider $f'(x) = |\cos(2\pi n)| = 1 > \epsilon$.
\newline
Therefore when $\epsilon=0.1, x_0=0$, we have found an $x$ such that for any $\delta > 0$, though $|x-x_0|<\delta$, $|f'(x)-f'(x_0)|>\epsilon$
\newline
Thus the function is not continuous.
$\blacksquare$
\newpage

\section{Q4}

\subsection{a}
Base case:
\newline
$(x^1)'$ By the definition of the derivative we know that $f'(a)=\lim_{x \to a} \frac{x-a}{x-a} = 1$ since the numerator and the denominator cancel.
\newline
Inductive step:
\newline
Assume that $(x^n)' = nx^{n-1}$ for some $n \in \N$ Consider $x^{n+1}$
\newline
We write it as $x^nx$, now we apply can the product rule. We have shown in the base case that $x'=1$, so we have
$$(x^{n+1})' = nx^{n-1}x + 1x^{n} = (n+1)x^{n+1}$$
Thus we have shown the inductive step. $\blacksquare$

\subsection{b}
We rewrite $(\frac{f}{g})' = (f\frac{1}{g})'$ Now we attempt to apply the chain rule to $(\frac{1}{g})'$
\newline
Since $g$ has codomain of $\R$ and is differentiable at $a$, we can use the chain rule.
$$(\frac{1}{g})' = -\frac{1}{(g(a))^2}g'(a)$$
Now we multiply $f(a)$ in with the product rule
$$(\frac{f}{g})' = f'(a)\frac{1}{g(a)}-f(a)\frac{1}{(g(a))^2}g'(a)
= \frac{f'(a)g(a)}{(g(a))^2}-\frac{f(a)g'(a)}{(g(a))^2}$$
Thus it is proven.
\newpage

\section{Ross 29.5}
My key observation to this problem is that when $x,y$ are close to each other, $(x-y)^2$ becomes very small. So in order to be smaller than or equal to this, $|f(x)-f(y)|$ must also be able to get arbitrarily small.
\newline
Let $\epsilon > 0$, for any $a \in \R$, we can simply pick our delta to be $\min\{1, \sqrt(\epsilon/2)\}$ Let $|x-a|<\delta$, then $(x-a)^2 < \delta ^2 = \epsilon/2$. Now by the statement of the problem $|f(x)-f(a)|\leq (x-a)^2 <\epsilon/2$. Therefore $f$ is be continuous everywhere.
\newline
Now consider the absolute value of the derivative of this function:
$$|f'(a)| = \lim_{x \to a}\frac{|f(x)-f(a)|}{|x-a|}$$
We can slightly modify the specification of the problem. Since $x^2 = (-x)^2$, we can rewrite the function assumption to be $|f(x)-f(y)|\leq |x-y|^2$
\newline
Now we have $|f'(a)| \leq \lim_{x \to a}x - a = 0$. Since $|f'(a)| \geq 0$, we have $|f'(a)| = 0$. Thus we have shown that the function has a derivative of $0$ everywhere, and it is therefore constant.
$\blacksquare$
\newpage

\section{Ross 29.13}
Consider $h(x)=g(x)-f(x)$. $h(0) = g(0)-f(0)=0, h'(x)=g'(x)-f'(x)$. By the specification of the problem we know that $h'(x)\geq 0 $ when $x \geq 0$
\newline
Now consider $h(y)$ with $y\geq 0$. Assume that there is a point $z \geq 0$ such that $g(z)<f(z)$, so $h(z)= g(z)-f(z) < 0$. Since $h(0)=0$, and $h$ is differentiable, by the Mean Value Theorem we know that there exists a point $p \in (0,z)$ where $h'(z)=\frac{h(z)-0}{z-0}<0$. However by our assumption we know that $h'(z) \geq 0 \forall z \geq 0$. We have reached a contradiction.
\newline
Therefore our assumption is incorrect, $g(x)\geq f(x)$ for all $x \geq 0$.
\newpage

\section{Ross 29.17}

\subsection{$h$ is differentiable implies equal derivative and limit}
Let $h$ be differentiable at $a$. Then by the definition of the derivative we have $\lim_{x \to a} \frac{h(x)-h(a)}{x-a}$ exists, which implies that $\lim_{x \to a} h(x)$ exists.
Since the limit exists, for any $\epsilon > 0$, $\exists \delta > 0$ such that $|x-a|<\delta$ implies $|h(x)-h(a)|<\epsilon$. $h(a)=g(a)$ When $x$ is less than $a$, $|f(x)-g(a)|<\epsilon$. Since the difference is less than any positive number, $f(a)$ must equal $g(a)$.
\newline
Now consider the fact that  $\lim_{x \to a} \frac{h(x)-h(a)}{x-a}$ exists. Then we must have
$$\lim_{x \to a^+} \frac{h(x)-h(a)}{x-a} = \lim_{x \to a^-} \frac{h(x)-h(a)}{x-a}$$
By the definition of $h$ we can now break the above statement down into:
$$\lim_{x \to a^+} \frac{f(x)-f(a)}{x-a} = \lim_{x \to a^-} \frac{g(x)-g(a)}{x-a}$$
Since $f'(a), g'(a)$ exist, then the above equation must imply $\lim_{x \to a^+} f'(a) = \lim_{x \to a^-} g'(a)$, thus $f'(a)=g'(a)$

\subsection{Converse}
Let $f(a)=g(a), f'(a)=g'(a)$. Since $f(a)=g(a)$, and $f,g$ are differentiable and therefore continuous, the function $h$ is continuous at $a$.
Now consider the derivative: $\lim_{x \to a} \frac{h(x)-h(a)}{x-a}$.
\newline
We break it down into two cases: $x$ converging from the negative and the positive
$$\lim_{x \to a^+} \frac{h(x)-h(a)}{x-a} = \lim_{x \to a^+} \frac{f(x)-f(a)}{x-a} = f'(a)$$
$$\lim_{x \to a^-} \frac{h(x)-h(a)}{x-a} = \lim_{x \to a^-} \frac{g(x)-g(a)}{x-a} = g'(a)$$
Now since $f'(a)=g'(a)$, we know that $\lim_{x \to a^+} \frac{h(x)-h(a)}{x-a} = \lim_{x \to a^-} \frac{h(x)-h(a)}{x-a}$.
Thus the limit exists, and therefore $h$ is differentiable at $a$
\newline
$\blacksquare$

\end{document}


\documentclass{llncs}

\usepackage{latexsym}
\usepackage{times}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{cite}
\usepackage{booktabs}
\usepackage{adjustbox}
\usepackage{marvosym}
%% Colored hyperlink
\newcommand{\cref}[2]{\href{#1}{\color{blue}#2}}
%% Colored hyperlink showing link in TT font
% \newcommand{\chref}[1]{\href{#1}{\small\tt \color{blue}#1}}
\newcommand{\hcref}[1]{\cref{#1}{\small\tt #1}}

\bibliographystyle{splncs04}

\newcommand{\one}{\mbox{\bf 1}}
\newcommand{\zero}{\mbox{\bf 0}}
\newcommand{\leafone}{L_1}
\newcommand{\leafzero}{L_0}
\newcommand{\booland}{\land}
\newcommand{\boolor}{\lor}
\newcommand{\boolxor}{\oplus}
\newcommand{\boolnot}{\neg}
%\newcommand{\tautology}{\top}
%\newcommand{\nil}{\bot}
\newcommand{\tautology}{1}
\newcommand{\nil}{0}
\newcommand{\obar}[1]{\overline{#1}}
\newcommand{\ite}{\mbox{\it ITE}}

\newcommand{\opname}[1]{\mbox{\sc #1}}
\newcommand{\andop}{\opname{And}}
\newcommand{\implyop}{\opname{Imply}}

\newcommand{\turnstile}{\vdash}

\newcommand{\fname}[1]{\mbox{\small\sf #1}}

\newcommand{\lo}{\fname{Lo}}
\newcommand{\hi}{\fname{Hi}}
\newcommand{\var}{\fname{Var}}
\newcommand{\val}{\fname{Val}}

\newcommand{\interp}{\sigma}
\newcommand{\interpset}[1]{\Sigma_{#1}}
\newcommand{\mcount}{\mu}

\newcommand{\ifarg}{\textbf{I}}
\newcommand{\thenarg}{\textbf{T}}
\newcommand{\elsearg}{\textbf{E}}
\newcommand{\depend}{{\it D}}

\newcommand{\subs}[2]{[#2/#1]}
\newcommand{\substrue}[1]{\subs{#1}{\tautology}}
\newcommand{\subsfalse}[1]{\subs{#1}{\false}}
\newcommand{\subsflip}[1]{\subs{#1}{\obar{#1}}}

\title{Notes on Validated Model Counting \\ Version of \today}

\author{Randal E. Bryant}

\institute{
Computer Science Department \\
Carnegie Mellon University, Pittsburgh, PA, United States
}

\begin{document}

\maketitle

\section{Notation}

Consider Boolean formulas over a set of variables $X$.  An
assignment $\interp$ is a function mapping each variable to
truth value $\tautology$ (true) or $\nil$ (false).  We can extend
$\interp$ to Boolean formulas in the normal way, such that
$\interp(F)$ will be $\tautology$ (respectively, $\nil$) if the
evaluation of formula $F$ yields $\tautology$ (resp., $\nil$) when its
variables are assigned values according to $\interp$.

For Boolean formula $F$, we define its set of {\em models} $\interpset{F}$ as
\begin{eqnarray}
\interpset{F} & = & \{ \interp | \interp(F) = \tautology \}
\end{eqnarray}
The task of model counting is, given formula $F$, to determine the size of its set of models
$\mcount(F) = |\interpset{F}|$.  Ideally, this should be done without actually enumerating the set.

For interpretation $\interp$ and a Boolean formula $E$ over $X$, we
use the notation $\interp\subs{x}{E}$ to denote the intepretation
$\interp'$, such that $\interp'(y) = \interp(y)$ for all $y \not = x$
and $\interp'(x) = \interp(E)$.  In particular, the notation
$\interp\subsflip{x}$ indicates the interpretation in which the value
assigned to $x$ is complemented, while others remain unchanged.

A Boolean formula $F$ is said to be {\em independent} of variable $x$
if every $\interp \in \interpset{F}$ has $\interp\subsflip{x} \in
\interpset{F}$.

\begin{lemma}
\label{lemma:independent:split}
If Boolean formula $F$ is independent of variable $x$, then:
\begin{equation}
|\interpset{F}| \; = \; 2\cdot |\interpset{F\land x}| = \; 2\cdot |\interpset{F\land \obar{x}}| \label{eqn:independent}
\end{equation}
\end{lemma}
That is, the set of models of $F$ is split evenly between
those assigning $\tautology$ to $x$ and those assigning $\nil$ to $x$.

\section{ITE Operation}

We consider a single Boolean operation, known as
``If-Then-Else,'' or simply ``ITE.''  For Boolean values $a, b, c$,
the operation is defined as $\ite(a,b,c) = (a \land b) \lor (\obar{a}
\land c)$.  This single operation can be used to express several common Boolean operations:
\begin{eqnarray*}
\obar{a} & = & \ite(a, \nil, \tautology) \\
a \land b & = & \ite(a, b, \nil) \\
a \lor b & = & \ite(a, \tautology, b) \\
a \rightarrow b & = & \ite(a, b, \tautology) \\
\end{eqnarray*}

For variable $x$ and Boolean formulas $F$ and $G$, the set of models
for $\ite(x, F, G)$ is given by the formula:
\begin{eqnarray}
  \interpset{\ite(x, F, G)} & = & \{ \interp \in \interpset{F} | \interp(x) = \tautology \}
  \cup \{ \interp \in \interpset{G} | \interp(x) = \nil \} \label{eqn:ite:interp}
\end{eqnarray}

\section{ITE Graphs}
Model counting is especially simple for a class of formulas we
call ``free ITE graphs.''  This representation can be obtained
directly from an Ordered Binary Decision Diagram (OBDD) representation
of a Boolean function~\cite{Bryant:1986}.

An {\em ITE graph} is defined to be a directed acyclic graph,
consisting of three node types:
\begin{description}
\item[Constant:] Corresponds to value $\tautology$ or $\nil$.  A constant node has no incoming arcs.
\item[Input:] Corresponds to one of the variables in $X$.   A variable node has no incoming arcs.
\item[Operator:] Represents an application of the $\ite$ operation.  An operator node has three incoming arcs, labeled \ifarg{}, \thenarg{}, and \elsearg{}, corresponding to the three arguments of the $\ite$ operation.
\end{description}

An {\em input-controlled} ITE graph is one for which the incoming \ifarg{} arc for every operator node is
from an input node.  That is, the result of one $\ite$ operation can
serve as the ``then'' or ``else'' argument to another $\ite$
operation, but not for the ``if'' argument.

When describing input-controlled ITE graphs, we refer to constant and input
nodes by their associated value or variable.  An operator node is described by an expression of the form
$\ite(x, v_t, v_e)$, where $v_t$ and $v_e$ are the
nodes corresponding to the \thenarg{} and \elsearg{} inputs, respectively.

We can define the function $\depend$ mapping each node
$v$ in an input-controlled ITE graph to the set of variables on which it logically depends.  This can be expressed recursively as
\begin{eqnarray}
\depend(\tautology) & = & \emptyset \label{eqn:depend:true}\\
\depend(\nil) & = & \emptyset \label{eqn:depend:false}\\
\depend(x) & = & \{x \} \label{eqn:depend:variable}\\
\depend(\ite(x, v_t, v_e)) & = & \{ x \} \cup \depend(v_t) \cup \depend(v_e) \label{eqn:depend:ite}
\end{eqnarray}

\begin{lemma}
\label{lemma:independent:dset}
Any node $v$ in an ITE graph is independent of any variable $y \in X$ such that $y \not \in \depend(v)$.
\end{lemma}
\noindent
{\em Proof:} Follows by the recursive definition of $\depend$ (\ref{eqn:depend:true}--\ref{eqn:depend:ite}) and by (\ref{eqn:ite:interp}).

A {\em free} ITE graph is an input-controlled ITE graph satisfying the added
property that for every operator node $v = \ite(x, v_t, v_e)$, its
incoming \thenarg{} and \elsearg{} arcs must be from nodes such that
$x \not \in \depend(v_t)$ and $x \not \in \depend(v_e)$.

Free ITE graphs correspond directly to a class of binary decision diagrams known as {\em free} BDDs (FBDDs)~\cite{gergov:ieeetc:1994}.  Ordered BDDs are a subclass of FBDDs.

\section{Model Counting Free ITE Graphs}

While model counting is a difficult problem for arbitrary
formulas, it readily be computed when $F$ takes the form of a free
ITE graph.

\begin{theorem}
\label{theorem:count}
Letting $n = |X|$, the following recursive formula holds for any node in a free ITE graph
\begin{eqnarray}
\mcount(\tautology) & = & 2^{n} \label{eqn:count:true}\\
\mcount(\nil) & = & 0 \label{eqn:count:false}\\
\mcount(x) & = & 2^{n-1} \label{eqn:count:variable}\\
\mcount(\ite(x, v_t, v_e)) & = & \frac{\mcount(v_t) + \mcount(v_e)}{2} \label{eqn:count:ite}
\end{eqnarray}
\end{theorem}
\noindent
{\em Proof:} The key result here is (\ref{eqn:count:ite}).  It follows
from Lemma~\ref{lemma:independent:split}, (\ref{eqn:ite:interp}),
Lemma~\ref{lemma:independent:dset}, and (\ref{eqn:depend:ite}).  That is, both
$v_t$ and $v_e$ are independent of $x$.  That implies that half of the
satisfying assignments $\interp$ for $v_t$ have have $\interp(x) =
\tautology$.  Similarly, half of the satisfying assignments $\interp$
for $v_e$ have $\interp(x) = \nil$.  Equation (\ref{eqn:count:ite}) then follows by (\ref{eqn:ite:interp}).

\bibliography{references}

\end{document}


\section{Task C}
\label{sec:task-c}

The same plots as in section~\ref{sec:task-b} were obtained for the case
when the truss angle \(\alpha = \operatorname{atan} (1/3)\).
The plots are given in figure~\ref{fig:plots-C} for the left truss.
\begin{figure}[th]
  \centering
  % Force - deflection:
  \begin{subfigure}[t]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[
        width = 0.95\textwidth,
        height=\axisdefaultheight,
        tick label style={/pgf/number format/fixed},
        try min ticks=6,
        minor tick num=1,
        grid=both,
        xlabel = {\( (Y - y) / L \), [-]},
        ylabel = {\( F / (E A) \), [-]},
        xmin = 0,
        xmax = 2,
        ymin = -0.15,
        ymax = 0.2,
        legend cell align=left,
        legend style={anchor=south west, at={(0,0)}}
        ]
        \addplot+ table[skip first n=1] {data/force_deflection_elastic_C.dat}; % + = same automatically determined styles, but in addition it uses [options]
        \addlegendentry{elastic}
        \addplot+ table[skip first n=1] {data/force_deflection_plastic_C.dat};
        \addlegendentry{plastic}
      \end{axis}
    \end{tikzpicture}
    \caption{Force - deflection}
  \end{subfigure}

  % Stress - strain:
  \begin{subfigure}[t]{0.48\textwidth}
    \begin{tikzpicture}[baseline,yscale=1.0, xscale=1.0]
      \begin{axis}[
        try min ticks=5,
        minor tick num=1,
        grid=both,
        xlabel = {strain, [-]},
        ylabel = {Kirchhoff stress, [kN/mm\textsuperscript{2}]},
        % xmin = -0.4,
        xmax = 0.4,
        % xtick={-0.4, -0.2, 0, 0.2},
        % ymin = -30,
        % ymax = 30,
        legend cell align=left,
        legend style={anchor=south east, at={(1,0)}}
        ]
        \addplot+ table[skip first n=1] {data/stress_strain_elastic_C.dat};
        \addlegendentry{elastic}
        \addplot+ table[skip first n=1] {data/stress_strain_plastic_C.dat};
        \addlegendentry{plastic}
      \end{axis}
    \end{tikzpicture}
    \caption{Constitutive behaviour}
  \end{subfigure}
  % Total - plastic strain:
  \begin{subfigure}[t]{0.48\textwidth}
    \begin{tikzpicture}[baseline,yscale=1.0, xscale=1.0]
      \begin{axis}[
        tick label style={/pgf/number format/fixed},
        % try min ticks=8,
        minor tick num=1,
        grid=both,
        xlabel = {total strain, [-]},
        ylabel = {plastic strain, [-]},
        % xmin = -0.4,
        % xmax = 0.2,
        % ymin = -0.1,
        ymax = 0.01
        ]
        \addplot+ [mark=none] table[skip first n=1] {data/tot_pl_strain_C.dat};
        \addplot+  table[skip first n=1] {data/tot_pl_strain_C.dat}; % Plot again using style of 2nd element in cycle list.
      \end{axis}
    \end{tikzpicture}
    \caption{Plastic - total strain}
  \end{subfigure}
  \caption{Plots of large deflection elasto-plastic behaviour of the fist truss when \(A_{2} = 1.05 A_{1}\).}
  \label{fig:plots-C}
\end{figure}

In addition, the displacement of the initially upper joint is shown in
figure~\ref{fig:node-displ-C}.
It is evident that for such small angle the solution is sensitive to the
difference in cross-section area between the trusses.
\begin{figure}[th]
  \centering
  \begin{tikzpicture}[baseline,yscale=1.0, xscale=1.0]
    \begin{axis}[
      % try min ticks=8,
      width = 0.7\textwidth,
      minor tick num=1,
      grid=both,
      xlabel = {horizontal displacement, [mm]},
      ylabel = {vertical displacement, [mm]},
      legend cell align=left,
      legend style={anchor=center, at={(0.5,0.5)}},
      every axis plot/.append style={line width=0.8pt}, % 0.4 - default
      % xmin=-12,
      % xmax=30,
      % ymin=-300,
      ymax=0
      ]
      \addplot+ [mark=none] table[skip first n=1] {data/node_displ_elastic_C.dat};
      \addlegendentry{elastic, \(A_{2} = 1.05 A_{1}\)}
      \addplot+ [mark=none] table[skip first n=1] {data/node_displ_plastic_C.dat};
      \addlegendentry{plastic, \(A_{2} = 1.05 A_{1}\)}
      \addplot+ [mark=none, dashed] table[skip first n=1] {data/node_displ_plastic_C_1percent.dat};
      \addlegendentry{plastic, \(A_{2} = 1.01 A_{1}\)}
      \addplot+ [mark=none, densely dotted] table[skip first n=1] {data/node_displ_plastic_C_10percent.dat};
      \addlegendentry{plastic, \(A_{2} = 1.1 A_{1}\)}
    \end{axis}
  \end{tikzpicture}
  \caption{Displacement of the initially upper node of the first (left) truss.}
  \label{fig:node-displ-C}
\end{figure}

Obtaining the convergence for this task was the most problematic.
Even in the case of elastic material response, a situation depicted
in the figure~\ref{fig:no-newton} was occurring, where the local
minimum prevented the Newton-Raphson method from finding the
correct solution.
In those situations, the false position method was used instead.
\begin{figure}[th]
  \centering
  \begin{tikzpicture}[baseline,yscale=1.0, xscale=1.0]
    \begin{axis}[
      width = 0.7\textwidth,
      max space between ticks=50,
      minor tick num=1,
      grid=both,
      xlabel = {displacement, [mm]},
      ylabel = {residual, [N]},
      ]
      \addplot+ table[skip first n=1] {data/residual_vs_du.dat};
    \end{axis}
  \end{tikzpicture}
  \caption{Example of situation, where Newton-Raphson method can fail.}
  \label{fig:no-newton}
\end{figure}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../main"
%%% End:


\chapter{Differential Algebraic Equation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Singular perturbed problems and differential algebraic
	equation of index 1}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{example}[Van der Pol equation]
  Given is the following equation:

  \begin{equation}\begin{split}
    \epsilon z'' + (z^2 - 1) z' + z & = 0 \\
    z(0) & = z_0 \\
  \end{split}\end{equation}

	It is also called Van der Pol equation. If one now chooses $\epsilon$
	very small, then one obtains a stiff IVP.

  \begin{equation}
    \epsilon z'' + (z^2 - 1) z' = \diffq[t] \underbrace{(\epsilon z' + (\frac{z^3}{3} - z))}_{=: y}
  \end{equation}

  \begin{equation}\begin{split}
    \diffq[t] y & = -z \\
    \epsilon \diffq[t] z & = y - (\frac{z^3}{3} - z) \\
  \end{split}\end{equation}

  \begin{equation}\begin{split}
    y' & = f(y,z) \\
    \epsilon z' & = g(y,z) \\
  \end{split}\end{equation}

	This is an example of a singular perturbed problem (SDP)

  For $\epsilon = 0$ we receive:

  \begin{equation}\begin{array}{lll}
    y' & = f(y,z) & \leftarrow \text{ordinary differential equation (ODE)} \\
    0 & = g(y,z) & \leftarrow \text{algebraic equation (AE)}\\
  \end{array}\end{equation}

	The upper system consists of an ordinary differential equation and an algebraic
	equation. Therefore one also calls this a differential algebraic equation
	(DAE).
\end{example}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "notes"
%%% End:


\problemname{Hole in One}

Janine recently went to her local game store and bought ``Hole in One'', a new mini-golf game for her computer. As indicated by the name, the objective of the game is to shoot a ball into a hole using just one shot. The game also borrows elements from brick breaker style games: in the playing field, several walls are placed that will be destroyed upon being hit by the ball.  The score of a successful shot depends on the number of destroyed walls, so Janine wonders: what is the maximum number of walls that can be hit while performing a ``Hole in One''?

For the purposes of this problem you can think of the playing field as a cartesian plane with the initial position of the ball at the origin. The walls are non-intersecting axis-parallel line segments in this plane (i.e., parallel to either the $x$ axis or the $y$ axis).  The diameter of the ball is negligible so it is represented as a single point.
\begin{figure}[h!]
	\centering
	\includegraphics[width=0.55\textwidth]{sample-illustration-new.png}
	\begin{minipage}[c]{0.82\textwidth}
		\caption{Illustration of the first sample input: The ball first bounces off two walls
		at points 1 and 2. When it passes point 3 the wall has already vanished.}
	\end{minipage}
\end{figure}

Whenever the ball hits a wall, two things happen:
\begin{itemize}
\item The direction of the ball changes in the usual way: the angle of incidence equals the angle of reflection.
\item The wall that the ball touched is destroyed. Following common video game logic, no rubble of the wall remains; it will be as though it vanished.
\end{itemize}

The behaviour of the ball is also affected by the power of the
shot.  In particular, an optimal shot may need to first roll over the
hole, then hit some more walls, and only later drop into the hole.

\section*{Input}

The input consists of:
\begin{itemize}
\item one line with one integer $n$ ($0 \le n \le 8$), the number of walls;
\item one line with two integers $x$ and $y$, the coordinates of the hole;
\item $n$ lines each with four integers $x_1$, $y_1$, $x_2$, and $y_2$ (either $x_1 = x_2$, or $y_1 = y_2$, but not both), representing a wall with end points $(x_1, y_1)$ and $(x_2, y_2)$.
\end{itemize}

The hole is not at the origin and not on a wall.  The walls do not
touch or intersect each other.  No wall lies completely on the $x$
axis or the $y$ axis.
All coordinates in the input are integers with absolute value at most $1\,000$.

\section*{Output}

If there is no way to shoot the ball such that it reaches the hole, print ``\texttt{impossible}''.
Otherwise, print the maximum number of walls that can be destroyed in a single ``Hole in One'' shot.


\documentclass{article}

\usepackage[colorlinks=true]{hyperref}
\usepackage[cmex10]{amsmath}
\usepackage{bbm}
\usepackage{graphicx}
\usepackage{subfig}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{comment}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{multirow}

\newcommand{\BigO}[1]{\ensuremath{\operatorname{O}\left(#1\right)}}
\DeclareMathOperator*{\argmin}{\mathrm{argmin}}

\begin{document}

\title{SubKit: Subspace Clustering Library}
\author{Stephen Tierney, Yi Guo and Junbin Gao}
\maketitle

\tableofcontents

\newpage
\section{Motivation}

SubKit is a library with implementations for subspace clustering learning algorithms. This library was created during the writing of the OSC journal article to ensure fair comparison between different methods. In the original OSC CVPR paper we used implementations provided by authors of SSC and LRR. However these implementations used different noise models to what we suggested with OSC. For example the original LRR implementation expects column wise noise while we expect Gaussian noise in all columns. Therefore we required new implementations with the same noise model for fair comparison in our journal article.

\section{Preliminaries}

We introduce required notation. We assume the following noise model
\begin{align}
\mathbf{X = A + N}
\end{align}
where $\mathbf A$ is the noise free data (column wise data samples), $\mathbf N$ is some noise and $\mathbf X$ is the observed data. We assume that each data sample in $\mathbf A$ lies exactly on its corresponding subspace. In an ideal case subspace clustering uses the self expressive model with the noise free data i.e.\
\begin{align}
\mathbf{A = AZ}
\end{align}
where $\mathbf Z$ is the matrix of coefficients. However this is rarely the case since we do not observe noise free data therefore the self expressive model becomes
\begin{align}
\mathbf{X = XZ + E}
\end{align}
where $\mathbf E$ is a fitting error. Once $\mathbf Z$ has been estimated one performs spectral clustering to obtain the final segmentation.

\newpage
\section{Function Listing}

\begin{table}[!h]
{\small{
\centering

\begin{tabular}{c | c | c}
\hline
Objective & Function & Section \\
\hline

$\begin{array}{c} \min_{\mathbf Z} \; \|\mathbf Z\|_{1} \\
\text{s.t.} \; \mathbf{A = AZ} \end{array}$
	& ssc\_noisefree	& 4.1  \\
\hline

\multirow{4}{*}{$\begin{array}{c} \min_{\mathbf Z} \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F + \lambda\|\mathbf Z\|_{1} \\
\text{s.t.} \quad \mathbf{X = XZ + E}\text{, diag}(\mathbf Z) = \mathbf 0 \end{array}$}
		& ssc\_relaxed	& 4.2 \\
		& ssc\_relaxed\_lin	& 4.3.1 \\
		& ssc\_relaxed\_lin\_ext	& 4.3.2 \\
		& ssc\_relaxed\_lin\_acc	& 4.3.3 \\
\hline

$\begin{array}{c} \min_{\mathbf{Z, E}}  \frac12\|\mathbf E\|^2_F+  \lambda\|\mathbf Z\|_{1} \\
\text{s.t.} \quad \mathbf{X = XZ + E}\text{, diag}(\mathbf Z) = \mathbf 0 \end{array}$
	& ssc\_exact\_fro	& 4.4  \\
\hline

$\begin{array}{c} \min_{\mathbf{Z, E}}  \|\mathbf E\|_1+  \lambda\|\mathbf Z\|_{1} \\
\text{s.t.} \quad \mathbf{X = XZ + E}\text{, diag}(\mathbf Z) = \mathbf 0\end{array}$
	& ssc\_exact\_l1	& 4.4  \\
\hline

$\begin{array}{c} \min_{\mathbf{Z, E}}  \|\mathbf E\|_{1,2}+  \lambda\|\mathbf Z\|_{1} \\
\text{s.t.} \quad \mathbf{X = XZ + E}\text{, diag}(\mathbf Z) = \mathbf 0 \end{array}$
	& ssc\_exact\_l1l2	& 4.4  \\
\hline

$\begin{array}{c} \min_{\mathbf Z} \; \|\mathbf Z\|_{*} \\
\text{s.t.} \; \mathbf{A = AZ} \end{array}$
	& lrr\_noisefree	& 5.1  \\
\hline

\multirow{4}{*}{$\begin{array}{c} \min_{\mathbf Z} \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F + \lambda\|\mathbf Z\|_{*} \end{array}$}
		& lrr\_relaxed	& 5.2 \\
		& lrr\_relaxed\_lin	& 5.3 \\
		& lrr\_relaxed\_lin\_ext	& 5.3 \\
		& lrr\_relaxed\_lin\_acc	& 5.3 \\
\hline

$\begin{array}{c} \min_{\mathbf{Z, E}}  \frac12\|\mathbf E\|^2_F+  \lambda\|\mathbf Z\|_{*} \\
\text{s.t.} \quad \mathbf{X = XZ + E} \end{array}$
	& lrr\_exact\_fro	& 5.4  \\
\hline

$\begin{array}{c} \min_{\mathbf{Z, E}}  \|\mathbf E\|_1+  \lambda\|\mathbf Z\|_{*} \\
\text{s.t.} \quad \mathbf{X = XZ + E} \end{array}$
	& lrr\_exact\_l1	& 5.4  \\
\hline

$\begin{array}{c} \min_{\mathbf{Z, E}}  \|\mathbf E\|_{1,2}+  \lambda\|\mathbf Z\|_{*} \\
\text{s.t.} \quad \mathbf{X = XZ + E} \end{array}$
	& lrr\_exact\_l1l2	& 5.4  \\
\hline

\multirow{3}{*}{$\begin{array}{c} \min_{\mathbf {Z, A, N}} \; \lambda \|\mathbf{N}\|_{q} + \|\mathbf Z\|_{*} \\
\text{s.t.} \quad \mathbf{A = AZ, X = A + N} \end{array}$}
	& r\_lrr\_fro	& 5.5  \\
	& r\_lrr\_l1		& 5.5  \\
	& r\_lrr\_l2		& 5.5  \\
\hline

$\begin{array}{c} \min_{\mathbf Z} \frac12\|\mathbf{X - X Z}|^2_F +\lambda_1\|\mathbf Z\|_{1}+\lambda_2\|\mathbf Z\mathbf R\|_{1} \\
\text{s.t.} \; \text{diag}(\mathbf Z) = \mathbf 0 \end{array}$
	& spatsc\_noisefree	& 6.1  \\
\hline

$\begin{array}{c} \min_{\mathbf Z} \frac12\|\mathbf{X - X Z}|^2_F +\lambda_1\|\mathbf Z\|_{1}+\lambda_2\|\mathbf Z\mathbf R\|_{1,2} \\
\text{s.t.} \; \text{diag}(\mathbf Z) = \mathbf 0 \end{array}$
	& osc\_noisefree	& 7.1  \\
\hline

\end{tabular}
}}
\end{table}

\newpage
\section{Sparse Subspace Clustering}

\subsection{Noise Free}

\begin{align}
\min_{\mathbf Z} \lambda\|\mathbf Z\|_{1} \\
\text{s.t.} \quad \mathbf{A = AZ} \nonumber
\end{align}
The solution to the above objective has a fast approximate solution, given by the Shape Interaction Matrix. See the low-rank clustering section for further details. Alternatively one can calculate the sample correlation matrix $\mathbf X^T \mathbf X$ and select points with the maximum correlation as neighbours on the graph.

\subsection{Relaxed SSC ADMM}

\begin{align}
\min_{\mathbf Z} \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F + \lambda\|\mathbf Z\|_{1}
\end{align}
To solve the relaxation via ADMM one must incorporate an auxiliary variable
\begin{align}
\min_{\mathbf{Z, J}} \frac12\|\mathbf X - \mathbf X\mathbf J\|^2_F + \lambda\|\mathbf Z\|_{1} \\
\text{s.t.} \quad \mathbf{Z = J} \nonumber
\end{align}
\begin{align}
\min_{\mathbf{Z, J}} \frac12\|\mathbf X - \mathbf X\mathbf J\|^2_F + \lambda\|\mathbf Z\|_{1} + \langle \mathbf{Y, Z - J} \rangle + \frac{\mu}{2} \| \mathbf{Z - J} \|_F^2
\end{align}
Then iterate the following
\begin{enumerate}
\item Fix others and solve for $\mathbf J$
\[
\min_{\mathbf J} \frac12\|\mathbf X - \mathbf X\mathbf J\|^2_F - \langle \mathbf{Y, J} \rangle + \frac{\mu}{2} \| \mathbf{Z - J} \|_F^2
\]
\[
\min_{\mathbf J} \frac12\|\mathbf X - \mathbf X\mathbf J\|^2_F + \frac{\mu}{2} \| (\mathbf Z + \frac{1}{\mu} \mathbf Y ) - \mathbf J \|_F^2
\]
\[
\mathbf X^T(\mathbf X - \mathbf X\mathbf J) + \mu ( (\mathbf Z + \frac{1}{\mu} \mathbf Y ) - \mathbf J )
\]
\[
\mathbf X^T\mathbf X\mathbf J  + \mu \mathbf J = \mathbf X^T\mathbf X + \mu \mathbf Z + \mathbf Y
\]
\[
(\mathbf X^T\mathbf X + \mu \mathbf I )\mathbf J = \mathbf X^T\mathbf X + \mu \mathbf Z + \mathbf Y
\]
\[
\mathbf J = (\mathbf X^T\mathbf X + \mu \mathbf I )^{-1} (\mathbf X^T\mathbf X + \mu \mathbf Z + \mathbf Y)
\]

\item Fix others and solve for $\mathbf Z$
\[
\min_{\mathbf Z} \lambda\|\mathbf Z\|_{1} + \langle \mathbf{Y, Z} \rangle + \frac{\mu}{2} \| \mathbf{Z - J} \|_F^2
\]
\[
\min_{\mathbf Z} \lambda\|\mathbf Z\|_{1} + \frac{\mu}{2} \| \mathbf Z - (\mathbf J - \frac{1}{\mu} \mathbf Y) \|_F^2
\]

\item Update $\mathbf Y$
\[
\mathbf Y = \mathbf Y + \mu (\mathbf{Z - J})
\]

\item Check stopping criteria
\[
\| \mathbf Z - \mathbf J \|_F  < \epsilon_1, \;
\mu \; \textrm{max} ( \| \mathbf Z_{k+1} - \mathbf Z_{k}  \|_F  , \|  \mathbf J_{k+1} - \mathbf J_{k} \|_F) < \epsilon_2
\]

\end{enumerate}

\subsection{Relaxed SSC Linearised}

Instead of using ADMM one can linearise the Frobenius norm term instead. This removes the need to do an expensive matrix inversion.
\begin{align}
\min_{\mathbf Z} L = \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F + \lambda\|\mathbf Z\|_{1}
\end{align}

Let $F = \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F$ and $\partial F = - \mathbf X^T \mathbf X + \mathbf X^T \mathbf{X Z}$.

\subsubsection{Gradient Descent}

This version consists on iterating the following until convergence

\[
\min_{\mathbf Z} \widetilde{L}_{\rho}(\mathbf{Z, Z_k}) =  \lambda\|\mathbf Z\|_{1} + \frac{\rho}{2} \| \mathbf Z - ( \mathbf Z_{k-1} - \frac{1}{\rho} \partial F(\mathbf Z_{k-1})) \|_F^2
\]

\subsubsection{Extended Gradient Algorithm}

It has been shown that through the correct choice of step size $\rho$ the gradient method can achieve a convergence rate of $\BigO{\frac{1}{k}}$ \cite{ji2009accelerated}. More specifically we require that $\rho \geq C$ where $C$ is the Lipschitz constant. However we do not know the value of $C$ in advance. Fortunately we know that if $L(\mathbf B) \leq \widetilde{L}_{\rho}(\mathbf B, \mathbf Z_{k-1})$ where $\mathbf B = \mathcal S_{\frac{\lambda}{\rho}}(\mathbf Z_{k-1} - \frac1{\lambda}\partial F(\mathbf Z_{k-1}))$ then $\rho \geq C$.

\begin{algorithm}
\caption{Extended Gradient Descent for Robust MC}
\begin{algorithmic}

\REQUIRE $k = 1$, $r_0 = \infty$, $\mathbf Z_0 = \mathbf 0$, $\lambda$, $\rho$, $\gamma$, $\epsilon$

\WHILE{$r_k - r_{k-1} \geq \epsilon$ }

	\WHILE{$L(\mathbf B) \geq \widetilde{L}_{\rho}(\mathbf B, \mathbf Z_{k-1})$}

		\STATE $\rho = \gamma \rho$

	\ENDWHILE

	\STATE $\mathbf Z_k = \mathcal S_{\frac{\lambda}{\rho}}(\mathbf Z_{k-1} - \frac1{\rho}\partial F(\mathbf Z_{k-1}))$
	\STATE $r_k = \frac12\|\mathbf X - \mathbf X\mathbf Z_k\|^2_F + \lambda\|\mathbf Z_k\|_{1}$
	\STATE $k = k + 1$

\ENDWHILE

\end{algorithmic}
\end{algorithm}

\subsubsection{Accelerated Gradient Algorithm}

We can further improve our convergence rate to $\BigO{\frac{1}{k^2}}$ by adopting Nesterov's accelerated gradient algorithm. This is similar to the algorithm described in \cite{ji2009accelerated}. Let $\mathbf B = \mathcal S_{\frac{\lambda}{\rho}}(\mathbf J_{k-1} - \frac1{\rho}\partial F(\mathbf J_{k-1}))$.

\begin{algorithm}
\caption{Accelerated Gradient Descent for Robust MC}
\begin{algorithmic}

\REQUIRE $k = 1$, $r_0 = \infty$, $\mathbf Z_0 = \mathbf 0$,  $\mathbf J_0 = \mathbf 0$, $\alpha_0 = 1$, $\lambda$, $\rho$, $\gamma$, $\epsilon$

\WHILE{$r_k - r_{k-1} \geq \epsilon$ }

	\WHILE{$L(\mathbf B) \geq \widetilde{L}_{\rho}(\mathbf B, \mathbf J_{k-1})$}

		\STATE $\rho = \gamma \rho$

	\ENDWHILE

	\STATE $\mathbf Z_k = \mathcal S_{\frac{\lambda}{\rho}}(\mathbf J_{k-1} - \frac1{\rho}\partial F(\mathbf J_{k-1}))$
	\STATE $\alpha_{k} = \frac{1 + \sqrt{1 + 4 \alpha_{k-1}^2)}}{2}$
	\STATE $\mathbf J_{k} = \mathbf Z_k + \left ( \frac{\alpha_{k-1} - 1}{\alpha_{k}} \right ) (\mathbf Z_k - \mathbf Z_{k-1})$
	\STATE $r_k = \frac12\|\mathbf X - \mathbf X\mathbf Z_k\|^2_F + \lambda\|\mathbf Z_k\|_{1}$
	\STATE $k = k + 1$

\ENDWHILE

\end{algorithmic}
\end{algorithm}

\subsection{Exact SSC LADM}

Using LADM and exact constraints allows us to modify the noise term from Gaussian noise only to any other norm. The objective becomes
\begin{align}
\min_{\mathbf{E, Z}} \frac12\|\mathbf E \|^2_q + \lambda\|\mathbf Z\|_{1} \\
\text{s.t.} \quad \mathbf{X = XZ + E} \nonumber
\end{align}
where $q$ is a placeholder for norms such as $\ell_1, F, \ell_{1,2}$ etc.
\[
\min_{\mathbf{E, Z}} \frac12\|\mathbf E \|^2_q + \lambda\|\mathbf Z\|_{1} + \langle \mathbf{Y, XZ - X + E } \rangle + \frac{\mu}{2} \|\mathbf{XZ - X + E } \|_F^2
\]

Iterate the following
\begin{enumerate}
\item Fix others  solve for $\mathbf Z$
\[
\min_{\mathbf{Z}} \lambda\|\mathbf Z\|_{1} + \langle \mathbf{Y, XZ} \rangle + \frac{\mu}{2} \|\mathbf{XZ - (X - E) } \|_F^2
\]
\[
\min_{\mathbf{Z}} \lambda\|\mathbf Z\|_{1} + \frac{\mu}{2} \|\mathbf{XZ - (X - E} - \frac{1}{\mu} \mathbf Y ) \|_F^2
\]
Let $F = \frac{\mu}{2} \|\mathbf{XZ - (X - E} - \frac{1}{\mu} \mathbf Y ) \|_F^2$ and $\partial F =  \mu \mathbf X^T (\mathbf{XZ - (X - E} - \frac{1}{\mu} \mathbf Y ))$
\[
\min_{\mathbf{Z}} \lambda\|\mathbf Z\|_{1} + \frac{\rho}{2} \| \mathbf Z - (\mathbf Z_{k} - \frac{1}{\rho} \partial F(\mathbf Z_{k}) \|_F^2
\]

\item Fix others and solve for $\mathbf E$
\[
\min_{\mathbf{E}} \frac12\|\mathbf E \|^2_q + \langle \mathbf{Y, E} \rangle + \frac{\mu}{2} \|\mathbf{XZ - X + E } \|_F^2
\]
\[
\min_{\mathbf{E}} \frac12\|\mathbf E \|^2_q + \langle \mathbf{Y, E} \rangle + \frac{\mu}{2} \|\mathbf{E - (X - XZ)} \|_F^2
\]
\[
\min_{\mathbf{E}} \frac12\|\mathbf E \|^2_q  + \frac{\mu}{2} \|\mathbf{E - (X - XZ} - \frac{1}{\mu} \mathbf Y) \|_F^2
\]

\item Update $\mathbf Y$
\begin{align*}
\mathbf Y =& \mathbf Y + \mu (\mathbf{XZ - X + E})
\end{align*}

\item Update $\mu$
\begin{align*}
 \mu = \textrm{min}( \mu_{\text{max}}, \gamma \mu)
\end{align*}
where $\gamma$ is defined as
\[
\gamma =
\begin{cases}
\gamma_0 & \text{if} \;\; \mu_k \sqrt{\rho} \frac{\textrm{max} (   \| \mathbf Z_{k+1} - \mathbf Z_{k}  \|_F  , \|  \mathbf E_{k+1} - \mathbf E_{k} \|_F)}{\| \mathbf X \|_F} < \epsilon \\
1 & \text{otherwise,}
\end{cases}
\]
and $\rho > \| \mathbf X \|_F^2$,  $\mu_{\text{max}} >>  \mu_0$ and $\epsilon > 0$.

\item Check stopping criteria
\[
\frac{\| \mathbf X \mathbf Z_{k+1} - \mathbf X + \mathbf E_{k+1}  \|_F}{ \| \mathbf X \|_F} < \epsilon_1, \;
 \mu_k \sqrt{\rho} \frac{\textrm{max} ( \| \mathbf Z_{k+1} - \mathbf Z_{k}  \|_F  , \|  \mathbf E_{k+1} - \mathbf E_{k} \|_F)}{\| \mathbf X \|_F} < \epsilon_2
\]

\end{enumerate}

\subsection{Diagonal Constraint}

In some cases it may be desirable to enforce the constraint $\textrm{diag}(\mathbf Z) = \mathbf 0$ i.e.\ we should not allow each data point to be represented by itself. To enforce such a constraint it is not necessary to significantly alter the aforementioned optimisation schemes. This constraint only affects the step involving $\mathbf Z$. Since this step is the soft shrinkage operator and is separable at the element level one can simply set the diagonal entries to $0$ afterwards.

\newpage
\section{Low-Rank Subspace Clustering}

\subsection{Noise Free}

\begin{align}
\min_{\mathbf{Z}} \;  \| \mathbf{Z} \|_* \\
\text{s.t.} \quad \mathbf{A = AZ} \nonumber
\end{align}

The solution to the above objective is given by the Shape Interaction Matrix (SIM) which is defined as $\mathbf{V  V^T}$ where $\mathbf V$ is the right singular vectors of $\mathbf A$ i.e.\
\[
\mathbf{A = U \Sigma V^T}, \;\; \mathbf \Sigma = \text{diag}(\{\sigma_i\}_{i=1}^r)
\]

\subsection{Relaxed LRR ADMM}

\begin{align}
\min_{\mathbf Z} \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F + \tau \|\mathbf Z\|_{*}
\end{align}
This is solved similarly to Relaxed SSC ADMM. Instead of the $\ell_1$ shrinking operator in the update step for $\mathbf Z$ we must use the singular value shrinking operator which is defined as
\begin{align}
\mathcal D_{\tau}(\mathbf Y) = \mathbf U S_{\tau}(\mathbf \Sigma) \mathbf V^T, \;\; S_{\tau}(\mathbf \Sigma) = \text{diag}(\{\text{max}(\sigma_i - \tau, 0)\}).
\end{align}

\subsection{Relaxed LRR Linearised}

This is solved similarly to Relaxed SSC ADMM, again replacing the $\ell_1$ with the singular value shrinking operator.

\subsection{Exact LRR LADM}

This is solved similarly to Exact SSC LADM, again replacing the $\ell_1$ with the singular value shrinking operator.

\subsection{Robust LRR}

An extension of LRR called Robust-LRR (R-LRR) aims to directly deal with noisy data using the previously described data generation model
\begin{align}
\min_{\mathbf {Z, A, N}} \; \lambda \|\mathbf{N}\|_{q} + \|\mathbf Z\|_{*} \\
\text{s.t.} \quad \mathbf{A = AZ, X = A + N} \nonumber
\end{align}
This objective can be decomposed into two steps. The first step solves a variant of RPCA [CITE] and the second uses the estimated $\mathbf A$ to compute $\mathbf Z$ using the SIM. However it is unclear as to whether such a procedure will be capable of exactly removing all noise, thus the SIM may give poor results. Fortunately further work in nuclear norm based subspace clustering by Vidal and Favaro \cite{Vidal201447} demonstrated a number of other closed form solutions for noisy data.

\newpage
\section{Spatial Subspace Clustering}

In \cite{Guo.Y;Gao.J;Li.F-2013} the following spatial subspace clustering (SpatSC) objective was proposed
\begin{align}
\label{YiGuo1}
\min_{\mathbf Z, \mathbf E} \frac12\|\mathbf E|^2_F +\lambda_1\|\mathbf Z\|_{1}+\lambda_2\|\mathbf Z\mathbf R\|_{1} \\
\text{s.t.} \quad \mathbf{X = XZ + E}\text{, diag}(\mathbf Z) = \mathbf 0 \nonumber
\end{align}

\subsection{Implementation}

SpatSC can be implemented similarly to OSC. However the $\ell_{1,2}$ shrinkage operator is replaced with $\ell_1$ shrinkage operator. Please see the following section for details.

\newpage
\section{Ordered Subspace Clustering}

\begin{align}
\label{objective}
\min_{\mathbf Z, \mathbf E} \frac12\|\mathbf E \|^2_F +\lambda_1\|\mathbf Z\|_{1}+\lambda_2\|\mathbf Z\mathbf R\|_{1,2} \\
\text{s.t.} \quad \mathbf{X = XZ + E} \nonumber
\end{align}

\subsection{Relaxed Constraints}

First we remove the variable $\mathbf E$ by using the constraint and thus the objective \eqref{objective} can be re-written as follows,
\begin{align}
\label{objective_relaxed}
\min_{\mathbf Z} \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F +\lambda_1\|\mathbf Z\|_{1}+\lambda_2\|\mathbf J \|_{1,2}\\
\text{s.t.} \quad \mathbf{J = ZR} \nonumber
\end{align}

Then the Augmented Lagrangian for the two introduced constraints is
\begin{align}
\mathcal{L}(\mathbf Z, \mathbf J) = & \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F + \lambda_1 \|\mathbf Z\|_{1} + \lambda_2\|\mathbf J\|_{1,2} \notag\\
& + \langle \mathbf Y_1, \mathbf J - \mathbf Z\mathbf R\rangle +\frac{\mu^k}2\|\mathbf J - \mathbf Z\mathbf R\|^2_F
\label{objectiveADMM}
\end{align}

We can solve \eqref{objectiveADMM} for $\mathbf Z$ and $\mathbf J$ in an alternative manner when fixing the others, respectively.

\begin{enumerate}

\item Set $\mathbf J = \mathbf J^k$, solve for $\mathbf Z^{k+1}$ by
\begin{align*}
\lambda_1 \|\mathbf Z\|_{1} + \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F + \langle \mathbf Y, \mathbf J - \mathbf Z\mathbf R\rangle +\frac{\mu^k}2\|\mathbf J - \mathbf Z\mathbf R\|^2_F
\end{align*}
which is equivalent to
\begin{align*}
\lambda_1 \|\mathbf Z\|_{1} + \frac12\|\mathbf X - \mathbf X\mathbf Z\|^2_F + \frac{\mu^k}2\|\mathbf J - \mathbf Z\mathbf R + \frac{1}{\mu} \mathbf Y \|^2_F
\end{align*}
We linearise the last two terms which we call $F$
\begin{align*}
\min_{\mathbf Z} \lambda_1 \| \mathbf Z \|_1 + \frac{\rho}{2} \| \mathbf Z - (\mathbf Z_{k-1} - \frac{1}{\rho} \partial F) \|_F^2
\end{align*}
and $\partial F = \mathbf X^T(\mathbf{X - XZ}) + \mu (\mathbf J - \mathbf Z\mathbf R + \frac{1}{\mu} \mathbf Y)\mathbf R^T$.

\item Set $\mathbf Z = \mathbf Z^k$, solve for $\mathbf J^{k+1}$ by
\begin{align*}
\lambda_2\|\mathbf J\|_{1,2} + \langle \mathbf Y_1, \mathbf J - \mathbf Z\mathbf R\rangle +\frac{\mu^k}2\|\mathbf J - \mathbf Z\mathbf R\|^2_F
\end{align*}
which is equivalent to
\begin{align*}
\lambda_2\|\mathbf J\|_{1,2} + \frac{\mu^k}2\|\mathbf J - \mathbf Z\mathbf R + \frac{1}{\mu} \mathbf Y \|^2_F
\end{align*}
We linearise the last term which we call $F$
\begin{align*}
\min_{\mathbf J} \lambda_2\|\mathbf J\|_{1,2}+ \frac{\rho}{2} \| \mathbf J - (\mathbf J_{k-1} - \frac{1}{\rho} \partial F) \|_F^2
\end{align*}
and $\partial F = \mu (\mathbf J - \mathbf Z\mathbf R + \frac{1}{\mu} \mathbf Y)$.

\item Update $\mathbf Y$
\[
\mathbf Y = \mathbf Y + \mu (\mathbf J - \mathbf Z\mathbf R)
\]

\item Update $\mu$
\begin{align*}
 \mu^{k+1} = \textrm{min}( \mu_{\text{max}_1}, \gamma \mu^k)\\
\end{align*}
where $\gamma$ is defined as
\[
\gamma =
\begin{cases}
\gamma^0 & \text{if} \;\; \mu^k \textrm{max} ( \| \mathbf Z^{k+1} - \mathbf Z^{k}  \|_F  , \|  \mathbf J^{k+1} - \mathbf J^{k} \|_F) < \epsilon_2 \\
1 & \text{otherwise,}
\end{cases}
\]

and $\mu^{\text{max}} \gg  \mu^0$.

\item Check stopping criteria
\[
\|\mathbf J^{k+1} - \mathbf Z^{k+1} \mathbf R \|_F < \epsilon_1, \;
\mu^k \sqrt{\rho} \; \textrm{max} ( \| \mathbf Z^{k+1} - \mathbf Z^{k}  \|_F  , \|  \mathbf J^{k+1} - \mathbf J^{k} \|) < \epsilon_2
\]

\end{enumerate}

\subsection{Exact Constraints}

Similar to the relaxed version we begin by introducing auxiliary variables
\begin{align}
\label{objective_exact}
\min_{\mathbf Z, \mathbf E, \mathbf J} \frac12\|\mathbf E\|^2_F +\lambda_1\|\mathbf Z\|_{1}+\lambda_2\| \mathbf J \|_{1,2}\\
\text{s.t.} \quad \mathbf{X = XZ + E}, \mathbf{J = ZR} \nonumber
\end{align}
We then form the Augmented Lagrangian to incorporate our constraints
\begin{align}
\mathcal{L}(\mathbf E, \mathbf Z, \mathbf J) = & \frac12\|\mathbf E\|^2_F + \lambda_1 \|\mathbf Z\|_{1} + \lambda_2\|\mathbf J\|_{1,2} \notag\\
& + \langle \mathbf Y_1, \mathbf{XZ} - \mathbf X + \mathbf E \rangle +\frac{\mu}2\|\mathbf{XZ} - \mathbf X + \mathbf E\|^2_F \notag\\
& + \langle \mathbf Y_2, \mathbf J - \mathbf Z\mathbf R\rangle +\frac{\mu}2\|\mathbf J - \mathbf Z\mathbf R\|^2_F
\end{align}
Then we solve for $\mathbf{E, Z, J}$ alternatively while fixing others in the following way

\begin{enumerate}

\item Set $\mathbf E = \mathbf E^k$ and $\mathbf J = \mathbf J^k$, solve for $\mathbf Z^{k+1}$ by
\begin{align*}
\min_{\mathbf Z} \lambda_1 \| \mathbf Z \|_1 + \langle \mathbf Y^k_1, \mathbf{XZ} - \mathbf X + \mathbf E \rangle +\frac{\mu^k}2\|\mathbf{XZ} - \mathbf X + \mathbf E\|^2_F\\
+ \langle \mathbf Y^k_2, \mathbf J - \mathbf Z\mathbf R\rangle +\frac{\mu^k}2\|\mathbf J - \mathbf Z\mathbf R\|^2_F
\end{align*}
\begin{align*}
\min_{\mathbf Z} \lambda_1 \| \mathbf Z \|_1 + \frac{\mu^k}2\|\mathbf{XZ} - (\mathbf X - \mathbf E - \frac{1}{\mu^k} \mathbf Y^k_1)\|^2_F \\ + \frac{\mu^k}2\| \mathbf Z\mathbf R - (\mathbf J - \frac{1}{\mu^k} \mathbf Y^k_2)\|^2_F
\end{align*}
We linearise the last two terms which we call $F$
\begin{align*}
\min_{\mathbf Z} \lambda_1 \| \mathbf Z \|_1 + \frac{\rho}{2} \| \mathbf Z - (\mathbf Z_{k-1} - \frac{1}{\rho} \partial F) \|_F^2
\end{align*}
and $\partial F = \mu^k \mathbf X^T (\mathbf{XZ_{k-1}} - (\mathbf X - \mathbf E - \frac{1}{\mu^k} \mathbf Y^k_1)) + \mu^k(\mathbf{Z_{k-1}R} - (\mathbf J - \frac{1}{\mu^k} \mathbf Y^k_2))\mathbf R^T$.

\item Set $\mathbf Z = \mathbf Z^k$ and $\mathbf J = \mathbf J^k$ solve for $\mathbf E^{k+1}$ by
\begin{align*}
\min_{\mathbf E} \frac12\|\mathbf E\|^2_F + \langle \mathbf Y^k_1, \mathbf{XZ} - \mathbf X + \mathbf E \rangle \\ + \frac{\mu^k}2\|\mathbf{XZ} - \mathbf X + \mathbf E\|^2_F
\end{align*}
\begin{align*}
\min_{\mathbf E} \frac12\|\mathbf E\|^2_F + \frac{\mu^k}2\| \mathbf E - (\mathbf{XZ} - \mathbf X + \frac{1}{\mu^k} \mathbf Y^k_1) \|^2_F
\end{align*}

\item Set $\mathbf Z = \mathbf Z^k$ and $\mathbf E = \mathbf E^k$ solve for $\mathbf J^{k+1}$ by
\begin{align*}
\min_{\mathbf J} \lambda_2\|\mathbf J\|_{1,2} + \langle \mathbf Y^k_2, \mathbf J - \mathbf Z\mathbf R\rangle +\frac{\mu^k}2\|\mathbf J - \mathbf Z\mathbf R\|^2_F
\end{align*}
\begin{align*}
\min_{\mathbf J} \lambda_2\|\mathbf J\|_{1,2} + \frac{\mu^k}2\|\mathbf J - (\mathbf Z\mathbf R - \frac{1}{\mu^k} \mathbf Y^k_2) \|^2_F
\end{align*}

\item Update $\mathbf Y_1$ and $\mathbf Y_2$
\begin{align*}
\mathbf Y^{k+1}_1 =& \mathbf Y^k_1 + \mu^k (\mathbf{XZ - X + E})\\
\mathbf Y^{k+1}_2 =& \mathbf Y^k_2 + \mu^k (\mathbf{J - ZR})
\end{align*}

\item Update $\mu$
\begin{align*}
 \mu^{k+1} = \textrm{min}( \mu_{\text{max}_1}, \gamma \mu^k)
\end{align*}
where $\gamma$ is defined as
\[
\gamma_1 =
\begin{cases}
\gamma^0 & \text{if} \;\; \mu^k \sqrt{\rho} \frac{ \textrm{max} ( \| \mathbf Z^{k+1} - \mathbf Z^{k}  \|_F  , \|  \mathbf E^{k+1} - \mathbf E^{k} \|, \| \mathbf J^{k+1} - \mathbf J^{k}  \|_F  , \|  \mathbf Z^{k+1} \mathbf R - \mathbf Z^{k} \mathbf R \|_F)}{\| \mathbf X \|_F}  < \epsilon_2 \\
1 & \text{otherwise,}
\end{cases}
\]

where $\rho > \|\mathbf X\|_F^2$ and $\mu^{\text{max}} \gg  \mu^0$.

\item Check stopping criteria
\[
\frac{\|\mathbf{XZ^{k+1} - X + E^{k+1}} \|_F}{\| \mathbf X \|_F} < \epsilon_1, \frac{\|\mathbf{J^{k+1} - Z^{k+1}R}\|_F}{\| \mathbf X \|_F} < \epsilon_1
\]
\[
\mu^k \sqrt{\rho} \frac{ \textrm{max} ( \| \mathbf Z^{k+1} - \mathbf Z^{k}  \|_F  , \|  \mathbf E^{k+1} - \mathbf E^{k} \|, \| \mathbf J^{k+1} - \mathbf J^{k}  \|_F  , \|  \mathbf Z^{k+1} \mathbf R - \mathbf Z^{k} \mathbf R \|_F)}{\| \mathbf X \|_F}  < \epsilon_2
\]

\end{enumerate}

\newpage
\bibliographystyle{plain}
\bibliography{references}

\end{document}