\chapter{Discrete Markov Chains}
\emph{“和女朋友在商场走散了，是在原地等碰到的概率大还是随机走碰到的概率大？急，在线等。”}
\newpage

\section{Finite Space Markov Chain}

    \subsection{Basic definitions}
    % State Space
    \begin{definition}[State Space]
        A \textbf{state space} $\mathcal{S}$ is a finite or countable set of states, i.e. the values that random variables $X_i$ may take on.
    \end{definition}
    % Initial Distribution
    \begin{definition}[Initial Distribution]
        The \textbf{initial distribution} $\pi_0$ is the probability distribution of the Markov Chain at time $0$. Denote $\mathbb{P}[X_0 = i]$ by $\pi_0(i)$.
    \end{definition}
    \begin{remark}
        Formally, $\pi_0$ is a function from $\mathcal{S}$ to $[0,1]$ s.t.
        \[ \pi_0(i) \ge 0 \text{ for all $i\in\mathcal{S}$} \]
        \[ \sum_{i\in\mathcal{S}} \pi_0(i) = 1\]
    \end{remark}
    % Probability Transition Matrix
    \begin{definition}[Probability Transition Matrix]
        The \textbf{Transition Matrix} is a matrix $P = (p_{ij})$, where
        \[ p_{ij} = \mathbb{P}[X_{n+1} = j | X_n = i] \]
        i.e. the probability given that the chain is at state $i$ at $T=n$ that jumps to $j$ at $T=n+1$
    \end{definition}
    \begin{remark}
        ~{}
        \begin{itemize}
            \item The \emph{rows} of $P$ sum up to $1$.
            \item The entries in $P$ are all non-negative.
        \end{itemize}
    \end{remark}

    %%%%%%%% Markov Property
    \subsection{The Markov Property}
    % Markov Property
    \begin{definition}[Markov Property]
        We say a stochastic process $X_1, X_2, \dots $ satisfies the \textbf{Markov property} if
        \[ \mathbb{P}[X_{n+1} = i_{n+1} | X_n = i_n, \dots, X_0 = i_0] = \mathbb{P}[X_{n+1} = i_{n+1} | X_n = i_n] \]
    \end{definition}
    That is, the \emph{next} state $X_{n+1}$ depends only on the \emph{current} state $X_n$.
    % Time-homogeneity
    \begin{definition}[Time-homogeneous Markov Chain]
        A Markov Chain is said to be \textbf{time homogeneous} if
        \[ \forall t \quad \mathbb{P}[X_{t+1} = j | X_t = i] = P(i,j) \]
    \end{definition}
    For now, we only consider \emph{time homogeneous} Markov Chains.

    %%%%%%%% Multistep Transition by Matrix Algebra
    \subsection{Matrix Interpretation of Markov Chains}
    We now computes the probability distribution at $T=n+1$, denoted by $\pi_{n+1}$.
    \[ \pi_{n+1}(j) = \mathbb{P}[X_{n+1} = j] = \sum_{i=1}^N \mathbb{P}[X_n=i]\mathbb{P}[X_{n+1} = j | X_n = i] = \sum_{i=i}^N \pi_n(i)P(i,j)\]
    Therefore
    \[ \pi_{n+1}^T = \pi_n^T P \]
    \[ \pi_n^T = \pi_0^T P^n \]
    We use $P(i,j)$ to denote the element $(i,j)$ of $P$, we use $P^n$ to denote the $n$-th power of $P$, and we assume that $\pi_i$'s are column vectors.
    % Chapman-Kolmogorov
    \begin{theorem}[Chapman-Kolmogorov Equality]\label{thm:ChapmanKolmogorovEquality}
        \[ P^{m+n}(i,j) = \sum_k P^m(i,k)P^n(k,j) \]
    \end{theorem}

\section{Stationary Distribution}

    % Stationary Distribution
    \begin{definition}[Stationary Distribution]
        $\pi$ is called a \textbf{stationary distribution} of a Markov Chain if
        \[ \pi^TP = \pi \]
    \end{definition}
    \begin{remark}
        A Markov Chain may have 0, 1 or infinitely many stationary distribution.
    \end{remark}

    \emph{Here comes the question: When does a stationary distribution exist? If it exists, is it unique? If it is unique, does the chain converges to it?}

\section{Irreducibility, Aperiodicity and Recurrence}
For convenience, we will use $\mathbb{P}_i[A]$ to denote $\mathbb{P}[A | X_0 = i]$, and use $\mathbb{E}_i$ to denote expectation in an analogous way.

    %%%%%%%% Irreducibility
    \subsection{Irreducibility}
    % Accessible
    \begin{definition}[Accessibility]
        Let $i$, $j$ be two states, we say $j$ is \textbf{accessible from} $i$ if it is possible (with positive probability) for the chain to ever visit $j$ if the chain starts from $i$.
        \[ \mathbb{P}_i[\bigcup_{n=0}^\infty\{X_n = j\}] > 0 \]
        or equivalently
        \[ \sum_{n=0}^\infty P^n(i,j) = \sum_{n=0}^\infty \mathbb{P}_i[X_n = j] > 0 \]
    \end{definition}
    % Communicate
    \begin{definition}[Communication]
        We say $i$ \textbf{communicates with} $j$ if $j$ is accessible from $i$ and $i$ is accessible from $j$.
    \end{definition}
    % Irreducible
    \begin{definition}[Irreducibility]
        We say a Markov Chain is \textbf{irreducible} if all pairs of states communicate. And it is reducible otherwise.
    \end{definition}
    The relation \emph{communicate with} is an equivalent relation, and irreducible simply means the number of equivalent classes is 1.

    %%%%%%%% Aperiodicity
    \subsection{Aperiodicity}
    % Period
    \begin{definition}[Period]
        Given a Markov Chain, its \textbf{period} of state $i$ is defined to be the greatest common divisor $d_i$ of the lengths of loops starting from $i$.
        \[ d_i = \gcd\{n|P^n(i,i) > 0\} \]
    \end{definition}
    % Period is a class property
    \begin{theorem}
        If states $i$ and $j$ communicate, then $d_i = d_j$
    \end{theorem}
    \begin{sketchproof}
        ~{}
        \begin{itemize}
            \item $P^{n_1}(i,j) > 0$ and $P^{n_2}(j,i) > 0$.
            \item $P^{n_1+n_2}(i,i)>0 \Rightarrow d_i | n_1 + n_2$.
            \item Suppose $P^n{j,j} > 0$, then $P^{n+n_1+n_2}(i,i) > 0 \Rightarrow d_i | n + n_1 + n_2$.
            \item $d_i | n \Rightarrow d_j \ge d_i$.
            \item Similarly $d_i \ge d_j$. We are done.
        \end{itemize}
    \end{sketchproof}
    \begin{remark}
        Therefore all states in a communicating class have the same period, and all states in an irreducible Markov chain have the same period.
    \end{remark}
    % Aperiodic
    \begin{definition}[Aperiodicity]
        An irreducible Markov chain is said to be \textbf{aperiodic} if its period is 1, and periodic otherwise.
    \end{definition}
    \begin{proposition}
        \normalfont
        If $P(i,i) > 0$, then the Markov Chain is aperiodic.
    \end{proposition}
    \begin{remark}
        This is a sufficient but not necessary condition.
    \end{remark}

    %%%%%%%% Recurrence
    \subsection{Recurrence}
    We define the \textbf{First Hitting Time} $T_i$ of the state $i$ by
    \[ T_i = \inf\{n>0 | X_n = i\} \]
    and we can define recurrence as follows
    % Recurrent
    \begin{definition}[Recurrence]
        The state $i$ is \textbf{recurrent} if $\mathbb{P}_i[T_i<\infty] = 1$, and is transient if it is not recurrent.
    \end{definition}
    \begin{remark}
        Recurrence means that starting from state $i$ at $T=0$, the chain \emph{is sure to} return to $i$ eventually.
    \end{remark}
    % Recurrence is a class property
    \begin{theorem}
        \normalfont
        Let $i$ be a recurrent state, and let $j$ be accessible from $i$, then all of the following hold:
        \begin{enumerate}
            \item $\mathbb{P}_i[T_j < \infty] = 1$.
            \item $\mathbb{P}_j[T_i < \infty] = 1$.
            \item The state $j$ is recurrent.
        \end{enumerate}
    \end{theorem}
    \begin{sketchproof}
        ~{}
        \begin{itemize}
            \item The paths starting from $i$ can be seen as infinitely many \emph{cycles}.
            \item Whether the chain visits $p$ in the cycles can be seen as a Bernoulli distribution with probability $p>0$.
            \item The probability of not visiting $j$ in the first $n$ cycles is $(i-p)^n$, which goes to $0$ as $n\to\infty$. So (1)hold.
            \item (2) can be proved by contradiction. $\mathbb{P}_j[T_i < \infty] < 1$ will lead to contradiction against the fact that $i$ is recurrent.
            \item (1)(2) implies (3).
        \end{itemize}
    \end{sketchproof}
    \begin{corollary}
        If $\mathbb{P}_i[T_j < \infty] > 0$ but $\mathbb{P}_j[T_i < \infty] < 1$, then $i$ is transient.
    \end{corollary}
    % Equivalent Statement of Recurrence
    \begin{theorem}[Equivalent Statement of Recurrence]
        \normalfont
        The state $i$ is recurrent if and only if $\mathbb{E}_i[N_i] = \infty$, where $N_i = \sum_{i=0}^\infty\mathbb{I}\{X_n = i\}$.
    \end{theorem}
    \begin{sketchproof}
        ~{}
        \begin{itemize}
            \item Recurrence $\Rightarrow \mathbb{P}_i[N_i = \infty] = 1 \Rightarrow \mathbb{E}_i[N_i] = \infty$.
            \item The converse is proved by contradiction.
            \item If $i$ is transient, there is a chance of $p$ that the chain never return to $i$.
            \item So $N_i$ is distributed geometrically, and $\mathbb{E}_i[N_i]$ will be finite. Contradiction.
        \end{itemize}
    \end{sketchproof}
    \begin{remark}
        By taking expectation on $N_i$, we have:
        \[ \mathbb{E}_i[N_j] = \sum_{i=0}^\infty P^n(i,j) \]
    \end{remark}

    \begin{corollary}\label{TransientStateGoesToZero}
        If $j$ is transient, then $\lim_{n\to\infty}P^n(i,j) = 0$ for all states $i$.
    \end{corollary}
    \begin{sketchproof}
        ~{}
        \begin{itemize}
            \item $\mathbb{E}_j[N_j] < \infty$.
            \item $\mathbb{E}_i[N_j] = \mathbb{P}_i[T_j < \infty]\mathbb{E}_i[N_j | T_j < \infty]$.
            \item $\mathbb{E}_i[N_j] \le \mathbb{E}_i[N_j | T_j] = \mathbb{E}_j[N_j] < \infty$ since the probability \emph{restarts} once the chain visits $j$ again.
            \item $\mathbb{E}_i[N_j] = \sum_{i=0}^\infty P^n(i,j) < \infty$ and this implies our conclusion.
        \end{itemize}
    \end{sketchproof}

    \begin{corollary}
        If $i$ is recurrent, then $\sum_{n=1}^{\infty}p^n(i,i) = \infty$;

        If $i$ is transient, then $\sum_{n=1}^{\infty}p^n(i,i) < \infty$.
    \end{corollary}

    \begin{proposition}
        Suppose a Markov Chain has a stationary distribution $\pi$, if the state $j$ is transient, then $\pi(j) = 0$.
    \end{proposition}
    The last proposition follows from Corollary \ref{TransientStateGoesToZero}.

    \begin{corollary}
        If an irreducible Markov Chain has a stationary distribution, then the chain is recurrent.
    \end{corollary}
    \begin{sketchproof}
        The chain cannot be transient, or otherwise all $\pi(j)$ would be 0 and the sum of $\pi$ does not equal to 1.
    \end{sketchproof}
    \begin{remark}
        The converse is not true!
    \end{remark}

    \begin{proposition}
        A drunk man will find his way home, but a drunk bird may get lost forever.
    \end{proposition}
    This is because the random walk on $\mathbb{Z}$ and $\mathbb{Z}^2$ is recurrent, while the walks on higher dimensions are transient.

    %%%%%%%% More on Recurrence
    \subsection{More on Recurrence}
    % Null Recurrence
    \begin{definition}[Null Recurrence]
        The state $i$ is \textbf{null recurrent} if it is recurrent and $\mathbb{E}_i[T_i] = \infty$.
    \end{definition}
    % Positive Recurrence
    \begin{definition}[Positive Recurrence]
        The state $i$ is \textbf{positive reccurent} it is recurrent and $\mathbb{E}_i[T_i] < \infty$.
    \end{definition}

    \begin{proposition}
        Given an irreducible Markov Chain, it is either transient, null recurrent or positive recurrent.
    \end{proposition}

\section{Strong Law of Large Numbers of Markov Chains}
% SLLN of Markov Chains
\begin{theorem}[SLLN of Markov Chains]\label{SLLN}
    Let $X_0, X_1, \dots$ be a Markov Chain starting in the state $X_0=i$. Suppose state $i$ communicates with state $j$. The limiting fraction of time that the chain spends in $j$ is $\frac{1}{\mathbb{E}_j[T_j]}$.
    \[ \mathbb{P}_i[\lim_{n\to\infty}\frac{1}{n}\sum_{t=1}^n\mathbb{I}\{X_t=j\} = \frac{1}{\mathbb{E}_j[T_j]}] = 1 \]
\end{theorem}
\begin{sketchproof}
    Read the book. I'm not going to write this because I am lazy.
\end{sketchproof}

\section{Basic Limit Theorem}\label{BasicLimitTheorem}
aka. Foundamental Theorem of Markov Chains

    %%%%%%%% Finite Case
    \subsection{Finite State Space}
    % Spectral Radius
    \begin{definition}[Spectral Radius]
        Given a non-negative matrix $A$, the spectral radius $\rho(A)$ is the maximum norm of its eigenvalues
        \[ \rho(A) = \max\{ \lambda(A) \} \]
    \end{definition}
    \begin{proposition}
        Let $A$ be a non-negative matrix, then
        \[ \min_{1\le i \le n} \sum_{j=1}^N a_{i,j} \le \rho(A) \le \max_{1\le i \le n} \sum_{j=1}^N a_{i,j} \]
    \end{proposition}
    % Perron-Frobenius Theorem
    \begin{lemma}[Perron-Frobenius Theorem]\label{PFT}
        Let $A$ be a non-negative matrix with spectral radius $\rho(A) = \alpha$, then $\alpha$ is an eigenvalue of $A$, and has both left and right non-negative eigenvectors.
    \end{lemma}
    \begin{remark}
        Lemma \ref{PFT} implies that for a finite probability transition matrix $P$, it always has at least one stationary distribution, because it always has eigenvalue $1$ ($P\mathbf{1} = \mathbf{1}$) and a corresponding eigenvector $\pi$ s.t. $\pi^TP = \pi$.
    \end{remark}

    \begin{lemma}
        Suppose a $k \times k$ matrix $P$ is irreducible. Then there exists a unique solution to $\pi P = \pi$.
    \end{lemma}
    % Foundamental Theorem, Finite Case
    \begin{theorem}[Foundamental Theorem, Finite Case] If a finite Markov Chain is \emph{irreducible} and \emph{aperiodic}, then it has a \emph{unique stationary distribution} $\pi$ any initial distribution \emph{converges} to it:
        \[ \forall \pi_0,\quad \lim_{t\to\infty}\pi_0^T P^t = \pi^T \]
    \end{theorem}

    %%%%%%%% Countable Case
    \subsection{Countable Case}
    % Basic Limit Theorem
    \begin{theorem}[Basic Limit Theorem]
        Let $X_1, X_2, \dots$ be an \emph{irreducible aperiodic} Markov Chain that has a \emph{stationary distribution} $\pi$. Then
        \[ \lim_{n\to\infty}\pi_n(i) = \pi(i) \]
        for any state $i$ and for any initial distribution $\pi_0$.
    \end{theorem}
    % Bounded Convergence Theorem
    \begin{lemma}[Bounded Convergence Theorem]\label{BCT}
        If $X_n \to X$ with probability $1$ and there is a finite number $b$ such that $|X_n| < b$ for all $n$, then
        \[ \mathbb{E}[X_n] \to \mathbb{E}[X] \]
    \end{lemma}
    % Corollary of SLLN
    Recall SLLN \ref{SLLN}, and the following is a corollary.
    \begin{corollary}\label{SLLNCoro}
        For an \emph{irreducible} Markov Chain, we have
        \[ \lim_{n\to\infty} \frac{1}{n}\sum_{t=1}^n P^t(i,j) = \frac{1}{\mathbb{E}_j[T_j]} \]
    \end{corollary}
    \begin{sketchproof}
        Take expectations on both sides of Theorem \ref{SLLN} yields the conclusion.
    \end{sketchproof}
    % Cesaro Average
    \begin{proposition}[Cesaro Average]
        If a sequence of numbers $a_n$ converges to a value $a$, then the \textbf{Cesaro Average} $(1/n)\sum_{t=1}^na_t$ also converges to it.
    \end{proposition}
    Corollary \ref{CesaroCoro} follows immediately from the proposition,
    \begin{corollary}\label{CesaroCoro}
        For an \emph{irreducible aperiodic} Markov Chain with a \emph{stationary distribution},
        \[ \lim_{n\to\infty}\frac{1}{n}\sum_{t=1}^nP^t(i,j) \to \pi(j) \]
    \end{corollary}
    \begin{theorem}
        An \emph{irreducible, aperiodic} Markov Chain with a \emph{stationary distribution} has a stationary distribution given by
        \[ \pi(j) = \frac{1}{\mathbb{E}_j[T_j]} \]
    \end{theorem}
    \begin{sketchproof}
        The proof of the theorem is trivial: compare Corollary \ref{SLLNCoro} and \ref{CesaroCoro}.
    \end{sketchproof}
    % Foundamental Theorem, Countable Case
    \begin{theorem}[Foundamental Theorem, Countable Case]\label{theorem:FTMC-Countable}
        An \emph{irreducible} Markov Chain has a \emph{unique stationary distribution} given by
        \[ \pi(j) = \frac{1}{\mathbb{E}_j[T_j]} \]
        if and only if it is \emph{positive recurrent}.
    \end{theorem}

    %%%%%%%% Other Conclusions
    \subsection{Other Conclusions}
    \emph{Since they are not covered in class, no proof will be provided. I'm lazy you know.}
    \begin{definition}[Doubly Stochastic Chains]
        A transition matrix $P$ is said to be \textbf{doubly stochastic} if all of its columns sum up to 1.
    \end{definition}
    \begin{optTheorem}
        For a doubly stochastic Markov Chain with $N$ states, the uniform distribution $\pi(i) = 1/N$ is a stationary distribution.
    \end{optTheorem}
    \begin{optTheorem}
        For a Markov Chain with symmetric $P$ and $N$ states, the uniform distribution $\pi(i) = 1/N$ is a stationary distribution.
    \end{optTheorem}
    \begin{optTheorem}
        Let $T_j^k = \min\{ n > T_y^{k-1} | X_n = y \}$ be the time of the $k$-th visit to $j$, then by the Markov property,
        \[ \mathbb{T}_x[T_y^k < \infty] = \mathbb{P}_x[T_y < \infty]\cdot\mathbb{P}_y[T_y<\infty] \]
        Notice that
        \[ \mathbb{E}[X] = \sum_{k=0}^{\infty}\mathbb{P}[X \le k] \]
        Using the two equations, we have
        \[ \mathbb{E}_x[N_y] = \frac{\mathbb{P}_x[T_y < \infty]}{1-\mathbb{P}_y[T_y < \infty]} \]
    \end{optTheorem}
    \begin{remark}
        This theorem gives us more insights into the results in Section \ref{BasicLimitTheorem}.
    \end{remark}

\section{Examples}

    %%%%%%%% Galton-Watson Process
    \subsection{Galton-Watson Process}\label{sub:BranchingProcess}
    \emph{aka. Branching Process}
    \paragraph*{Problem.} What is the probability that a family name eventually extincts?
    \paragraph*{Notations.}
    \begin{itemize}
        \item $G_t$ denotes the number of males in generation $t$.
        \item $X_{tk}$ denotes the number of sons fathered by the $k$-th father in the $t$-th generation. Assume $X_{tk}$ are \emph{iid} with $\mathbb{P}[X_{tk}=i]=p(i)$.
        \item $\rho = \mathbb{P}[extinction] = \mathbb{P}[\cup_{k\ge 1}\{ G_k=0 \}]$ denotes the probability that the family name eventually goes extinct.
        \item $q_t = \mathbb{P}[G_t = 0]$ denotes the probability that the family name goes extinct at the $t$-th generation.
    \end{itemize}
    The problem is trivial when $p(0) = 0$ or $p(0) + p(1) = 1$. Either the family name never goes extinct or it goes extinct almost surely. We consider $p(0) > 0$ and $p(0) + p(1) < 1$.

    By conditioning on what happens at the first step, we can calculate $\rho$ by
    \[ G_{t+1} = \sum_{i=1}^{G_t} X_{ti} \]
    \begin{align}
    \rho &= \sum_{k=0}^{\infty}\mathbb{P}[extinction \wedge  G_1 = k] \notag \\
    &= \sum_{k=0}^{\infty}\mathbb{P}[extinction | G_1 = k]\cdot\mathbb{P}[G_1 = k | G_0 = 1] \quad \text{(by 全概率公式)} \notag \\
    &= \sum_{k=0}^{\infty}p(k)\rho^k \triangleq \psi(\rho) \notag
    \end{align}
    The last step used the fact that all males have sons independently. $\psi(\rho)$ is called the \textbf{Probability Generating Function} of $p(\cdot)$.

    \begin{definition}[Probability Generating Function]
        The probability generating function of $X$ is given by
        \[ G_X(z) \triangleq \mathbb{E}[z^X] = \sum_{k=0}^{\infty}\mathbb{P}[X=k]z^k \]
    \end{definition}

    The probability of eventual extinction satisfies $\psi(\rho) = \rho$.
    \[ \psi'(z) = \sum_{k=1}^{\infty}kp(k)z^{k-1} > 0\]
    \[ \psi''(z) = \sum_{k=2}^{\infty}k(k-1)p(k)z^{k-2} > 0 \]
    So $\psi(z)$ is a strictly increasing convex function with the following properties.
    \begin{itemize}
        \item $\psi(1) = 1$. So $\rho = 1$ is always a solution.
        \item $\psi'(1) = \sum_{k=1}^{\infty}kp(k) = \mathbb{E}[X] \triangleq \mu$
    \end{itemize}
    \begin{enumerate}[(i)]
        \item If $\mu \le 1$, $\rho = 1$ is the only solution. Therefore the family name will go extinct.\footnote{This can be better understood by plotting a sketch of $\psi$.}
        \item If $\mu > 1$, there exists another solution $r<1$.

        Observe that $\{ G_t = 0 \} \subseteq \{ G_{t+1} = 0 \}$ and $q_t \le q_{t+1}$. Since $q_t$ has a upper bound $\rho$, it will always converge.
        \[ q_t \uparrow \rho \]
        So we only need to prove $\forall t \quad q_t < r$. By induction,
        \begin{itemize}
            \item[base.] $q_0 = 0 < r$.
            \item[hypo.] $q_t < r$.
            \item[step.] $q_{t+1} = \sum_{i=0}^{\infty}\mathbb{P}[G_1 = i]\mathbb{P}[G_{t+1} = 0| G_1 = i] = \sum_{i=0}^{\infty}p(i)(q_t)^i$. The last step used the iid assuption of $X$.
            Therefore $q_{t+1} = \psi(q_t) \le \psi(r) = r$. Done.
        \end{itemize}
    \end{enumerate}

    %%%%%%%% Gambler's Ruin
    \subsection{Gambler's Ruin}\label{GamblerRuin}
    \paragraph*{Problem.} Consider a gambler in a casino, who has probability $p$ to win \$$1$ and $1-p$ to lose \$$1$. The process stops when the gambler gets \$$N$ or loses all his money. Assume the gambler starts the game with \$$i$.
    \paragraph*{Notations.}
    \begin{itemize}
        \item $P_i$ denotes the probability of the gambler gets \$$N$ and wins, starting with \$$i$.
        \item $Z_i \in \{1, -1\}$ denotes whether the gambler wins or loses in round $i$.
        \[ X_t = X_0 + \sum_{i=0}^{t-1}Z_i \]
    \end{itemize}
    Obviously,
    \[ P_N = 1 \quad P_0 = 0 \]
    When $1 \le i \le N-1$, $P_i$ can be calculated by
    \begin{align*}
        P_i &= \mathbb{P}[win | X_0 = i] \\
        &= \mathbb{P}[win \wedge Z_0 = 1 | X_0 = i] + \mathbb{P}[win \wedge Z_0 = -1 | X_0 = i] \quad \text{(Again by 全概率公式 on $Z_i$)} \\
        &= \mathbb{P}[win | X_0=i, Z_0 = 1]\cdot\mathbb{P}[Z_0=1|X_0=i] + \mathbb{P}[win|X_0=1, Z_0=-1]\cdot\mathbb{P}[Z_0=-1|X_0=i] \\
        &= p\cdot P_{i+1} + (1-p)\cdot P_{i-1}
    \end{align*}
    Rearranging,
    \[ p(P_{i+1} - P_i) = (1-p)(P_i - P_{i-1}) \]
    \[ P_{i+1} -P_i = \frac{1-p}{p}(P_i - P_{i-1}) \]
    Let $\theta = \frac{1-p}{p}$, by high school mafs
    \[ P_{i+1} - P_i = \theta^i(P_1 - P_{0}) \]
    \begin{enumerate}[(i)]
        \item Assume for now that $p \neq \frac{1}{2}$ so $\theta \neq 1$, then summing over $i$ yields
        \[ P_N - P_0 = 1 = \frac{1-\theta^N}{1-\theta} P_1 \]
        Therefore
        \[ P_1 = \frac{1-\theta}{1-\theta^N} \]
        Summing from $0$ to $i$ yields
        \[ P_i = \frac{1-\theta^i}{1-\theta^N} \]
        \item If $p=\frac{1}{2}$,
        \[ P_{i+1} - P_i = P_i - P_{i-1} \]
        This is a arithmetic progress, and again by high school mafs
        \[ P_i = \frac{i}{N} \]
    \end{enumerate}
    To sum up
    \[
        P_i =
        \begin{cases}
            \frac{i}{N} &(i = \frac{1}{2})\\
            \frac{1-\theta^i}{1-\theta^N} &(\text{o.w.})
        \end{cases}
    \]

    \subsection{Drug Test}\label{DrugTest}
    This example is based on results from the previous subsection \ref{GamblerRuin}.
    \paragraph*{Problem.} We want to test the cure rate $p_1$ of a drug \textrm{Drug}1. We already have a \textrm{Drug}2 with known cure rate $p_2$. We want to know whether $p_1 > p_2$ or not. To do this, we find $t$ pairs of patients $(X_i, Y_i)$ and conduct tests using the two drugs on $X$ and $Y$ respectively. Once the number of patients who are cured by \textrm{Drug}1 but not \textrm{Drug}2 exceeds a certain threshold $M$, we can claim that $p_1 \ge p_2$.
    \paragraph*{Notations.}
    \begin{itemize}
        \item $X_i,Y_i \in \{0,1\}$ is a boolean denoting whether the first or the second drug cured patient $i$.
        \item $Z_i = X_i - Y_i$.
        \item $p_1, p_2$ denotes the probability that the two drugs cure a patient, respectively.
    \end{itemize}

    The value of $Z_i$ falls into 3 cases:
    \[ Z_i =
        \begin{cases}
            1 &p_1(1-p_2)\\
            -1 &p_2(1-p_1)\\
            0 &(\text{o.w.})
        \end{cases}
    \]
    If we ignore the cases where $Z_i = 0$, then we can model this problem as a gambler's ruin, with $p = \frac{p_1(1-p_2)}{p_1(1-p_2)+p_2(1-p_1)}$.

    And
    \[ \mathbb{P}[TestWrong] = 1 - \frac{1-\theta^M}{1-\theta^{2M}} = \frac{1}{\theta^{-M} + 1} \]
    The probability above drops exponentially with $M$, so the threshold does not need to be very large to achieve accurate results.

    %%%%%%%% Another random walk on N
    \subsection{Another Random Walk}\label{AnotherRandWalk}
    \paragraph*{Problem.} Consider a random walk on $\mathbb{N}$ where
    \[ P(0,1) = 1 \quad P(N,N) = 1 \]
    We want to compute how many steps we need to reach $N$ starting from $i$.
    \paragraph*{Notations.}
    \begin{itemize}
        \item $h_i$ denotes the expectation of steps to reach $N$ starting from $X_0=i$.
        \item $Y_i$ denotes the number of steps from $i$ to $i+1$ for the first time.
        \item $g_j \triangleq \mathbb{E}[Y_j]$.
    \end{itemize}
    Obviously
    \[ \mathbb{E}[h_0] = 1 + \mathbb{E}[h_1] \quad \mathbb{E}[h_N] = 0 \]
    Similar to subsection \ref{DrugTest},
    \[ h_i = 1+ (1-p)h_{i-1} + ph_{i+1} \]
    This can be calculated using \emph{the linearity of expectation}.

    Notice that
    \[ h_i = \sum_{j=i}^{N-1}Y_j \]
    Taking expectations on both sides
    \[ \mathbb{E}[h_i] = \sum_{j=i}^{N-1}\mathbb{E}[Y_j] \]
    So we only need to caculate $\mathbb{E}[Y_j]$.
    \begin{itemize}
        \item $g_0 = 1$.
        \item $g_i = 1 + 0 \cdot p + (1-p)(g_{i-1}+g_i)$.
    \end{itemize}
    \begin{enumerate}
        \item Again assume $p \neq \frac{1}{2}$.
        Rearranging yields
        \[ g_i = \frac{1}{p} + \theta g_{i-1} \]
        and after a few steps of arithmetics
        \[ g_i = \sum_{i=0}^{t-1}\frac{1}{p}\theta^i + \theta^t \]
        Summing over $g_i$ is a sum of geometric progress, easy.
        \item If $p = \frac{1}{2}$, then
        \[ g_{i} - g_{i-1} = \frac{1}{p} = 2 \]
        so
        \[ g_t = 2t + 1 \]
        and
        \[ \mathbb{E}[h_0] = \sum_{t=0}^{N-1}(2t+1) = N^2 \]
    \end{enumerate}
    \begin{remark}
        As the converse of the conclusion, if we take $N$ steps, the farthest distance we can go is $\sqrt{N}$.
    \end{remark}

    \subsection{Yet Another Random Walk}
    Consider a random walk on $\mathbb{N}$ with
    \begin{align*}
        p(i,i+1) &= p\\
        p(i,i-1) &= 1-p\\
        p(0,0) &= 1-p
    \end{align*}
    We are going to prove that if $p > 1/2$, the chain is transient, and if $p \le 1/2$, the chain is recurrent.
    Let $V_i = \min\{ n\le0 | X_n = i \}$ denote the time of the first visit to state $i$.
    Let $h(x) = \mathbb{P}_x(V_N<V_0) (N > 0)$ denote the probability of a Markov chain starting from the state $x$ to \emph{reach state $N$ before it reaches state $0$}.

    By definition of the random walk, for all $1 \le x \le N-1$:
    \begin{equation}
        \notag
        h(x) = ph(x+1) + (1-p)h(x-1)
    \end{equation}
    and
    \[ h(0) = 0 \quad h(N) = 1 \]

    Rearranging gives us:
    \begin{equation}
        \label{eq3.1}
        p[h(x+1) - h(x)] = (1-p)[h(x) - h(x-1)]
    \end{equation}
    \[ h(x+1) - h(x) = \frac{1-p}{p}[h(x) - h(x-1)] \]

    So $a(x) = h(x) - h(x-1)$ is a geometric series with $a(1) = h(1) - h(0) = h(1)$.
    \[ a(x) = h(x+1) - h(x) = a(1) \cdot (\frac{1-p}{p})^{x-1} \]

    Summing over $x = 1$ to $N$ yields:
    \[ h(N) - h(0) = a(1) \cdot \sum_{i = 1}^{N}(\frac{1-p}{p})^i \]

    Notice that $h(N) - h(0) = 1$, we can solve for $a(1)$:
    \[ a(1) = \frac{1-\theta}{1-\theta^N} \]
    where $\theta = \frac{1-p}{p}$.

    Therefore:
    \begin{equation}
        \notag
        h(x) = a(1)\sum_{i=1}^{x}(\theta^i) = \frac{1-\theta}{1-\theta^N} \cdot \frac{1-\theta^x}{1-\theta} = \frac{1-\theta^x}{1-\theta^N}
    \end{equation}

    To briefly sum up:
    \begin{equation}
        \label{eq3.2}
        \mathbb{P}_x[V_N < V_0] = \frac{1-\theta^x}{1-\theta^N}
    \end{equation}
    where $\theta = \frac{1-p}{p}$.

    We now consider different cases of $p$.
    \begin{enumerate}
        \item $p > \frac{1}{2}$.\\
        The case where $p=1$ is trivial because the chain simply keeps going right and never returns, so any state in the chain is transient.

        When $\frac{1}{2} < p < 1$, the chain is irreducible. If we can prove that one state in the chain is transient, then all states in the chain must also be transient. WLOG, we will prove that the state $0$ is transient.

        Suppose the chain starts from $0$, it cannot stay in state $0$ forever, because $(1-p)^n \to 0$ as $n\to\infty$, so it will leave $0$ and visit $1$. We are going to prove that once the chain reaches $1$ from $0$, there is a probability that it will never return to $0$ again, and $0$ is therefore transient.

        Plug in $x=1$ into (\ref{eq3.2}) and let $N \to \infty$. Since $p > \frac{1}{2}$, $\theta < 1$, $\theta^N \to 0$.
        \[ \mathbb{P}_1[V_0 = \infty] = 1-\theta \]
        That is, there is a probability of $1-\theta$ that the chain reaches infinity before it returns to $0$, i.e. the chain is never returning to $0$ again. Therefore it is transient.

        \item $p \le \frac{1}{2}$.\\
        We first consider the case where $p < \frac{1}{2}$.

        Similarly, we only need to prove that $0$ is recurrent, since the chain is irreducible. Again, the chain cannot stay in $0$ forever, and once it reaches $1$ from $0$, we can compute $\mathbb{P}_1[V_0 = \infty]$ by letting $N$ go to infinity. This time, since $p < \frac{1}{2}$, $\theta > 1$ and $\theta^N \to \infty$, therefore,
        \[ \mathbb{P}_1[V_0 = \infty] = 0 \]
        That is, the chain will return to $0$ with probability $1$, and it is therefore recurrent.

        When $p = \frac{1}{2}$, $\theta = 1$, (\ref{eq3.2}) has a division by zero and cannot be used. We now consider (\ref{eq3.1}). Since $p=\frac{1}{2}$, $p = (1-p)$, they cancelled each other,
        \[ h(x+1) - h(x) = h(x) - h(x-1) \]
        So $h(x)$ is an arithmetic progression. Since $h(0)=0$ and $h(N)=1$, it must has the form of
        \[ h(x) = \frac{x}{N} \]

        Again plugging in $x=1$ and letting $N \to \infty$,
        \[ \mathbb{P}_1[\infty < V_0] = 0 \]
        Therefore it is still recurrent when $p = \frac{1}{2}$.

        \item $p = \frac{1}{2}$.\\
        Let $\tau$ be the \emph{exit time} defined as $\tau = \min\{ n | X_n \notin (0,N) \}$, i.e. the minimum time for the chain to be either in state $0$ or state $N$. EOSP claimed that
        \begin{equation}
            \label{claim}
            \mathbb{E}_x[\tau] = x(N-x)
        \end{equation}

        Let $g(x) = \mathbb{E}_x[\tau]$, since $p = \frac{1}{2}$:
        \begin{equation}
            \label{gx}
            g(x) = 1 + \frac{1}{2}g(x+1) + \frac{1}{2}g(x-1)
        \end{equation}

        We can verify (\ref{claim}) by plugging $g(x) = x(N-x)$ into (\ref{gx}).
        \begin{align}
            RHS &= 1 + 0.5(x+1)(N-x-1) + 0.5(x-1)(N-x+1) \notag \\
            &= 1 + Nx - x^2 -1 \notag \\
            &= Nx - x^2 = LHS \notag
        \end{align}

        Therefore $\mathbb{E}_x[\tau] = x(N-x)$. Then we can again follow our previous proofs: plug in $x=1$ and let $N\to\infty$, which yields
        \[ \mathbb{E}_1[\tau] \to \infty \quad n \to \infty \]
        That is, $\mathbb{E}_1[V_0] = \infty$.

        Then we can calculate $\mathbb{E}_0[T_0]$:
        \[ \mathbb{E}_0[T_0] = \frac{1}{2}\times 1 + \frac{1}{2}\mathbb{E}_1[V_0] = \infty \]
        Therefore the chain is null recurrent when $p=\frac{1}{2}$.

        \item $p<\frac{1}{2}$.\\
        EOSP claimed that
        \[ \mathbb{E}_x[\tau] = \frac{x}{1-2p}-\frac{N}{1-2p}\cdot\frac{1-\theta^x}{1-\theta^N} \]
        This claim can be verified by plugging it into and checking that the two sides of the equation are equal.
        \[ g(x) = 1 + pg(x+1) + (1-p)g(x-1) \]

        Plug in $x=1$ and let $N\to\infty$.
        \[ \frac{N}{1-\theta^N} \to 0\]
        \[ \mathbb{E}_1[\tau] \to \frac{1}{1-2p} \quad n \to \infty \]

        Similarly,
        \[ \mathbb{E}_0[T_0] = (1-p) \times 1 + p \cdot \frac{1}{1-2p} < \infty \]
        $\mathbb{E}_0[T_0] < \infty$. Therefore the chain is positive recurrent when $p < \frac{1}{2}$.
    \end{enumerate}

\section{Coupling and Stochastical Dominance}
In this section we assume the sample space $\Omega$ is at most countable, but the results in this section can be generalized to continuous case.

    %%%%%%%% Motivating Example: 2-SAT
    \subsection{Motivating Example: 2-SAT}
        Recall the SAT problem in \textsc{AI2615 Design and Analysis of Algorithms}. A 2-SAT is a special case of SAT where each \verb|or| expression has at most 2 terms
        \[ \varphi = (x_1 \vee y_1) \wedge (x_2 \vee y_2) \cdots \]
        The problem is RP and can be solved using the following algorithm.

        \verb|Expected Algorithm, but not found.| It's here: Algorithm \ref{algo:2SATAlgo}.

        \begin{algorithm}
            \caption{2-SAT Solver}
            \KwIn{CNF with $n$ terms}
            Random intialize a solution $\sigma$.\\
            \For{$i = 1:100n^2$}
            {
                \If{$\sigma$ satisfies CNF}
                {
                    \KwRet{$\sigma$}
                }
                \Else
                {
                    Randomly choose one term $(X_i \vee Y_i)$ that is false.\\
                    Randomly flip $X_i$ or $Y_i$.
                }
            }
            \KwRet{Not satisfiable}
            \label{algo:2SATAlgo}
        \end{algorithm}

        To prove its correctness, we only need to consider the cases where the CNF is indeed satisfiable with solution $\sigma$. We denote the sequence of attempts produced by the algorithm by
        \[ \sigma_0 \to \sigma_1 \to \cdots \to \sigma_{100n^2} \]
        Let $X_i$ denotes the number of terms in $\sigma$ and $\sigma_i$ that are exactly the same.

        You should convince yourself that $X_0, X_1, \dots$ is not a Markov Chain. \sout{So we get stuck}. However we can still informally show the correctness.

        Notice that if the CNF is not satisfiable, the algorithm will always give the correct answer. So we only consider the case where the CNF is indeed satisfiable, but the algorithm fails to find the solution.

        We first introduce some basic concepts.

    %%%%%%%% Stochastic Dominance
    \subsection{Stochastic Dominance}
        \begin{definition}[Stochastic Dominance]
            A distribution $\mu$ is said to \textbf{stochasitcally dominate} $\nu$ if
            \[ \forall x\quad \mathbb{P}_{X\sim\mu}[X \ge x] \ge \mathbb{P}_{Y\sim\nu}[Y \ge x] \]
        \end{definition}
        \begin{remark}
            Let $F$ and $G$ be the cumulative distribution function of $X$ and $Y$, we have $F(x) \le G(x)$, i.e. the probability of $X$ concentrates on larger values.
        \end{remark}
        \paragraph{Some Examples.}
        \begin{itemize}
            \item 2-SAT.
            \item Binomial distribution. $B(n,p)$ and $B(n,q)$.
            \item Erd$\mathrm{\ddot{o}}$s-R$\mathrm{\acute{e}}$nyi Random Graph $\mathcal{G}(n,p)$ is a graph with $n$ nodes and the edges have a probability of $p$ to exist. We sample $G$ randomly from $\mathcal{G}$ and test if $G$ is connected. If $p>q$ then $G_p$ stocahstically dominates $G_q$.
        \end{itemize}

    %%%%%%%% Coupling
    \subsection{Coupling}
        \begin{definition}[Coupling]
            $(X',Y')$ is the coupling of random variables $(X,Y)$ if $X'$ has the same distribution of $X$ and $Y'$ has the same distribution as that of $Y$. i.e. A coupling $C$ of $X$ and $Y$ is a joint distribution of $X$ and $Y$.
        \end{definition}
        \begin{remark}~{}
            \begin{itemize}
                \item The coupling of $X$ and $Y$ is not unique.
                \item If $X$ has distribution $\mu$ and $Y$ has distribution $\nu$, then any joint distribution $(X,Y)$ having marginal distribution $X\sim\mu$ and $Y\sim\nu$ is a coupling of $X$ and $Y$.
                \[ \forall x, \quad \mathbb{P}_{(X,Y) \sim C}[X=x] = \mu(x) \]
                \[ \forall y, \quad \mathbb{P}_{(X,Y) \sim C}[Y=y] = \nu(y) \]
                \item Constructing a coupling can be intuitively seen as a process of filling a table of the joint distribution of $X$ and $Y$.
            \end{itemize}
        \end{remark}
        \begin{definition}[Monotone Coupling]
            Let $C$ be a coupling of $X$ and $Y$. $C$ is a \textbf{monotone coupling} if
            \[ \mathbb{P}_{(X,Y)\sim C}[X\ge Y] = 1\footnote{“它总是大”——Prof.Zhang} \]
        \end{definition}
        \begin{theorem}\label{MonoCouplingAndDominance}
            There is a monotone coupling of $X$ and $Y$ \emph{if and only if} $X$ stochastically doniminates $Y$.
        \end{theorem}
        \begin{proof}
            ~{}\\
            $\Rightarrow$.
            By definition of coupling
            \[ \mathbb{P}_{Y\sim\nu}[Y \ge a] = \mathbb{P}_{(X,Y)\sim C}[Y\ge a] \]
            By the Law of Total Probability
            \[ \mathbb{P}_{Y\sim\nu}[Y \ge a] = \mathbb{P}_{(X,Y)\sim C}[Y \ge a \wedge X \ge Y]\mathbb{P}[X \ge Y] + \mathbb{P}_{(X,Y)\sim C}[Y \ge a \wedge X < Y]\mathbb{P}[X < Y] \]
            The latter term is $0$ by definition of monotone coupling, so
            \[ \mathbb{P}_{Y\sim\nu}[Y \ge a] = \mathbb{P}_{(X,Y)\sim C}[X \ge Y \ge a] \le \mathbb{P}_{(X,Y)\sim C}[X \ge a] = \mathbb{P}_{X\sim\mu}[X \ge a] \]

            $\Leftarrow$. Suppose $X$, $Y$ are two random variables having  distribution $F(x)$ and $G(x)$ respectively. By stochastic dominance we have $F(x) \le G(x)$ for any $x$. Let $U$ be a random variable having a uniform distribution over $(0,1)$, since $F(x)$ is increasing
            \[ F^{-1}(U) \le x \Leftrightarrow F(F^{-1}(U)) \le F(x) \]
            Therefore
            \[ \mathbb{P}[F^{-1}(U) \le x] = \mathbb{P}[F(F^{-1}(U)) \le F(x)] = \mathbb{P}[U \le F(x)] = F(x) \]
            Similarly
            \[ \mathbb{P}[G^{-1}(U) \le x] = G(x) \]
            Since $F(x) \le G(x)$, we have $F^{-1}(x) \ge G^{-1}(x)$ and therefore we obtained a coupling $C = (X' = F^{-1}(U), Y' = G^{-1}(U))$ for which $X' \ge Y'$.

            \paragraph{Alternative proof.} $\Leftarrow$. As is mentioned in class, let us think of constructing a coupling of $\mu$ and $\nu$ as the process of filling out a joint distribution table of $\mu$ and $\nu$. In order for the coupling to be a monotone coupling, all positive values in the table should only be in the bottom left part. Following the hint in the lecture note, we will show that we can always construct such a table in a ``greedy'' manner, as long as $\mu$ stochastically dominates $\nu$.

            Suppose we are going to fill a $N$ by $N$ table $C$ whose rows are $\mu(i)$ and columns are $\nu(i)$.
            For the $i$-th row, our goal is to fill out the row, in a way that the values in this row sum up to $\mu(i)$, and that all the values in and after the $(i+1)$-th column are $0$, i.e.
            \begin{equation}\label{Eq:constr1}
                \forall i \quad \sum_{j=1}^N C(i, j) = \mu(i)
            \end{equation}
            \begin{equation}\label{Eq:constr2}
                \forall i \forall j > i \quad C(i, j) = 0
            \end{equation}
            Meanwhile, we are also subjected to the constraint that the $j$-th column sums up to $\nu(j)$.
            \begin{equation}\label{Eq:constr3}
                \forall j \quad \sum_{i=1}^N C(i,j) = \nu(j)
            \end{equation}

            Let $F$ and $G$ denote the cumulative distribution function of $\mu$ and $\nu$.
            \[ F(x) = \sum_{i=1}^x\mu(i) \quad G(x) = \sum_{i=1}^x\nu(i) \]
            Notice that since $\mu$ stochastically dominates $\nu$, we have
            \[ \forall x \quad F(x) \le G(x) \]

            \textbf{We first briefly explains the main idea.}

            When $i=1$, constrained by (\ref{Eq:constr2}), we can only put positive values in the first column, i.e. only $C(1,1)$ in the first row can be positive. Therefore, $\nu(1)$ cannot be smaller than $\mu(1)$, or otherwise we can only put at most $\nu(1)$ in $C(1,1)$, and we ``run out of budget'' for the positive remainder $\mu(1)-\nu(1)$. Fortunately, $\mu$ stochastically dominates $\nu$, so we have $\mu(1) = F(1) \le G(1) = \nu(1)$. Therefore we can set $C(1,1) = \mu(1)$, and set other elements in the first row to $0$.

            Similarly, when $i=2$, we have a total ``budget'' of $(\nu(1) - \mu(1)) + \nu(2)$. The first term comes from the extra ``budget'' that remains when filling the first row, and the second term comes from the new column $\nu(2)$. We have to ``divide and assign'' $\mu(2)$ into the first two columns. Again $\mu(2)$ cannot be larger than our ``budget'', or we will have a positive remainder $\mu(2) - \left(\nu(1) - \mu(1) + \nu(2)\right)$ that has no place to assign. Fortunately, since $\mu(1) + \mu(2) = F(2) \le G(2) = \nu(1) + \nu(2)$, we have
            \[ \mu(2) \le \nu(1) + \nu(2) - \mu(1) \]
            so we have enough ``budget'', and can assign $\mu(2)$ in the following way
            \begin{enumerate}
                \item $C(2,1) = \min\{\mu(2), \nu(1) - \mu(1)\}$
                \item $C(2,2) = \mu(2) - C(2,1)$
            \end{enumerate}

            \textbf{Based on this idea, we can then fill out the table $C$ inductively.}

            \textbf{Induction Base.} As is argued above, when $i=1$, we can always successfully fill out the first row, as long as $\mu$ stochastically dominates $\nu$.

            \textbf{Induction Hypothesis.} Now suppose we have successfully filled out the first $k$ rows of $C$.

            \textbf{Induction Step.} When filling out the $(k+1)$-th row, the ``total budget'' for this row will be
            \[ \sum_{i=1}^{k}(\nu(i) - \mu(i)) + \nu(i+1) \]
            again the former term is the remaining budget and the latter is from the new column $\nu(k+1)$. The budget is always enough because
            \[ F(k+1) = \sum_{i=1}^{k+1}\mu(i) \le G(k+1) = \sum_{i=1}^{k=1} \nu(i) \]
            so
            \[ \mu(k+1) \le \sum_{i=1}^{k}(\nu(i) - \mu(i)) + \nu(i+1) \]
            Therefore we can assign values on the $k+1$-th row in the following way
            \begin{itemize}
                \item $C(k+1, 1) = \min\{ \mu(k+1), \nu(1) - \sum_{i=1}^k C(i,1) \}$
                \item $C(k+1, 2) = \min\{ \mu(k+1) - C(k+1,1), \nu(2)-\sum{i=1}^k C(i,2) \}$
                \item \dots
                \item $C(k+1, k+1) = \mu(k+1) - \sum_{i=1}^k C(k+1, i)$
            \end{itemize}
            In this way we can fill $C$ such that all the positive values only lie in the lower left part of $C$, and therefore the coupling $C$ is a monotone coupling.
        \end{proof}

    %%%%%%%% 2-SAT Revisited
    \subsection{2-SAT: Proof of Correctness}
        \begin{theorem}[Markov's Inequality]\label{MarkovIneq}
            Let $X$ be a nonnegative random varialbe and let $a > 0$, then
            \[ \mathbb{P}[X \ge a] \le \frac{\mathbb{E}[X]}{a} \]
        \end{theorem}
        We can now prove the correctness of Algorithm \ref{algo:2SATAlgo}.
        \begin{sketchproof}
            Note that $X_{i+1}$ differs from $X_i$ in at most 1 operand, and that
            \[ \mathbb{P}[X_{i+1} = X_i + 1] \ge \frac{1}{2} \]
            \[ \mathbb{P}[X_{i+1} = X_i - 1] \le \frac{1}{2} \]
            We can introduce a random walk described in subsection \ref{AnotherRandWalk} $Y_1, Y_2, \dots$ with $p = \frac{1}{2}$.

            $X_i$ stochatically dominates $Y_i$. This is intuitive, and can be proved using Theorem \ref{MonoCouplingAndDominance} and constructing a monotone coupling:

            We introduce a random variable $U$ that has a uniform distribution $U \sim U(0,1)$. Let $q$ be the probability of $X_{i+1} = X_i + 1$, from the arguments above we already have $p < q$. Then we construct a coupling of $X$ and $Y$ by letting $X_{t+1} = X_t + 1$ if $U \ge q$ and $X_{t+1} = X_t - 1$ otherwise; and by letting $Y_{t+1} = Y_t + 1$ if $U \ge p$ and $Y_t - 1$ otherwise. Then it follows that $X_t \ge Y_t$, and therefore $X_t$ stochastically dominiates $Y_t$.

            Since $Y_i$ has an expectation of $n^2$ to get to state $n$ (starting from $0$), it follows intuitively that $X_i$ has a smaller expectation of steps to get to $n$.

            Actually,
            \[ \mathbb{E}[\text{First Hitting Time of $n$}|Y_0] = n^2 - Y_0^2 \]

            We have chosen the max iteration to be $100n^2$, by Markov's Inequality \ref{MarkovIneq}, the probability that we take $100n^2$ iterations without finding a feasible solution is at most $1/100$.
        \end{sketchproof}
        \begin{remark}~{}
            \begin{itemize}
                \item \sout{A formal proof of correctness will be given in the next lecture.}
                \item The rigorous proof has been updated.
                \item The idea of coupling can be used to prove other examples of stochastic dominance.
            \end{itemize}
        \end{remark}

    %%%%%%%% Maximum Couplings
    \subsection{Maximum Couplings}
        \begin{definition}[Maximum Coupling]
            Suppose $X$ has distribution function $\mu$, $Y$ has distribution function $\nu$. Let $C$ denote the set of all couplings of $(X,Y)$. $(\hat{X},\hat{Y})$ is a \textbf{maximum $\mu$,$\nu$ couple} if
            \[ \mathbb{P}[\hat{X}=\hat{Y}] = \max_{(X,Y)\in C}\mathbb{P}[X=Y] \]
            i.e. among all such couples its components are most likely to be equal.
        \end{definition}
        \begin{proposition}
            A maximum coupling always exists.
            \[ \mathbb{P}[\hat{X} = \hat{Y}] = \sum_{z\in\Omega}m(z) \]
            where $m(z) = \min\{ \mu(z), \nu(z) \}$.
        \end{proposition}
        \begin{remark}
            For continuous cases, the sum becomes integral.
        \end{remark}

    %%%%%%%% Total Variance Distance
    \subsection{Total Variance Distance}
        \begin{definition}[Total Variation Distance]
            Given two distributions $\mu$ and $\nu$ defined on a sample space $\Omega$, the \textbf{total variance distance} of $\mu$ and $\nu$ is defined as
            \[ \|\mu - \nu \|_{TV} \triangleq \frac{1}{2}\sum_{x\in\Omega}|\mu(x)-\nu(x)| = \max_{A\subseteq\Omega} \mu(A)-\nu(A) = 1-\min_{z\in\Omega}\{ \mu(z), \nu(z) \} \]
            Geometrically, this is the half of size of the area between the pdf of $\mu$ and $\nu$.
        \end{definition}
        \begin{lemma}[The Coupling Lemma]\label{CouplingLemma}
            For any coupling $C$ of $\mu$ and $\nu$,
            \[ \mathbb{P}_{(X,Y)\sim C}[X \neq Y] \ge \|\mu - \nu \|_{TV} \]
            and there exists a coupling $C^{*}$ that achieves the equality.
        \end{lemma}
        \begin{proof}
            If we view the coupling as a way to fill the table, then $\mathbb{P}_{(X,Y) \sim C}[X=Y]$ is the sum of all values on the diagonal of the table. Intuitively, the value on $i$-th row and $j$-th column is upper bounded by $\mu(i)$ and $\nu(j)$, so the sum is upper bounded by $\sum_{z\in\Omega}\min\{\mu(z), \nu(z)\}$.
            \begin{align*}
                \mathbb{P}[X \neq Y] &= 1 - \mathbb{P}_{(X,Y) \sim C}[X = Y] \\
                &= 1 - \sum_{z \in \Omega}\mathbb{P}[X=Y=z] \\
                &\ge 1 - \sum_{z \in \Omega}\min \{\mu(z), \nu(z)\}\\
                &= \sum_{z \in \Omega}\left( \mu(z) - \min\{\mu(z),\nu(z)\} \right)\\
                &= \| \mu - \nu \|_{TV}
            \end{align*}
            The previous part showed that $\mathbb{P}_{(X,Y)\sim C}[X \neq Y]$ is always lower bounded by $\|\mu - \nu\|_{TV}$.

            We then complete the proof by showing that there exists a coupling $\mathcal{C}^*$ that achieves the equality.

            Intuitively, if a coupling achieves the upper bound $\mathbb{P}[X = Y = z] = \min\{ \mu(z), \nu(z) \}$, it will also achieve the equality in the Coupling Lemma.

            Let $U$ be a random variable with uniform distribution $U \sim Uniform(0,1)$.

            Let $m(z) = \min\{\mu(z), \nu(z)\}$, and $p = \sum_{z\in\Omega}m(z)$.

            Let $V_1$, $V_2$, $V_3$ be three random variables with distributions
            \begin{enumerate}
                \item $V_1$: $f_{V_1}(x) = \frac{m(z)}{p}$
                \item $V_2$: $f_{V_2}(x) = \frac{\mu(x) - m(x)}{1-p}$
                \item $V_3$: $f_{v_3}(X) = \frac{\nu(x) - m(x)}{1-p}$
            \end{enumerate}
            Then we can construct the coupling as follows
            \begin{itemize}
                \item $X=Y=V_1$ if $U \le p$
                \item $X=V_2 \quad Y=V_3$ if $U > p$
            \end{itemize}
            Since $\mathbb{P}[V_2 = V_3] = 0$, we have that $\mathbb{P}[X = Y] = p = \sum_{z\in\Omega}\min\{\mu(z), \nu(z)\}$. Now we only need to show that the distribution we created is a coupling of $X$ and $Y$.
            \[ p_X(x) = pf_{V_1}(x) + (1-p)f_{V_2}(x) = m(z) + \mu(z) - m(z) = \mu(z) \]
            \[ p_Y(x) = pf_{V_1}(x) + (1-p)f_{V_3}(x) = m(z) + \nu(z) - m(z) = \nu(z) \]
            Therefore it is indeed a coupling of $X$ and $Y$, and it achieves the upper bound of $\mathbb{P}[X=Y] = \sum_z\min\{\mu(z),\nu(z)\}$, therefore it also achieves the equality in the Coupling Lemma.
        \end{proof}
        \begin{remark}
            The $C^*$ here is the maximal coupling.
        \end{remark}

\section{Proof of the Foundamental Theorem}
    We are going to prove that:
    \paragraph{}
        \emph{An irreducible, aperiodic, positive recurrent Markov Chain has a unique stationary distribution, and it converges to the distribution.}

    \begin{proof}
        We only consider the finite case. The proof of Theorem \ref{theorem:FTMC-Countable} (which does not exist in this note) has already shown that an Irreducible Positive Recurrent Markov chain has a Unique Stationary Distribution, so we only prove the convergence part here. Suppose there are two Markov Chains $X$ and $Y$
        \[ X_0 \to X_1 \to \dots \to X_t \to \dots \]
        \[ Y_0 \to Y_1 \to \dots \to Y_t \to \dots \]
        Let the initial distribution of $Y$ be the stationary distribution $\pi$, so each $Y_i$ has distribution $\pi$. Let the initial distribution of $X$ be $\mu_0$, and $X_i$ has the distribution given by $\mu_{i} = P^T\mu_{i-1}$.

        We define \textbf{convergence} by $\lim_{i\to\infty}\|\mu_{i} - \pi\|_{TV} = 0$.

        For each $(X_t, Y_t)$, we construct a coupling $C_t$ such that each chain is runned independently, and once $X_t = Y_t$ for some $t$, then $X_{t'} = Y_{t'}$ for all $t' \ge t$.

        Irreducibility guarantees that
        \[ \forall i,j \quad \exists n \quad P^n(i,j) > 0 \]
        And we first claim a fact: Aperiodicity further guarantees that
        \begin{equation}\label{FTMCEq1}
            \exists n \quad \forall i,j \quad P^n(i,j) > 0
        \end{equation}
        To prove (\ref{FTMCEq1}), we are going to show that
        \begin{equation}\label{FTMCEq2}
            \forall i,j \quad \exists t_{ij} \quad \forall t>t_{ij} \quad P^t(i,j) > 0
        \end{equation}
        Then (\ref{FTMCEq1}) can be proved by setting $n = \max_{i,j}\{t_{ij}\}$. Note that this result only holds in finite case.

        We visualize the Markov Chain from $i$ to $j$ as many cycles starting from and ending at $i$, of length $c_1,c_2,\dots$, plus a path from $i$ to $j$ (this path must exist, guaranteed by irreducibility). Since the chain is Aperiodic,
        \[ \gcd(c_1, c_2,\dots,c_s) = 1 \]
        And by Lemma \ref{theorem:Bezout}, there exist integers $x_1,x_2,\dots,x_s$ such that
        \[ c_1x_1 + c_2x_2 + \dots + c_sx_s = 1 \]
        And it follows immediately that there exist $y_1,y_2,\dots,y_s$ such that
        \[ c_1y_1 + c_2y_2 + \dots + c_sy_x = b \]
        for all sufficiently large $b$. Additionally we also need to add the finite length of path from $i$ to $j$.

        \begin{lemma}[B$\mathrm{\acute{e}}$zout's Theorem]\label{theorem:Bezout}
            Let $a,b$ be two integers, then there exist integer $\mu, \nu$ such that
            \[ a\mu + b\nu = \gcd(a,b) \]
        \end{lemma}

        Now consider the coupling.
        \[ \mathbb{P}[X_n = Y_n] \ge \mathbb{P}[X_n=Y_n=x] \ge SomeConstant~{} \alpha > 0 \]
        This is guaranteed by the irreducibility of the chain.
        So
        \[ \mathbb{P}[X_n \neq Y_n] \le 1 - \alpha \]
        Now condider the $2n$-th step, by the Law of Total Probability,
        \[ \mathbb{P}[X_{2n} \neq Y_{2n}] = \mathbb{P}[X_{2n} \neq Y_{2n} \wedge X_n \neq Y_n] + \mathbb{P}[X_{2n} \neq Y_{2n} \wedge X_n = Y_n] \]
        The latter is $0$ because it is impossible by our construction of coupling.

        By conditional probability and the Markov Property
        \[ \mathbb{P}[X_{2n} \neq Y_{2n} \wedge X_n \neq Y_n] \le \mathbb{P}[X_{2n} \neq Y_{2n} | X_n \neq Y_n](1-\alpha) \le (1-\alpha)^2 \]

        It follows immediately that $\mathbb{P}[X_t \neq Y_t] \to 0$ as $t \to \infty$. So $\lim_{t\to\infty}\|\mu_t - \pi\|_{TV} = 0$, and we are done.
    \end{proof}

\section{Applications of Markov Chain}
    We assume the state space $\mathcal{S}$ is finite, but in most cases the applications can be generalized to countably infinite cases. And we assume that the Markov chains are Aperiodic and Irreducible.

    \subsection{Time Reversibility}
        \begin{definition}[Time Reversibility]
            A Markov chain is said to be time reversible if there exists a distribution $\pi$ such that
            \begin{equation}\label{def:DetailedBalanceCondition}
                \forall x,y \in \mathcal{S} \quad \pi(x)P(x,y) = \pi(y)P(y,x)
            \end{equation}
        \end{definition}
        \begin{remark} ~{}
            \begin{itemize}
                \item The $\pi$ here is actually the stationay distribution. because
                \[ (\pi^TP)(y) = \sum_{x\in\mathcal{S}}\pi(x)P(x,y) = \sum_{x\in\mathcal{S}}\pi(y)P(y,x) = \pi(y)\sum_{x\in\mathcal{S}}P(y,x) = \pi(y) \]
                \item (\ref{def:DetailedBalanceCondition}) is also called the \textbf{detailed balance condition}.
                \item The detailed balance condition implies that $(X_0,X_1,\dots,X_n)$ and $X_n,X_{n-1},\dots,X_0$ have \emph{the same distribution}, and this is the reason why such chains are called time-reversible.
            \end{itemize}
        \end{remark}

    \subsection{Metropolis Algorithm}
        If we want to sample from some distribution $\mu$, one way of doing this is to design a random walk (Markov Chain) with stationary distribution $\mu$ and run the random walk for a sufficiently long period of time. We now discuss how to design such random walks. Recall the following proposition:
        \begin{proposition}
            A random walk on graph $\mathcal{G}$ with probability transition matrix
            \[P(i,j) =
            \begin{cases}
                \frac{1}{d_i} \quad &j \in \mathcal{N}(i)\\
                0 \quad &j otherwise
            \end{cases}
            \]
            has stationary distribution
            \[ \pi(i) = \frac{d_i}{\sum_{j\in\mathcal{S}}d_j} \]
        \end{proposition}
        Consider a random walk on graph $\mathcal{G} = (V, E)$. Given a distribution $\mu$, we want to design a random walk such that the stationay distribution is $\mu$. Let $d_i$ denote the degree of the $i$-th node in $\mathcal{G}$.\\
        Let
        \[ \Delta = \max_{i \in V} d_i \]
        For any node $i$, for all $j \in \mathcal{N}(i)$, we ``propose'' to move to $j$ with probability $1/\Delta$. And $j$ ``accepts'' $i$ with probability $\min \{\frac{\mu(j)}{\mu(i)}, 1\}$. If the move is rejected, we stay in $i$. We define the probability transition matrix as follows
        \begin{equation}\notag
            P(i, j) =
            \begin{cases}
                0 \quad & j \notin \mathcal{N}(i)\\
                \frac{1}{\Delta}\min\{1, \frac{\mu(j)}{\mu(i)}\} \quad &j \in \mathcal{N}(i)\\
                1 - \sum_{k\in\mathcal{N}(i)} P(i,k) \quad & i = j
            \end{cases}
        \end{equation}
        \begin{proposition}
            The $P$ defined above satisfies
            \[ \mu P = \mu \]
            i.e. it is a Markov chain with stationay distribution $\pi$.
        \end{proposition}
        \begin{sketchproof}~{}
            \begin{itemize}
                \item Trivial if $j \notin \mathcal{N}(i)$.
                \item If $j \in \mathcal{N}(i)$. WLOG assume $\mu(j) \ge \mu(i)$.
                \[ \mu(i)P(i,j) = \mu(i)\frac{1}{\Delta}\min\{1,\frac{\mu(j)}{\mu(i)}\} = \frac{\mu(i)}{\Delta} \]
                and
                \[ \mu(j)P(j,i) = \mu(j)\frac{1}{\Delta}\min\{1,\frac{\mu(i)}{\mu(j)}\} = \frac{\mu(i)}{\Delta} \]
                So we are done.
            \end{itemize}
        \end{sketchproof}
        \begin{remark}
            If we run the algorithm sufficiently long, the sequence we get will have a distribution that is approximately the same as $\mu$. What's more, in this algorithm, we only need to know $\frac{\mu(j)}{\mu(i)}$, instead of knowing $\mu$. And this is useful in practice.
        \end{remark}

        In Chang's note, if we assume the desired distribution is proportional to
        \[ \pi(i) \propto d(i)f(i) \]
        where $f(i)$ is some additional weight on the degree of node $i$, then we can define the matrix by
        \begin{equation}\notag
            P(i, j) =
            \begin{cases}
                0 \quad & j \notin \mathcal{N}(i)\\
                \frac{1}{d(i)}\min\{1, \frac{f(j)}{f(i)}\} \quad &j \in \mathcal{N}(i)\\
                1 - \sum_{k\in\mathcal{N}(i)} P(i,k) \quad & i = j
            \end{cases}
        \end{equation}

        \begin{remark}
            The Metropolis method simulates a random walk on a \emph{graph}, whose nodes are different states. However, the choice of edges depends on us, and different choices of edges will lead to different choices of $P$.
        \end{remark}

    \subsection{Simulated Annealing}
        \par Simulated Annealing is a stochastic optimization algorithm. Given a set $\mathcal{S}$, and a function $w(x)$. We want to minimize (or maximize) $w(x)$.
        \paragraph{Notations.}
        \begin{itemize}
            \item $\mathcal{S}^* = \{x\in\mathcal{S}|w(x) = \min_{y\in\mathcal{S}} w(y)\}$: the set of global minimizers of $w(x)$.
            \item $\mu^*$: a uniform distribution on $\mathcal{S}^*$.
            \item $T$: The ``Temperature'' parameter.
        \end{itemize}
        \par We define a distribution on $\mathcal{S}$, and we want to make the probability concentrate on $x$ where $w(x)$ is small. However sampling from $\mathcal{S}$ is usually infeasible because $\mathcal{S}$ is too large.
        \par Instead, we introduce a parameter $T$, and let
        \[ \mu_T(x) \sim e^{-\frac{w(x)}{T}} \]
        \begin{proposition}
            As $T \to 0$, $\mu_T(x) \to \mu^*(x)$.
        \end{proposition}
        \par However, if we start with a very small $T$, we may get stuck in a local extremum. So we start with a large $T_1$ and gradually decrease $T$.
        \begin{theorem}
            If $T_1,T_2,\dots$ satisify
            \begin{enumerate}
                \item $\{T_i\} \to 0$.
                \item $T_i$ decreases slow enough. (``Slow'' means that the sum of a certain series defined by $\mathcal{S}$ diverges, but it is so complicated that it is beyond the scope.)
            \end{enumerate}
            Then let $P^{(n)} \triangleq \prod_{k=1}^n P_{T_k}$.
            \[ \forall \mu \quad \mu^TP^{(n)} \to \mu^* \]
        \end{theorem}
        That is, as long as $T$ decreases slowly enough, we only need to run each temperature $T_i$ \emph{for one step with the Metropolis algorithm}.

\section{Mixing Times}
    By designing a proper coupling $C$, we can upper bound the convergence speed of a Markov Chain.

    \subsection{Mixing Time}
        \begin{definition}[Mixing Time]\label{def:MixingTime}
            The \textbf{mixing time} $\tau_{mix}(\varepsilon)$ is defined as
            \[ \tau_{mix}(\varepsilon) = \max_{a} \min_{t} \| \mu_t^a - \pi \|_{TV} \le \varepsilon \]
            It represents the minimal time for $\mu$ to converge to $\pi$ starting from the worst initial distribution $\mu^a$.
        \end{definition}

    \subsection{Some Examples}

        \subsubsection{Random Walk on Hypercube.}
            A \textbf{hypercube} is a $\{0,1\}$ string $\{0,1\}^n$, and there exists an edge between $x$ and $y$ if and only if $\sum_{i=1}^n|x(i)-y(i)| = 1$, i.e. $x$ and $y$ differs in only one bit.
            Let
            \[
                X_{t+1} =
                \begin{cases}
                    X_t \quad & w.p.\frac{1}{2}\\
                    \textit{Move into one of the neighbours} & w.p.\frac{1}{2}
                \end{cases}
            \]
            i.e. we have probability $1/2$ to stay still and probability $1/2$ to move into a neighbour.

            If we start from the worst case, then $\mathbb{P}[X_t \neq Y_t]$ can give an upper bound of $\tau_{mix}$.

            We construct the coupling as follows. Suppose we are at $X_t$,
            \begin{enumerate}
                \item Uniformly pick $i \in n$.
                \item Uniformly pick $c \in \{0, 1\}$.
                \item Set the $i$-th bit of $X_t$ to $c$ and yield $X_{t+1}$.
            \end{enumerate}
            We choose \emph{the same} $i$ and $c$ for both $X_t$ and $Y_t$.

            It can be proved that if we run $n\log n + cn$ iterations, the probability that $X_t \neq Y_t$ is at most $e^{-c}$. i.e.
            If
            \[ t \ge n\log n + cn \]
            Then
            \[ \mathbb{P}[X_t \ne Y_t] \le e^{-c} = \varepsilon \]
            Therefore
            \[ \tau_{mix} \le n\log n + n\log \frac{1}{\varepsilon} \le \varepsilon\]
            So we need approximately at most $n\log n$ steps.

        \subsubsection{Coupon Collection}
            Given a set of $n$ coupons, we pick $X$ times. And we want to know the total time we need to collect all $n$ coupons.

            Let $X_i$ denotes the number of gachas needed to get an $i$-th new coupon.
            \[ \mathbb{E}[X] = \mathbb{E}[\sum_{i=1}^n X_i] = \sum_{i=i}^n \mathbb{E}[X_i] \]
            Notice that $X_i$ has a geometric distribution with parameter $\frac{n-i+1}{n}$. Therefore
            \[ \mathbb{E}[X] = \sum_{i=1}^n \frac{n}{n-i+1} = n\sum_{i=1}^n \frac{1}{i} \approx n(\log n + \gamma) \]

        \subsubsection{Card Shuffling}
            Suppose we shuffle a deck of 52 cards using a ``top-in-at-random'' method: we pick the card on the top and insert it into one of 52 positions (including the top) at random. This Markov chain has the stationary distribution $\pi = Uniform(\mathcal{S}_{52})$. By FTMC, $\|\pi_n - pi\|_{TV} \to 0$ as $n \to \infty$.

            It is possible to find a random variable $T$ which denotes the time at which the deck becomes exactly uniformly distributed. We first introduce some notations.

            \begin{itemize}
                \item $U_i$ is uniformly distributed on $1,2,\dots,52$ representing that the $i$-th shuffle moves the card on the top to position $U_i$.
                \item $T_1 = \inf\{ n: U_n = 52 \}$ denotes the first time a card is moved below card 52.
                \item $T_2 = \inf\{ n>T_1: U_n \ge 51 \}$ denotes the second time a card is moved below card 52.
                \item $T_{51} = \inf\{ n>T_50: U_n \ge 2 \}$ denotes the 51st time a card is moved below card 52.
            \end{itemize}

            Then $T = T_{52} = T_{51} + 1$.

            Clearly we have
            \begin{itemize}
                \item $T_1 \sim Geometry(1/51)$
                \item $T_2 - T_1 \sim Geometry(2/52)$
                \item \dots
                \item $T_{52} - T_{51} \sim Geometry(52/52) = 1$
            \end{itemize}

            We can calculate $\mathbb{E}[T]$. By linearity of Expectation,
            \[ \mathbb{E}[T] = \mathbb{E}[T_1] + \mathbb{E}[T_2-T_1] + \dots + \mathbb{E}[T_52 - T_51] \approx 52\log 52 \]
            Similarly, if we are shuffling $d$ cards, we have
            \[ \mathbb{E}[T] = d\log{d} \]

    \subsection{Strong Stationary Times}
        \begin{definition}[Stopping Time]
            The time $T$ is called the \textbf{stopping time} if for each $n$, one can tell whether $T=n$ by just looking at $X_0, X_1, \dots, X_n$, i.e. it is not necessary to know any other future values of $X_{n+1}, X_{n+2}, \dots$.
        \end{definition}
        \begin{remark}
            In the card-shuffling example, $T$ is a stopping time.
        \end{remark}
        \begin{definition}[Strong Stationary Time]
            A random variable $T$ is called a \textbf{strong stationary time} if it satisfies
            \begin{enumerate}
                \item $T$ is a stopping time
                \item $X_T$ is distributed as $\pi$
                \item $X_T$ is independent of $T$
            \end{enumerate}
        \end{definition}
        \begin{lemma}\label{Lemma:StrongStationaryTime}
            If $T$ is a strong stationary time for the Markov Chain $\{X_n\}$, then $\|\pi_n - \pi\| \le \mathbb{P}[T > n]$ for all $n$.
        \end{lemma}

    \subsection{Threshold Pheonomenon}
        In card shuffling and many other Markov Chains, $\|\pi_n - \pi\|$ stays close to $1$ for a while, and suddenly drop to nearly $0$ at approximately $n = d\log{d}$

        \begin{theorem}
            For $T$ defined in the card shuffling, let $\Delta(n) = \|\pi_n - \pi\|$, we have
            \[ \Delta(d\log{d} + cd) \le \mathbb{P}[T > d\log{d} + cd] \le e^{-c} \]
            for all $c \ge 0$
        \end{theorem}
        \begin{sketchproof} ~{}
            \begin{enumerate}
                \item The first inequality follows from Lemma \ref{Lemma:StrongStationaryTime}.
                \item To prove the second inequality, we introduce the coupon collector's problem.
                \item Suppose we want to collect $d$ uniforly distributed coupons, the \sout{gachas} boxes we need to open to collect all $d$ coupons have the distribution
                \[ 1 + Geometry((d-1)/d) + \dots + Geometry(1/d) \]
                \item Let $B_i = \{ \text{coupon $i$ does not appear in the first $n$ cereal boxes} \}$
                \item Then $T_i > n$ is the union $\bigcup_{i=1}^dB_i$
                \item $\mathbb{P}[T>n] \le \sum_{i=1}^d\mathbb{P}[B_i] = d\left(1 - \frac{1}{d}\right)^n \le de^{-n/d}$
                \item Take $n = d\log{d} + cd$ yield the result
            \end{enumerate}
        \end{sketchproof}


Binary search trees suffer from a problem which is known as ``degeneration''. A binary tree is said to be \textit{degenerate} if for each node there is only one non-empty child. This phenomenon happens when the elements inserted do not have a substantially random distribution, that is they are partially ordered. The limit case is the insertion of a sorted sequence such as the numbers from 1 to 10, which results in the following tree:

\begin{lstlisting}
1
 \
  2
 / \
    3
   / \
      5
     / \
        6
       / \
          7
         / \
            8
           / \
              9
             / \
               10
              /  \
\end{lstlisting}

Search and removal operations in this sort of tree will have $O(N)$ complexity, which is significantly worse than the $O(\log N)$ that we expect from a balanced tree.

In the rest of this section, we will present a new sort of binary trees, 2-3-4 trees, which always remain balanced by only adding elements to the root node and therefore increasing the height of the tree uniformly by one.

There are multiple possible implementations of balanced binary trees. We present here 2-3-4 trees as they are very widely used in many common implementation, and are also slightly simpler to describe and relatively simple to implement in our language. A widely used alternative to 2-3-4 trees is \textit{red-black} trees, which are \textit{isomorph} to 2-3-4. In other words, any 2-3-4 tree can be translated to an equivalent red-black tree without loss of structure or information, and vice-versa.

\paragraph{Structure of 2-3-4 trees}
A 2-3-4 tree does not only have nodes with one value\footnote{\textit{values} are often also called \textit{keys}.} and two children. In a 2-3-4 tree we distinguish three kinds of nodes:
\begin{inparaenum}[\itshape i\upshape)]
\item 2-nodes have one value $k$ and two children $l$ and $r$ such that $\forall k' \in l . k' < k$ and $\forall k' \in r . k' > k$;
\item 3-nodes have two value $k_1$ and $k_2$ and three children $l$, $m$, and $r$ such that $\forall k' \in l . k' < k_1$, $\forall k' \in r . k' > k_2$, and $\forall k' \in m . k_1 < k' < k_2$;
\item 4-nodes have three values $k_1$, $k_2$, and $k_3$ and four children $l$, $m_1$, $m_2$, and $r$ such that $\forall k' \in l . k' < k_1$, $\forall k' \in r . k' > k_3$, $\forall k' \in m_1 . k_1 < k' < k_2$, and $\forall k' \in m_2 . k_2 < k' < k_3$.
\end{inparaenum}

An example of such a tree with a depth of only two could be:

\begin{lstlisting}
    10 20
 /   |    \
5   17   22 24 29
\end{lstlisting}

Here the root \texttt{10 20} is a 3-node with two elements, \texttt{10} and \texttt{20}, and three children:
\begin{inparaenum}[\itshape i\upshape)]
\item 2-node \texttt{5};
\item 2-node \texttt{17};
\item 4-node \texttt{22 24 29}.
\end{inparaenum}

Our goal is to insert elements in such a manner that the tree maintains its balance, independent of the order of insertion of the elements.

Our implementation of 2-3-4 trees is based on lists of elements. Lists are defined as usual, either a list is empty (\texttt{nil}) or it is the append of an element to a list (\texttt{;}):

\begin{lstlisting}
Data [] "nil" [] Priority 0 Type List
Data [ListElem] ";" [List] Priority 100 Type List
\end{lstlisting}

The elements of the list can either be values (in our case just integers) or children, thus:

\begin{lstlisting}
<<int>> is ListElem
BTree is ListElem
\end{lstlisting}

The tree is defined as either an \texttt{empty} node without values and children, or a proper \texttt{node} that contains a list of values and children:

\begin{lstlisting}
Data [] "empty" [] Priority 0 Type BTree
Data [] "node" [List] Priority 0 Type BTree
\end{lstlisting}

The above definition means that a 2-node with value \texttt{k} and children \texttt{l} and \texttt{r} will be represented as:

\begin{lstlisting}
node l;k;r;nil
\end{lstlisting}

a 3-node with values \texttt{k1} and \texttt{k2} and children \texttt{l}, \texttt{m}, and \texttt{r} will be represented as:

\begin{lstlisting}
node l;k1;m;k2;r;nil
\end{lstlisting}

and similarly for 4-nodes.

\paragraph{Insertion}
The main idea behind insertion into a \texttt{2-3-4} tree is that for each node, we:
\begin{inparaenum}[\itshape i\upshape)]
\item find the appropriate child where we wish to insert by checking the values inside the node;
\item insert into that child;
\item if the child becomes too big (a 5-node, which is not allowed) then we split the node into two smaller nodes and insert the excess value into the current node.
\end{inparaenum}

Insertion is thus a function that given a tree and an element to add returns the transformed tree:

\begin{lstlisting}
Func [BTree] "insert" [<<int>>] Priority 0 Class Expr => BTree
\end{lstlisting}

Insertion into an empty tree is quite simple, in that we add the element between two empty nodes, therefore creating a trivial 2-node. This will only happen during the first insertion, that is when the tree is empty:

\begin{lstlisting}
----------------------------------------------
empty insert kv => node (empty;(kv;(empty;nil)))
\end{lstlisting}

For non-empty trees we will always stop right before the empty children, and we will always insert into leaf nodes. If the node is not a leaf, then we find the appropriate position and insert the new value there with function \texttt{insertInto}. This yields a new tree which needs to be \texttt{split} if its root is too big (a 5-node):

\begin{lstlisting}
isLeaf l => no
kv insertInto l => l'
split l' => res
----------------------------------------
(node l) insert kv => res
\end{lstlisting}

If the node is indeed a leaf, then we find the appropriate position within its list of content and add the element plus a new \texttt{empty} child with the \texttt{insertSorted} function. The node, just like in the previous case, might become too big (a 5-node) as a result of insertion, and may thus need to be \texttt{split}:

\begin{lstlisting}
isLeaf l => yes
l insertSorted kv empty => l1
split l1 => res
-------------------------------
(node l) insert kv => res
\end{lstlisting}

The function that checks whether or not a tree is a leaf takes as input its list of values and children and returns a boolean value:

\begin{lstlisting}
Func [] "isLeaf" [List] Priority 0 Type Expr => YesNo
\end{lstlisting}

\texttt{isLeaf} returns \texttt{yes} when the children of the node are all \texttt{empty}. Since we know that this is a balanced tree, then we know that if one child (for example the first) is \texttt{empty}, then all other children will be as well:

\begin{lstlisting}
---------------------------------
isLeaf (empty;rest) => yes
\end{lstlisting}

If the first child of the tre is not \texttt{empty}, then we can be sure that all other children are not empty as well and thus we are not in presence of a leaf:

\begin{lstlisting}
l != empty
---------------------------------
isLeaf (l;rest) => no
\end{lstlisting}

In case of leaves, we simply\footnote{For a large enough value of ``simply''.} need to find the proper position within the list where to insert our element. The \texttt{insertSorted} will therefore take a list of values and trees plus a value to insert as input, and return another such list:

\begin{lstlisting}
Func [List] "insertSorted" [<<int>>] Priority 0 Type Expr => List
\end{lstlisting}

If we reach the end of the list, then we simply add the value followed by the \texttt{empty} tree. The function \texttt{insertSorted} is only called on leaves, thus all children are already \texttt{empty}:

\begin{lstlisting}
--------------------------------------
l;nil insertSorted k => l;k;empty;nil
\end{lstlisting}

If we find the element already in the tree, then we do nothing and return the original tree untouched:

\begin{lstlisting}
x == k
--------------------------------
l;x;xs insertSorted k => l;x;xs
\end{lstlisting}

If the element to insert is bigger than the current value in the list, then we have found the point of insertion. We put the new element right before the bigger element and between them we put an \texttt{empty} child:

\begin{lstlisting}
x > k
----------------------------------------
l;x;xs insertSorted k => l;k;empty;x;xs
\end{lstlisting}

If the element to insert is smaller than the current value in the list, then we will have to go on looking in the rest of the list. We perform the insertion in the rest of the list, which then becomes the updated tail\footnote{\textit{Tail} is a common name given to ``the rest of a recursively defined list''.}:

\begin{lstlisting}
x < k
xs insertSorted k => xs'
--------------------------------
l;x;xs insertSorted k => l;x;xs'
\end{lstlisting}

Insertion into non-leaf elements is very similar to \texttt{insertSorted}. The first aspect of \texttt{insertInto} is that it looks for the right position in the list of values and children where the insertion needs to happen. Whereas \texttt{insertSorted} at that point simply adds two elements to the list, \texttt{insertInto} recursively calls \texttt{insert} into the appropriate child. We know that the child where we perform the recursive insertion is not \texttt{empty} because otherwise \texttt{insertSorted} would have been called instead of \texttt{insertInto}.

\texttt{insertInto} takes as input the value to insert and the list where the insertion needs to take place, and returns the list with the value added:

\begin{lstlisting}
Func [<<int>>] "insertInto" [List] Priority 0 Type Expr => List
\end{lstlisting}

The base case of \texttt{insertInto} happens when we reach the last child of the input list. In this case we cannot proceed forward, and therefore we will have to perform the insertion into the last child itself. After the recursive insertion step is performed, the child is updated and we need to \texttt{merge} it back into the original list:

\begin{lstlisting}
l insert k => l'
merge l' nil => l''
---------------------------
k insertInto l;nil => l''
\end{lstlisting}

If the value we are trying to insert is smaller than the current value, then we have found the position of insertion and perform the \texttt{insert} and the \texttt{merge} steps precisely like we did for the previous case:

\begin{lstlisting}
x > k
l insert k => l'
merge l' x;xs => l''
---------------------------
k insertInto l;x;xs => l''
\end{lstlisting}

If we find the value we are trying to insert, then we simply return the original list unchanged:

\begin{lstlisting}
x == k
------------------------------
k insertInto l;x;xs => l;k;xs
\end{lstlisting}

If the value we are trying to insert is bigger than the current value, then we try to insert it at a later position with a recursive call to \texttt{insertInto} the tail of the list \texttt{xs}:

\begin{lstlisting}
x < k
k insertInto xs => xs'
-------------------------------
k insertInto l;x;xs => l;x;xs'
\end{lstlisting}

The role of \texttt{split} is extremely important. After insertion, we may get a 5-node, which is not allowed. Split therefore takes as input a list of values and nodes, and returns a 2-3-4 tree where the eventual 5-nodes have been split:

\begin{lstlisting}
Func [] "split" [List] Priority 0 Class Expr => BTree
\end{lstlisting}

If \texttt{split} encounters a 2-node (list of three elements), 3-node (list of five elements), or 4-node (list of seven elements), then it simply ``wraps'' the list into a single node and returns the node as the resulting tree:

\begin{lstlisting}
------------------------------------------
split l;(k;(r;nil)) => node l;(k;(r;nil))

-------------------------------------------------------------
split l;(k1;(m;(k2;(r;nil)))) => node l;(k1;(m;(k2;(r;nil))))

------------------------------------------------------------------------
split l;(k1;(l_m;(k2;(r_m;(k3;(r;nil)))))) => node l;(k1;(l_m;(k2;(r_m;(k3;(r;nil))))))
\end{lstlisting}

If the list is a 5-node, then it contains nine elements. We take the middle value as a \textit{pivot}, and use the first three elements of the list as a 2-node and the remaining five elements of the list as a 3-node. We then return a 2-node where the newly created elements are the left and right child and the \textit{pivot} is the value:

\begin{lstlisting}
l' := node l;(k1;(l_m;nil))
r' := node m;(k3;(r_m;(k4;(r;nil))))
----------------------------------------------------------------------------
split l;(k1;(l_m;(k2;(m;(k3;(r_m;(k4;(r;nil)))))))) => node l';(k2;(r';nil))
\end{lstlisting}

After insertion we always perform a \texttt{split}. The fact that we just performed an insertion means that the sub-tree is now bigger by one element. The fact that we just \texttt{split} means that if we find a 2-node then this node was just split and therefore its single element should be added to the current list of elements, thereby promoting it by one level and therefore potentially propagating the excess size to the parent node. This way, if we need to create a new node, we keep sending new elements to parent nodes until we reach the root. When we split the root, then \textit{the whole tree} has grown by one level. This will always keep the tree perfectly balanced.

\texttt{merge} takes as input a tree (where the insertion just took place) and the list of elements where the node needs to be merged, and returns a new list of elements where the input tree has been added:

\begin{lstlisting}
Func [] "merge" [BTree List] Priority 0 Type Expr => List
\end{lstlisting}

If we find a 2-node, then we know that it has just been split. This means that we effectively promote its value to the current level:

\begin{lstlisting}
---------------------------------------
merge (node l;(k;(r;nil))) xs => l;(k;(r;xs))
\end{lstlisting}

If we find a 3-node or a 4-node (in general a node where the input list is longer than four elements) then we simply add this node as a single child of the list to return:

\begin{lstlisting}
--------------------------------------------------
merge (node l;(k1;(m;(k2;rs)))) xs => (node l;(k1;(m;(k2;rs))));xs
\end{lstlisting}

Let us now try to visualize\footnote{``Try'' is the keyword here.} this process. We will consider the most interesting path of insertion, that is insertion into a tree which needs to grow in height by one.

The initial tree, where we shall insert the value 9, is:

\begin{lstlisting}
   1  3  5
 /  |   |  \
0   2   4  6 7 8
\end{lstlisting}

The initial call to \texttt{insert} processes the root of the tree. Since the root is not a leaf (all its children are non-empty), then it will perform an \texttt{insertInto} followed by a \texttt{split}.

\texttt{insertInto} finds that 9 is bigger than all elements of the root, thus insertion needs to happen in the right-most child: \texttt{6 7 8}. This leads us to another call to \texttt{insert} of value 9 into tree, which will then be followed by a call to \texttt{merge}.

\begin{lstlisting}
  6 7 8
 /  |  \
\end{lstlisting}

Since the node is a leaf, then \texttt{insert} calls \texttt{insertSorted}, followed by a \texttt{split}. \texttt{insertSorted} simply places the new value into the right position in the tree:

\begin{lstlisting}
  6 7 8 9
 / | | | \
\end{lstlisting}

The first \texttt{split} creates a new 2-node:

\begin{lstlisting}
     7
   /   \
  6    8 9
 / \  / | \
\end{lstlisting}

This tree is sent back to as the result of \texttt{insertInto}, which merges it with the list of values which was the original root of the tree (that is \texttt{|2|4|5|}). \texttt{merge} adds the value of the 2-node as a sibling of the elements of the root, since it recognizes the result of a succesful split. The resulting tree (not yet valid) is thus:

\begin{lstlisting}
   1  3  5  7
 /  |   |  |  \
0   2   4  6  8 9
\end{lstlisting}

This is returned as the result of \texttt{insertInto}, which is then processed by the last call to \texttt{split}. Since \texttt{split} finds a 5-node, it proceeds with its splitting and returns the result as the final tree:

\begin{lstlisting}
         3
      /     \
     /       \
   1        5  7
 /   \     /  |  \
0     2   4   6  8 9
\end{lstlisting}

This elegant algorithm has thus managed to push the excess elements back up until the root was reached, therefore performing the final split at the root instead of one of the leaves and thus all nodes increase by one level of height instead of just a few, which would progressively unbalance the tree. Notice that the nodes of the resulting tree are all 2-nodes or 3-nodes, therefore it will take a few more insertions (at least three) before we can increase the height of the tree yet again.

\paragraph{A coward's approach to search}
We now quickly conclude the section by showing a ``lazy'' (in the sense that we use very simple predicates and some code duplication) approach to lookup. We define function \texttt{contains} which determines whether or not a value is present within the tree:

\begin{lstlisting}
Func [] "contains" [BTree <<int>>] Priority 0 Type Expr => Bool
\end{lstlisting}

The trivial case is that the search has ended up in an empty tree. In this case we simply return a negative result:

\begin{lstlisting}
-----------------------
contains empty k => no
\end{lstlisting}

If we have found a two-node, then we check whether the value of the node is the one we are looking for. If this is the case, then we can return a positive result:

\begin{lstlisting}
x == k
-----------------------------------
contains (node l;x;r;nil) k => yes
\end{lstlisting}

If the value of the node is bigger (smaller) than the searched value, then we proceed with a recursive search in the left (right) child:

\begin{lstlisting}
k < x
contains l key => res
----------------------------------
contains node(l;x;r;nil) k => res

k > x
contains r key => res
----------------------------------
contains node(l;x;r;nil) k => res
\end{lstlisting}

Similarly, for 3-nodes we need to find the proper interval where search must recursively continue:

\begin{lstlisting}
key > x1
key < x2
contains m k => res
------------------------------------------
contains node(l;x1;m;x2;r;nil) k => res
\end{lstlisting}

The rest would need to be written by hand.

\paragraph{A courageous approach to search}
The above is perhaps useful for illustration purposes. As an alternative, we could write a recursive search functions similar to \texttt{insertSorted} or \texttt{insertInto}.

Containment fails immediately on an \texttt{empty} sub-tree:

\begin{lstlisting}
-----------------------
contains empty k => no
\end{lstlisting}

On a non-empty \texttt{node} we invoke auxiliary function \texttt{containsIterate}, which simply takes a list of elements instead of a tree as a parameter:

\begin{lstlisting}
Func [] "containsIterate" [List <<int>>] Priority 0 Type Expr => Bool
\end{lstlisting}

\begin{lstlisting}
containsIterate l k => res
---------------------------
contains node l k => res
\end{lstlisting}

\texttt{containsIterate} defaults to searching in the right-most child if no other suitable children have been found so far:

\begin{lstlisting}
contains r k => res
-------------------------
containsIterate r;nil k => res
\end{lstlisting}

If the current value of the list is equal to the value we are searching, then search has been concluded positively:

\begin{lstlisting}
k = x
--------------------------------
containsIterate l;x;xs k => yes
\end{lstlisting}

If the current value of the list is bigger than the value we are searching, then we have found the child where search needs to continue recursively:

\begin{lstlisting}
k < x
contains l k => res
--------------------------------
containsIterate l;x;xs k => res
\end{lstlisting}

If the current value of the list is smaller than the value we are searching, then we have not yet found the child where search needs to recurse, and we must continue iterating the elements of the list:

\begin{lstlisting}
k > x
containsIterate xs k => res
--------------------------------
containsIterate l;x;xs k => res
\end{lstlisting}

This second formulation of search is quite shorter than the previous, and illustrates nicely how a well-though out recursive procedure is often much shorter to write and read \footnote{but perhaps not invent} than lots of repeated, boilerplate code.


\subsection{Shooting method}
Final time is free so:
$$
\vec{F} = \begin{bmatrix}
	x_1(t_f) - x_{1_f}\\
	x_2(t_f) - x_{2_f}\\
	\left.(\mathcal{H} - h_t)\right\vert_{t_f}
\end{bmatrix}
$$
$$
\vec{y}_{k+1} = \vec{y}_k - \left.\dfrac{\partial \vec{F}}{\partial \vec{y}}
\right\vert_{\vec{y}_k}
\vec{F}(\vec{y}_k)
$$
\begin{figure}[H]
	\caption{Shooting method}
	\centering
	\includegraphics[width=11.5cm]{../Figure/Q3/Shooting.png}
\end{figure}


\subsection{Density of the rationals}

\subsubsection{Rationals are dense in rationals}

For any pair of rationals, there is another rational between them:

\(a=\dfrac{p}{q}\)

\(b=\dfrac{m}{n}\)

Where \(b>a\).

We define a new rational:

\(c=\dfrac{a+b}{2}\)

\(c=\dfrac{pn+qm}{2qn}\)

This is a rational number.

We can write:

\(a=\dfrac{2pn}{2qn}\)

\(b=\dfrac{2qm}{2qn}\)

As \(b>a\) we know \(2qm>2pn\)

So: \(a < c < b\)


\subsection{Space shuttle and Corvette}
The space shuttle accelerates from zero to 17{,}000 miles per hour
in 8 minutes.
A Corvette accelerates from zero to 60 miles per hour in 4.5 seconds.
The following script compares the two.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
vs = 17000*"mile"/"hr"
ts = 8*"min"/(60*"min"/"hr")
as = vs/ts
as
vc = 60*"mile"/"hr"
tc = 4.5*"sec"/(3600*"sec"/"hr")
ac = vc/tc
ac
"Time for Corvette to reach orbital velocity:"
vs/ac
vs/ac*60*"min"/"hr"
\end{Verbatim}

Here is the result when the script runs.
It turns out that the space shuttle accelerates more than twice as fast as a
Corvette.

$\displaystyle a_s={\hbox{127500 mile}\over(\hbox{hr})^2}$

$\displaystyle a_c={\hbox{48000 mile}\over(\hbox{hr})^2}$

\verb$Time for Corvette to reach orbital velocity:$

$\displaystyle 0.354167\;\rm hr$

$\displaystyle 21.25\;\rm min$


\subsection{Angles}

\subsubsection{Recap: Cauchy-Schwarz inequality}

This states that:

\(|\langle u,v\rangle |^2 \le \langle u, u\rangle \dot \langle v, v\rangle \)

Or:

\(\langle v,u\rangle\langle u,v\rangle \le \langle u, u\rangle \dot \langle v, v\rangle \)

\subsubsection{Introduction}

\(\langle v,u\rangle\langle u,v\rangle \le \langle u, u\rangle \dot \langle v, v\rangle \)

\(\dfrac{\langle v,u\rangle\langle u,v\rangle}{||u||.||v||} \le ||u||.||v||\)

\(\dfrac{||u||.||v||}{\langle v,u\rangle} \ge \dfrac{\langle u,v\rangle}{||u||.||v||}\)

\(\cos (\theta )=\dfrac{\langle u,v\rangle }{||u||.||v||}\)


\section{Covariance and Uncertainty}

Covariance is the relationship that every variable has with every other variable, and it is very important in multidimensional systems. We have our main covariance $\boldsymbol{P}$, also called the estimate uncertainty, which is modified both in the \textit{Predict} and in the \textit{Update} phases.

In the \textit{Predict} step, we extrapolate the estimate uncertainty to create a prediction for $\boldsymbol{P}$ at the next timestep. This is done by using our state transition matrix previously obtained from the motion model.
\begin{equation}
    \boldsymbol{P}_{k+1} = \boldsymbol{F} \cdot \boldsymbol{P} \cdot \boldsymbol{F}^T + \boldsymbol{Q}
\end{equation}
where $\boldsymbol{Q}$ is a matrix representing the process noise. This is hard to nail down and understand physically, but think of it as the uncertainty in the process of the EKF itself. There are formulas and ways to find more complicated versions of all entries, but it suffices in our case to simply use the 8x8 identity matrix multiplied by a constant that we can tune. This constant tends to remain in the range (1.1, 1.3) in our application.

In addition to these uncertainties in the estimations and in the process, we of course have uncertainties in our measurements. If sensors give direct values for their variance, those should be used. We define the measurement uncertainty as a 6x6 matrix, $\boldsymbol{R}$, with the variance of each measurement as the entry on the diagonal corresponding to that measurement's position in $\boldsymbol{z}$. We know that our approximations for the positions using GPS coordinates are fairly scuffed and have a standard deviation around 5 meters, whereas our IMU and encoders can easily be more accurate than a tenth of a unit. As such, we used the following matrix as a starting point. We frequently modify these values on the fly based on which sensors are performing well and which ones are having more interference than expected. Check the competition build of the code to see what we finalized at the venue.

\begin{equation}
    \boldsymbol{R} =
    \begin{pmatrix}
    35 & 0 & 0 & 0 & 0 & 0 \\
    0 & 25 & 0 & 0 & 0 & 0 \\
    0 & 0 & 0.1 & 0 & 0 & 0 \\
    0 & 0 & 0 & 0.1 & 0 & 0 \\
    0 & 0 & 0 & 0 & 0.01 & 0 \\
    0 & 0 & 0 & 0 & 0 & 0.01
    \end{pmatrix}
\end{equation}

Note that if a sensor is being absolutely horrible or breaks it can be essentially disabled by setting its uncertainty value in this matrix to something huge like 100,000. Even if the filter never receives data from a sensor (i.e., it is unplugged or not functioning), it will use the starting value, 0, weighted with a massive uncertainty; this means the filter will still work if a sensor is disconnected, albeit not as well as with all the expected data.


\abbreviations

\section*{Symbols}

\begin{symbollist}
  \item[$P\left(X \mid Y\right)$] (Conditional) Probability of $X$ Given $Y$
  \item[$\expval{X}$ or $\expvalE{X}$] Expectation Value of $X$
  \item[$\sigma_{X}^{2}$ or $\variance{X}$] Variance of $X$
  \item[$\sigma_{u,v}^{2}$ or $\cov{u}{v}$] Covariance of $u$ and $v$
  \item[$\vb*{\beta}$] Model Parameters, $n \times 1$ Column Vector
  \item[$\mathbf{X}$] Input Features, $m \times n$ Matrix
  \item[$y$] Dependent Feature
  \item[$m$] Number of Data Points or Rows
  \item[$n$] Number of Input Features or Columns\footnote{$n$ is also sometimes used for the number of rows, if it is the standard terminology for a method.}
  \item[$\nu$] Number of Degrees of Freedom
  \item[$S\left(\vb*{\beta}\right)$] Objective Function
  \item[$L\left(\vb*{\beta}\right)$] Loss Function
  \item[$\Omega\left(\vb*{\beta}\right)$] Regularization Function
  \item[$L$] Likelihood Function
  \item[$\yhat$] Estimated Dependent Feature or Classification Score, Prediction % Leave as $\yhat$ to get the proper spacing
  \item[$Z$] Significance
  \item[$S\left(t\right)$] Survival Function
  \item[$\lambda\left(t\right)$] Hazard Function
  % \item[\ZB] Significance, Incomplete Beta Function Approximation
  % \item[\CLs] Signal Confidence Level
\end{symbollist}

\clearpage
\section*{Abbreviations}
% TODo keep updated

\begin{symbollist}
  \item[\iid] Independent and Identically Distributed
  \item[\kNN] $k$-Nearest Neighbors
  \item[\tSNE] $t$-Distributed Stochastic Neighbor Embedding
  \item[ABC] Approximate Bayesian Computation
  \item[ACF] Auto-Correlation Function
  \item[ADF] Augmented Dickey--Fuller Test for Stationarity
  \item[AIC] Akaike Information Criterion
  \item[AICc] Akaike Information Criterion with corrections
  \item[ANCOVA] Analysis of Covariance
  \item[ANOVA] Analysis of Variance
  \item[AR] Autoregressive (Models)
  \item[AUC] Area Under Curve
  \item[BDT] Boosted Decision Tree
  \item[BFS] Breadth First Search
  \item[BIC] Bayesian Information Criterion
  \item[BLUE] Best Linear Unbiased Estimator
  \item[CART] Classification and Regression Tree
  \item[CDF] Cumulative Distribution Function
  \item[CLT] Central Limit Theorem
  \item[CNN] Convolutional Neural Network
  \item[DFS] Depth First Search
  \item[DID] Difference in Differences
  \item[EM] Expectation Maximization
  \item[FWHM] Full Width at Half Maximum
  \item[GLM] Generalized Linear Model
  \item[GLS] Generalized Least Squares
  \item[GMM] Gaussian Mixture Model
  \item[GNB] Gaussian N{a\"i}ve Bayes (Classification)
  \item[GP] Gaussian Process
  \item[HR] Hazard Ratio
  \item[IQR] Interquartile Range
  \item[KDE] Kernel Density Estimation
  \item[LDA] Linear Discriminant Analysis
  \item[LSTM] Long Short Term Memory
  \item[LVQ] Learning Vector Quantization
  \item[MA] Moving Average (Models)
  \item[MANOVA] Multivariate Analysis of Variance
  \item[MAP] Maximum \aposteriori (Estimation)
  \item[MAPE] Mean Absolute Percent Error
  \item[MC] Monte Carlo
  \item[MCMC] Markov Chain Monte Carlo
  \item[MI] Mutual Information
  \item[ML] Machine Learning
  \item[MLE] Maximum Likelihood Estimation (or Estimator)
  \item[MMSE] Minimum Mean Square Error (Estimator)
  \item[MSE] Mean Squared Error
  \item[NLP] Natural Language Processing
  \item[NMI] Normalized Mutual Information
  \item[NN] Neural Network
  \item[OLS] Ordinary Least Squares
  \item[PACF] Partial Auto-Correlation Function
  \item[PCA] Principle Component Analysis
  \item[PCR] Principal Component Regression
  \item[PDF] Probability Density Function
  \item[PR] Prevalence Ratio
  \item[QDA] Quadratic Discriminant Analysis
  \item[RDBMS] Relational Database Management System
  \item[RMSD] Root Mean Squared Deviation
  \item[RMSE] Root Mean Squared Error
  \item[RNN] Recurrent Neural Network
  \item[ROC] Receiver Operating Characteristic
  \item[RSF] Random Survival Forest
  \item[SGD] Stochastic Gradient Descent
  \item[SMBO] Sequential Model-Based Optimization
  \item[SQL] Structured Query Language
  \item[SVD] Singular Value Decomposition
  \item[SVM] Support Vector Machine
  \item[TF-IDF] Term Frequency-Inverse Document Frequency
  \item[TPE] Tree-Structured Parzen Estimator
  \item[VIF] Variance Inflation Factor
%  \item[]
\end{symbollist}
%  \item[GAN] Generational Adversarial Network