\documentclass[11pt]{article}
\usepackage{amsmath,amssymb}
\usepackage{lmodern}
\usepackage{cite}
\usepackage{listings}
\usepackage{graphicx}
\usepackage{minted}
\usepackage{url}
\usepackage{hyperref}

\title{sympy-nondim}
\author{Christoph Heindl \\ \url{https://github.com/cheind} }
\date{\today}

\begin{document}
\maketitle

\section{Introduction}
This Python package addresses physical dimensional analysis. In particular, \texttt{sympy-nondim} calculates from an unknown relation of (dimensional) variables, a new relation of (usually fewer) dimensionless variables.

\section{Example}
Suppose that you are asked to find an equation for the period of a simple frictionless pendulum (example taken from \cite{lemons2017student}). Unaware of the solution, you may assume that the period $t$ of the pendulum depends somehow on the (massless) string length $l$, the point mass $m$, the initial release angle $\theta$ and gravitational acceleration $g$ as shown in the following diagram.
\begin{center}
\includegraphics[width=0.3\textwidth]{pendulum.png}
\end{center}
Hence, you need to find an unknown relation $$t = f(l,m,g,\theta).$$ You plan to carry out experiments to study the value of $t$ for different values of all the independent variables in various combinations. Assuming $N$ values per variable, you will perform on the order of $N^4$, i.e $10,000$ experiments when $N=10$.

Using dimensional analysis we can a) reduce the number of experiments and b) gain insights into the unknown functional relationship of the variables. Dimensional analysis applies the principle dimensional homogeneity to manipulate a functional relationship of dimensional variables $$y = f(a,b,c,...)$$ into a new function $F$ of (usually fewer) nondimensional variables $$Y = F(A,B...).$$

\subsection{Problem setup}
In the pendulum case we first define the relevant symbols, their dimensions, and define the abstract equation we would like to analyze.
\begin{minted}{python}
import sympy
from sympy.physics import units

t, m, l, g, theta = sympy.symbols('t m l g theta')
dimmap = {
    t:units.time,
    m:units.mass,
    l:units.length,
    g:units.acceleration,
    theta:units.Dimension(1)
}

eq = sympy.Eq(t, sympy.Function('f')(m,l,g,theta))
print(sympy.latex(eq))
\end{minted}
$$t = f{\left(m,l,g,\theta \right)}.$$

\subsection{Result}
Next, we apply dimensional analysis
\begin{minted}{python}
import nondim

r = nondim.nondim(eq, dimmap)
print(sympy.latex(r))
\end{minted}
Which returns a new equation
\begin{equation}
    \sqrt{\frac{g}{l}}t = F{\left(\theta \right)}. \label{eq:A}
\end{equation} Note, all variable products appearing on the LHS and RHS are dimensionless. Solving for $t$ yields
\begin{minted}{python}
f = sympy.Eq(t, sympy.solve(r, t)[0])
print(sympy.latex(f))
\end{minted}
$$t = \sqrt{\frac{l}{g}}F{\left(\theta \right)}.$$
Dimensional analysis provides us with the following valuable insights
\begin{enumerate}
    \item The mass $m$ is irrelevant in the given problem.
    \item There is no need to consider an unknown function $f$ of four independent variables, instead we can reduce the search to unknown function $F$ of a single variable (initial release angle $\theta$). Few experiments according to Equation~\ref{eq:A} will quickly reveal that $F(\theta)\approx 2\pi$ for small angles.
    \item Keeping $F{\left(\theta \right)}$ constant, the period $t \propto  \sqrt{\frac{l}{g}}$.
\end{enumerate}

To learn more about dimensional analysis and how it might be helpful, consider \cite{szirtes2007applied, santiago2019first, sonin2001dimensional, lemons2017student,schetz1999fundamentals}. The method implemented in this library is based on the Buckingham-Pi theorem and the Rayleigh algorithm as explained in \cite{szirtes2007applied}. The method implemented here frames the problem in linear algebra terms, see \texttt{buckpi.py} for details.

\section{Mathematical Notes}
A dimensional equation of $N_v$ variables
\begin{equation}
    f(x_1,x_2,\ldots,x_{N_v-1}) = x_{N_v} \label{eq:fnc}
\end{equation}
is dimensionally homogeneous, if it can be written as
\begin{equation}
    g(\pi_1,\pi_2,\ldots,\pi_{N_p}) = 0.
\end{equation}
Here $\left\{\pi_1,\pi_2,\ldots,\pi_{N_p}\right\} = \Pi$ is a complete (and independent) set of $|\Pi|={N_p}$ dimensionless variable products.

Without loss of generality, consider 3 variables $x,y,z$. Let $\left[\cdot\right]$ denote the dimension, we require
\begin{align}
    \left[\pi_i\right] &= \left[x^{\alpha_i}y^{\beta_i}z^{\gamma_i} \right] \\
    &=\left[x^{\alpha_i}\right]\left[y^{\beta_i}\right]\left[z^{\gamma_i}\right] \\
    &=\left[x\right]^{\alpha_i}\left[y\right]^{\beta_i}\left[z\right]^{\gamma_i} = 1 \label{eq:pi}
\end{align}
for unknown rational scalars $\left\{\alpha_i,\beta_i,\gamma_i\right\}.$ In a dimensional system, each dimension is defined as product of $N_k$ base-dimensions. Assuming base-dimensions mass $M$, length $L$ and time $T$, we write
\begin{equation}
    \left[x\right] = M^mL^lT^t, \label{eq:xinmlt}
\end{equation}
where the values of the scalars $\left\{m,l,t\right\}$ depend variable dimension. For example, gravitational acceleration $g$ in the $MLT$-system is given by
\begin{equation}
    \left[g\right] = M^0L^1T^{-2}.
\end{equation}

\subsection{Physical dimensions as vector spaces}
Physical dimensions form a commutative (abelian) group $G$ under multiplication~\cite{wiki:Dimensional_analysis}: if $M,L \in G$, then $M^m,L^l \in G,$ assuming $m,l \in \mathbb{Q}$ and also $M^m \times L^l \in G$. The identity is $1 = M^0$.

The group $G$ can be described as a vector space over the field of rational numbers, mapping $[x] = M^0L^1T^{-2}$ to $(0,1,-2)^T$. Dimensional multiplication corresponds then to vector addition and scalar multiplication to raising dimensional symbols to a rational power. The basis is spanned by $N_k$ canonical unit directions corresponding to the physical base dimensions. For example, mass $M$ is mapped to $(1,0,0)^T$ in a three-dimensional vector space associated with the base dimensions $MLT$. The origin $\mathbf{0}$ of the vector space $(0,0,0)^T$ corresponds to a dimensionless quantity.

Using vector notation, we may rewrite Equation~\ref{eq:xinmlt}
\begin{equation}
    \alpha_i[x] + \beta_i[y] + \gamma_i[z] = \mathbf{0}, \label{eq:zeroeq}
\end{equation}
where we assume that $\left[\cdot\right]$ returns the vector space mapping of the variable's dimensions.

\subsection{Determining $\pi$}

We may rewrite Equation~\ref{eq:zeroeq} as a matrix-vector product
\begin{equation}
    \mathbf{D}\mathbf{v} =\begin{pmatrix}[x] & [y] & [z]\end{pmatrix}\begin{pmatrix} \alpha_i \\ \beta_i \\ \gamma_i\end{pmatrix} = \mathbf{0},
\end{equation}
where $\mathbf{D}$ is the $N_k \times N_v$ dimensional matrix. Hence, determining $\Pi$ becomes equivalent to the problem of finding a set of vectors $\mathbf{v}$ for which
\begin{equation}
    \mathbf{D}\mathbf{v}=\mathbf{0}.
\end{equation}
The set of vectors $\{\mathbf{v} \mid \mathbf{D}\mathbf{v}=\mathbf{0}\}$ span a sub-space of the domain of the linear map $\mathbf{D}$: the null-space. The dimensionality of the null-space is given by the rank-nullity theorem
\begin{equation}
    N_p = N_v - \text{rank}(\mathbf{M}) = \text{nullity}(\mathbf{M}) = |\Pi|,
\end{equation}
and determines the number of independent dimensionless products. The span of the null-space is not unique and this leads potentially different $\Pi$ sets. For practical purposes one should try to find a basis in which variables of interest will appear in only one of the $\pi$ terms. That's always possible as long of the variables of interest are 'free' variables.

\subsection{Specific solutions}
When the non-dimensionalization of Equation~\ref{eq:fnc} results in a single $\pi$ term
\begin{equation}
    g(\pi_1) = 0,
\end{equation}
then $\pi_1$ is a root of $g$. Assuming $g$ has only a single root (or discrete number of them), we see $\pi_1$ itself must be an (unknown) constant
\begin{equation}
    \pi_1 = c.
\end{equation}
When $g$ is a function of more than one dimensionless product
\begin{equation}
    g(\pi_1, \pi_2) = 0,
\end{equation}
we may invoke the Implicit Function Theorem to solve for one of the arguments and write instead
\begin{equation}
    \pi_1 = h_1(\pi_2).
\end{equation}
Similarly for three arguments $g(\pi_1, \pi_2, \pi_3) = 0$ we have
\begin{equation}
    \pi_1 = h_2(\pi_2, \pi_3).
\end{equation}

\subsection{References}
\bibliographystyle{alpha}
\begingroup
\renewcommand{\section}[2]{}%
\bibliography{biblio}
\endgroup

\end{document}


\documentclass{article}
\usepackage{amsmath}
\usepackage[margin=1.0in]{geometry}
\usepackage{xcolor}

\begin{document}

\noindent
Does $\displaystyle \sum_{n=1}^\infty \frac{1}{(n+1)^n}$
diverge, converge absolutely, or converge conditionally?

\subsection*{Solution 1}

\begin{align*}
L&=\lim_{n \to \infty} \sqrt[n]{|a_n|}\\
&= \lim_{n \to \infty} \sqrt[n]{\left| \frac{1}{(n+1)^n} \right|}\\
&= \lim_{n \to \infty} \frac{1}{n+1}\\
&= 0.
\end{align*}
Since $L < 1$, the series $\displaystyle \sum_{n=1}^\infty \frac{1}{(n+1)^n}$ converges absolutely by the Root Test.

\subsection*{Solution 2: much longer than Solution 1}

In an attempt to use the Ratio Test, you'd have to consider the limit
\begin{align*}
L&=\lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|\\
&= \lim_{n \to \infty} \left| \frac{1}{(n+2)^{n+1}}\frac{(n+1)^n}{1}\right|\\
&= \lim_{n \to \infty} \frac{(n+1)^n}{(n+2)^{n+1}},
\end{align*}
but it looks like there's not any really nice limit here, and while L'Hopital's rule applies, it will be quite a mess. Actually, it is possible to evaluate this limit using some algebra and some limit laws:
\begin{align*}
&= \lim_{n \to \infty} \frac{(n+1)^n}{(n+2)^{n+1}}\\
&= \lim_{n \to \infty} \frac{(n+1)^n}{(n+2)^n(n+2)}\\
&= \lim_{n \to \infty} \left[\left(\frac{n+1}{n+2}\right)^n \cdot \frac1{n+2}\right].
\end{align*}
We consider the limit
\[\lim_{n \to \infty} \left(\frac{n+1}{n+2}\right)^n\]
by defining $y = \left(\frac{n+1}{n+2}\right)^n$. So $\ln y = n \ln \frac{n+1}{n+2}$. So
\[ \ln y = \frac{\ln \frac{n+1}{n+2}}{\frac1n}\]
By applying a limit to both sides,
\[ \lim_{n \to \infty}\ln y = \lim_{n \to \infty} \frac{\ln \frac{n+1}{n+2}}{\frac1n}\]
and the limit on the right uses L'Hopital's rule. (Be sure to pay attention to the Chain Rule and the Quotient Rule.)
\begin{align*}
&=\lim_{n \to \infty} \frac{ \frac{n+2}{n+1} \cdot \frac{(n+2)\cdot 1 - (n+1) \cdot 1}{(n+2)^2}}{\frac{-1}{n^2}}\\
&=\lim_{n \to \infty} \frac{-n^2}{(n+1)(n+2)}\\
&=-1.
\end{align*}
after two more applications of L'Hopital's rule. Thus,
\[ \lim_{n \to \infty} y = e^{-1} = \frac1e.\]
We were originally considering the limit:
\begin{align*}
&\lim_{n \to \infty} \left[\left(\frac{n+1}{n+2}\right)^n \cdot \frac1{n+2}\right]
&= \left[ \lim_{n \to \infty} \left(\frac{n+1}{n+2}\right)^n \right] \cdot \left[\lim_{n \to \infty} \frac1{n+2}\right]
&= \frac1e \cdot 0
&= 0.
\end{align*}
and we could treat the limit of the product as a product of the limits because both limits existed. Since $L < 1$, the series  $\displaystyle \sum_{n=1}^\infty \frac{1}{(n+1)^n}$ converges absolutely by the Ratio Test.

\end{document}%%%%%%%%%%%%%%%%%

\begin{align*}
L&=\lim_{n \to \infty} \sqrt[n]{|a_n|}\\
&= \lim_{n \to \infty} \sqrt[n]{\left| \right|}\\
\end{align*}

\begin{align*}
L&=\lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|\\
&= \lim_{n \to \infty} \left| \right|\\
\end{align*}

\begin{align*}
\lim_{n \to \infty} a_n
&= \lim_{n \to \infty} \\
\end{align*}

Since $\sum |a_n| = \sum a_n$, the series $\displaystyle \sum_{n=1}^\infty AAAAAAAAAAAAAA$ converges absolutely.

Since $|r| < 1$, the series ...  converges by the Geometric Series Test.

Since $|r| \geq 1$, the series ...  diverges by the Geometric Series Test.

The function $f(x)=\frac{}{}$ is continuous, positive, and decreasing on $[1,\infty)$.

\subsection*{Solution}


\documentclass[a4paper,11pt]{article}
\title{A Small Collection Of Mathematical Formulas For The
Prospective Statistician}
\author{Joshua Simon  \\
	Otto-Friedrich-University Bamberg \\
	}

\date{\today}

% Packages.
\usepackage{amssymb}
\usepackage{amsmath}

\begin{document}

\maketitle

%\begin{abstract}
%Short introduction to subject of the paper \ldots
%\end{abstract}

\section{Arithmetic}
\paragraph{Logarithms.} In the following let $a$, $b > 0$ and $n \in \mathbb{R}$. Then
\begin{gather*}
    ln(a \cdot b) = ln(a) + ln(b) \\
    ln(\frac{a}{b}) = ln(a) - ln(b) \\
    ln(a^n) = n \cdot ln(a) \\
    e^{ln(a)} = ln(e^{a}) = a.
\end{gather*}

\paragraph{Binomial Coefficient.} Suppose $0 < k \leq n$. Then
\begin{gather*}
    \binom{n}{k} = \frac{n!}{k! \cdot (n-k)!}
\end{gather*}
where $n! = 1 \cdot 2 \cdot \ldots \cdot n$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Differential Calculus} \label{DifferentialCalculus}
\paragraph{Elementary Derivatives.} In the following let $f$ be a real-valued function of $x$ and $a, b, r \in \mathbb{R}$. Then the derivative of $f$ is given by $f'$ as

\begin{align*}
    f(x) = x^r &\rightarrow f'(x) = r \cdot x^{r - 1}
    &f(x) = e^x &\rightarrow f'(x) = e^x \\
    f(x) = r^x &\rightarrow f'(x) = r^x \cdot ln(r)
    &f(x) = e^{a \cdot x + b} &\rightarrow f'(x) = a \cdot e^{a \cdot x + b} \\
    f(x) = \frac{1}{x^r}  &\rightarrow f'(x) = - r \cdot x^{-r - 1}
    &f(x) = sin(x) &\rightarrow f'(x) = cos(x) \\
    f(x) = ln(x) &\rightarrow f'(x) = \frac{1}{x}
    &f(x) = cos(x) &\rightarrow f'(x) = - sin(x).
\end{align*}

\paragraph{Differentiation rules.} In the following let $h, u, v$ be real-valued functions of $x$.
\begin{itemize}
    \item Product Rule: $h(x) = u(x) \cdot v(x) \rightarrow h'(x) = u'(x) \cdot v(x) + u(x) \cdot v'(x)$
    \item Chain Rule: $h(x) = u(v(x)) \rightarrow  h'(x) = u'(v(x)) \cdot v'(x)$
    \item Reciprocal Rule: $h(x) = \frac{1}{v(x)} \rightarrow  h'(x) = \frac{v'(x)}{(v(x))^2}$
    \item Quotient Rule: $h(x) = \frac{u(x)}{v(x)} \rightarrow  h'(x) = \frac{v(x)u'(x) - u(x)v'(x)}{(v(x))^2}$
\end{itemize}

\paragraph{Gradient.} Let $f$ be a differentiable fuction of $n$ variables. Then
\begin{gather*}
    \mathbf{grad}(f) = \nabla f = \left( \frac{\partial}{\partial x_1} f(x) \dots \frac{\partial}{\partial x_n} f(x) \right)^T.
\end{gather*}

\paragraph{Jacobian Matrix.} Let $f: \mathbb{R}^{n} \rightarrow \mathbb{R}^{m}$ be a differentiable function of $n$ variables. Then the Jacobian matrix $\mathbf {J}$ of $f$ is an $m \times n$ matrix whose $(i,j)$th entry is ${\mathbf {J}_{ij}={\frac {\partial f_{i}}{\partial x_{j}}}}$. Which is
\begin{gather*}
    {\mathbf {J} ={\begin{bmatrix}{\dfrac {\partial f }{\partial x_{1}}}&\cdots &{\dfrac {\partial f }{\partial x_{n}}}\end{bmatrix}}={\begin{bmatrix}\nabla ^{T}f_{1}\\\vdots \\\nabla ^{T}f_{m}\end{bmatrix}}={\begin{bmatrix}{\dfrac {\partial f_{1}}{\partial x_{1}}}&\cdots &{\dfrac {\partial f_{1}}{\partial x_{n}}}\\\vdots &\ddots &\vdots \\{\dfrac {\partial f_{m}}{\partial x_{1}}}&\cdots &{\dfrac {\partial f_{m}}{\partial x_{n}}}\end{bmatrix}}}
\end{gather*}
where ${\nabla ^{T}f_{i}}$ is the transpose (row vector) of the gradient of the $i$th component.

\paragraph{Symmetry Of Second Derivatives (Schwarz's theorem).} The order of taking partial derivatives of a function $f(x) = f(x_1, x_2, \dots , x_n)$ of $n$ variables is interchangeable
\begin{gather*}
    \frac{\partial}{\partial x_i} \left( \frac{\partial}{\partial x_j} f(x) \right) = \frac{\partial}{\partial x_j} \left( \frac{\partial}{\partial x_i} f(x) \right).
\end{gather*}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Integral Calculus} \label{integralCalculus}

\paragraph{Fundamental Theorem Of Calculus.}
Let  $f$ be a real-valued function on a closed interval $[a,b]$ and $F$ an antiderivative of $f$ in $[a,b]$: $F'(x)=f(x)$. If $f$ is Riemann integrable on $[a,b]$ then
\begin{gather*}
    \int _{a}^{b}f(x)\,dx=F(b)-F(a).
\end{gather*}

\paragraph{Elementary Integrals.}
\begin{align*}
    \int x^r dx &= \frac{x^{r + 1}}{r+1} + C \text{ with } (r \neq -1)
    &\int e^x dx &= e^x + C \\
    \int r^x dx &= \frac{r^{x}}{ln(r)} + C \text{ with } (r > 1, r \neq 1)
    &\int e^{a \cdot x + b} dx &= \frac{e^{a \cdot x + b}}{a} + C \\
    \int \frac{1}{x} dx &= ln(x) + C
    &\int ln(x) dx &= x \cdot (ln(x) - 1) + C \\
    \int \frac{1}{x^2} dx &= - \frac{1}{x} + C
    &\int sin(x) dx &= - cos(x) + C \\
    \int \frac{1}{x^3} dx &= - \frac{1}{2x^2} + C
    &\int cos(x) dx &= sin(x) + C
\end{align*}

\paragraph{Integration By Parts.} Let $u$ and $v$ be two continuously differentiable functions of $x$ in $[a,b]$. Then
\begin{gather*}
   \int_{a}^{b} u(x)v'(x) dx = \left[ u(x)v(x) \right]_a^b - \int_{a}^{b} u'(x)v(x) dx.
\end{gather*}

\paragraph{Integration By Substitution.} Let $I$ be an real interval, $f: I \rightarrow \mathbb{R}$ a continuous function and $\varphi: [a,b] \rightarrow I$ continuously differentiable. Then
\begin{gather*}
    \int _{a}^{b}f(\varphi (x))\varphi '(x)\,dx=\int _{\varphi (a)}^{\varphi (b)}f(u)\,du.
\end{gather*}
The substitution $u = \varphi(x)$ yields ${\frac {du}{dx}}=\varphi '(x)$ which leads to $du=\varphi '(x) \,dx$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{thebibliography}{9}
    \bibitem[Bron]{Bron} \emph{Taschenbuch der Mathematik.}
    I. N. Bronstein, K. A. Semendjajew, G. Musiol, H. Mühlig, 11. Auflage, 2020.
\end{thebibliography}

\end{document}


\section{Background}

\subsection{Hidden Markov Model}
A hidden Markov model assumes that the system follows a Markov process. For this
paper, we will assume a finite number of states and observations. The
probability of transitioning from state $i$ to state $j$ only depends on the
current state, and we define transition matrix $A$ such that element $a_{ij}$ is
the probability  of transitioning from $i$ to $j$. In an HMM, we cannot observe
states, but we can observe outputs generated by the states. Given $N$ states and
$M$ observations, we capture this with a $N \times M$ emission matrix $B$, where
each element $b_{ij}$ is the probability of observation $j$ given the state is $i$. Finally, we define a $N$ length vector as $\pi$ as the prior, where $\pi_i$ is the probability of initial state being $i$.

We can thus capture all the information about an HMM with $\theta = (A, B, \pi)$. In an HMM, we are given a sequence of $T$ observations, and we will use a series of forward and backwards propagations to estimate the parameters $\theta = (A, B, \pi)$.

\subsection{Baum-Welch}

The following explanation is based on Wikipedia's presentation of the
algorithm~\cite{bwwiki}.

Let $(X_1, X_2, \ldots, X_N)$ denote the $N$ states and $\bold{O} = \{O_1, O_2, \ldots, O_T\}$ the $T$ observed outputs. The Baum-Welch algorithm uses expectation-maximization to find the maximum likelihood estimate.

Suppose we begin with some initial parameters, $\theta  =  (A, B, \pi)$.  Given the observation sequence, our task is to update these parameters based on insights from the training sequence.

Given our training sequence, Baum-Welch first figures out the probability of being in a state at some time in the ``run" of the observation sequence.  Let's call this $\gamma_{i}(t)$, the ``probability of being in state $i$ at time $t$ of this observation sequence given our current model parameters," $P(X_t = i | \bold{O}, \theta)$.

How can we figure this out? To do this, the Baum-Welch algorithm performs two sub-calculations, for the ``forward probabilities" and the ``backward probabilities."  Like $\gamma_i(t)$ values, the forward probability are computed for each ($\textit{state}, \textit{time}$) pair: given some observation sequence and our current moel, the forward probability $\alpha_i(t)$ is the probability of seeing the observation sequence up to time $t$ and landing in state $i$.  The backward probability is defined analogously: it's the probability of starting in some state $i$ and seeing the observation sequence \textit{from} time $t$ until the end.

We can define the forward and backward probabilities recursively:

\begin{equation} \label{alpha}
\alpha_i(t+1) =  b_{jo_1}\sum_{i=1}^{N} \alpha_i(t)a_{ij}
\end{equation}

\begin{equation} \label{beta}
\beta_i(t) = \sum_{i=1}^N \beta_j(t+1)a_{ij}b_j(o_{t+1})
\end{equation}

We can put the alpha and beta probabilities together to find $\gamma_i(t)$.  Some intuition for this formula: the bottom is the probability of being in \textit{some state} at time $t$ given our \textit{entire} observation sequence.  The top is the probability of being in some particular state $i$ at time $t$ given our \textit{entire} observation sequence.  For both numerator and denominator, you have to get to the state at that time (forward probability) and leave from the state next and finish emitting the observation sequence  (backward probability).

\begin{equation} \label{gamma}
\gamma_i(t) = P(X_t = i | \bold{O}, \theta) = \frac{\alpha_i(t) \beta_i(t)}{\sum_{j=1}^N \alpha_j(t) \beta_j(t)}
\end{equation}

From here, we can figure out our updates for the $B$ matrix and the new prior.

$B$ can be updated by:
\begin{equation}\label{b}
b^*_{i,v_k} = \frac{\sum_{t=1}^{T}1_{o_t = v_k} \gamma_i(t)}{\sum_{t=1}^T \gamma_i(t)}
\end{equation}
where $1_{o_t = v_k}$ is $1$ if $o_t = v_k$ and $0$ otherwise.

Intuition: the update for  $b^*_{i,v_k}$ is just the frequency across the entire observation sequence of being in the state $i$ and seeing $v_k$.

The new prior can be determined by

\begin{equation}\label{pi}
\pi_i^* = \gamma_i(0)
\end{equation}

Intuition: this is the probability of being in state $i$ before the observation sequence begins.

To update the $A$ matrix (each $a_{ij}$ entry), we need to figure  out the
probability of being in state $i$ and $\textit{moving}$ to state $j$ next.
Fortunately, this calculation can be done in a straightforward way, and we can
think of it as a ``relative frequency calculation" of the transitions that begin
at state $i$ and go to state $j$ compared to the transitions through state $i$.
This ``relative frequency intuition" is provided by Rabiner and Juang~\cite{rabiner1986introduction}.

Define $\xi_{ij}(t)$ to be the probability of being in state $i$ at time $t$ and state $j$ at time $t+1$ given $\bold{O}$ and parameter $\theta$.
\begin{equation}\label{xi}
\begin{aligned}
\xi_{ij}(t) &= P(X_t = i, X_{t+1} = j | \bold{O}, \theta)  \\
&=\frac{\alpha_i(t) a_{ij} \beta_j(t+1)b_{j}(o_{t+1})}{\sum_{k=1}^N \alpha_k(t) \beta_k(t)}
\end{aligned}
\end{equation}
Finally, $A$ can be updated by:

\begin{equation}\label{a}
a^*_{ij} = \frac{\sum_{t=1}^{T-1}\xi_{ij}(t)}{\sum_{t=1}^{T-1}\gamma_i(t)}
\end{equation}

These steps are repeated until the parameters converge.

\subsection{Prior Work}

Here we review prior work related to \textit{parallelization} of hidden Markov
models specifically.  In 2009, Chuan Liu conducted an independent project aiming
to implement HMM training in parallel on CUDA.  The project's parallelization
forward-backward probabilities computation inspired our design.  The rest of his
work frames computations in terms of matrix multiplication operations.  His work
also utilizes ``parallel reduction" for computing the sum of a vector, something
we've seen in CS205~\cite{cuda-hmm}.

Also, we noted the approach of Li et. al in 2010, applying an earlier
parallelization approach---``cut and stitch''---to the problem of hidden Markov
model training.  The ``cut and stitch" approach parallelizes along the length of
the input observation sequence (along the $T$-axis).  The graph parallel scheme
in this paper is orthogonal to the ``cut and stitch" approach, and we are
excited about the prospect of applying these two parallelization schemes to the
problem of HMM training~\cite{cut-stitch}.


\subsection{Delta rule}

\subsubsection{Introduction}

We want to train the parameters \(\theta \).

We can do this with gradient descent, by working out how much the loss function falls as we change each parameter.

The delta rule tells us how to do this.

\subsubsection{The loss function}

The error of the network is:

\(E=\sum_j\dfrac{1}{2}(y_j-a_j)^2\)

We know that \(a_j=a(\theta x_j)\) and so:

\(E=\sum_j\dfrac{1}{2}(y_j-a(\theta x_j))^2\)

\subsubsection{Minimising loss}

We can see the change in error as we change the parameter:

\(\dfrac{\delta E}{\delta \theta_i }=\sum_j \dfrac{\delta E}{\delta a_j}\dfrac{\delta a_j}{\delta z_j}\dfrac{\delta z_j}{\delta \theta_i}\)

\(\dfrac{\delta E}{\delta \theta_i }=-\sum_j(y_j-a_j)a'(z_j)x_{ij}\)

\subsubsection{Delta}

We define delta as:

\(\delta_i=-\dfrac{\delta E}{\delta z_j}=\sum_j(y_j-a_j)a'(z_j)\)

So:

\(\dfrac{\delta E}{\delta \theta_i }=\delta_i x_{ij}\)

\subsubsection{The delta rule}

We update the parameters using gradient descent:

\(\Delta \theta_i=\alpha \delta_i x_{ij}\)


% !TEX root = main.tex

\chapter[Complex Numbers and Functions]{Preliminaries on Complex Numbers and Functions}

\section{Basic Definitions and Algebraic Operations}

Formally, a \emph{complex number} is an ordered pair $(x,y)$ of real numbers.  We denote the set of all complex numbers by $\mathbb{C}$, and define addition and multiplication on $\C$ as follows:
\begin{enumerate}
\item[(i)] $(x_1,y_1)+(x_2,y_2) = (x_1+x_2,y_1+y_2)$,
\item[(ii)] $(x_1,y_1)(x_2,y_2) = (x_1x_2-y_1y_2,x_1y_2+x_2y_1)$.
\end{enumerate}
With this definition, $\C$ and $\R^2$ are equal \emph{as sets}, however, we have also defined an operation of multiplication on $\C$.

The subset of $\C$ given by
\[
\set{(x,0): x \in \R }
\]
is called the \emph{real axis}.  For complex numbers in this subset we have
\begin{align*}
(x_1,0)+(x_2,0) & = (x_1+x_2,0) \\
(x_1,0)(x_2,0) &= (x_1x_2,0),
\end{align*}
so these complex numbers behave exactly like real numbers.  For this reason, we shall use $\R$ to refer to the real axis and denote the complex number $(x,0)$ by $x$.

Writing $i=(0,1)$, the definition of multiplication in $\C$ gives $i^2=-1$. With this notation, we get a more familiar `definition': a complex number $z=(x,y)$ can be written as
\begin{align*}
z = (x,y) &= (x,0)+(y,0)(0,1) \\
& = x + iy.
\end{align*}

The real numbers $x$ and $y$ are the \emph{real} and \emph{imaginary} parts of $z$ respectively, and we often write
\[
\Re (z) = x,\quad \Im (z)=y.
\]
Note that the imaginary part of $z$ is \emph{real}.

With this more familiar notation, the algebraic operations of addition and multiplication on $\mathbb{C}$ can be expressed as follows: for $z_1=x_1+iy_1$ and $z_2=x_2+iy_2 \in \mathbb{C}$, we have
\begin{align*}
z_1 +z_2  &= (x_1+x_2) + i (y_1+y_2) \\
z_1z_2 & = (x_1+iy_1)(x_2+iy_2) \\
& = x_1x_2 + i (x_1y_2) + i (y_1x_2) + (i)^2(y_1y_2) \\
& = (x_1x_2-y_1y_2) + i (x_1y_2+x_2y_1).
\end{align*}

Note in particular that for $r \in \R$ and $z = x+iy \in \C$ we have
\[
rz = r(x+iy) = rx+iry.
\]

It is convenient to identify the complex number $z \in \C$ with the point (or sometimes, the vector) $(\Re (z), \Im (z) ) \in \R^2$.  Obviously the map $z \mapsto ( \Re (z), \Im (z))$ is a bijection $\C \to \R^2$, and thus geometrically, we view $\C$ as $\R^2$.

\begin{definition}[Complex Conjugate]
Given $ z \in \mathbb{C}$, the \emph{complex conjugate} $\overline{z}$ of $z$ is defined as
\[
\overline{z} := \Re (z) - i\ \Im (z).
\]
\end{definition}

\begin{definition}[Modulus]
Given $z \in \mathbb{C}$, we define the modulus of $z$ to be
\[
| z | : = \sqrt{ \Re (z)^2 + \Im (z)^2 }.
\]
\end{definition}
\
In other words, for $z=x+iy$, $\conj{z}=x-iy$ and $\abs{z} = \sqrt{x^2+y^2}$.

Geometrically, the complex conjugate of $z$ describes the reflection of $z$ through the real axis, and the modulus of $z$ is the distance of $z$ from the origin.

\begin{proposition}
\label{p:absmod}
Let $z,z_1,z_2 \in \C$ be given.
Then:
\begin{enumerate}
%\item $i^2 = -1$.
\item[(i)] $\conj{\conj{z}} = z$ and $\abs{\conj{z}} = \abs{z}$.
\item[(ii)] $\abs{z} = \sqrt{\conj{z}z}$
\item[(iii)] $\conj{z_1+z_2} = \conj{z_1} + \conj{z_2}$ and $\conj{z_1z_2} = \conj{z_1}\ \conj{z_2}$.
\item[(iv)] $\conj{z} = z$ if and only if $z$ is real, i.e., if and only if $\Im (z)=0$.
\item[(v)] $\Re (z) = \displaystyle\frac{z + \conj{z}}{2}$ and $\Im (z) = \displaystyle\frac{z - \conj{z}}{2i}$.
\item[(vi)] $\abs{z} = 0$ if and only if $z = 0$.
\item[(vii)] $\abs{z_1 \pm z_2} \leq \abs{z_1} + \abs{z_2}$ \emph{(Triangle Inequality)}.
\item[(viii)] $\abs{z_1 \pm z_2} \geq \abs{\abs{z_1} - \abs{z_2}}$ \emph{(Reverse Triangle Inequality)}.
\item[(ix)] $\abs{z_1z_2} = \abs{z_1} \abs{z_2}$.
\end{enumerate}
\end{proposition}
You really should be familiar with all of these properties.  I would suggest trying to prove at least some of them, or verifying them for some specific choices of $z_1$ and $z_2$. Most of the proofs are short.

\begin{question}
The rules of multiplication allow us to divide a complex number $z$ by a nonzero real number $r$; i.e.
\[
\frac{z}{r} = \left( \frac{1}{r} \right) z.
\]
How do we define division of complex numbers?
\end{question}
\begin{answer}
For a complex number $z \neq 0$, we first define the multiplicative inverse of $z$, denoted $z^{-1}$ or $\frac{1}{z}$.  Note that  by Proposition~\ref{p:absmod} (ii), we have $\abs{z}^2 = z \conj{z}$.  Since $\abs{z}$ is real, we may  define the complex number
\[
\frac{\conj{z}}{\abs{z}^2}
\]
which must satisfy
\[
z \brac{ \frac{\conj{z}}{\abs{z}^2} } = \frac{z \conj{z}}{z \conj{z}} = \frac{\abs{z}^2}{\abs{z}^2} = 1.
\]
It follows that the multiplicative inverse of $z$ is
\[
z^{-1} = \frac{\conj{z}}{\abs{z}^2}.
\]
With this in mind, we may divide any complex number $z_1$ by any nonzero complex number $z_2$ as follows:
\[
\frac{z_1}{z_2} = z_1 z_2^{-1} = \frac{z_1\conj{z_2}}{\abs{z_2}^2}.
\]
\end{answer}

\begin{definition}[Argument]
The \emph{argument} of a complex number $z \neq 0$ is the angle $\arg (z)$ from the positive real axis to the vector representing $z$ in the anticlockwise direction.
\end{definition}
Of course negative values of $arg(z)$ are allowed: these values represent angles taken in the clockwise direction.  Note that there are typically many ways in which we can represent $\arg (z)$, since we identify any two angles that differ by an integer multiple of $2\pi$ with one another.  For instance, for (the complex number) $z=1$ any of the angles
\[
\ldots,-4\pi, -2\pi, 0, 2 \pi , 4\pi,\ldots
\]
are valid choices for $\arg (z)$.   Similarly, if we look at $z=-1-i$:
%\begin{center}
%\includegraphics[scale=1]{arg}
%\end{center}
We can write $\arg (z) = \frac{5\pi}{4}$ or $\arg (z)=- \frac{3\pi}{4}$ (or indeed an integer multiple of $2\pi$ added to either of these angles).

By convention we usually take $\arg (z) \in (-\pi , \pi]$.
\begin{definition}
For $z \in \C$, $z \neq 0$, we define the \emph{Principal argument of $z$} (or the \emph{Principal value of $\arg(z)$}) to be the value of $\arg (z)$ that lies in $(-\pi,\pi]$. We shall denote this value by $\Arg (z)$.
\end{definition}
So for $z=-1-i$ we have $\Arg (z) = - \frac{3 \pi}{4}$, while $\arg (z)$ can be taken to be any of the values
\[
\ldots, - \frac{11\pi}{4}, - \frac{3\pi}{4}, \frac{5\pi}{4}, \frac{13\pi}{4}, \ldots.
\]

\begin{definition}
The \emph{polar form} of a complex number $z$ is given by writing $z$ in the form
\[
z = r \left( \cos (\theta) + i \sin (\theta) \right),
\]
where $r , \theta \in \R$ and $r \geq 0$.
\end{definition}
The polar form of $z$ is found by setting $r = \abs{z}$ and $\theta = \arg (z)$ (any value of $\arg(z)$ will of course suffice).  Equivalently, we may write
\[
z = r e^{i \theta}
\]
(though we have yet to define the complex exponential function).  For completeness, we should specify that the polar form of $0$ is simply $0$, since $\arg (0)$ is not defined.

\begin{definition}
Let $w=r \left( \cos( \theta) + i \sin ( \theta) \right)$ be (the polar form of) a complex number.  Then the $n^{th}$ \emph{complex roots} of $w$ are defined to be the $n$ solutions $z_0, z_1, \ldots , z_{n-1}$ of the equation $z^n = w$.  These roots are given by
\[
z_k = \sqrt[n]{r} \left( \cos \left( \frac{\theta+2k \pi}{n} \right) + i \sin \left( \frac{\theta + 2k\pi}{n} \right) \right),
\]
for $k=0,1,\ldots,n-1$, where $\sqrt[n]{r}$ is the (positive) $n^{th}$ real root of $r$.
\end{definition}

\begin{figure}[H]
\centering
\begin{tabular}{cc}
\includegraphics{ch1_sqrt1} & \includegraphics[scale=1]{ch1_sqrt2}
\end{tabular}
\caption{The 5\textsuperscript{th} roots of $z=-1+i$ (left), and the two square roots of $-i$ (right).}.
\end{figure}

\begin{theorem}
Let $z_1$ and $z_2$ be nonzero complex numbers.  Then
\begin{enumerate}
\item[(i)] $\abs{z_1z_2}=\abs{z_1} \ \abs{z_2}$, and
\item[(i)] $\arg (z_1z_2)=\arg(z_1)+\arg(z_2)$.
\end{enumerate}
\end{theorem}
{\bf Proof:} Exercise.

\section{Complex Functions}

We now start our investigation of functions from the complex plane to itself.  First of all, note that a function $\mathbf{f}:\R^2 \to \R^2$ can be written in the form
\[
\mathbf{f}(x,y)=\left( u(x,y), v (x,y) \right)
\]
where $u,v:\R^2 \to \R$.  For example, if
\[
\mathbf{f}(x,y)=\left(x^2+y^2-2x,2y-6 \right),
\]
we have
\[
u(x,y)=x^2+y^2-2x \text{ and } v(x,y)= 2y-6.
\]

Since we have a formal definition of $\C$ as $\R^2$, we can express a function $f : \C \to \C$ as
\begin{equation*}
f(z) = f(x+iy) = u(x,y) +i v(x,y)
\end{equation*}
for all $z =x+iy \in \C$, where $u, v : \R^2 \to \R$.  In this way, we can extend our definition of real and imaginary parts from complex numbers to functions $f:\C \to \C$, writing $\Re (f) = u$ and $\Im (f) = v$.

Going in the other direction, given two functions $u:\R^2 \to \R$ and $v:\R^2 \to \R$, we can create a function $f : \C \to \C$ by \emph{defining} $f(x+iy) := u(x,y) +iv(x,y)$.
So,
\begin{center}
\emph{
 There is a correspondence between functions from the complex plane to itself, and pairs of functions from the real plane to the real line.}
 \end{center}

\begin{example}
The function \[ f: \C \to \C,\quad f(z) = \conj{z} \]
corresponds to a function
\[ \mathbf{f}: \R^2 \to \R^2, \quad \mathbf{f}(x,y) = \left( u(x,y), v(x,y) \right)
\]
\begin{blankbox}
\[
u(x,y) = x\ \qquad\text{and}\qquad v(x,y) = -y
\]
since
\[
\conj{z} = \conj{x+iy} = x-iy = \underbrace{x}_{u(x,y)} + i \underbrace{(-y)}_{v(x,y)}.
\]
\end{blankbox}
\end{example}

\begin{example}

Let us determine the functions $u:\R^2 \to \R$ and $v:\R^2 \to \R$ corresponding to
\begin{equation*}
h: \C \to \C, z \mapsto z^2.
\end{equation*}
\end{example}
\begin{solution}
With $z=x+iy$ we have
\[
z^2 = (x+iy)^2 = x^2-y^2+i2xy
\]
Thus we identify $h:\C \to \C$ with $\mathbf{h}:\R^2 \to \R^2$, $\mathbf{h}(x,y) = (u(x,y), v(x,y))$  where
\begin{gather*}
u: \R^2 \to \R, \quad u(x,y) = x^2-y^2, \\
v: \R^2 \to \R, \quad v(x,y) = 2xy.
\end{gather*}

\end{solution}

\begin{comment}
\begin{example} Let us find the corresponding complex function, $f:{\mathbb C} \rightarrow {\mathbb C}$, for the function
\begin{equation*}
\mathbf{f} : \R^2 \to \R^2, (x,y) \mapsto (2y,-x)
\end{equation*}
and express it in terms of $z \in \C$.
\end{example}

%\begin{master}
\begin{solution}
The corresponding function $f:\C \to \C$ is given by
\begin{align*}
f(z)= f(x+iy)&=2y+i(-x) \\
& = 2 \Im (z) +i ( - \Re (z) ) \\
& = 2 \left( \frac{z-\overline{z}}{2i} \right)-i \left( \frac{z+\overline{z}}{2} \right) \\
&=\frac{-3i z}{2}+\frac{i\overline{z}}{2}.
\end{align*}

Thus the corresponding complex function is
\begin{equation*}
f: \C \to \C, \ f(z)= \frac{-3i z}{2}+\frac{i\overline{z}}{2}.
\end{equation*}
(Here we have used the identities $\Re (z) = \frac{1}{2} (z+\conj{z})$ and $\Im (z) = \frac{1}{2i}(z-\conj{z})$, together with the fact that $i^{-1}=-i$)
\end{solution}
%\end{master}
%\vspace*{5cm}
\end{comment}

\begin{example}
Let find the complex function $f$ corresponding to the function $\mathbf{f}:\R^2 \to \R^2$ where
\begin{enumerate}
\item[(i)] $\mathbf{f} (x,y) = (-2y+3,2x),$ and
\item[(ii)] $\mathbf{f}(x,y) = (x^2+y^2-2x,2y-6)$.
\end{enumerate}

\end{example}

\begin{solution}
We could use the fact that if $z=x+iy$ then
\begin{align*}
x & = \Re(z) = \frac{1}{2} \left( z+ \conj{z} \right) \\
y & = \Im (z) = \frac{1}{2i} \left( z- \conj{z} \right).
\end{align*}
Substituting these expressions for $x$ and $y$, we could then simplify and find $f(z)$. However, this is time consuming, and it is sometimes easier to look for familiar expressions in the definition of $f$.

In part (i), $\mathbf{f}$ corresponds to $f: \C \to \C$ defined by
\begin{align*}
f(z) = f(x+iy) &= -2y+3+i(2x) \\
& = 2(ix-y) + 3 \\
& = 2(ix+i^2y)+3 \\
& = 2i(x+iy)+3 \\
& = 2iz+3.
\end{align*}

In part (ii), $\mathbf{f}$ corresponds to $f:\C \to \C$ where
\[
f(x+iy) = x^2+y^2-2x+i2y-6i.
\]
With $z=x+iy$, notice that
\[
x^2+y^2=\abs{z}^2 = z \conj{z},
\]
and that
\[
-2x+i2y = -2 (x-iy) = -2 \conj{z},
\]
hence
\[
x^2+y^2-2x+i2y-6i = z \conj{z} - 2 \conj{z} -6i.
\]
Hence $f(z) = z \conj{z}-2\conj{z}-6i$ is the corresponding complex function.

\end{solution}

It is worth recalling that some familiar functions defined on $\R$ extend to $\C$.

\begin{definition}
\label{d:exp}
The \emph{exponential function} is the function $\exp : \C \to \C$ defined by
\[
\exp (x+iy) = \exp(x) \left( \cos (y) + i \sin (y) \right)
\]
for all complex numbers $x+iy \in \C$, where $\exp (x) (=e^x)$ is the usual (real) exponential of $x$.
\end{definition}
We shall often write $e^z$ in place of $\exp (z)$.   The trigonometric functions also extend to $\C$ via
\begin{align}
\cos (z) &= \frac{1}{2} \left( \exp (iz) + \exp (-iz) \right) \\
\sin (z) & = \frac{1}{2i} \left( \exp(iz)- \exp (-iz) \right).
\end{align}

\begin{full}
\begin{remark}
We conclude this section with some remarks about how you might go about `visualising' a complex function.  For functions $f:\R \to \R$, we usually do so by drawing the graph of $f$, that is, the subset $\set{(x,f(x)): x \in \R}$ of $\R^2$.

We cannot draw the `graph' of a function $f:\C \to \C$, as to do so would require four coordinates
\[
\set{ \Re (z), \Im (z) , \Re \left( f(z) \right), \Im \left( f(z) \right) }
\]
and thus four dimensions.  We can however view $f$ as a `transformation' of the complex plane, and examine the effect of applying $f$ to
\begin{itemize}
\item Regions (subsets) of $\C$
\item Curves in $\C$ (lines, circles etc)
\item A combination of the two.
\end{itemize}

\begin{figure}[H]
\centering
\includegraphics[scale=0.25]{ch1_z2quad}
\caption{The image of the first quadrant under $z \mapsto z^2$.}
\label{f:z2}
\end{figure}
Figure~\ref{f:z2} indicates that the image of the first quadrant under the map $z \mapsto z^2$ is the region consisting of the first and second quadrants.  This follows from the fact that for any $z_1,z_2 \in \C$,
\[
\arg (z_1z_2)=\arg(z_1)+\arg(z_2).
\]
Hence $\arg(z^2)=2\arg(z)$, and so if $0 \leq \arg (z) \leq \pi/2$ (i.e. $z$ is in the first quadrant), we have $0 \leq \arg (z^2) \leq \pi$ ($z^2$ lies in either the first or second quadrant).  Of course, Figure~\ref{f:z2} does not tell us anything about the modulus of $z^2$.

Figure~\ref{f:z2c} illustrates how $z \mapsto z^2$ also squares the modulus; the circle of radius $r>0$ and centre $0$ is sent to the circle of radius $r^2$ and centre $0$.  Moreover, the anticlockwise arrows on the circles indicate that if $z_2$ lies (a small distance) anticlockwise of $z_1$, then $z_2^2$ lies anticlockwise of $z_1^2$.

%\vspace*{2cm}

\begin{figure}[H]
\centering
\includegraphics[scale=0.3]{ch1_z2circle}
\caption{The effect of the map $z \mapsto z^2$ on a circle of radius $r$ centred at the origin.}
\label{f:z2c}
\end{figure}

%\vspace{2cm}

\begin{figure}[H]
\centering
\includegraphics[scale=0.15]{ch1_expimage}
\caption{The geometric effect of the exponential map.}
\label{f:exp}
\end{figure}

Figure~\ref{f:exp} combines the approaches of the previous examples.  Here we see that the shaded region,
\[
\set{z \in \C: \Re (z) \leq x \text{ and } y_0 \leq \Im (z) < y_0+2\pi}
\]
is sent to the shaded disc of radius $e^x$ and centre $0$.  The vertical line $\Re(z)=x$ is sent to the circle of radius $e^x$ and centre $0$, while the dashed horizontal line is sent to the `infinite ray' from $0$ indicated with another dashed line.

Strictly speaking this diagram is inaccurate, as some of the region to the right of the line $\Re(z)=x$ is also shaded.

I will not ask you to produce diagrams like this in the exam, but it may be helpful to have the idea of complex functions as transformations in your mind throughout this module.
%\vspace{2cm}

%\vspace*{2cm}
Figures~\ref{f:z2},~\ref{f:z2c} and~\ref{f:exp} are from \emph{Basic Complex Analysis}, (Jerrold. E. Marsden, published by W.H. Freeman and Company, 1973).

\end{remark}
\end{full}

\section{Open Sets in $\C$}

\begin{definition}[Open Disc]
Let $z_0 \in \C$ and let $r>0$ be some real number.  The open disc of radius $r$ centred at $z_0$ is defined to be the set
\[
D(z_0,r):= \set{ z \in \C: \abs{z-z_0} <r}.
\]
\end{definition}
In other words, $D(z_0,r)$ is the set of all points that lie (strictly) inside the circle or radius $r$ with centre $z_0$.
\begin{definition}
The set of points
\[
D'(z_0,r) = \set{ z \in \C: 0 < \abs{z-z_0} < r},
\]
where $r>0$ and $z_0 \in \C$, is called the \emph{punctured open disc} of radius $r$, centre $z_0$.
\end{definition}

Note that $D'(z_0,r)$ is the set obtained by removing the centre $z_0$ from the open disc $D(z_0,r)$.
\begin{sfig}
\includegraphics[scale=1]{ch1_discs}
\caption{The open disc $D(z_0,r)$ and punctured open disc $D'(z_0,r)$.}
\label{f:discs}
\end{sfig}

\begin{notation}
This is a good place to introduce some conventions for sketching sets.  The broken circle in Figure~\ref{f:discs} around the disc indicates that the boundary is not included in this set.  The filled point $\bullet$ beside $z_0$ indicates that the point $z_0$ is included in $D(z_0,r)$, while the `hollow' point $\circ$ beside $z_0$ indicates that $z_0$ is not included in the punctured disc $D'(z_0,r)$.
\end{notation}

\begin{definition}
Let $U \subseteq \C$, then we say that $U$ is an \emph{open set} if given any $z \in U$ there is some $r_z>0$ with $D(z,r_{z}) \subseteq U$.
\end{definition}
\begin{sfig}
\includegraphics[scale=1]{ch1_openset}
\caption{An open set $U$ with two examples of open discs around points of $U$.  Note that the radius typically depends on the point $z$; points nearer the `edge' of $U$ will need smaller discs.}
\end{sfig}
Informally, we think of an open set as a set that does not include its `boundary.'  Thus a set $U$ is open if, given any $z \in U$, we can move a small distance in \emph{any} direction without leaving $U$.

\begin{example}
\begin{itemize}
\item[(i)] The open disc $D(0,1)$ is an open set.

\altgraphics[scale=1]{ch1_unit_disc_full}{ch1_unit_disc}

\begin{full}
Let $z \in D(0,1)$.  Our goal is to find some $r>0$ so that $D(z,r) \subseteq D(0,1)$. To this end, set $r= (1-\abs{z})/2$. Then if $w \in D(z,r)$,
\begin{align*}
\abs{w-0} & \leq \abs{w-z} + \abs{z} \\
& < \frac{1-\abs{z}}{2} +\abs{z} \\
&= \frac{1+\abs{z}}{2} < \frac{1+1}{2} = 1,
\end{align*}
which shows that $w \in D(0,1)$.

%\vspace*{2cm}
Hence $D(z,r) \subseteq D(0,1)$ and so $D(0,1)$ is open.
\end{full}

\item[(ii)] The closed disc $\overline{D} (0,1) := \set{ z \in \C: \abs{z} \leq 1 }$ is not an open set (note: \emph{closed} does not necessarily mean not open).
\leftimage{
\altgraphics[scale=1]{ch1_cldisc1}{ch1_cldisc1_full}
}
{
The solid circle in this image indicated that the boundary is included.
}

\begin{blankbox}
Given any point $z$ on the boundary (e.g. $z=1$) and any $r>0$, the open disc $D(z,r)$ contains points that do not belong to $\overline{D}(0,1)$.
\end{blankbox}
\begin{full}
 For example, $w=1+r/2$ belongs to $D(1,r)$, since
\[
\abs{w-z} = \abs{(1+r/2)-1} = r/2 < r
\]
but not to $\overline{D} (0,1)$, since
\[
\abs{w-0} = \abs{1+r/2} = 1+r/2 >1.
\]
\end{full}

\end{itemize}
\end{example}

\begin{example}
 The upper-half plane $H_+$, where
\[
H_+ := \set{ z \in \C: \Im (z) > 0 }
\]
is an open set, while the set $K_+$ defined by
\[
K_+:= \set{ z \in \C : \Im (z) \geq 0 }
\]
is not.
\begin{center}
\altgraphics[scale=1]{ch1_upperhalf_full}{ch1_upperhalf} \quad \altgraphics[scale=1]{ch1_upperhalf2_full}{ch1_upperhalf2}
\end{center}
\begin{blankbox}
If $z \in H_+$ then $\Im (z)>0$, so we set $r = \frac{1}{2} \Im (z)$.  Then $D(z,r)\subseteq H_+$, or in other words, given any $w \in D(z,r)$, we must have $\Im(w)>0$.

For the set $K_+$, if we choose $z$ on the real axis (e.g $z=-1$), then any open disc centred at $z$, no matter how small, contains points below the real axis.  Thus there is no $r>0$ with $D(z,r) \subset K_+$, so $K_+$ is not open.
\end{blankbox}
\end{example}

\begin{note}
Some more examples of open sets include $\C \backslash \set{0}$, or indeed $\C \backslash F$ where $F$ is any finite set.  Many regions determined by \emph{strict} inequalities of real numbers are also open, for example, sets of the form
$
\set{ z \in \C: c_1<\abs{z}<c_2 }$, $\set{ z \in \C: 0 < \Arg (z) < \pi/4 }$ or $\set{z \in \C: c_1<\Re(z) < c_2 },$
where $c_1<c_2 $ are real numbers.

More examples of sets that are not open include circles, lines, curves, single points or finite sets.  Do not use \emph{closed} to mean \emph{not open}.  In the context of analysis, closed has a different meaning.
\end{note}
Throughout this module, we shall be mostly concerned with functions $f:U \to \C$, where $U$ is an open subset of $\C$.

\section{Limits}
Limits in $\C$ are defined in an analogous way to those in $\R$ (and almost identically to those in $\R^2$).

First, fix some complex number $z_0 \in \C$.  For any $z \in \C$, the modulus $\abs{z-z_0}$ measures the distance between $z$ and $z_0$.  Note that if we write $z=x+iy$ and $z_0=x_0+iy_0$, then $\abs{z-z_0}$ is exactly the same as the Euclidean distance between $(x,y)$ and $(x_0,y_0)$ in $\R^2$.

 For a complex function $f$, what does it mean to say that $f(z)$ approaches $L \in \C$ as $z$ approaches $z_0$?  \ Intuitively, we want
 \begin{center}
 $\abs{f(z)-L}$ is small whenever  $\abs{z-z_0}$ is small.
 \end{center}
  More formally, this can be written as
 \begin{center}
 Given any $\epsilon>0$ there is a $\delta>0$ such that
 \[
 \abs{f(z)-L} < \epsilon\text{ whenever } \abs{z-z_0}<\delta.
 \]
 \end{center}

 This definition works whenever $f$ is defined on all of $\C$, but it is not so clear how it should work when $f(z)$ is not defined near $z_0$. In other words, we want to be able to exclude points $z_0$ where $\abs{z-z_0}$ small implies $f(z)$ does not exist.
 \begin{comment}
 Recall the following definition from Foundations II: given a set $A \subseteq \R$, a point $x_0$ is a \emph{limit point} of $A$ if whenever $I$ is an open interval with $x_0 \in I$, then $I$ contains an element of $A \backslash \set{x_0}$.

 We define limit points of subsets of $\C$ in almost exactly the same way, replacing open intervals with open discs.
 \end{comment}

 \begin{definition}
 A point $z_0$ is a \emph{limit point} of of a set $S \subseteq \C$  if for any $\delta>0$, we have
 \[
 D'(z_0,\delta) \cap S \neq \emptyset.
 \]
 In other words, any punctured disc centred at $z_0$, no matter how small, contains at least one point of $S$.
 \end{definition}

A limit point of a set $S$ may or may not belong to $S$.  Moreover, a point $z_0 \in S$ may or may not be a limit point of $S$.  If $S$ is an open set however, then any $z_0 \in S$ is necessarily a limit point of $S$.

\begin{example}
\begin{enumerate}
\item[(i)] The point $0$ is a limit point of the punctured plane $\C \backslash \set{0} = \set{ z \in \C : z \neq 0 }$.
\begin{center}
\altgraphics[scale=1]{ch1_pplane_full}{ch1_pplane}
\end{center}
\begin{blankbox}
Indeed, it is clear that every punctured disc $D'(0,r)$ contains points of $\C \backslash \set{0}$.
\end{blankbox}

In fact, every point $z$ of $\C$ is a limit point of $\C \backslash \set{0}$, since every disc $D'(z,r)$ must contain a point of $\C \backslash \set{0}$.

\item[(ii)] If $S= \set{z_0}$ is a one-point set, then there are no limit points of $S$.  %Indeed, for any $\delta>0$, $D'(z_0,\delta)$ does not contain any points of $S$ by definition.  Moreover, given any other point $z\in \C$ with $z \neq z_0$, the punctured disc $D'(z,\delta)$, with $\delta = \frac{1}{2} \abs{z-z_0}$, does not contain $z_0$.

\item[(iii)] The set of limit points of the open disc $S=D(z_0,r)$ is precisely the closed disc
\[
\set{z \in \C: \abs{z-z_0} \leq r }.
\]
\begin{center}
\altgraphics[scale=1]{ch1_limitpoints_disc_full}{ch1_limitpoints_disc}
%\includegraphics[scale=1]{cldisc_big}
\end{center}
\begin{blankbox}
For a point $z_1$ on the boundary, any punctured disc $D'(z_1,r)$ intersects $S$, while for a point $z_2$ outside of the boundary, there are punctured discs $D'(z_2,r_2)$ that do not.
\end{blankbox}

\item[(iv)] Let $L$ be the strictly positive real axis, regarded as a subset of $\C$, that is
\[
L = \set{ x+ iy \in \C: x>0 \text{ and } y=0 }.
\]
Then $z=0$ is a limit point of $L$.
\begin{center}
\altgraphics[scale=1]{ch1_halfline_full}{ch1_halfline}
\end{center}

%If $z_1$ is a point on the boundary of $D(w,r)$, then for any $\delta>0$, $D'(z_1,\delta)$ contains points of $D(w,r)$.  If $z_2$ lies outside the boundary of $D(w,r)$, the the disc $D(z_2,\delta_2)$, where $\delta_2 = \frac{1}{2} ( \abs{w-z_2} -r)$, does not contain any points of $D(w,r)$.

\end{enumerate}
\end{example}

\begin{note}
To confuse things further, some authors require only allow points $z_0 \in \C \backslash S$ to be limit points of $S$.  When reading textbooks or lecture notes, take care as to which definition is being used.

You need to be familiar with the concept of a limit point, though I will not ask you to prove that a given point $z_0$ is a limit point of a set $S$.
\end{note}

We are now in a position to define limits of complex functions.
\begin{definition}
\label{d:limit}
Let $f$ be a complex function, $S \subseteq \C$ the domain of $f$,  and let $z_0 \in \C$ be a limit point of $S$. Then we say that \[ \lim_{\substack{z \to z_0 \\ z \in S}} f(z) = \alpha \in \C\] if given any $\epsilon >0$ there is some $\delta >0$ such that
\[ \text{ if $z \in S$ and } 0<\abs{z-z_0} < \delta \text{ then }\abs{f(z)-\alpha}< \epsilon.\]
\end{definition}

\begin{note}

\begin{enumerate}
\item[(i)]  Note that this definition allows us to examine limits of functions at point that lie on the boundary of their domains.  For example, we can sensibly speak about things like
\[
\lim_{z \to 0} \frac{\sin(z)}{z}
\]
since this function is defined on $\C \backslash \set{0}$ and $0$ is a limit point of this set.
\item[(ii)]  Definition~\ref{d:limit} may equivalently be written in the language of open discs:
\[
\lim_{\substack{z \to z_0 \\ z \in S}} f(z) = \alpha
\]
if given any $\epsilon >0$ there is a $\delta >0 $ such that
\[
z \in D' (z_0, \delta ) \cap S \text{ implies } f(z) \in D( \alpha, \epsilon).
\]
\item[(iii)] When
\[ \lim_{\substack{z \to z_0 \\ z \in S}} f(z) = \alpha \]
we sometimes write
\[
f(z) \to \alpha \text{ as } z \to z_0.
\]
\end{enumerate}

\end{note}

 \begin{question}
 Is it possible to find a function $f$ for which $f(z) \to \alpha_1$ and $f(z) \to \alpha_2$ as $z \to z_0$, where $\alpha_1\neq\alpha_2$?
  \end{question}

\begin{answer}
The answer, unsurprisingly, is no.  Indeed, set $\epsilon = \frac{1}{3} \abs{\alpha_1-\alpha_2}>0$.  If $f(z) \to \alpha_1$ and $f(z) \to \alpha_2$ as $z \to z_0$, then there would be some $\delta >0$ such that
\[
z \in D'(z_0,\delta) \Longrightarrow f(z) \in D'(\alpha_1 , \epsilon ) \cap D'(\alpha_2 , \epsilon).
\]
But this is impossible since clearly $D'(\alpha_1,\epsilon) \cap D'(\alpha_2 , \epsilon) = \emptyset$.
\end{answer}

\begin{proposition}[Algebra of limits; proof non-examinable]
\label{p:alglimits}
Let $S\subseteq \C$ and consider functions $f,g:S \to \C.$  Suppose that $z_0$ is a limit point of $S$, and that $\displaystyle \lim_{\substack{z \to z_0 \\ z \in S}} f(z) = \alpha$ and $\displaystyle \lim_{\substack{z \to z_0 \\ z \in S}} g(z)= \beta$.  Then
\begin{enumerate}
\item[(i)] $\displaystyle \lim_{\substack{z \to z_0 \\ z \in S}} \left( f(z) + g(z) \right) = \alpha + \beta$,
\item[(ii)] $\displaystyle \lim_{\substack{z \to z_0 \\ z \in S}} \left( f(z)g(z) \right) = \alpha  \beta$,
\item[(iii)] If in addition $\beta \neq 0$, and $z_0$ is a limit point of the set $T=\set{ z \in S: g(z) \neq 0 }$, then
\[
\lim_{ \substack{z \to z_0 \\ z \in T}} \frac{f(z)}{g(z)} = \frac{\alpha}{\beta}
\]
\end{enumerate}
\end{proposition}
(The proof of this proposition is almost identical to the corresponding proof for real functions, except that $\abs{\cdot}$ refers to the modulus and not the absolute value, and is thus omitted.)

%\vspace*{3cm}

\begin{question}
Suppose that $T$ is a subset of $S$, $z_0$ is a limit point of both $T$ and $S$, and that $\rlim{z \to z_0}{z \in S} f(z) = \alpha$.
\begin{SCfigure}[2][h]
\includegraphics[scale=1]{ch1_subset1}
\caption{A subset $T$ of $S$ and a point $z_0$ that is a limit point of both $T$ and $S$.}
\end{SCfigure}
What can we say about $\rlim{z \to z_0}{z \in T} f(z)$?
\end{question}
\begin{answer}
Let $\epsilon >0$ be given, then we know that there exists $\delta >0$ such that
\[
z \in S,\ 0<|z-z_0|<\delta \ \Rightarrow |f(z)-\alpha| < \epsilon.
\]
Note that
\[
 z \in D'(z_0, \delta) \cap T \text{ implies } z \in D' (z_0,\delta) \cap S,
\]
or in other words,
\[
z \in T\ \text{ and } 0 < \abs{z -z_0} < \delta \Rightarrow z \in S \text{ and } 0 < \abs{z-z_0} < \delta.
\]

It follows that
\[
 \lim_{\substack{z \to z_0 \\ z \in S}} f(z) = \alpha \text{ implies }  \lim_{\substack{z \to z_0 \\ z \in T}} f(z) = \alpha
\]
\end{answer}
The second limit, where $f$ is restricted to the subset $T$ of $S$, is called a \emph{restricted limit}.   If a function has a limit at a point $z_0$ then all restricted limits of that function at $z_0$ must be the same.   In particular,

\emph{If we have two subsets $T_1,T_2 \subseteq S$ such that
\[
 \lim_{\substack{z \to z_0 \\ z \in T_1}} f(z) = \alpha_1 \text{ and } \lim_{\substack{z \to z_0 \\ z \in T_2}} f(z) = \alpha_2,
\]
with $\alpha_1 \neq \alpha_2$, then
\[
\lim_{\substack{z \to z_0 \\ z \in S}} f(z)
\]
does not exist.
}

Let $f$ be a complex function with domain $S$ and let $z_0$ be a limit point of $S$.  In what follows, we shall simply write $\displaystyle \lim_{z \to z_0} f(z)$ in place of \[
\lim_{\substack{z \to z_0 \\ z \in S}} f(z),
\]
and we shall reserve the second subscript for restricted limits.

\begin{example}
\label{e:rlim}
Consider the function
\[
f: \C \backslash \set{0} \to \C, \quad f(z) = \frac{\conj{z}}{z}.
\]
Does
$
\displaystyle \lim_{z \to 0} f(z)
$
exist?
\end{example}
\begin{note}
In future, we may simply write things like ``Let $f(z) = \dfrac{\conj{z}}{z}$'' without specifying the domain, since it is clear that this function $f$ is defined on $\C \backslash \set{0}$.
\end{note}
\begin{solution}
We shall look at two restricted limits of $f(z)$ as $z$ approaches $0$, namely the limits when $z$ is restricted to the subsets
\begin{itemize}
\item the nonzero real axis $\R \backslash \set{0}$, and
\item the nonzero imaginary axis $i \R \backslash \set{0}$ (Since every point on the imaginary axis is of the form $iy$ for some $y \in \R$, it makes sense to use the notation $i \R$ for this set.)
\end{itemize}
of the domain $\C \backslash \set{0}$.  Note that $0$ is a limit point of both $\R \backslash \set{0}$ and $i\R \backslash \set{0}$.

%\begin{center}
%\includegraphics[scale=1]{origin_full}
%\end{center}

If $z \in \R \backslash \set{0}$, then $\conj{z} = z$ and so $f(z) = \frac{\conj{z}}{z} = \frac{z}{z} =1$ on $\R \backslash \set{0}$.  Hence
\[
\rlim{z \to 0}{z \in \R \backslash \set{0} } f(z)=1.
\]
For $z \in i \R \backslash \set{0}$, we have $\conj{z}=-z$, so that $f(z)=-1$ for all such $z$, giving
\[
\rlim{z \to 0}{z \in i\R \backslash \set{0} }=-1.
\]
Since  these limits are not equal, it follows that the (unrestricted) limit $\lim_{z \to 0} f(z)$ does not exist.
%\vspace*{12cm}
\end{solution}

\section{Continuity}

\begin{definition}
Let $S \subseteq \C$ and let $f:S \to \C$ be given.  For a point $z_0 \in S$, we say that $f$ is \emph{continuous at $z_0$} if $\lim_{z \to z_0 } f(z)=f(z_0)$. If $f$ is continuous at all points $z_0 \in S$ then we say that $f$ is \emph{continuous on $S$}.
\end{definition}
\begin{note}
Note that the definition of continuity at $z_0$ only makes sense when $z_0$ belongs to the domain of $f$.
\end{note}

\begin{example}
\label{e:cts}

The functions
\begin{enumerate}
\item[(i)] $f(z) = \Re (z)$,
\item[(ii)] $f(z) = \Im (z)$, and
\item[(iii)] $f(z) = \conj{z}$
\end{enumerate}
are all continuous on $\C$.
\end{example}

\begin{solution}
Fix $z_0=x_0+iy_0 \in \C$ and let $\epsilon>0$ be given.  Note that for any $z=x+iy \in \C$ we have the following inequalities:
\begin{align*}
\abs{ \Re (z) - \Re (z_0)} = \abs{\Re (z-z_0)} & = \sqrt{(x-x_0)^2} \\
&\leq \sqrt{(x-x_0)^2+(y-y_0)^2} = \abs{z-w} \\
\abs{ \Im (z) - \Im (z_0)} = \abs{\Im (z-z_0)} & = \sqrt{(y-y_0)^2} \\
&\leq \sqrt{(x-x_0)^2+(y-y_0)^2} = \abs{z-z_0} \\
\abs{\conj{z}-\conj{z_0}} = \abs{\conj{z-z_0}} & = \abs{z-z_0}.
\end{align*}
Thus in all three cases, setting $\delta=\epsilon$, we get
\[
0< \abs{z-z_0} < \delta \Longrightarrow \abs{f(z)-f(z_0)} < \epsilon.
\]
\end{solution}

\begin{proposition}
\label{t:continuity}
Let $S \subseteq \C$ and let $f,g:S \to \C$ be functions that are continuous on $S$.  Then
\begin{enumerate}
\item[(i)] The function $f+g,$ where $(f+g)(z)=f(z)+g(z)$, is continuous on $S$,
\item[(ii)] The function $fg$, where $(fg)(z):=f(z)g(z)$, is continuous on $S$,
\item[(iii)] For any $\alpha \in \C$, the function $\alpha f$, where $(\alpha f)(z)=\alpha \left( f(z) \right),$ is continuous on $S$,
\item[(iv)] If $T=\set{z \in S: g(z) \neq 0 }$ then the function $ f/g$, where $ \left( \dfrac{f}{g} \right) (z)= \dfrac{f(z)}{g(z)},$ is continuous on $T$.
\end{enumerate}
\end{proposition}
{\bf Proof:} Immediate from Proposition~\ref{p:alglimits}.

\begin{note}
If we have
\[
f(x+iy) = u(x,y) + i v(x,y),
\]
and $z_0 = x_0 + i y_0$ is a point in the domain of $f$, then
\[
\lim_{z \to z_0} f(z) = \lim_{(x,y) \to (x_0,y_0)} u(x,y)+ i \lim_{(x,y) \to (x_0,y_0)} v(x,y),
\]
provided these limits exist.  Hence
\[
f \text{ continuous at }z_0 \Leftrightarrow u \text{ and } v \text{ continuous at } (x_0,y_0).
\]
This is useful when the real and imaginary parts of $f$ are familiar functions that we know to be continuous.
\end{note}

\newpage


\documentclass[12pt]{mmalatex}
\usepackage{examples}
\usepackage{caption}
\usepackage{pgfplots}

\begin{document}

\section*{Plotting Bessel functions}

\vspace{-5pt}

This simple example uses Mathematica to produce a plot of the first six Bessel functions. Two plots are shown, one created by Mathematica and a second created by LaTeX using the plotting package {\tt\small pgfplots} and the data exported from Mathematica.

\begin{mathematica}
   myData = Partition[Flatten[Table[{x, Table[BesselJ[n, x], {n, 0, 5}]}, {x, 0, 15, 0.1}]], 7];
   myPlot = Plot[Evaluate[Table[BesselJ[n, x], {n, 0, 5}]], {x, 0, 15}, PlotLegends -> "Expressions"];

   Export["example-04-fig.png", myPlot, "PNG"];
   Export["example-04-fig.pdf", myPlot, "PDF"];
   Export["example-04.txt", myData, "Table",  "FieldSeparators" -> " "];
\end{mathematica}

\vfill

\begin{minipage}{\textwidth}
   \centering
   \IfFileExists{example-04-fig.pdf}%
   {\includegraphics[width=6.4in]{example-04-fig.pdf}}{Failed to create pdf plot.}
   \captionof{figure}{The first six Bessel functions.}
\end{minipage}

\vfill

\clearpage

\pgfplotsset{compat=newest}
\pgfplotsset{width=0.45\textwidth,height=0.34\textwidth}

\subsection*{Using pgfplots}

\begin{minipage}[t]{\textwidth}
   \centering
   \begin{tikzpicture}
      \begin{axis}
         [xmin= 0.0,  xmax=15.0,
          ymin=-0.45, ymax=1.05,
          xlabel=$x$, ylabel=$J_n(x)$,
          grid=major, grid style={dashed,gray!30},
          legend entries = {$J_0$, $J_1$, $J_2$, $J_3$, $J_4$, $J_5$}]
          \addplot[blue]   table [x index=0, y index=1]{example-04.txt};
          \addplot[red]    table [x index=0, y index=2]{example-04.txt};
          \addplot[green]  table [x index=0, y index=3]{example-04.txt};
          \addplot[teal]   table [x index=0, y index=4]{example-04.txt};
          \addplot[orange] table [x index=0, y index=5]{example-04.txt};
          \addplot[purple] table [x index=0, y index=6]{example-04.txt};
      \end{axis}
   \end{tikzpicture}
   \captionof{figure}{The first six Bessel functions.}
\end{minipage}

\vfill

\begin{latex}
   \begin{tikzpicture} % requires \usepackage{pgfplots}
      \begin{axis}
         [xmin= 0.0,  xmax=15.0,
          ymin=-0.45, ymax=1.05,
          xlabel=$x$, ylabel=$J_n(x)$,
          grid=major, grid style={dashed,gray!30},
          legend entries = {$J_0$, $J_1$, $J_2$, $J_3$, $J_4$, $J_5$}]
          \addplot[blue]   table [x index=0, y index=1]{example-04.txt};
          \addplot[red]    table [x index=0, y index=2]{example-04.txt};
          \addplot[green]  table [x index=0, y index=3]{example-04.txt};
          \addplot[teal]   table [x index=0, y index=4]{example-04.txt};
          \addplot[orange] table [x index=0, y index=5]{example-04.txt};
          \addplot[purple] table [x index=0, y index=6]{example-04.txt};
      \end{axis}
   \end{tikzpicture}
   \captionof{figure}{The first six Bessel functions.} % requires \usepackage{caption}
\end{latex}

\end{document}


\documentclass[paper.tex]{subfiles}

\usepackage{tikz}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{tabularx}
\usepackage{multicol}
\usepackage{algpseudocode}
\usepackage{algorithm}

% Add vertical spacing to tables
\renewcommand{\arraystretch}{1.4}

% Begin Document
\begin{document}

\section{Cramer's Rule}

The second method used to solve this is the use of Cramer's Rule.
Let us first recap the original matrix equation.
\begin{equation*}
    \begin{bmatrix}
        2 & 1 & 0 & 0 & 0   \\
        1 & 1 & -1 & 0 & -1 \\
        1 & 0 & -1 & 0 & -1 \\
        0 & -1 & 0 & 1 & 1  \\
        0 & 1 & 1 & -1 & 1
    \end{bmatrix}
    \begin{bmatrix}
        x_1 \\
        x_2 \\
        x_3 \\
        x_4 \\
        x_5
    \end{bmatrix}
    =
    \begin{bmatrix}
        100 \\
        0 \\
        -50 \\
        120 \\
        0
    \end{bmatrix}
\end{equation*}

We must first find the determinant of $A$, as noted below:
\begin{equation*}
    \begin{vmatrix}
        A
    \end{vmatrix}
    = \begin{vmatrix}
        2 & 1 & 0 & 0 & 0   \\
        1 & 1 & -1 & 0 & -1 \\
        1 & 0 & -1 & 0 & -1 \\
        0 & -1 & 0 & 1 & 1  \\
        0 & 1 & 1 & -1 & 1
    \end{vmatrix}
    = -2
\end{equation*}

Following Cramer's rule then leads to finding the determinant of each $A_i(b)$, which indicates the matrix formed when replacing the $i^{th}$ column of $A$ with $b$.
\begin{align*}
    &\begin{vmatrix}A_1\end{vmatrix} = \begin{vmatrix}
        100 & 1 & 0 & 0 & 0   \\
        0 & 1 & -1 & 0 & -1 \\
        -50 & 0 & -1 & 0 & -1 \\
        120 & -1 & 0 & 1 & 1  \\
        0 & 1 & 1 & -1 & 1
    &\end{vmatrix} = -50\\
    &\begin{vmatrix}A_2\end{vmatrix} = -100 \\
    &\begin{vmatrix}A_3\end{vmatrix} = -60 \\
    &\begin{vmatrix}A_4\end{vmatrix} = -250 \\
    &\begin{vmatrix}A_5\end{vmatrix} = -90 \\
\end{align*}

Then for each determinant, division by the original determinant provides the solution to that particular $x$ value.
\begin{equation*}
    \frac{1}{-2} \begin{bmatrix}
        -50 \\
        -100 \\
        -60 \\
        -250 \\
        -90
    \end{bmatrix}
\end{equation*}

The result is then the answer:
\begin{equation*}
    x = \begin{bmatrix}
        25 \\
        50 \\
        30 \\
        125 \\
        45
    \end{bmatrix}
\end{equation*}

This answer is identical to the answer obtained through $LU$ Factorization.

\end{document}


\lab{PageRank Algorithm}{PageRank Algorithm}
\label{lab:PageRank}
\objective{Implement the PageRank Algorithm and understand the theory behind it.}

When you enter keywords into Google's search engine, Google finds every page containing your keywords and lists the pages in order of their \emph{rank}.
The rank of a page reflects many factors, including how often the page is visited and how connected it is to other pages.
As of 2013, the PageRank algorithm is one of over 200 algorithms that Google uses to determine the rank of a website.
Named for Larry Page, cofounder of Google, this algorithm ranks pages based on how many other pages link to them.

The PageRank algorithm is also used in applications other than internet search engines.
For example, it has been used to rank graduate institutions and the impact factor of journals, and it has been used in some biological applications.

\section*{The internet as a graph}
The PageRank algorithm models the internet with a directed graph.
Each webpage is a node, and there is an edge from node $i$ to node $j$ if page $i$ links to page $j$.
Let $\In(i)$ be the websites linking to page $i$ and let $\Out(i)$ be the websites that page $i$ links to.
That is, $\In(i)$ is the set of nodes with an arrow to node $i$, and $\Out(i)$ is the set of nodes with an arrow from node $i$.
An example is illustrated in Figure \ref{fig:network1}.

\begin{figure}
\centering
\begin{tikzpicture}[node distance=1.75cm, thick ]

\node[draw=none](2)[]{2};
\node[draw=none](3)[right of=2]{3};
\node[draw=none](4)[right of=3]{4};
\node[draw=none](5)[right of=4]{5};
\node[draw=none](6)[right of=5]{6};
\node[draw=none](1)[above of=3]{1};
\node[draw=none, node distance=2.5cm](0)[right of=1]{0};
\node[draw=none](dummy)[above right of=0]{};
\node[draw=none, node distance=.5cm](7)[below
	of=dummy]{7};

\foreach \x/\y in {3/2, 4/5, 5/6, 1/0, 3/0, 4/0, 5/0} \draw[->,
	>=stealth'](\x)--(\y);
\draw[->, >=stealth'](6)--(0);
\draw[->, >=stealth', shorten >= .1cm](7)edge[bend left=20](0);
\draw[->, >=stealth', shorten <= .1cm](0)edge[bend left=20](7);
\draw[->, >=stealth', shorten <= .1cm](3)edge[bend right=40](6.9,-.25);
\draw[->, >=stealth', shorten <= .1cm](4)edge[bend right](6);

\end{tikzpicture}

\caption{A directed graph can describe the links between webpages. In this example, the set of web pages linking to page 0 is $\In(0)=\{1,2,3,4,5,6,7\}$, and page 0 links to $\Out(0)=\{7\}$.}
\label{fig:network1}
\end{figure}

The PageRank algorithm ranks pages based on how many other pages link to them.
Moreover, a link from a more important page counts more than a link from a less important page.
For example, in Figure \ref{fig:network1} we would expect node 0 to have a very high rank because every other node links to it.
Consequently we would expect node 7 to have a fairly high rank because node 0 links to it, even though node 0 is the only node to do so.

\section*{The PageRank algorithm}
The PageRank algorithm assumes that a surfer chooses a starting webpage randomly.
Then, if the surfer is at page $i$, they randomly select a page from $\Out(i)$ to visit next.
This means that the surfer's chance of being on page $i$ at time $t$ is determined by where they were at time $t-1$.

Suppose the internet has $N$ webpages and let $P(i,t)$ be the likelihood that the surfer is on page $i$ at time $t$.
Then the probabilities $P(i,t)$ are given by
\begin{equation}\label{equ:pr1}
p(i,0)=\frac{1}{N} \qquad p(i,t+1) = \sum_{j \in \In(i)} \frac{p(j,t)}{\Out(j)}.
\end{equation}

For example, in Figure \ref{fig:network1} we have $N=8$, and
\[
p(6, t+1)=\frac{p(3,t)}{3}+\frac{p(4,t)}{3} + \frac{p(5,t)}{2}.
\]

\subsection*{Refining the model: pages with no outbound links}
A node with no outbound links is called a \emph{sink}.
For example, node 2 in Figure \ref{fig:network1} is a sink.
According to our model, if the surfer ever visits a sink, they will stay there forever.

This is not very realistic; in this situation, a person would likely select another webpage at random and begin surfing again.
Hence, in our model we replace sinks with nodes linking to every other page.
For example, we replace Figure \ref{fig:network1} with Figure \ref{fig:network2}.

\begin{figure}
\centering
\begin{tikzpicture}[node distance=1.75cm, >=stealth', thick]

\node[draw=none](2)[]{2};
\node[draw=none](3)[right of=2]{3};
\node[draw=none](4)[right of=3]{4};
\node[draw=none](5)[right of=4]{5};
\node[draw=none](6)[right of=5]{6};
\node[draw=none](1)[above of=3]{1};
\node[draw=none, node distance=2.5cm](0)[right of=1]{0};
\node[draw=none](dummy)[above right of=0]{};
\node[draw=none, node distance=.5cm](7)[below
	of=dummy]{7};

\draw[->, color=black!35!](2)--(1);
\draw[->, color=black!35!, shorten >= .2cm](2)--(0);
\foreach \x/\y in {2/3, 2/4, 2/5} \draw[->, color=black!35!](\x)
	edge[bend right](\y);

\draw[->, shorten <= .1cm, color=black!35!](2)edge[bend right=50](7,-.35);
\draw[->, shorten <= .1cm](3)edge[bend right=40](6.9,-.25);
\draw[->, shorten <= .1cm](4)edge[bend right](6);
\draw[->, shorten <=.1cm, color=black!35!](2)edge[bend left=40](7);

\draw[->, shorten >= .1cm](7)edge[bend left=20](0);
\draw[->, shorten <= .1cm](0)edge[bend left=20](7);

\foreach \x/\y in {3/2, 4/5, 5/6, 1/0, 3/0, 4/0, 5/0} \draw[->,
	>=stealth'](\x)--(\y);
\draw[-> ](6)--(0);

\end{tikzpicture}

\caption{This figure arises from Figure \ref{fig:network1} by adding a link from page 2 to every other page (the added links are grey). The new links guarantee that page 2 is no longer a sink, as required by the PageRank algorithm.}
\label{fig:network2}
\end{figure}

\subsection*{Refining the model: adding boredom}
The equations in \eqref{equ:pr1} describe the surfer's habits as a Markov chain.
However, the model is more realistic if we add the assumption that the surfer sometimes gets bored and randomly picks a new starting page.
We will denote the probability that a surfer stays interested at step $t$ by a constant $d$, called the \emph{damping factor}.
Then the probability that the surfer gets bored at time $t$ is $1-d$.
Accounting for boredom, the formulas in \eqref{equ:pr1} become
\begin{equation}\label{equ:pr2}
p(i,0)=\frac{1}{N} \qquad p(i,t+1) = \frac{1-d}{N} + d\sum_{j \in \In(i)} \frac{p(j,t)}{\Out(j)}.
\end{equation}

\subsection*{Matrix form of the PageRank algorithm}
We can rewrite \eqref{equ:pr2} as the matrix equation
\begin{equation}\label{equ:pr3}
\mathbf{p}(0)=\frac{1}{N}\mathbf{1} \qquad \mathbf{p}(t+1) = dK\mathbf{p}(t) + \frac{1-d}{N}\mathbf{1}
\end{equation}
where $\mathbf{p}(t)=(p(1,t), p(2,t), \ldots, p(N,t))^T$ and $\mathbf{1} = (1,\ldots, 1)^T$, and $K$ is defined by
\[K_{ij} = \begin{cases} \frac{1}{\Out(j)} & \mbox{ if j links to i} \\
	0 & \mbox{ otherwise.} \end{cases}\]

\subsection*{Defining page rank}
As given by the PageRank algorithm, the \emph{rank} of page $i$ is
\[p(i) = \lim_{t\to \infty} p(i,t).\]
In other words, the page ranks are the steady state of the modified Markov chain defined in \eqref{equ:pr3}.

\section*{Implementation in Python}
A good strategy for computing $K$ comes from writing
\[
K = (D^{-1}A)^T
\]
where $A$ is the adjacency matrix of the directed graph representing the internet and $B$ is a diagonal matrix with $B_{jj}=\Out(j)$.
Since $\Out(j)$ can be computed from the adjacency matrix, we will begin by creating $A$.

The adjacency matrix $A$ of a directed graph has $A_{ij}=1$ if there is an edge from node $i$ to node $j$, and $A_{ij}=0$ otherwise.
For example, the adjacency matrix of the graph in Figure \ref{fig:network1} is defined below.
We use a code environment to describe $A$ so you can easily use this example to debug the problems in this lab.
\begin{lstlisting}
A = np.array([[ 0,  0,  0,  0,  0,  0,  0,  1],
              [ 1,  0,  0,  0,  0,  0,  0,  0],
              [ 0,  0,  0,  0,  0,  0,  0,  0],
              [ 1,  0,  1,  0,  0,  0,  1,  0],
              [ 1,  0,  0,  0,  0,  1,  1,  0],
              [ 1,  0,  0,  0,  0,  0,  1,  0],
              [ 1,  0,  0,  0,  0,  0,  0,  0],
              [ 1,  0,  0,  0,  0,  0,  0,  0]])
\end{lstlisting}

\begin{problem}
Write the following function that creates an adjacency matrix from a file.
\begin{lstlisting}
def to_matrix( datafile, n ):
    ''' Return the nxn adjacency matrix described by datafile.

    INPUTS:
    datafile - A .txt file describing a directed graph. Lines
    			describing edges should have the form
				'<from node>\t<to node>\n'. The file may also
				include comments.
    n		- The number of nodes in the graph described by datafile

    RETURN:
    Return a SciPy sparse `dok_matrix'.
    '''
\end{lstlisting}
Hints:
\begin{enumerate}
\item The file \texttt{data.txt} included with this lab describes the matrix in Figure \ref{fig:network1}.
You may use it to test your function.

\item You can open a file in Python using the \li{with} syntax.
Then, you can iterate through the lines using a \li{for} loop.
Here is an example.
\begin{lstlisting}
# Open `data.txt' for read-only
with open('./data.txt', 'r') as myfile:
    for line in myfile:
        print line
\end{lstlisting}

\item Here is an example of how to process a line of the form in \li{datafile}.
\begin{lstlisting}
>>> line = '0\t4\n'
# strip() removes trailing whitespace from a line.
# split() returns a list of the space-separated pieces of the line.
>>> line.strip().split()
['0', '4']
\end{lstlisting}

\item Rather than testing for lines of \texttt{data.txt} that contain comments, put all your string operations in a \li{try} block with an \li{except} block following.
\end{enumerate}
\end{problem}

The next step is to compute $K$.
The matrix $D$ is easily obtained from $A$ (how?).
Then, we can use $D$ to identify sinks.
Modify $A$ so that rows corresponding to sinks have all ones instead of all zeros.
For Figure \ref{fig:network2}, the modified adjacency matrix is defined below.
\begin{lstlisting}
Am = np.array([[ 0,  0,  0,  0,  0,  0,  0,  1],
               [ 1,  0,  0,  0,  0,  0,  0,  0],
               [ 1,  1,  1,  1,  1,  1,  1,  1],
               [ 1,  0,  1,  0,  0,  0,  1,  0],
               [ 1,  0,  0,  0,  0,  1,  1,  0],
               [ 1,  0,  0,  0,  0,  0,  1,  0],
               [ 1,  0,  0,  0,  0,  0,  0,  0],
               [ 1,  0,  0,  0,  0,  0,  0,  0]])
\end{lstlisting}

Finally we compute $K$.
It is a \emph{very} bad idea to invert $D$ and perform the matrix multiplication $D^{-1}A$.
Rather, store the diagonal entries of $D$ in a vector and use array broadcasting to divide $A$ by $D$.
For Figures \ref{fig:network1} and \ref{fig:network2}, the matrix $K$ is as follows.

\begin{lstlisting}
K = np.array([[ 0   ,  1   ,  1./8,  1./3,  1./3,  1./2,  1   ,  1   ],
              [ 0   ,  0   ,  1./8,  0   ,  0   ,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  1./3,  0   ,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  0   ,  0   ,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  0   ,  0   ,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  0   ,  1./3,  0   ,  0   ,  0   ],
              [ 0   ,  0   ,  1./8,  1./3,  1./3,  1./2,  0   ,  0   ],
              [ 1   ,  0   ,  1./8,  0   ,  0   ,  0   ,  0   ,  0   ]])
\end{lstlisting}

\subsection*{Solving for the page ranks}
There are several ways to solve for $\lim_{t \to \infty} \mathbf{p}(t)$.
\subsubsection*{Algebraic method}
One possibility is to assume the modified Markov chain has a steady state $\mathbf{p}$ and solve for it algebraically:
\begin{equation}\label{equ:matrix_solve}
(I-dK)\mathbf{p} = \frac{1-d}{N} \mathbf{1}.
\end{equation}

We can implement this algorithm in SciPy to solve for the page ranks of the network in Figure \ref{fig:network2}.
\begin{lstlisting}
>>> from scipy import linalg as la
>>> I = np.eye(8)
>>> d = .85
>>> la.solve(I-d*K, ((1-d)/8)*np.ones(8))
array([ 0.43869288,  0.02171029,  0.02786154,  0.02171029,  0.02171029,
        0.02786154,  0.04585394,  0.39459924])
\end{lstlisting}
As expected, node 0 has the highest rank, approximately equal to .43.
Node 7 has a much higher rank than node 6, even though $\In(7)=1$ and $\In(6)=3$.
This is because node 7's single in-edge comes from a node that has a very high rank (node 0).

\subsubsection*{Iterative method}
Solving the system in \eqref{equ:matrix_solve} is feasible for our small working example, but this is not an efficient strategy for very large systems.

One option for large systems is to iterate the equation for $\mathbf{p}(t)$ until $\norm{\mathbf{p}(t)-\mathbf{p}(t-1)}$ is sufficiently small.

\begin{problem}
\label{prob:pagerank_dense_iter}
Implement the function below, using the iterative method to find the steady state of the PageRank algorithm.
\begin{lstlisting}
def iter_solve( data, N=None, d=.85, tol=1E-5):
    '''
    Return the page ranks of the network described by `data`.

    INPUTS:
    data - A NumPy array representing the adjacency matrix of a directed
            graph
    N     - Restrict the computation to the first `N` nodes of the graph.
            Defaults to N=None; in this case, the entire matrix is used.
    d     - The damping factor, a float between 0 and 1.
            Defaults to .85.
    tol  - Stop iterating when the change in approximations to the
            solution is less than `tol'. Defaults to 1E-5.

    OUTPUTS:
    Iterate through the PageRank algorithm until the error is less than
    `tol'. Return the approximation to the steady state.
    '''
\end{lstlisting}
Hints:
\begin{enumerate}
\item At each step, test your function against the example in this lab.
\item Don't forget to eliminate sinks.
It is possible (but not necessary) to do this in a single line.
\item When \li{n} is not \li{None}, you want to work with only the upper $n \times n$ portion of the array \li{data}.
\end{enumerate}
\end{problem}

\subsubsection*{Eigenvalue method}
Another way to solve this problem is to make it into an eigenvalue problem.
Let $E$ be an $N \times N$ matrix of ones; then $E\mathbf{p}(t) = \mathbf{p}(t)$.
Hence, the matrix equation \eqref{equ:pr3} for $\mathbf{p}(t+1)$ becomes
\[\mathbf{p}(t+1) = \Big(dK + \frac{1-d}{N}E\Big)\mathbf{p}(t).\]
If we write $B = dK + \frac{1-d}{N}E$, this simplifies to $\mathbf{p}(t+1) = B\mathbf{p}(t).$
Thus, the steady state $\mathbf{p}(t)$ is an eigenvector of $B$ corresponding to the eigenvalue 1.

The columns of $B$ sum to 1, and the entries of $B$ are strictly positive (because the entries of $E$ are all positive).
With these hypotheses, the Perron-Frobenius theorem says that 1 is the unique eigenvalue of B of largest magnitude, and the corresponding eigenvector is unique.
In this case, the ``iterative method'' described above is just the power method for finding the eigenvector corresponding to a dominant eigenvalue, introduced in Lab \ref{lab:EigSolve}.

We can also attempt to compute $\mathbf{p}$ using eigenvalue solvers in SciPy.

%TODO: the output from scipy.linalg.eig is not very accurate, even for the small example in this lab.
\begin{problem}
Implement the function below, using the eigenvalue method to find the steady state of the PageRank algorithm.
\begin{lstlisting}
def iter_solve( data, n=None, d=.85):
    '''
    Return the page ranks of the network described by `data`.

    INPUTS:
    data - A NumPy array representing the adjacency matrix of a directed
            graph
    N     - Restrict the computation to the first `N` nodes of the graph.
            Defaults to N=None; in this case, the entire matrix is used.
    d     - The damping factor, a float between 0 and 1.
            Defaults to .85.

    OUTPUTS:
    Use the eigenvalue solver in \li{scipy.linalg} to calculate the steady
    state of the PageRank algorithm.
    '''
\end{lstlisting}
Hint: Review the techniques from the Markov chain section of Lab \ref{lab:EigSolve}.
\end{problem}

\section*{SNAP datasets}
The SNAP graph library, located at \url{http://snap.stanford.edu/data/index.html}, provides a variety of medium sized data sets for public use.
The \li{matplotlib.pyplot.spy} command on the adjacency matrix from a SNAP data set yielded the plot shown in Figure \ref{fig:WebSparse}

\begin{figure}
\centering
\includegraphics[width=\textwidth]{sparse_web.png}
\caption{Output of the \li{spy} command on the adjacency matrix corresponding to the websites supported by Notre Dame University in 1999.
Data was taken from the SNAP datasets.}
\label{fig:WebSparse}
\end{figure}

\begin{problem}
Try running the functions you wrote in this lab on a data set downloaded from SNAP.
\begin{enumerate}
\item Begin by running your methods on the first 100 nodes of the data set.
\item Modify your solution to Probelm \ref{prob:pagerank_dense_iter} so that it uses only sparse matrices.
With this modification, you should be able to run the function on more nodes.
Hint: Convert the adjacency matrix to a \li{csc_matrix} or a \li{csr_matrix} to perform the computations.
\end{enumerate}
\end{problem}


\gotosection{1}{2}
\subsection{Introducing the actors: matrices}

\begin{exercise}{17}
  \begin{enumerate}
    \item
    $$A_T = \begin{xmatrix}
              0 & 1 & 1 \\
              1 & 0 & 1 \\
              1 & 1 & 0
            \end{xmatrix}$$

    $$A_S = \begin{xmatrix}
              0 & 1 & 0 & 1 \\
              1 & 0 & 1 & 0 \\
              0 & 1 & 0 & 1 \\
              1 & 0 & 1 & 0
            \end{xmatrix}$$

    Note: I label the square's vertexes in the same order as a person draws the square in one stroke.

    \item
    $$A_T^5 = (A_T^2)^2A_T
            = \begin{xmatrix}
                 2 & 1 & 1 \\
                 1 & 2 & 1 \\
                 1 & 1 & 2
               \end{xmatrix}^2A_T
            = \begin{xmatrix}
                 6 & 5 & 5 \\
                 5 & 6 & 5 \\
                 5 & 5 & 6
               \end{xmatrix}A_T
            = \begin{xmatrix}
                 10 & 11 & 11 \\
                 11 & 11 & 11 \\
                 11 & 10 & 10
               \end{xmatrix}$$

    $$A_S^5 = \begin{xmatrix}
                2 & 0 & 2 & 0 \\
                0 & 2 & 0 & 2 \\
                2 & 0 & 2 & 0 \\
                0 & 2 & 0 & 2
              \end{xmatrix}^2A_S
            = \begin{xmatrix}
                8 & 0 & 8 & 0 \\
                0 & 8 & 0 & 8 \\
                8 & 0 & 8 & 0 \\
                0 & 8 & 0 & 8
              \end{xmatrix}A_S
            = \begin{xmatrix}
                 0 & 16 &  0 & 16 \\
                16 &  0 & 16 &  0 \\
                 0 & 16 &  0 & 16 \\
                16 &  0 & 16 &  0
              \end{xmatrix}$$

    Each diagonal entry shows how many possible routes do there exist to go from that vertex to itself.

    \item Let $xy(n) = A^n{xy}$, or how many possible solutions of going from $x$ to $y$ in $n$ steps.
          Assigning the three vertexes of the triangle to $a, b, c$ randomly, one have:
          $$\left\{
            \begin{aligned}
              aa(n+1) = ab(n) + ac(n) \\
              ab(n+1) = aa(n) + ac(n) \\
              ac(n+1) = aa(n) + ab(n)
            \end{aligned}
          \right.$$

          , which can be derived using both the geometrical method and from the matrix multiplication. We need to prove $ab(n) = ac(n)$ and $|aa(n) - ab(n)| = 1$.

          First, we have the base case:

          $$\left\{
            \begin{aligned}
              aa(1) = 0 \\
              ab(1) = 1 \\
              ac(1) = 1
            \end{aligned}
          \right.$$

          $ab(1) = ac(1)$, and $|aa(1) - ab(1)| = 1$. Then, using the induction hypothesis, we have $ab(n+1) = aa(n) + ac(n) = aa(n) + ab(n) = ac(n+1)$, and,

          \begin{align*}
            |aa(n+1) - ab(n+1)| &= |2ab(n) - (ab(n) + aa(n))| \\
                                &= |ab(n) - aa(n)| \\
                                &= 1
          \end{align*}

          This completes the proof. \QED

    \item Label the square's vertexes $a, b, c, d$ in the same order as a person draws the square in one stroke. Assume that the route starts from $a$. With only one step, the route must end in $b$ or $d$. With two steps, the route must end in $a$ or $c$. With three steps, the route must end in $b$ or $d$, again. This forms a cycle: in odd steps, the route must end in $b$ or $d$; in even steps, the route must end in $a$ or $c$. In the matrix's terminology, this means that half entries are nonzero while the rest are zero. \QED
  \end{enumerate}
\end{exercise}

\begin{exercise}{20}
  \begin{enumerate}
    \item $z_1 + z_2 = x_1 + iy_1 + x_2 + iy_2 = (x_1 + x_2) + i(y_1 + y_2).$ Therefore, $M_{z_1 + z_2} = \begin{xmatrix}
                        x_1 + x_2 & y_1 + y_2 \\
                       -y_1 - y_2 & x_1 + x_2
                     \end{xmatrix} =
                     \begin{xmatrix}x_1 & y_1 \\ -y_1 & x_1\end{xmatrix} +
                     \begin{xmatrix}x_2 & y_2 \\ -y_2 & x_2\end{xmatrix} =
                     M_{z_1} + M_{z_2}$. \QED

    \item $M_{z_1}M_{z_2} = \begin{xmatrix}x_1 & y_1 \\ -y_1 & x_1\end{xmatrix}
                            \begin{xmatrix}x_2 & y_2 \\ -y_2 & x_2\end{xmatrix}
                          = \begin{xmatrix}
                               x_1x_2 - y_1y_2 &  x_1y_2 + y_1x_2 \\
                              -y_1x_2 - x_1y_2 & -y_1y_2 + x_1x_2
                            \end{xmatrix}$.
          The product matrix is associated with $x_1x_2 - y_1y_2 + i(x_1y_2 + y_1x_2) = x_1x_2 + iy_1iy_2 + x_1iy_2 + x_2iy_1 = (x_1 + iy_1)(x_2 + iy_2) = z_1z_2$. Since this association is bijective, we have $M_{z_1}M_{z_2} = M_{z_1z_2}$. \QED
  \end{enumerate}
\end{exercise}

\begin{exercise}{23}
  \begin{enumerate}
    \item $\begin{xmatrix}
            a & 1 & 0 \\
            b & 0 & 1
          \end{xmatrix}
          \begin{xmatrix}
            0 & 0 \\
            1 & 0 \\
            0 & 1
          \end{xmatrix} =
          \begin{xmatrix}
            1 & 0 \\
            0 & 1
          \end{xmatrix} = I_2$

    \item Suppose there exists $B$ such that $AB = I_3$. Let $I = I_3$.
          $I_{11} = 1$, $I_{11} = A_{11}B_{11} + A_{12}B_{12} = 0$, and $0 \neq 1$.
          Therefore, A does not have a right inverse.

    \item Let $A$ be the matrix in (a). Suppose $B$ be a left inverse of $A$, we have:
          $$BA = I_3 = I_3^{\top} = (BA)^{\top} = A^{\top}B^{\top}$$
          Since there is infinitely many possible $B$s, $A^{\top}$ has infinitely many right inverses $B^{\top}$.
  \end{enumerate}
\end{exercise}


\section{MCSSolver}
This test evaluates the MCSSolver model for a specific case found in literature (see pages 108-110
of N. J. McCormick, ``Reliability and Risk Analysis'', Academic Press inc. (1981)).
Provided this set of minimal cuts sets (MCSs) for the top event TopEvent:
\begin{equation}
  TopEvent = A + BD + BE + CD + CE
\end{equation}
the probability of TopEvent can be calculated as
\begin{equation}
  P(TopEvent) = [A + BD + BE + CD + CE]
              - [ABD + ABE + ACD + ACE + BCD + BCE + BDE + CDE + 2*BCDE]
              + [ABCD + ABCE + ABDE + ACDE + 4*BCDE + 2*ABCDE]
              - [BCDE + 4*ABCDE]
              + [ABCDE]
\end{equation}

Give these probability values for the basic events:
\begin{itemize}
  \item $P(A)=0.01$
  \item $P(B)=P(C)=P(D)=P(E)=0.1$
\end{itemize}
then the analytical values for $P(TopEvent)$ for each order are as follows:

\begin{table}
\begin{tabular}{cc}
Order &  $P(TopEvent)$\\
\hline
1 & 0.05     \\
2 & 0.0454   \\
3 & 0.045842 \\
4 & 0.045738 \\
5 & 0.045739 \\
\hline
\end{tabular}
\end{table}


\chapter{Expectation of a Random Variable}
\label{chapter:expectation}
To understand the behaviour of a random variable one may try to use the average
value; there are several ways to define the average value, but we are going to
concentrate on the definition based on the mean. Given a random variable $X$
in the finite discrete probability space $(\Omega, \Distribution{D})$, the
expected value $\expectation[\Distribution{D}]{X}$
(or $\expectation[\omega \gets \Distribution{D}]{X(\omega)}$) of $\chi$ is equal to
$\sum_{\omega \in \Omega} \Pr_{\Distribution{D}}(\omega) X(\omega)$.

Note that it is possible that there are several $\omega$'s with the same
$X(\omega)$. Hence, one may give an alternative definition of the expectation.
\begin{theorem}
  Let $(\Omega, \Distribution{D})$ be a finite discrete probability space, and
  let $X$ be a random variable in the probability space. Then
  $\expectation[\Distribution{D}]{X} =
  \sum_{a \in \Im X} a\Pr_{\Distribution{D}}(X = a)$.
\end{theorem}

An important property of the expectation that often allows simplifications in
computing the expected value of a random variable is its linearity.
\begin{theorem}
  Let $(\Omega, \Distribution{D})$ be a finite discrete probability space; $X$
  and $Y$ be random variables; and let $\lambda \in \R$.
  Then $\expectation[\Distribution{D}]{X + Y} =
    \expectation[\Distribution{D}]{X} + \expectation[\Distribution{D}]{Y}$ and
  $\expectation[\Distribution{D}]{\lambda X} = \lambda
  \expectation[\Distribution{D}]{X}$.
\end{theorem}
Let us give a simple example showing that this theorem can help to compute the
expected value of a random variable. Consider an experiment consisting of
tossing $n$ standard coins; i.e. consider the finite discrete probability space
$(\Omega, \Pr)$ such that $\Omega = \set{H, T}^n$ and $\Uniform{\Omega}$ is the
uniform distribution on $\Omega$. We would like to find the expected number $X$
of heads in the experiment. Let $X_i$ be the random variable that is equal to
$1$ if the $i$th flip yields heads, otherwise it is equal to $0$. It is clear
that $X = \sum_{i = 1}^n X_i$. Hence, $\expectation[\Uniform{\Omega}]{X} =
\sum_{i = 1}^n \expectation[\Uniform{\Omega}]{X_i}$.
However, $\expectation[\Uniform{\Omega}]{X_i} = 1 / 2$ which implies that
$\expectation[\Uniform{\Omega}]{X} = n / 2$.

Let us use this knowledge to study a game similar to the game discussed in
\Cref{chapter:structural-induciton}. Alice selected $500$ numbers from $1$ to
$1000$ and Bob would like to guess at least one of them. How many questions Bob
need to ask to do this?

Apparently, the situation is drastically different if Bob's algorithm can be
randomized and if it cannot be randomized. To show this we need to extend the
definition of $B$-decision trees so that they can operate not only with
integers.
\begin{definition}
  Let $X$ and $Y$ be some sets. We say that $T$ is a $B$-decision tree if
  \begin{description}
    \item [(base case)] either $T$ is equal to $\DTReturn{y}$ for $y \in Y$, or
    \item [(recursion step)] $T$ is equal to $\DTIf{f}{T_0}{T_1}$,
      where $f : X \to \set{0, 1}$, and $T_0$ and $T_1$
      are $B$-decision trees.
  \end{description}

  The number of queries $\DTHeight[x]{T}$ of $T$ at $x \in X$ can be defined as
  follows.
  \begin{description}
    \item [(base case)] Let $T$ be equal to $\DTReturn{y}$, where$y \in Y$. Then
      $\DTHeight[x]{T} = 0$ for all $x \in X$.
    \item[(recursion step)] Let $T$ be equal to $\DTIf{f}{T_0}{T_1}$. Then
      \[
          \DTHeight[x]{T} =
          \begin{cases}
            \DTHeight[x]{T_0} + 1 & \text{if } f(x) = 0 \\
            \DTHeight[x]{T_1} + 1 & \text{otherwise}
          \end{cases}
      \]
  \end{description}

  The value $\DTValue{T}{x}$ of a $B$-decision tree $T$ at $x \in X$
  can be defined as follows.
  \begin{description}
      \item [(base case)] Let $T$ be equal to $\DTReturn{y}$, where $y \in Y$.
        Then $\DTValue{T}{x} = y$.
      \item[(recursion step)] Let $T$ be equal to $\DTIf{f}{T_0}{T_1}$. Then
        \[
          \DTValue{T}{x} =
          \begin{cases}
            \DTValue{T_0}{x} & \text{if } f(x) = 0 \\
            \DTValue{T_1}{x} & \text{otherwise}
          \end{cases}.
        \]
  \end{description}
\end{definition}

\begin{theorem}
\label{theorem:guess-one-out-of-many}
  Let $\binom{\range{1000}}{500}$ denote the set of subsets of $\range{1000}$
  with $500$ elements.\footnote{%
    We are going to discuss such sets in \Cref{chapter:binomials}.
  }
  \begin{itemize}
    \item Let $T$ be a $B$-decision tree such that $\DTValue{T}{S} \in S$
      for all $S \in \binom{\range{1000}}{500}$. Then $\DTHeight{T} \ge 9$.
    \item There are a set $\Omega$ of $B$-decision trees and a probability
      distribution $\Distribution{D}$ on $\Omega$ such that
      $\DTValue{T}{S} \in S$ for all $S \in \binom{\range{1000}}{500}$ and
      $T \in \Omega$, but
      $\expectation[\Distribution{D}]{\DTValue{T}{S}} \le 2$ for all
      $S \in \binom{\range{1000}}{500}$.
  \end{itemize}
\end{theorem}
Informally, this theorem says that in the worst-case scenario Bob needs at least
$9$ questions to guess at least one element of Alice's set; however, there is
a strategy for Bob such that it would be enough to ask only $2$ questions, on
average.
\begin{proof}
  We prove only the second part of the statement, the first part can be proven
  similarly to \Cref{theorem:guess-the-number}.
  To prove this statement let us consider the following algorithm.
  \begin{itemize}
    \item Choose $x_1, \dots, x_n \in [1000]$ uniformly at random;
    \item Set $i = 1$;
    \item Query whether $x_i \in S$ or not.
    \item If yes, the output is $x_i$, otherwise increase $i$.
    \item If $i \le n$ go to step $3$, otherwise go to step $6$.
    \item Bruteforce all the numbers from $[1000]$ and check whether they belong
      to $S$ or not.
  \end{itemize}
  Let us compute the expected number of queries made by this algorithm. It is
  clear that the probability that the algorithm gets yes for $i = k$ is equal to
  $\left(\frac{1}{2}\right)^k$. Hence, the expected number of queries is equal
  to
  \[
    \left(\frac{1}{2}\right)^n 1000 +
    \sum_{k = 1}^n \left(\frac{1}{2}\right)^k k.
  \]

  The following claim gives the formula that allows to compute this
  sum.\footnote{%
    We are going to discuss a method to guess such formulas in
    \Cref{chapter:binomials}.
  }
  \begin{claim}
  \label{claim:guess-one-out-of-many}
      For any $k \in \N$, $\sum_{k = 1}^n \left(\frac{1}{2}\right)^k k =
      2 - \frac{n + 2}{2^n}$.
  \end{claim}

  Therefore, the average number of queries is at most
  $2 - \frac{n + 998}{2^n} \le 2$ for $n \ge 10$.

  It is also clear that the result of this algorithm is always correct.

  To finish the proof we need to prove \Cref{claim:guess-one-out-of-many}.
  We prove it using induction by $n$. It is clear that the statement is true for
  $n = 0$. Assume the statement is true for $n$; i.e.,
  $\sum_{k = 1}^n \left(\frac{1}{2}\right)^k k = 2^{-n} (-n + 2^{n + 1} - 2)$.
  This implies that
  \begin{multline*}
    \sum_{k = 1}^{n + 1} \left(\frac{1}{2}\right)^k k =
    2 - \frac{n + 2}{2^n} + \frac{n + 1}{2^{n + 1}} = \\
    2 - \frac{2n + 4 - n - 1}{2^{n + 1}} = 2 - \frac{(n + 1) + 2}{2^{n + 1}}.
  \end{multline*}
  In other words, the statement is true for $n + 1$. Hence, by the induction
  principle, we proved the statement for all $n$.
\end{proof}

\begin{chapterendexercises}
  \exercise Prove the first part of \Cref{theorem:guess-one-out-of-many}.
\end{chapterendexercises}