%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS622: Theory of Formal Languages
% Copyright 2014 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 5}

Let $L$ be a regular language over an alphabet $A$.
Prove that the set of all words $z \in A^*$ for which there exist $u,v \in L$ such that $u = zx$ and $v=yz$ for some $x,y \in A^*$ is a regular language.

\subsection*{Solution}

We assume $S_1$ as set of all words $z \in A^*$ for which there exist $u \in L$ such that $u=zx$ for some $x \in K \subseteq A^*$.
We can represent $S_1$ as shown in Equation \ref{Eq1}.

\begin{equation}\label{Eq1}
	S_1 = LK^{-1} = \{z\in A^* | \exists x\in K, zx\in L\}
\end{equation}

Also, we assume $S_2$ as set of all words $z \in A^*$ for which there exists $v \in L$ such that $v = yz$ for some $y \in K \subseteq A^*$.
We can represent $S_2$ as shown in Equation \ref{Eq2}.

\begin{equation}\label{Eq2}
	S_2 = K^{-1}L = \{z\in A^* | \exists y\in K, yz\in L\}
\end{equation}

It has been proven \cite{simovici1999theory} that for every language $K$, both the right and the left quotients $LK^{-1}$ and $K^{-1}L$ are regular, if $L$ is a regular language over alphabet $A$.
Based on this theorem, $S_1$ and $S_2$ would both be regular.

Since any word $z \in A^*$ for which there exist $u,v \in L$ such that $u = zx$ and $v=yz$ for some $x,y \in A^*$ should be an element of $S_1$ and $S_2$, set of all such words will be the intersection of $S_1$ and $S_2$.
As regular languages $\mathcal{R}$ are closed with respect to intersection, it is proven that the desired set is a regular language.


\addcontentsline{toc}{chapter}{Useful Formulae}
\chapter*{Useful Formula}
\begin{itemize}
	%%%%%%%%%%
	\item Equations of a Line:
\begin{center}\begin{tabular}{rl}
		point $(x_1,y_1)$, slope $m$ form:&$y-y_1 = m(x - x_1)$\\
		slope $m$, intercept $b$ form:&$y=mx+b$
	\end{tabular}\end{center}
	%%%%%%%%%%
	\item Quadratic Formula:
	If $ax^2+bx+c=0$,\,$a\ne0,\quad$ then $\displaystyle\quad x = \frac{-b\pm \sqrt{b^2-4ac}}{2a}$
	%%%%%%%%%%
	\item Sine Law: $\frac{\sin A}{a}= \frac{\sin B}{b}=\frac{\sin C}{c}$; \hspace{0.2cm}Cosine Law: $a^2=b^2+c^2-2bc\cos(A)$
\end{itemize}
{\hspace*{0.55cm}
		\renewcommand{\arraystretch}{1.2}
		\begin{tabular}{lrclcrcl}
			&\multicolumn{3}{l}{\phantom{aaaa}$\bullet$ Logarithms}& &\multicolumn{3}{c}{$\bullet$ Exponents}\\
			\cmidrule{1-4}\cmidrule{6-8}
			Law 1: &$ \log_x(ab) $&$=$&$ \log_x(a)+\log_x(b)  $&\hspace{1cm} &$x^a x^b$&$=$&$x^{a+b} $\\
			Law 2: &$ \log_x\left(\frac{a}{b}\right) $&$=$&$  \log_x(a)-\log_x(b) $& &$\dfrac{x^a}{x^b}$&$=$&$x^{a-b}  $\\
			Law 3: &$ \log_x(a^b) $&$=$&$ b\cdot\log_x(a)   $& &$(x^a)^b $&$=$&$x^{ab} $\\
			\cmidrule{1-4}\cmidrule{6-8}
			&$  \log_{x}\left(\frac{1}{x^a}\right) $&$=$&$-a   $& &$x^{-a} $&$=$&$\frac{1}{x^a} $\\
			&$ \log_{x}1  $&$=$&$ 0  $& &$x^0 $&$=$&$1 $\\
			&$ \log_x(x) $&$=$&$  1 $& &$x^1 $&$=$&$x $\\
			\bottomrule
			&$ \log_a(x) $&$=$&$y$\hspace{0.5cm} converts to& $\Leftrightarrow$ &$x $&$=$&$a^y $\\
			\bottomrule
		\end{tabular}\\
}
\newline\phantom{blank line here please}\\
{\hspace*{0.55cm}
	\renewcommand{\arraystretch}{1.2}
		\begin{tabular}{llll}
			\multicolumn{4}{c}{$\bullet$ Standard Derivatives}\\\midrule
			$f(x)$ & $f'(x)$  &&Notes\\ \midrule
			$A$&0&&$A$ is constant\\
			$x$&$1$&&power rule for $x^1$\\
			$Ax$&$A$&&$A$ is a constant multiple\\ \midrule
			$x^n$ & $nx^{n-1}$ &&power rule general form\\
			$e^x$ & $e^x$  && exponential\\
			$\ln(x)$ & $\frac{1}{x}$ &&logarithmic\\ \midrule
			$\sin(x)$ & $\phantom{-}\cos(x)$  &&trigonometric \\
			$\cos(x)$ & $-\sin(x)$ && \\
			$\tan(x)$ & $\phantom{-}\sec^2(x)$ && \\ \bottomrule
		\end{tabular}
}
\begin{itemize}
	%%%%%%%%%%
	\item Product Rule: $[f(x)g(x)]' = f'(x)g(x) + f(x)g'(x)$
	%%%%%%%%%%
	\item Quotient Rule: $\displaystyle \left[\frac{f(x)}{g(x)}\right]' = \frac{f'(x)g(x) - f(x)g'(x)}{[g(x)]^2}$
	%%%%%%%%%%
	\item Chain Rule: $f[g(x)]' = f'[g(x)] \cdot g'(x)$
	%%%%%%%%%%
\clearpage
\thispagestyle{empty}
		\renewcommand{\arraystretch}{1.2}
		\begin{tabular}{llcl}
			\multicolumn{4}{c}{$\bullet$ Standard Integrals}\\\midrule
			$f(x)$  &  $\int f(x)dx$&&Notes \\ \midrule
			$1$&$x+C$&&constant\\
			$A$&$Ax+C$&&$A$ is constant\\\midrule
			$x^n, n \neq -1$ & $\displaystyle\frac{x^{n+1}}{n+1}+C$&&power rule general form\\
			$e^x$ & $e^x+C$&&exponential\\
			$\frac{1}{x}$ & $\ln|x|+C$&&special case: $x^{-1}$\\
			$\ln x$&$x\ln |x|-x+C$&&natural logarithm: $\log_e$\\\midrule
			$\sin(x)$ & $-\cos(x)+C$&& trigonometric \\
			$\cos(x)$ & $\sin(x)+C$\\
			$\tan (x)$&$\ln |\sec x|+C$\\
			$\sec^2(x)$ & $\tan(x)+C$ \\ \bottomrule
		\end{tabular}
	%%%%%%%%%%
	\item Integration By Parts: $\displaystyle \int fg' = fg-\int f'g \qquad \text{or} \qquad \int udv = uv - \int vdu$
	%%%%%%%%%%
	\item Area between the curve $f(x)$ and the $x$-axis: $\displaystyle A=\int_{a}^{b} f(x)dx$\\
	%%%%%%%%%%
	\item Area between two curves $f(x)$ and $g(x)$: $\displaystyle A=\int_{a}^{b} \left[f(x)-g(x)\right]dx$\\
	%%%%%%%%%%
	\item Volume of revolution about the $x$-axis:
	$\displaystyle V= \int_{a}^{b} \pi y^2 dx = \pi\int_{a}^{b}  [f(x)]^2 dx$
	\item Volume of revolution about the $y$-axis:
	$\displaystyle V= \int_{a}^{b} \pi x^2 dy = \pi\int_{a}^{b}  [f(y)]^2 dy$
\end{itemize}


\subsubsection{Cosine}
\noindent
Let $a$ be a constant.\\
By definitions of a Laplace transform and an improper integral,
\begin{equation*}
\Laplace{\cos{(at)}} = \lim\limits_{n\to\infty}{\int_{0}^{n}{\cos{(at)}e^{-st}\mathrm{d}t}}
\end{equation*}
\begin{equation*}
	= \frac{1}{s^2 + a^2}\lim\limits_{n\to\infty}{\left[e^{-st}\left(a\sin{(at)} - s\cos{(at)}\right)\right]_{0}^{n}}
\end{equation*}
\begin{equation*}
	 = \frac{1}{s^2 + a^2}\left(\lim\limits_{n\to\infty}{\left(e^{-sn}\left(a\sin{sn} - s\cos{(an)}\right)\right)} - \left(e^{-s\cdot 0}\left(a\sin{(a\cdot 0)} - s\cos{(a\cdot 0)}\right)\right)\right)
\end{equation*}
Both $\sin$ and $\cos$ have maximum values of 1 and minimum values of -1, so we can way that the left part of the expression has a maximum value of at most $a + s$. For positive $s$, the exponential dominates and the expression goes to 0 in the limit.
\begin{equation*}
	 = \frac{1}{s^2 + a^2}\left(0 + s\right) \text{, } s > 0
\end{equation*}
\begin{equation*}
	 = \frac{s}{s^2 + a^2} \text{, } s > 0
\end{equation*}
So,
\begin{equation*}
	\Laplace{\cos{(at)}} = \frac{s}{s^2 + a^2} \text{, } s > 0
\end{equation*}


\chapter{Multigrid Methods and Residual Neural Networks}
We are going to investigate the connections and difference between one of the most efficient solvers in numerical PDEs ``Multigrid Methods" and one of the most successful models ``Residual Neural Networks" especially for the one of the most updated ResNet \cite{he2016identity} for computer vision.

\section{Multigrid methods}\label{cs:mg}
\subsection{Multigrid methods for numerical PDEs}

Let us first briefly describe a geometric multigrid method used to
solve the following boundary value problem:
% \begin{equation}
%   \label{laplace}\left\{
%   \begin{array}{rccl}
%     -\Delta u &=& f & \mbox{ in } \Omega,\\
%     \frac{\partial u}{\partial n} &=& 0 & \mbox{ on } \partial\Omega,\\
%   \end{array}
%   \right.\quad \Omega=(0,1)^2.
% \end{equation}
\begin{equation}
\label{laplace}
-\Delta u = f,  \mbox{ in } \Omega,\quad
u =0  \mbox{ on } \partial\Omega,\quad
\Omega=(0,1)^2.
\end{equation}

\subsection{Multigrid methods as network}
\input{6DL/v-cycle}

We consider a continuous linear finite element discretization of
\eqref{laplace} on a nested sequence of grids of sizes $n_\ell\times
n_\ell$ with $n_{\ell-1}=(n_\ell-1)/2$, as shown in the left part of
Fig. \ref{fig:V-cycle} and the corresponding sequence of finite
element spaces:
$$
V_1\supset V_2\supset\cdots\supset V_J.
$$
The discretized system is
\begin{equation}
\label{laplace-h}
Au=f.
\end{equation}
In correspondence with the two-dimensional grid, here we treat $f\in \mathbb R^{n\times n}$.  We need to find $u\in \mathbb R^{n\times n}$ satisfying \eqref{laplace-h}.
Here,  $A:\mathbb R^{n\times n}\mapsto \mathbb R^{n\times n}$ is a tensor satisfying
\begin{equation}
\label{uniform-laplace}
(Au)_{i,j}=4u_{i,j}-u_{i+1,j}-u_{i-1,j}-u_{i,j+1}-u_{i,j-1},
\end{equation}
which holds for $1\le i,j \le n$ whereas a slightly different formula holds for
other $i, j$. Here we notice that, there exist a $3\times 3$ kernel as
\begin{equation}\label{eq:kernel-A}
K_A = \begin{pmatrix}
0 & -1 & 0 \\
-1 & 4 & -1 \\
0 & -1 & 0
\end{pmatrix},
\end{equation}
with
$$
Au = K_A \ast u.
$$
Where $\ast$ is the stander convolution operation with zero padding like \cite{goodfellow2017deep}.
We now briefly describe a two-level multigrid method by
a mixed use of the terminologies from deep learning \cite{goodfellow2017deep} and multigrid methods.

The first main ingredient in GMG is a smoother.  A commonly used smoother is a
damped Jacobi,  which can be written as $S:\mathbb R^{n\times n}\mapsto
\mathbb R^{n\times n}$ satisfying
\begin{equation}
\label{jacobi1}
(Sf)_{i,j}={\omega\over 4}f_{i,j}.
\end{equation}
If we apply the Jacobian iteration twice, then
\begin{equation}
\label{jacobi2}
(S^2f)_{i,j}
={1\over 4}\omega(2-\omega)f_{i,j}+{\omega^2\over 16}(f_{i+1,j}+f_{i-1,j}+f_{i,j+1}+f_{i,j-1}).
\end{equation}
Then we have
\begin{equation}\label{eq:kernel-S}
K_S = {\omega \over 4},
\end{equation}
and
\begin{equation}\label{eq:kernel-S2}
K_{S^2} = \begin{pmatrix}
0 & \frac{\omega^2}{16} & 0 \\
\frac{\omega^2}{16} & {\omega(2-\omega) \over 4} & \frac{\omega^2}{16}  \\
0 & \frac{\omega^2}{16}  & 0
\end{pmatrix}
\end{equation}
Another ingredient is the restriction matrix $R: \mathbb R^{n_1\times
	n_1}\mapsto \mathbb R^{n_2\times n_2}$, which can be obtained by two
different steps. First, we carry out a convolution operation $\tilde
R: \mathbb R^{n_1\times n_1}\mapsto \mathbb R^{n_1\times n_1}$:
\begin{equation}
\label{linear-restriction}
(\tilde Rr)_{i,j}=r_{i,j}+{1\over 2}(r_{i-1,j}+r_{i+1,j}+r_{i,j-1}+r_{i,j+1}+r_{i-1,j-1}+r_{i+1,j+1}),
\end{equation}
Then, we define
\begin{equation}
\label{linear-restriction-stride}
(Rr)_{i,j}=(\tilde R r)_{2i-1,2j-1}, \quad 1\le i, j \le n_2.
\end{equation}
Using the terminology from deep learning \cite{goodfellow2017deep},
we note that \eqref{linear-restriction} can be written as a
convolution with a $3\times3$ kernel:
\begin{equation}
\label{convolution}
\tilde Rr=K*r,\quad K=\left ( \begin{array}{ccc}
0 &\frac{1}{2}&\frac{1}{2}\\
\frac{1}{2}& 1&\frac{1}{2}\\
\frac{1}{2}&\frac{1}{2}& 0
\end{array}\right ).
\end{equation}
Similarly, \eqref{jacobi1} is a convolution with a $1\times1$ kernel,
\eqref{jacobi2} is a convolution with a $3\times 3$ kernel, and
\eqref{linear-restriction} and \eqref{linear-restriction-stride} is a
convolution with a stride.

In general, we have restriction $R_{\ell-1}^\ell:\mathbb
R^{n_{\ell-1}\times n_{\ell-1}}\mapsto R^{n_{\ell}\times n_{\ell}}$ on
level $\ell$, and the corresponding prolongation is given by
$P_\ell^{\ell-1}=[R_{\ell-1}^\ell]^T$.  This restriction gives rise
to a restriction of $A=A_1$ on the coarse mesh:
$A_\ell=R_{\ell-1}^\ell A_{\ell-1}P_\ell^{\ell-1}: \mathbb
R^{n_\ell\times n_\ell}\mapsto \mathbb R^{n_\ell\times n_\ell}$
($\ell=2:J$).

Using the smoother $S$ and restriction $R$ above, a two-level
multigrid method for solving \eqref{laplace-h} can be written as
$u\leftarrow u_0, r\leftarrow f-Au$:
\begin{equation}
\label{2level}
u\leftarrow u+ Sr, \quad r\leftarrow f-Au, \quad u\leftarrow u+ B_2Rr,
\quad r\leftarrow f-Au, \quad u\leftarrow u+S R^Tr.
\end{equation}
In a two-level setting, we can take $B_2=A_2^{-1}$, or we can apply
\eqref{2level} recursively to obtain a multilevel V-cycle.     More
specifically, given $u_0\in V$,  this multilevel V-cycle can be
described as follows

From the fine to the coarse level,
\begin{equation}\label{multi-backslash}
r_\ell = F_\ell (R^\ell_{\ell - 1} r_{\ell-1}), \quad
w_\ell =[(I+S_{\ell}(I - A_{\ell} )]R^{\ell}_{\ell-1} r_{\ell-1}\quad \ell = 2:J,
\end{equation}
where
\begin{equation}\label{eq:backslash}
F_\ell (x) = x - A_\ell S_\ell x, \quad \ell = 1:J-1
\end{equation}
with
$$
r_0=f-Au_0, r_1 = F_1(r_0),  F_J(x) = A_J^{-1}x.
$$
From the coarse to the fine level,
\begin{equation}\label{multi-slash}
v_{\ell-1} = G_{\ell-1} (P^{\ell-1}_{\ell} v_{\ell}), \quad \ell = J:2,
\end{equation}
where
\begin{equation}\label{eq:slash}
G_{\ell-1} (x) = w_{\ell-1} + x - S_\ell A_\ell x
\end{equation}
with
$$
v_J = r_J = F_J(R^J_{J-1} r_{J-1}).
$$
An example of a $4$-level ($J=4$) V-cycle is illustrated in
Fig. \ref{fig:V-cycle}.

In a more general setting, \eqref{laplace-h} can be a general linear
algebraic system, with a slight abuse of notation here, such that
$u\in \mathbb R^N$ and $A: \mathbb R^N\mapsto \mathbb R^N$.  (In the
uniform grid case $N=n^2$.)  In this case, the AMG method can still be
written as in \eqref{2level}, but the smoother and restriction
matrices must be designed to maximize the algorithm efficiency.  In
multigrid methods, we almost always choose the prolongation
$P=R^T$. Thus the design of restriction is the same as the design of
prolongation.  Most of the existing AMG literature focuses on
designing an appropriate prolongation for a given smoother such as the
Jacobi iterative method.  In this proposal, we advance the field by
also considering the design of an appropriate smoother for a given
restriction.

\section{Convolutional neural networks}\label{sc:cnn}
Convolutional neural network, first introduced by LeCun
\cite{lecun1998gradient}, has been successfully applied to many
problems.  One of the most successful applications is for image
classification
\cite{lecun1998gradient,krizhevsky2012imagenet,simonyan2014very,he2016deep}.
Here,  for simplicity of exposition, we introduce CNN for image
de-noising
\cite{NIPS2008_3506,bengio2013generalized,alain2014regularized,xie2012image,zhang2017beyond}.
We emphasize that our introduction may not follow the standard
notation or terminology used in the deep learning literature. Instead
we borrow some notation and terminology from multigrid methods
for our descriptions.

Given an image $x \in \mathbb{R}^{n\times n}$ stained with
noise (often assumed to be Gaussian), our goal is to use a deep neural
network to find a de-noised image such that $ y=f(x, \Theta) $.  The
function $f$, which depends on a set of parameters $\Theta$ and can be
viewed as a vector in $\mathbb R^{\mathcal N}$, will be given
in terms of a convolutional neural network  briefly described
next.  The parameter $\Theta$ can be obtained, for example, by
solving the following least squares problem:
\begin{equation}  \label{min-LSQ}
\min_{\Theta\in\mathbb R^{\mathcal N}}\mathcal L(\Theta), \quad
\mathcal L(\Theta)={1\over N}\sum_{i=1}^N \mathcal L_i(\Theta),
\quad \mathcal L_i(\Theta)=
\|f(x_i,\Theta)-\bar x_i)\|_{F}^2.
\end{equation}
Here, $x_i \in \mathbb{R}^{n\times n}$ ($i=1:N$), often referred to as training
data, are images obtained by adding noise to the corresponding
``clean'' images $\bar x_i$.

An image can be viewed as a function defined on a uniform grid.  We
assume that this uniform grid can be coarsened in a uniform fashion,
as shown in Fig. \ref{fig:V-cycle}.  As in multigrid methods, we
designate these grids as multilevel.  Every grid
represents a different level.  This is in contrast with the concept of
``layers'' in CNN in the deep learning literature
\cite{goodfellow2017deep}.  In Fig. \ref{fig:V-cycle}, the number of
levels is $J=4$. A CNN involves operations among different levels of
grids together with layers of nonlinear operations within each level
of grid.

A key ingredient of an artificial neural network is the
activation function, which is always nonlinear.  A commonly used
activation function is ReLU defined as
\begin{equation}
\label{ReLU}
\sigma(t)=\max(0,t).
\end{equation}

\subsection{Operations from level to level}\label{ssc:level}
Given an image $x$, we first consider operations from fine to coarse
levels as follows:
\begin{itemize}
	\item Classical CNN model
\begin{equation}\label{multi-encode-classic}
f^{\ell} = F^{\ell}(R^{\ell}_{\ell -1} f^{\ell-1}), \quad \ell=2:J,
\end{equation}
	\item ResNet model
\begin{equation}\label{multi-encode}
f^{\ell} = R^{\ell}_{\ell -1} f^{\ell-1} + F^{\ell}(R^{\ell}_{\ell -1} f^{\ell-1}), \quad \ell=2:J,
\end{equation}
\end{itemize}
with $f^1 = F^1(x)$, $R^{\ell}_{\ell-1}: \mathbb R^{\hat
	c_{\ell-1}\times n_{\ell-1}\times n_{\ell-1}}\mapsto \mathbb
R^{c_\ell \times n_{\ell}\times n_{\ell}}$ is a restriction often
given by convolution with stride (or by pooling), and $F^\ell: \mathbb
R^{c_\ell \times n_{\ell}\times n_{\ell}}\mapsto \mathbb R^{\hat
	c_\ell \times n_{\ell}\times n_{\ell}}$ consists of a sequence of
convolution and nonlinear activation operations. \eqref{multi-encode}
is often referred as an encoding process by convolution.

Operations from coarse to fine levels can be defined similarly:
\begin{itemize}
	\item Classical CNN model
	\begin{equation}\label{multi-decode-classic}
	g^{\ell-1} = G^{\ell-1}(P^{\ell-1}_{\ell}g^{\ell}), \quad \ell=J:2,
	\end{equation}
	\item ResNet model
	\begin{equation}\label{multi-decode}
	g^{\ell-1} =P^{\ell-1}_{\ell}g^{\ell} + G^{\ell-1}(P^{\ell-1}_{\ell}g^{\ell}), \quad \ell=J:2,
	\end{equation}
\end{itemize}
with $g^{J} = f^J$.  Here, $P^{\ell-1}_{\ell}: \mathbb R^{c_{\ell}
	\times n_{\ell}\times n_{\ell}}\mapsto R^{\hat n_{\ell-1}\times
	n_{\ell-1}\times n_{\ell-1}}$ is a prolongation matrix of the same
size as the transpose of $R^{\ell}_{\ell-1}$ in \eqref{multi-encode}.
$G^{\ell-1}$ consists of a sequence of de-convolution and nonlinear
activations.  Further,  \eqref{multi-decode} is often known as a decoding
process by de-convolution.

\subsection{Multilayer operations within a given  level}\label{ssc:layer}

\subsubsection{Classical CNN model.}
For general CNN case, the $F^\ell$ and $G^\ell$ are defined for general CNN structure as: $F^\ell (y^{\ell,0}) = y^{\ell, m_\ell}$ with
\begin{equation}\label{eq:gcnnblock}
y^{\ell, j} = \sigma \circ \theta^{\ell,j}(y^{\ell,j-1}),
\quad j = 1:m_\ell,
\end{equation}
where $y^{\ell,0} =R^{\ell}_{\ell-1} f^{\ell-1}$. We can have a similar formula for $G^j$ as $y^{\ell,0} =
P^{\ell}_{\ell+1} g^{\ell+1}$.
Here,  one key point is that $\theta^{\ell,j}$
are convolutional operations in the following form:
\begin{equation}\label{eq:conv-1}
[\theta^{\ell,j}(y)]_{i} = \sum_k^{c^{\ell,j-1}}K^{\ell, j}_{i,k} \ast y_k + b^{\ell,j}_i \bm{1} \in \mathbb{R}^{n_\ell \times n_\ell}, \quad i = 1:c_{\ell,j},
\end{equation}
where $\ast$ denotes the standard convolution operation (see
\eqref{linear-restriction} and \eqref{convolution} for example), and
$c^{\ell,j}$ denotes the number of channels or multiple smoothed
(coarse) spaces in the $j$-th layer in level $\ell$.

\subsubsection{New ResNet model.}
We now describe a special CNN model using a special form of ResNet
developed in \cite{he2016deep,he2016identity};  that is,  we need to
specify how $F^\ell$ and $G^\ell$ are defined.  Specifically, we can
define $F^\ell (y^{\ell,0}) = y^{\ell, m_\ell} -  y^{\ell, 0}$, with
$y^{\ell,0} =R^{\ell}_{\ell-1} f^{\ell-1}$ and,
\begin{equation}\label{eq:resblock}
y^{\ell, j} = y^{\ell, j-1} + \mathcal{F}^{\ell, j}(y^{\ell,j-1}),
\quad j = 1:m_\ell.
\end{equation}
Here
\begin{equation}\label{eq:basic}
\mathcal{F}^{\ell,j}(y) = \xi^{\ell,j} \circ \sigma \circ \eta^{\ell,j}(y),
\end{equation}
as $\xi^{\ell,j}$ and $\eta^{\ell,j}$ are both general convolutions
with multi-channels. There are two observations for the basic structure \eqref{eq:basic}:
\begin{itemize}
	\item $ \xi \circ \sigma \circ \eta$ have the universal approximation for any continuous functions in compact sets for general fully connected deep neural networks.
	\item In the V-cycle method, especially for \eqref{eq:backslash} it is exact the case for \eqref{multi-encode} with $j=1$ for \eqref{eq:resblock}.
\end{itemize}

The same as general CNN models, we can have a similar formula for $G^j$ as $y^{\ell,0} =
P^{\ell}_{\ell+1} g^{\ell+1}$. This shows that, for this kind of model, we do residual for between both levels and layers as:
\medskip
\begin{table}[!htbp]
	\centering
	\footnotesize
	\begin{tabular}{|c|c|c|}
		\hline
		&level to level & layer to layer  \\
		\hline
		Iterative Formula  & $f^{\ell} = R^{\ell}_{\ell -1} f^{\ell-1} + F^{\ell}(R^{\ell}_{\ell -1} f^{\ell-1})$ & $y^{\ell, j} = y^{\ell, j-1} + \mathcal{F}^{\ell, j}(y^{\ell,j-1})$\\
%		\hline
%		Residual Form	 & $ F^{\ell}$ &$y^{\ell,1} = \mathcal{F}^{\ell, 1}(y^{\ell,0})$ \\
%		\hline
%		feature extractors &smoother (designed a priori) & convolution (trained) \\
%		\hline
%		grid transfer & restriction (designed a priori) & convolution with stride  (trained) \newline or pooling (given)\\
		\hline
	\end{tabular}
	\caption{\footnotesize A brief comparison between levels and layers.}
	\label{Level-Layes}
\end{table}

%So, now all the parameter as collected as
%\begin{equation}\label{para-level}
%	\Theta = \{\theta^1, \cdots, \theta^J, \theta^J, \theta^1\},
%\end{equation}
%as
%\begin{equation}\label{para-layer}
%	\theta^\ell = \{\eta^{\ell,0}, \xi^{\ell,0}, \cdots, \eta^{\ell,m_\ell}, \xi^{\ell,m_\ell}\}.
%\end{equation}

In comparing \eqref{multi-encode}-\eqref{multi-decode} for CNN with
\eqref{multi-backslash}-\eqref{eq:slash} for MG and the convolutions
\eqref{eq:conv-1} for CNN with the smoother \eqref{convolution} for
MG, we note that the CNN model described above is strikingly similar
to the structure of the V-cycle multigrid method as illustrated by
Fig.~\ref{fig:V-cycle}.

In our terminology, the number of layers in deep learning literature
is associated with the total number of applications of the activation
and is equal to $\sum_{\ell=1}^Jm_\ell$.  Within a given level, the
number of layers can be compared with the number of smoothings used in
multigrid methods.

%CNN used for purpose of de-noising has been studied extensively in the
%literature, including in
%\cite{NIPS2008_3506,bengio2013generalized,alain2014regularized,xie2012image,zhang2017beyond}.
%We also have performed some experiments of this kind by adding
%Gaussian noise to a MNIST dataset and then using the aforementioned CNN
%procedure for de-noising.  The results are remarkably good as observed
%in many existing studies referenced above, and,  as shown in
%Fig.~\ref{noise-denoise} in which the bottom images are the de-noised
%images of the ones above.

\section{Connections between multigrid method and CNN}
We now offer a comparison between geometric multigrid methods (GMG)
and convolutional neural networks (CNN).  It is this comparison
between these two distinctive methodologies that motivates much of the
research described in this proposal.  A brief summary of our comparison is
given in Table~\ref{MG-CNN}.
\medskip
\begin{table}[!htbp]
	\centering
	\footnotesize
	\begin{tabular}{|l|l|p{40mm}|c|}
		\hline
		&Multigrid &CNN  \\
		\hline
		goals  & linear system solver & image denoising or classification \\
		\hline
		structure	 &multilevel grids with multiple smoothings &multilevel grids with multiple convolutions and activations \\
		\hline
		feature extractors &smoother (designed a priori) & convolution (trained) \\
		\hline
		grid transfer & restriction (designed a priori) & convolution with stride  (trained) \newline or pooling (given)\\
		\hline
	\end{tabular}
	\caption{\footnotesize A brief comparison between MG and CNN.}
	\label{MG-CNN}
\end{table}

First we note that MG and CNN are both based on a hierarchy level of
``grids'' although the notion of a ``grid'' is not often discussed in
the CNN literature.   MG is a linear algorithm for solving linear system of equations
whereas CNN is a  nonlinear model designed for image de-noising or
classification.  On each level, MG uses a given method such as Jacobi or
Gauss-Seidel as a smoother, whereas CNN uses multiple ``channels'' of
convolutions.    The similarity here is that a smoother such as the damped
Jacobi can be interpreted as a convolution of the size $1\times 1$ or
$3\times 3$.  The difference between the two methods on this point is
that in MG the smoother is designed a priori
and usually only one smoother is used on each level, whereas in CNN,
the convolution is trained and multiple channels of convolutions are
used.   From fine to coarse (or vice-versa), restriction  matrices are used in both
methods, but again the difference remains: the restriction in MG is
designed a priori,  whereas in CNN the restriction is trained (as a
convolution with stride) or sometimes is given a priori (by an
operation called ``pooling'').

\subsection{V-cycle and U-net}
If we compare the V-cycle and new ResNet model, we may mention formula,
U-net \cite{ronneberger2015u} structure likes the V-cycle very much. Here we will give more comparison on those two structures.
%Let us show the U-net model with ResNet structure in mathematically.
\subsubsection{Segmentation problem}
Here we introduce some basic background about segmentation problem.....
\subsubsection{Classical U-net model}
The U-net was proposed firstly in \cite{ronneberger2015u} based on the classical CNN model as introduced in \S \ref{sc:cnn}. A mathematical expression of this model is:
\begin{itemize}
	\item Encode phase
	\begin{equation}\label{U-multi-encode-classic}
	f^{\ell} = F^{\ell}(R^{\ell}_{\ell -1} f^{\ell-1}), \quad \ell=2:J,
	\end{equation}
	for $f^1 = F^1(x)$ and
	$F^\ell (y^{\ell,0}) = y^{\ell, m_\ell}$ with
	\begin{equation}\label{u-eq:gcnnblock}
	y^{\ell, j} = \sigma \circ \theta^{\ell,j}(y^{\ell,j-1}),
	\quad j = 1:m_\ell,
	\end{equation}
	where $y^{\ell,0} =R^{\ell}_{\ell-1} f^{\ell-1}$.

	\item Decode phase
	\begin{equation}\label{U-multi-decode-classic}
	g^{\ell-1} = G^{\ell-1}([P^{\ell-1}_{\ell}g^{\ell},C^{\ell-1}f^{\ell-1}]), \quad \ell=J:2,
	\end{equation}
	for $g^{J} = f^J$ and $G^\ell (y^{\ell,0}) = y^{\ell, m_\ell}$ with
	\begin{equation}\label{eq:gcnnblock}
	y^{\ell, j} = \sigma \circ \theta^{\ell,j}(y^{\ell,j-1}),
	\quad j = 1:m_\ell,
	\end{equation}
	where $y^{\ell,0} = [P^{\ell-1}_{\ell}g^{\ell},C^{\ell-1}f^{\ell-1}]$. Here $[P^{\ell-1}_{\ell}g^{\ell},C^{\ell-1}f^{\ell-1}]$ means to collect the prolongated result $P^{\ell-1}_{\ell}g^{\ell}$ and the rescale cut for $f^{\ell-1}$ (the dimension reduction for segmentation results) as a new image with double size channels(One comes form $f$ one comes form $g$).
\end{itemize}

\subsubsection{Connections between V-cycle and U-net}
\begin{itemize}
	\item Structure of coarse and prolongation.
	\item Basic structure in universal approximation theory.
	\item PDE-based model expression.
\end{itemize}

\subsubsection{New U-net model motivated by V-cycle}
Now we proposed a new U-net model as:
\begin{itemize}
	\item Encode phase
	\begin{equation}\label{U-multi-encode}
	f^{\ell} = R^{\ell}_{\ell -1} f^{\ell-1} + F^{\ell}(R^{\ell}_{\ell -1} f^{\ell-1}), \quad \ell=2:J,
	\end{equation}
	for $f^1 = F^1(x)$.
	$F^\ell (y^{\ell,0}) = y^{\ell, m_\ell} -  y^{\ell, 0}$, with
	$y^{\ell,0} =R^{\ell}_{\ell-1} f^{\ell-1}$ and,
	\begin{equation}\label{u-eq:resblock}
	y^{\ell, j} = y^{\ell, j-1} + \mathcal{F}^{\ell, j}(y^{\ell,j-1}),
	\quad j = 1:m_\ell,
	\end{equation}
	for
	\begin{equation}\label{u-eq:basic-f}
	\mathcal{F}^{\ell,j}(y) = \xi^{\ell,j} \circ \sigma \circ \eta^{\ell,j}(y).
	\end{equation}

	\item Decode phase
	\begin{equation}\label{U-multi-decode}
	g^{\ell-1} = C^{\ell-1}f^{\ell-1} + P^{\ell-1}_{\ell}g^{\ell} + G^{\ell-1}(P^{\ell-1}_{\ell}g^{\ell}), \quad \ell=J:2,
	\end{equation}
	for $g^{J} = f^J$.
	$G^{\ell-1} (y^{\ell-1,0}) = y^{\ell-1, m_{\ell-1}} - y^{\ell-1,0}$ with
	$y^{\ell-1,0} = P^{\ell-1}_{\ell}g^{\ell}$
	\begin{equation}\label{u-eq:rcnnblock}
	y^{\ell-1, j} = y^{\ell-1, j-1} + \mathcal{G}^{\ell-1, j}(y^{\ell,j-1}),
	\quad j = 1:m_{\ell-1},
	\end{equation}
	for
	\begin{equation}\label{u-eq:basic-g}
	\mathcal{G}^{\ell-1,j}(y) = \phi^{\ell-1,j} \circ \sigma \circ \psi^{\ell-1,j}(y).
	\end{equation}
\end{itemize}

\section{Some general CNN structure motivated by V-cycle}
Here we will talk about more observations for V-cycle method writing as a ResNet model.


\section{3-D DP}

%%%%%%%%%%
\begin{frame}{3-D DP}
  \begin{exampleblock}{Floyd-Warshall algorithm}
    \begin{itemize}
      \item subproblem $\text{dist}[i,j,k]$: the length of the shortest path from $i$ to $j$ via only nodes $v_{1} \cdots v_{k}$
      \item goal: $\text{dist}[i,j,n], \forall i,j$
      \item question: Is $v_{k}$ in $\text{ShortestPath}[i,j,k]$?
      \item recurrence:
	\fignocaption{width = 0.40\textwidth}{fig/floyd-warshall.png}
	\[
	  \text{dist}[i,j,k] = \min \set{\text{dist}[i,j,k-1], \text{dist}[i,k,k-1] + \text{dist}[k,j,k-1]}
	\]
    \end{itemize}
  \end{exampleblock}
\end{frame}


\section{Approximation Error}
Although knowing these infinite Taylor series is nice, if we want to use them to find function values, we'll need to approximate.
This means only using some finite number of terms in the Taylor series for our approximation.
Because we're not using all the terms, we'll naturally have some truncation error.
We'd like to be able to say how big this truncation error is so we can be sure that our approximation is good enough.

\subsection{Alternating Series Estimation Theorem}
\begin{theorem}[Alternating Series Estimation Theorem]
	Let $s$ be a convergent, alternating (i.e the sign of each term alternates) series where the terms of $\abs{s}$ are strictly decreasing.
	Then the $n$th term truncation error is the same sign as and less than in absolute value the $(n+1)$th term.
\end{theorem}

\begin{example}
	Give a bound for the truncation error of using the first 10 terms of the Maclaurin series of $\ln{(1+x)}$ to approximate $\ln{2}$.
\end{example}
\begin{answer}
	The Maclaurin series for $\ln{(1+x)}$ is
	\begin{equation*}
		\ln{(1+x)} = x - \frac{x^2}{2} + \frac{x^3}{3} - \ldots = \sum_{k=0}^{\infty}{(-1)^k\frac{x^{k+1}}{k+1}}, \abs{x} \leq 1,
	\end{equation*}

	which is an alternating series.
	We see that to approximate $\ln{2}$, we'd use $x=1$, which is in the interval of convergence, so the series converges.
	The first missing term of the series is $1/11$.
	Let $s_{10}$ be the partial sum of the first 10 terms when $x=1$.
	By the Alternating Series Estimation Theorem,
	\begin{equation*}
		0 < \ln{2} - s_{10} < \frac{1}{11}.
	\end{equation*}
\end{answer}

\subsection{Taylor's Theorem}
Although the Alternating Series Estimation Theorem is useful for quickly bounding the error of alternating series, we'd like something that can apply more generally to all Taylor series.
\begin{theorem}[Taylor's Theorem]
	Let $f$ be a $k+1$ times differentiable function on an open interval $I$ containing $a$.
	Then for all $x$ in $I$,
	\begin{equation*}
		f(x) = f(a) + f^\prime(a)(x-a) + \frac{f^{\prime\prime}(a)}{2!}(x-a)^2 + \ldots + \frac{f^{(n)}(a)}{n!}(x-a)^n + R_n(x).
	\end{equation*}
	where
	\begin{equation*}
		\abs{R_n(x)} = \frac{\abs{f^{(n+1)}(c)}}{(n+1)!}\abs{x-a}^{n+1}
	\end{equation*}
	for some $c$ between $x$ and $a$.
\end{theorem}

We can use this theorem to find the maximum value of $\abs{R_n(x)}$ over some interval.
We can also be more precise about what it means for a Taylor series to converge to some function over some interval.
\begin{definition}
	Let $R_n(x)$ be the remainder of truncating after the degree $n$ term in the Taylor series for $f$ centered at $x=a$.
	If for all $x$ in some interval $I$ containing $a$,
	\begin{equation*}
		\lim_{n\to\infty}{R_n(x)} = 0,
	\end{equation*}
	then we say the Taylor series for $f$ at $x=a$ converges to $f$ on $I$.
\end{definition}

\begin{example}
	Show that the Maclaurin series for $\sin{x}$ converges to $\sin{x}$ for all real $x$.
\end{example}
\begin{answer}
	We need to find the remainder and show that in the limit it goes to 0 as $n$ grows large.
	By Taylor's Theorem,
	\begin{align*}
		\abs{R_n(x)} &= \frac{\abs{f^{(n+1)}(c)}}{(n+1)!}\abs{x-a}^{n+1} \\
		&\leq \frac{\abs{x}^{n+1}}{(n+1)!}.
	\end{align*}

	The numerator is an exponential function, while the denominator is a factorial function, so using $n^n$FEPL,
	\begin{align*}
		0 \leq \lim_{n\to\infty}{R_n(x)} &\leq \lim_{n\to\infty}{\frac{\abs{x}^{n+1}}{(n+1)!}} = 0. \\
		\lim_{n\to\infty}{R_n(x)} &= 0.
	\end{align*}

	So, the Maclaurin series converges to $\sin{x}$ for all real $x$.
\end{answer}

\subsubsection{Remainder Estimation Theorem}
Notice that we didn't have to actually find the value of $f^{(n+1)}(c)$.
We just had to find a suitable upper bound where the limit would still go to 0.
\begin{theorem}[Remainder Estimation Theorem]
	If there are positive constants $M$ and $r$ such that
	\begin{equation*}
		\abs{f^{(n+1)}(t)} \leq Mr^{(n+1)}
	\end{equation*}
	for all $t$ between $a$ and $x$, then $R_n(x)$ satisfies the inequality
	\begin{equation*}
		\abs{R_n(x)} \leq M\frac{r^{n+1}\abs{x-a}^{n+1}}{(n+1)!}.
	\end{equation*}
\end{theorem}

\begin{example}
	Give a maximum error bound for using $\ln{(1+x)} = x - x^2/2$ when $\abs{x} \leq 0.1$.
\end{example}
\begin{answer}
	We are using the second-order Taylor polynomial, so we need to find $R_2(x)$.
	By Taylor's Theorem,
	\begin{align*}
		\abs{R_2(x)} &= \frac{\abs{f^{(2+1)}(c)}}{(2+1)!}\abs{x-0}^{2+1}  = \frac{\abs{f^{(3)}(c)}}{3!}\abs{x}^3\\
		f^{(3)}(x) &= \dd{^3}{x^3}\ln{(1+x)} = \frac{2}{(1+x)^3} \\
	\end{align*}

	On $-0.1 \leq x \leq 0.1$, $\abs{f^{(3)}(x)}$ is maximal at $(-0.1, \frac{2000}{729})$.
	\begin{equation*}
		\abs{R_2(x)} \leq \frac{\frac{2000}{729}}{6}\abs{x}^3.
	\end{equation*}

	On $-0.1 \leq x \leq 0.1$, $\abs{x}^3$ is maximal at $(-0.1, \frac{1}{1000})$.
	\begin{equation*}
		\abs{R_2(x)} \leq \frac{\frac{2000}{729}}{6}\frac{1}{1000} = \frac{1}{2187} \approx 4.572 \times 10^{-4}.
	\end{equation*}

	So, the error of approximating $\ln{(1+x)}$ with $x-x^2/2$ when $-0.1 \leq x \leq 0.1$ is at most $4.572 \times 10^{-4}$.
\end{answer}


\subsection{Curvature}
Curvature is 1 divided by the radius of the circle that best approximates the curve at a point.
Tighter turns have smaller radii and higher curvature.
We can use $\hat{T}$ to find the curvature at a point on $\vec{r}(t)$.
\begin{equation*}
	\kappa(t) = \norm{\dd{\hat{T}}{s}} = \norm{\dd{\hat{T}}{t}\left(\dd{s}{t}\right)^{-1}} = \norm{\dd{\hat{T}}{t}}\frac{1}{v(t)}
\end{equation*}

\noindent
For example, let's find $\kappa(t)$ for the circle in the yz-plane: $\vec{r}(t)=\langle 7, R\sin{t}, R\cos{t} \rangle$.
\begin{align*}
	\vec{r^\prime}(t) &= \langle 0, R\cos{t}, -R\sin{t}\rangle
	\text{ and }
	v(t) = \sqrt{0^2+(R\cos{t})^2+(-R\sin{t})^2} = R \\
	\hat{T}(t) &= \frac{1}{R}\vec{r}(t) = \langle0,\cos{t},-\sin{t}\rangle \\
	\frac{\mathrm{d}\hat{T}}{\mathrm{d}t} &= \langle 0,\sin{t},-\cos{t}\rangle \\
	\norm{\frac{\mathrm{d}\hat{T}}{\mathrm{d}t}} &= 1 \\
	\kappa(t) &= \frac{1}{R}.
\end{align*}
This relationship is true for all circles.


\subsection{The pseudo-inverse}

The definition of the inverse  is that:

\(MM^{-1}=I\)

\(M^{-1}M=I\)

We also have:

\(MM^{-1}M=M\)

\(M^{-1}MM^{-1}=M^{-1}\)

\subsubsection{The inverse of a homomorphism}

Generally we don't have inverses of homomorphisms as the number of dimensions are different.

We can, however, find a matrix \(M^+\) which satisfies:

\(MM^+M=M\)

\(M^+MM^+=M^+\)

This is the pseudo-inverse.


\subsection{The Gradient \& C-Level Curves}
\noindent
Let $\vec{r}(t)$ be the C-level curve of $f(x, y)$.
\begin{align*}
f\circ\vec{r} &= C  \text{ and } \frac{\mathrm{d}}{\mathrm{d}t}(f\circ\vec{r}) = 0 \\
	&\implies \frac{\mathrm{d}}{\mathrm{d}t}(f\circ\vec{r}) = \nabla f\cdot\vec{r^\prime}(t) = 0 \\
	&\implies \nabla f\perp\vec{r^\prime}(t) \\
	&\implies \nabla f \text{ is perpendicular to the C-level curve of } f.
\end{align*}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.33\textwidth]{./Images/differentialMultivariableCalculus/grad_c_level.png}
	\caption{The gradient is perpendicular to C-level curves.}
\end{figure}


\subsection{Osculating Plane/Circle \& B-Hat $\left(\hat{B}\right)$}
\noindent
The osculating plane in the plane containing $\vec{r}(t)$, $\hat{T}$ and $\hat{N}$. It is only defined when $\hat{N}\neq 0$. This means straight lines do not have an osculating plane.
The osculating circle lives in the osculating plane, is centered at $\vec{r}(t) + \frac{\hat{N}(t)}{\kappa(t)}$, and has radius $\frac{1}{\kappa(t)}$.
The tangent line at $\vec{r}(t)$ is also tangent to the osculating circle because both points have the same curvature.

\begin{figure}[H]
	\label{osculating_plane_circle}
	\centering
	\includegraphics[width = 0.33\textwidth]{./vectorValuedFunctions/osculatingCircle.png}
	\includegraphics[width = 0.33\textwidth]{./vectorValuedFunctions/osculatingPlane.png}
	\caption{Osculating circle and plane}
\end{figure}

\noindent
The vector that is normal to the plane is
\begin{equation*}
	\hat{B}(t) = \hat{T}\times\hat{N},
\end{equation*}
which is called the ``binormal'' vector because it is perpendicular to both $\hat{T}$ and $\hat{N}$.
Together, $\hat{T}$, $\hat{N}$, and $\hat{B}$ form the Frenet Serret Frame, also called the TNB frame.

\begin{figure}[H]
	\label{osculating_plane_circle}
	\centering
	\includegraphics[width = 0.33\textwidth]{./vectorValuedFunctions/TNB1.png}
	\includegraphics[width = 0.5\textwidth]{./vectorValuedFunctions/TNB2.png}
	\caption{TNB frame}
\end{figure}

\noindent
We can write the equation for the osculating plane as
\begin{equation*}
	\hat{B}(t)\cdot\left(\langle x,y,z\rangle - \vec{r}(t)\right) = 0.
\end{equation*}


\documentclass{article}
\usepackage{amsmath}

\newcommand{\flow}{\rightleftharpoons}

\begin{document}
\title{Typesetting Mathematics in \LaTeX}
\author{Dr. Milaan Parmar \\ milaanparmar9@gmail.com}
\maketitle

\section{Introduction}
\LaTeX\ is extremely powerful when it comes to typesetting mathematics. It's one of the core strengths of this system.

\section{Displaying Mathematics}
There are two ways of displaying maths. One is \emph{inline} and the other is \emph{display} format -- in which the whole math sits on its own set of lines.

\subsection{Inline Mode}
We are going to insert a mathematics equation inline here using a pair of \$ signs:  $E=mc^2$    . As you can see, the display (such as line spacing) does not get messed up by the mathematics as it does with word processing softwares.

\subsection{Display Mode}
We can also display equations in their own set of lines. To do this, we can use the equation environment.

\begin{equation}\label{eq:emc}
E=mc^2
\end{equation}

As you can see, \LaTeX\ inserts the equation number automatically. We can refer to it using the \verb|\ref| command just as we referred to sections, figures and tables. (E.g. Equation~\ref{eq:emc}.) To get rid of the equation number, simply use the \emph{star variant} of the equation environment. (For this, you need the \texttt{amsmath} package.)

\begin{equation*}
E=mc^2
\end{equation*}

Alternatively, we can use the shorthand keys \verb|\[| and \verb|\]|

\[
E=mc^2
\]

\section{Mathematical Features}
\LaTeX\ has many builtin features and you can get many more easily. Here, we'll see some of these features:

Addition, subtraction, multiplication and division:

\[
x+2 - 25 \times 35 \div 98
\]

Superscripts and subscripts:

\[ x^2  \]
\[ x_{(i)} \]

Summation, union, intersection, big-union, integral:

\[ \sum_{i=1}^{n}{i^2} \]
\[ x \cup y \cap z \]
\[ \bigcup_{i=1}^{n}{x_i} \]
\[ \int_0^n{x^2} \]

Fractions, brackets, square root:

\[ \frac{x}{y} \]
\[ \frac{\sum_i{x^2}}{\int_0^n{x^2}} \]
\[ \sqrt{\frac{\sqrt{36}} {x^5}} \]

\[ 2 \times \left( \frac{34}{\frac{124}{356}}    \right)  \]

Greek letters:

\[
\alpha + \beta + \gamma^* + \Sigma + \Theta + 2_\epsilon
\]

Matrices and vectors. For this, you need to include the \texttt{amsmath} package and then use the \texttt{bmatrix} or \texttt{pmatrix} environment:

\[
\begin{pmatrix}
\frac{a}{44} & b \\
c & \sqrt{d}
\end{pmatrix}
\]

Accents:

\[ \hat{x} \]
\[ \hat{\imath} \]
\[ \dot{x} \]

See the \texttt{Math} menu in the IDE for other operations. You can refer to ``Short Math Guide for \LaTeX'' for a lot more examples.

\section{Using Symbols}
You might come across situations where you need to find new symbols. For this, you can refer to the ``The Comprehensive \LaTeX Symbols List''.

\[ x \rightleftharpoons  y \]

(Optional) Since this is a long command, we might want to create a shortcut using the \verb|\newcommand| command in the preamble. This also allows us to later change the symbol without having to change the equations.

\[ x \flow y \]

\end{document}


\chapter{Search}

\section{Binary Search}
\runinhead{Variants:}
\begin{enumerate}
\item get the idx equal or just lower (floor)
\item get the idx equal or just higher (ceil)
\item \pyinline{bisect_left}
\item \pyinline{bisect_right}
\end{enumerate}
\subsection{idx equal or just lower}
Binary search, get the idx of the element equal to or just lower than the target. The returned idx is the $A_{idx} \leq target$. It is possible to return $-1$. It is different from the \pyinline{bisect_lect}.

\runinhead{Core clues:}
\begin{enumerate}
\item To get ``equal'', \pyinline{return mid}.
\item To get ``just lower'', \pyinline{return lo-1}.
\end{enumerate}
$A_{idx} \leq target$.
\begin{python}
def bin_search(self, A, t, lo=0, hi=None):
    if hi is None: hi = len(A)

    while lo < hi:
        mid = (lo+hi)/2
        if A[mid] == t:  return mid
        elif A[mid] < t: lo = mid+1
        else:            hi = mid

    return lo-1
\end{python}
\subsection{idx equal or just higher}
$A_{idx} \geq target$.
\begin{python}
def bin_search(self, A, t, lo=0, hi=None):
    if hi is None: hi = len(A)

    while lo < hi:
        mid = (lo+hi)/2
        if A[mid] == t:  return mid
        elif A[mid] < t: lo = mid+1
        else:            hi = mid

    return lo
\end{python}
\subsection{bisect\_left}
Return the index where to insert item x in list A. So if t already appears in the list,
A.insert(t) will insert just before the \textit{leftmost} t already there.
\runinhead{Core clues:}
\begin{enumerate}
\item Move \pyinline{lo} if $A_{mid} < t$
\item Move \pyinline{hi} if $A_{mid} \geq t$
\end{enumerate}

\begin{python}
def bisect_left(A, t, lo=0, hi=None):
    if hi is None: hi = len(A)

    while lo < hi:
        mid = (lo+hi)/2
        if A[mid] < t: lo = mid+1
        else:          hi = mid

    return lo
\end{python}

\subsection{bisect\_right}
Return the index where to insert item x in list A. So if t already appears in the list, A.insert(t) will insert just after the \textit{rightmost} x already there.
\runinhead{Core clues:}
\begin{enumerate}
\item Move \pyinline{lo} if $A_{mid} \leq t$
\item Move \pyinline{hi} if $A_{mid} > t$
\end{enumerate}
\begin{python}
def bisect_right(A, t, lo=0, hi=None):
    if hi is None: hi = len(A)

    while lo < hi:
        mid = (lo+hi)/2
        if A[mid] <= t: lo = mid+1
        else:           hi = mid

    return lo
\end{python}
\section{Applications}
\subsection{Rotation}
\runinhead{Find Minimum in Rotated Sorted Array.} Three cases to consider:
\begin{enumerate}
\item Monotonous
\item Trough
\item Peak
\end{enumerate}

If the elements can be duplicated, need to detect and skip.
\begin{python}
def findMin(self, A):
    lo = 0
    hi = len(A)
    mini = sys.maxint
    while lo < hi:
        mid = (lo+hi)/2
        mini = min(mini, A[mid])
        if A[lo] == A[mid]:  # JUMP
            lo += 1
        elif A[lo] < A[mid] <= A[hi-1]:
            return min(mini, A[lo])
        elif A[lo] > A[mid] <= A[hi-1]:  # trough
            hi = mid
        else:  # peak
            lo = mid+1

    return mini
\end{python}
\section{Combinations}
\subsection{Extreme-value problems}\label{extremeValueProblem}
\runinhead{Longest increasing subsequence.} Array $A$.

Clues:
\begin{enumerate}
\item \pyinline{MIN}: \textit{min} of index \textit{last} value of LIS of a particular \textit{len}.
\item \pyinline{RET}: result table, store the $\pi$'s idx (predecessor); (optional, to build the LIS, no need if only needs to return the length of LIS)
\item \pyinline{bin_search}: For each currently scanning index \pyinline{i}, if it smaller (i.e. $\neg$ increasing), to maintain the \pyinline{MIN}, binary search to find the position to update the min value. The \pyinline{bin_search} need to find the element $\geq$ to \pyinline{A[i]}.
\end{enumerate}
\newpage
\begin{python}
def LIS(self, A):
    n = len(A)
    MIN = [-1 for _ in xrange(n+1)]

    l = 1
    for i in xrange(1, n):
        if A[i] > A[MIN[l]]:
            l += 1
            MIN[l] = i
        else:
            j = self.bin_search(MIN, A, A[i], 1, l+1)
            MIN[j] = i

    return l
\end{python}
If need to return the LIS itself.
\begin{python}
    for i in xrange(1, n):
        if A[i] > A[MIN[l]]:
            l += 1
            MIN[l] = i

            RET[i] = MIN[l-1]  # (RET)
        else:
            j = self.bin_search(MIN, A, A[i], 1, l+1)
            MIN[j] = i

            RET[i] = MIN[j-1] if j-1 >= 1 else -1  # (RET)

    # build the LIS (RET)
    cur = MIN[l]
    ret = []
    while True:
        ret.append(A[cur])
        if RET[cur] == -1: break
        cur = RET[cur]

    ret = ret[::-1]
    print ret
\end{python}

\section{High dimensional search}
\subsection{2D}
\runinhead{2D search matrix I.} $m\times n$ mat. Integers in each row are sorted from left to right. The first integer of each row is greater than the last integer of the previous row.
$$
\begin{bmatrix}
1 & 3 & 5 & 7 \\
10 & 11 & 16 & 20 \\
23 & 30 & 34 & 50 \\
\end{bmatrix}
$$

Row column search: starting at top right corner: $O(m+n)$.

Binary search: search rows and then search columns: $O(\log m + \log n)$.

\runinhead{2D search matrix II.} $m\times n$ mat. Integers in each row are sorted from
left to right. Integers in each column are sorted in ascending from top to bottom.
$$
\begin{bmatrix}
1&   4&  7& 11& 15 \\
2&   5&  8& 12& 19 \\
3&   6&  9& 16& 22 \\
10& 13& 14& 17& 24 \\
18& 21& 23& 26& 30 \\
\end{bmatrix}
$$

Row column search: starting at top right corner: $O(m+n)$.

Binary search: search rows and then search columns, but upper bound row and lower bound row:

$$O\big(\min(n\log m, m\log n)\big)$$


\documentclass[]{simple}

\title{Sum of the Natural Numbers and the Riemann-Zeta Function}
\date{January, 2016}
\author{Matt McCarthy\\\href{mailto:matthew.mccarthy.12@cnu.edu}{matthew.mccarthy.12@cnu.edu}}

\addbibresource{naturals-and-zeta.bib}

\begin{document}

\maketitle

\section{Introduction}

We begin by introducing a rather interesting result, namely that the sum of the natural numbers is $-1/12$.
\begin{thm}\label{wrong}
	\[
		\sum_{n=1}^\infty n = -\frac{1}{12}
	\]
\end{thm}
We now provide the canonical proof as shown by \cite{wrong-pf}.
\begin{proof}[Proof (\autoref{wrong})]
	Define $s:=1+2+3+4+\ldots$, $s_1:=\sum_{n=0}^\infty(-1)^n$, and $s_2:=1-2+3-4+\ldots$.
	If we consider $s_1$, when we stop after an odd number of terms, the partial sum is $1$ but when we stop after an even number of terms, the partial sum is zero.
	Obviously, the series converges to the average of the two, thus $s_1=1/2$.
	Consider $2s_2$.
	\[
	\begin{array}{rrrrrrr}
		 &1 & -2 &+3 &-4&+\ldots\\
		+&  &1   &-2 &+3&-4&+\ldots\\\hline
		 &1 &-1  &+1 &-1&+\ldots
	\end{array}
	\]
	Since $2s_2=s_1=1/2$, $s_2=1/4$.
	Consider $s-s_2$.
	\[
		\begin{array}{rrrrrr}
			&1&+2&+3&+4&+\ldots\\
			-&1&-2&+3&-4&+\ldots\\\hline
			&0&+4&+0&+8&+\ldots
		\end{array}
	\]
	Moreover, we can factor out the four to get $s-s_2=4[1+2+3+\ldots]=4s$
	Solving for $s$ then yields $s=-1/12$ and completes the proof.
\end{proof}
\begin{analysis}
	The trick to this proof lies in exploiting the commutativity of addition.
	By doing so, we can manipulate the subtraction of two infinite series into a single alternating series.
	After that step, we then use the Taylor series we found in the lemma to complete the proof.
\end{analysis}

Now that we have ``shown'' that the natural numbers sum up to $-1/12$ the only things left to do are show that it is wrong and how this incorrect result came to be so prolific.

\section{Refutation}

We begin refuting this result by stating two of the limit laws and by providing the definition of an infinite series.
\begin{thm}[Limit Laws]
	Let $\set{x_n}_{n=1}^\infty$ and $\set{y_n}_{n=1}^\infty$ be real-valued sequences that converge to $x$ and $y$ respectively.
	Then all of the following hold.
	\begin{enumerate}
		\item The sequence $\set{x_n+y_n}_{n=1}^\infty$ converges to $x+y$.
		\item Let $k\in\RR$ then $\set{k\cdot x_n}_{n=1}^\infty$ converges to $k\cdot x$.
	\end{enumerate}
\end{thm}
These laws should seem familiar, considering that we used both in our ``proof'' of \autoref{wrong} however we may have invoked them without respecting their hypotheses.

\begin{definition}[Infinite Series]
	Let $\set{x_k}_{k=1}^\infty$ be a sequence of real numbers and let $x\in\RR$.
	An \textit{infinite series} is an expression of the form
	\[
		\sum_{k=1}^\infty x_k=x_1+x_2+\ldots.
	\]
	The corresponding \textit{sequence of partial sums} is defined by
	\[
		s_n:=\sum_{k=1}^n x_k.
	\]
	The series, $\sum_{k=1}^\infty x_k$ \textit{converges} to $x$ if and only if the sequence $\set{s_n}_{n=1}^\infty$ converges to $x$.
\end{definition}
This definition tells us that an infinite sum exists if and only if the its corresponding sequence of partial sums converges, which leads us to our biggest error: assuming the sum exists.

To determine whether or not the sum exists, we need to consider the sequence of partial sums,
\[
	s_n:=\sum_{k=1}^n k.
\]
If we inspect the sequence long enough, we see that the sum only increases and is what we call a \textit{monotone increasing sequence}.
\begin{definition}[Monotone Increasing Sequence]
	Let $\set{x_n}_{n=1}^\infty$ be a real-valued sequence.
	We say $\set{x_n}$ is \textit{monotone increasing} if and only if $x_n\leq x_{n+1}$ for each $n\in\NN$.
\end{definition}
If we consider the $(n+1)$ partial sum, we see
\[
	s_{n+1}=\sum_{k=1}^{n+1} k = \paren{\sum_{k=1}^{n} k} + (n+1) \geq \sum_{k=1}^{n} k=s_n.
\]
Thus, $\set{s_n}$ is a monotone increasing sequence.

Since $s_n$ is monotone increasing, we have a very nice theorem that turns our limit problem into a boundedness problem.

\begin{definition}[Upper Bound]
	Let $\set{x_n}_{n=1}^\infty$ be a real valued sequence and let $b\in\RR$.
	We say $b$ is an \textit{upper bound} of $\set{x_n}$ if and only if $x_n\leq b$ for each $n\in\NN$.
	If any such $b$ exists, we say $\set{x_n}$ is \textit{bounded above}.
\end{definition}

\begin{thm}[Monotone Convergence Theorem]
	Let $\set{x_n}_{n=1}^\infty$ be a real-valued monotone increasing sequence. Then $\set{x_n}$ converges if and only if $\set{x_n}$ is bounded above.
\end{thm}

With this theorem in our arsenal, we will now show that this infamous sum does not converge.

\begin{proposition}
	The sequence defined by
	\[
		s_n:=\sum_{k=1}^n k
	\]
	is unbounded.
\end{proposition}
\begin{proof}
Assume that $s_n$ is bounded above.
Ergo there exists a natural number $N > 0$ such that $s_n:=\sum_{k=1}^n k \leq N$ for every $n\in\NN$.
We know that
\[
	s_n:=\sum_{k=1}^n k = \frac{n(n+1)}{2}.
\]
Consider $s_{2N}$.
\[
	s_{2N} = \frac{2N(2N+1)}{2}=N(2N+1)=2N^2+N>N
\]
The previous statement contradicts our assumption and thus $s_n$ is an unbounded sequence.
\end{proof}
\begin{analysis}
	We perform a standard contradiction proof.
	The only tricks we use are the fact that
	\[
		1+2+\ldots+n=\frac{n(n+1)}{2}
	\]
	and a careful selection of $n$.
	Essentially, the proof boils down to picking a candidate for a bound and showing that we have a term in the sequence that is greater than our bound.
\end{analysis}

Since $s_n$ is unbounded, by the monotone convergence theorem, $s_n$ does not converge and neither does our sum.
Thus, we cannot invoke limit laws in order to compute $-4s$ and $s-4s$ in our proof of \autoref{wrong}.
In turn, this invalidates our derivation of $-3s$ which destroys the rest of the proof.

\section{The Riemann-Zeta Function}

To figure out why people believe this nonsense, we turn to the Riemann-Zeta function.
Historically, the Riemann-Zeta function arose as a way to extend the Euler-Zeta function, defined on real numbers greater than 1
\[
	\zeta(s)=\sum_{n=1}^\infty \frac{1}{n^s}
\]
to the rest of the complex plane \cite{edwards}.
When viewed in this light, one may think that
\[
	\zeta(-1)=\sum_{n=1}^\infty n.
\]
However, this is not the case since the Riemann-Zeta function is defined as follows.
\begin{definition}[Riemann-Zeta Function from \cite{edwards}]
	The \textit{Riemann-Zeta function} is the map $\zeta:\CC\rightarrow\CC$ defined by the contour integral
	\[
		\zeta(s):=\frac{\Pi(-s)}{2\pi i}\int_\gamma\frac{(-x)^s}{(e^x-1)x}dx
	\]
	where $\gamma$ is a curve that starts at $+\infty$, moves towards the origin along the positive real axis, circles the origin in a counterclockwise direction, and returns to $+\infty$ along the positive real axis.
	Furthermore, $\Pi(s)$ is defined as
	\[
		\Pi(s):=\int_0^\infty e^{-x}x^s dx.
	\]
\end{definition}

As we can see from the definition, this function does not resemble the geometric series in any way.
However, if $s$ is real and greater than $1$ (note: strictly greater than, not greater than or equal to) the zeta function can be written as
\begin{equation}\label{zeta-series}
	\zeta(s)=\sum_{n=1}^\infty\frac{1}{n^s}.
\end{equation}
Furthermore, since $-1$ is less than $1$,
\[
	\zeta(-1)\neq\sum_{n=1}^\infty n.
\]

Another interesting property of the Riemann-Zeta function pops up when we let $s$ be a negative integer.
\begin{proposition}[From \cite{edwards}]
	For any natural number $n$,
	\[
		\zeta(-n)=(-1)^n\frac{B_{n+1}}{n+1}
	\]
	where $B_{n+1}$ is the $n+1$th Bernoulli number.
\end{proposition}
By using this property we get
\begin{equation}\label{zeta-1}
	\zeta(-1)=(-1)\frac{B_2}{2}=(-1)\paren{\frac{1}{6}}\paren{\frac{1}{2}}=-\frac{1}{12}.
\end{equation}

Very likely, the incorrect conclusion that $1+2+3+\ldots=-1/12$ came out of some confusion arising from \autoref{zeta-series}, \autoref{zeta-1}, and the history of the zeta function.

\printbibliography

\end{document}


\section*{Exercises}

\begin{ex}
  True or false? ``Cramer's rule is useful for finding solutions
    to systems of linear equations in which there is an infinite set
    of solutions.''
  \begin{sol}
    False. Cramer's rule only works when the coefficient matrix is
    invertible. In these cases, the solution is always unique.
  \end{sol}
\end{ex}

\begin{ex}
  Use Cramer's rule to find the solution to
  \begin{equation*}
    \begin{array}{c}
      x+2y=1 \\
      2x-y=2
    \end{array}
  \end{equation*}
  \begin{sol}
    Solution is: $(x,y) = (1,0)$.
  \end{sol}
\end{ex}

\begin{ex}
  Use Cramer's rule to find the solution to
  \begin{equation*}
    \begin{array}{c}
      x+2y+z=3 \\
      2x-y-z=1 \\
      x+z=1
    \end{array}
  \end{equation*}
  \begin{sol}
    Solution is: $(x,y,z) = (1,1,0)$. For example,
    \begin{equation*}
      y=\frac{\begin{absmatrix}{rrr}
          1 & 3 & 1 \\
          2 & 1 & -1 \\
          1 & 1 & 1
        \end{absmatrix}}{\begin{absmatrix}{rrr}
          1 & 2 & 1 \\
          2 & -1 & -1 \\
          1 & 0 & 1
        \end{absmatrix}}=1
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Use Cramer's rule to solve the system of equations
  \begin{equation*}
    \begin{mymatrix}{rrr}
      2 & 1 &  1 \\
      2 & 2 & -1 \\
      2 & 1 &  2 \\
    \end{mymatrix} \begin{mymatrix}{c}
      x \\
      y \\
      z \\
    \end{mymatrix} =\begin{mymatrix}{r}
      3 \\
      -2 \\
      5 \\
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    The solution is $(x,y,z) = (1,-1,2)$.
  \end{sol}
\end{ex}

\begin{ex}
  Find the value of $y$ in the following system of equations:
  \begin{equation*}
    \begin{mymatrix}{ccc}
      1 & 1 &  1   \\
      1 & t &  t^2 \\
      1 & s &  s^2 \\
    \end{mymatrix} \begin{mymatrix}{c}
      x \\
      y \\
      z \\
    \end{mymatrix} =\begin{mymatrix}{c}
      t \\
      s \\
      1 \\
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    By Cramer's rule, we have
    \begin{equation*}
      y = \frac{
        \begin{absmatrix}{ccc}
          1 & t &  1   \\
          1 & s &  t^2 \\
          1 & 1 &  s^2 \\
        \end{absmatrix}
      }{
        \begin{absmatrix}{ccc}
          1 & 1 &  1   \\
          1 & t &  t^2 \\
          1 & s &  s^2 \\
        \end{absmatrix}
      }
      = \frac{s^3+t^3+1-2-ts^2-t^2}{ts^2+t^2+s-t-st^2-s^2}.
    \end{equation*}
  \end{sol}
\end{ex}


\section{Convergence}
We have described what it means for a Taylor series to converge to a function over some interval.
What if we're given an infinite series that's not a function, just an infinite sum of numbers?
Can we still check if the series converges or diverges?
We'll develop several tests that we can apply to check for convergence or divergence.

\subsection{nth Term Test for Divergence}
\begin{lemma}
	Let $a_n$ be the nth term of a series $s$.
	If
	\begin{equation*}
		\lim_{n\to\infty}{a_n} \neq 0,
	\end{equation*}
	then the series diverges.
\end{lemma}
\begin{proof}
	Assume not.
	Let $a_n$ be the nth term and $s_n$ the nth partial sum of a convergent series $s$ whose terms do not tend to 0.
	Since $s$ converges to some value $L$, there exists some positive integer $m$ such that for all  $n > m$ and $\epsilon > 0$,
	\begin{equation*}
		\abs{s_n - L} < \epsilon.
	\end{equation*}
	We can also say the same for $s_{n+1}$.
	\begin{equation*}
		\abs{s_{n+1}-L} < \epsilon.
	\end{equation*}
	So, subtracting one inequality from the other,
	\begin{equation*}
		\abs{s_{n+1}-s_n} < 2\epsilon.
	\end{equation*}
	The difference between the two partial sums is just $a_{n+1}$.
	So,
	\begin{equation*}
		\abs{a_{n+1}} < 2\epsilon.
	\end{equation*}
	Since the terms of $s$ don't tend to 0, there exists some positive integer $h$ and real value $\delta > 0$ such that for all $n > h$,
	\begin{equation*}
		\abs{a_n} > \delta.
	\end{equation*}
	We can also say the same for $a_{n+1}$.
	\begin{equation*}
		\abs{a_{n+1}} > \delta.
	\end{equation*}
	Combining the two inequalities involving $a_{n+1}$, for all $n > \max{(m,h)}$,
	\begin{equation*}
		\delta < \abs{a_{n+1}} < 2\epsilon.
	\end{equation*}
	However, for $\epsilon \leq \delta/2$, the inequality creates a contradiction.
\end{proof}

Although the proof has to be a bit specific to cover the case of alternating series, the idea behind the test makes sense.
If you're adding on terms that don't get smaller in absolute value, then you can't ``zero in'' on a particular value and converge.

\begin{example}
	Show that the following series diverges:
	\begin{equation*}
		\sum_{k=0}^{\infty}{(-1)^k} = 1 - 1 + 1 - 1 + \ldots.
	\end{equation*}
\end{example}
\begin{answer}
	We see that
	\begin{equation*}
		\lim_{n\to\infty}{a_n} = \text{DNE} \neq 0.
	\end{equation*}

	So, by the nth Term Test for Divergence, the series diverges.
\end{answer}

\subsection{Geometric Series with $\abs{r} < 1$}
\begin{lemma}
	If $s$ is a geometric series with common ratio $r$, then $s$ converges if and only if $\abs{r} < 1$.
\end{lemma}

We already proved this when talking about power series and gave a formula to find its value.

\begin{example}
	Show that the following series diverges:
	\begin{equation*}
		\sum_{k=0}^{\infty}{\frac{2^k}{3}}.
	\end{equation*}
\end{example}
\begin{answer}
	This is a geometric series with initial term $1/3$ and common ratio $r=2$.
	Since $\abs{r} > 1$, the series diverges.
\end{answer}

\subsection{P-Series Test}
\begin{lemma}
	The p-series
	\begin{equation*}
		\sum_{k=1}^{\infty}{\frac{1}{k^p}}
	\end{equation*}
	converges if and only if $p > 1$.
\end{lemma}

\begin{example}
	Show that the Harmonic Series diverges:
	\begin{equation*}
		\sum_{k=1}^{\infty}{\frac{1}{k}}.
	\end{equation*}
\end{example}
\begin{answer}

	This is a p-series with $p=1$.
	So, by the P-Series Test, the sum diverges.
\end{answer}

\subsection{Direct Comparison Test}
\begin{lemma}
	Let $s = \sum_{k=0}^{\infty}{a_k}$ be a series where all $a_k \geq 0$.
	$s$ converges if there exists some convergent series $c = \sum_{k=0}^{\infty}{c_k}$ and positive integer $m$ such that for all $n > m$,
	\begin{equation*}
		c_n \geq a_n.
	\end{equation*}
	$s$ diverges if there exists some divergent series $d = \sum_{k=0}^{\infty}{d_k}$ and positive integer $m$ such that for all $n > m$,
	\begin{equation*}
		a_n \geq d_n.
	\end{equation*}
\end{lemma}

\begin{example}
	Show that the following series converges:
	\begin{equation*}
		\sum_{k=0}^{\infty}{\frac{1}{2 + 3^k}}.
	\end{equation*}
\end{example}
\begin{answer}
We see that this series looks really similar to a geometric series with common ratio 1/3, just with an extra 2 in the denominator.
For all $n > 0$,
\begin{equation*}
	\frac{1}{2+3^n} \leq \frac{1}{3^n}.
\end{equation*}

So, by the Direct Comparison Test, the series converges.
\end{answer}

\begin{example}
	Show that the following series diverges:
	\begin{equation*}
		\sum_{k=0}^{\infty}{\frac{1}{2+\sqrt{k}}}.
	\end{equation*}
\end{example}
\begin{answer}
	Although the sum looks like a p-series with $p=1/2$, we can use that series because our series has smaller terms.
	Instead, we can compare to a p-series where $p=1/2$.
	\begin{align*}
		0 \leq \frac{1}{n} &\geq \frac{1}{2+\sqrt{n}} \\
		0 \leq 2 + \sqrt{n} &\leq n \\
		n & \geq 4.
	\end{align*}

	So, for all $n \geq 4$, our series has larger values than the p-series with $p=1$.
	We know the p-series diverges by the P-Test, so by the Direct Comparison Test, our series also diverges.
\end{answer}

\subsection{Limit Comparison Test}
\begin{lemma}
	Let $a_n$ and $b_n$ be the nth terms of two series $a$ and $b$ that have all positive terms after some point.
	If
	\begin{equation*}
		\lim_{n\to\infty}{\frac{a_n}{b_n}}
	\end{equation*}
	converges to a finite value greater than 0, then $a$ and $b$ either both converge or both diverge.
	If the limit converges to 0 and $b$ converges, then $a$ also converges.
	If the limit goes to $\infty$ and $b$ diverges, then $a$ also diverges.
\end{lemma}

A common tactic is to select one of $a$ or $b$ to be a geometric or p-series.

\begin{example}
	Show that the following series diverges:
	\begin{equation*}
		\sum_{k=2}^{\infty}{\frac{2k}{k^2-k+a}}.
	\end{equation*}
\end{example}
\begin{answer}
	Looking at this rational function, we see a degree 1 term in the numerator and a degree 2 term in the denominator.
	So, we might expect that the series behaves similarly to $1/n$.
	\begin{equation*}
		\lim_{n\to\infty}{\frac{\frac{2n}{n^2-n+a}}{\frac{1}{n}}} = \lim_{n\to\infty}{\frac{2n^2}{n^2-n+a}} = 2.
	\end{equation*}

	Since the limit converges to a finite value greater than 0, and we know that $1/n$ diverges by the P-Test, then the series must also diverge by the Limit Comparison Test.
\end{answer}

\begin{example}
	Show that the following series converges:
	\begin{equation*}
		\sum_{k=1}^{\infty}{\frac{1}{2^k - 1}}.
	\end{equation*}
\end{example}
\begin{answer}
	This series looks similar to the geometric series $1/2^n$.
	\begin{equation*}
		\lim_{n\to\infty}{\frac{\frac{1}{2^n}}{\frac{1}{2^n-1}}} = \lim_{n\to\infty}{\frac{2^n - 1}{2^n}} = 1.
	\end{equation*}

	Since the limit converges to a finite value greater than 0, and we know that $1/2^n$ converges because it's a geometric series with $r=1/2$, then the series must also converge the the Limit Comparison Test.
\end{answer}

\subsection{Integral Test}
\begin{lemma}
	Let $a$ be a sequence of positive terms where $a_n = f(n)$.
	If $f$ is continuous, positive after some $m$, and decreasing, then the series
	\begin{equation*}
		\sum_{k=m}^{\infty}{a_k}
	\end{equation*}
	converges if and only if the integral
	\begin{equation*}
		\int_{m}^{\infty}{f(x)\d{x}}
	\end{equation*}
	converges.
\end{lemma}

\begin{example}
	Show that the following series diverges:
	\begin{equation*}
		\sum_{k=1}^{\infty}{\frac{2k}{k^2+1}}.
	\end{equation*}
\end{example}
\begin{answer}
	The function $f(x)=2x/(x^2+1)$ is continuous, positive for all $x \geq 1$, and decreasing.
	\begin{equation*}
		\int_{1}^{\infty}{\frac{2x}{x^2+1}\d{x}} = (u = x^2+1) \int_{2}^{\infty}{\frac{\d{u}}{u}} = \ln{u}\biggr\rvert_{2}^{\infty} = \text{diverges}.
	\end{equation*}

	So, by the Integral Test, the series also diverges.
\end{answer}

\begin{example}
	Show that the following series converges:
	\begin{equation*}
		\sum_{k=1}^{\infty}{\frac{1}{k^2+1}}.
	\end{equation*}
\end{example}
\begin{answer}
	The function $f(x)=1/(x^2+1)$ is continuous, positive for all $x \geq 1$, and decreasing.
	\begin{equation*}
		\int_{1}^{\infty}{\frac{\d{x}}{x^2+1}} = \arctan{x}\biggr\rvert_{1}^{\infty} = \frac{\pi}{2} - \frac{\pi}{4} = \frac{\pi}{4}.
	\end{equation*}

	So, by the Integral Test\footnote{We omitted evaluating the improper integral with limits, but the value is what you would get from doing that.} the series also converges.
\end{answer}

\subsection{Ratio Test}
\begin{lemma}
	Let $a$ be a series with only positive terms after some index.
	If the limit
	\begin{equation*}
		\lim_{n\to\infty}{\frac{a_{n+1}}{a_n}}
	\end{equation*}
	is less than 1, then the series converges.
	If the limit is greater than 1, then the series diverges.
	If the limit is equal to 1, then the test is inconclusive.
\end{lemma}

\begin{example}
	State whether the following series converges or diverges:
	\begin{equation*}
		\sum_{k=1}^{\infty}{\frac{k\ln{k}}{2^k}}.
	\end{equation*}
\end{example}
\begin{answer}
	Taking the limit of the ratio of subsequent terms,
	\begin{equation*}
		\lim_{n\to\infty}{\frac{\frac{(n+1)\ln{(n+1)}}{2^{n+1}}}{\frac{n\ln{n}}{2^n}}} = \lim_{n\to\infty}{\frac{(n+1)\ln{n+1}}{2n\ln{n}}} = \lim_{n\to\infty}{\frac{n+1}{2n}} = \frac{1}{2}.
	\end{equation*}

	Since the limit is less than 1, the series converges by the Ratio Test.
\end{answer}

\subsection{nth Root Test}
\begin{lemma}
	Let $a$ be a series with all positive terms after some index.
	If the limit
	\begin{equation*}
		\lim_{n\to\infty}{\sqrt[n]{a_n}}
	\end{equation*}
	is less than 1, then the series converges.
	If the limit is greater than 1, the series diverges.
	If the limit is equal to 1, the test is inconclusive.
\end{lemma}

This test is most useful for series that looks like geometric series, but the common ratio is not a constant.

\begin{example}
	State whether the following series converges or diverges:
	\begin{equation*}
		\sum_{k=1}^{\infty}{\left(\frac{k}{2k-1}\right)^k}.
	\end{equation*}
\end{example}
\begin{answer}
	Taking the limit of the nth root,
	\begin{equation*}
		\lim_{n\to\infty}{\sqrt[n]{\left(\frac{n}{2n-1}\right)^n}} = \lim_{n\to\infty}{\frac{n}{2n-1}} = \frac{1}{2}.
	\end{equation*}

	Since the limit is less than 1, the series converges by the nth Root Test.
\end{answer}

\subsection{Alternating Series Test}
All of the previous 8 tests have tested whether a series converges absolutely.
That is, all terms in the series could be made positive and the series would still converge or diverge.
There are some series that do converge but don't converge absolutely.
We say that these series converge conditionally.
You've already seen  few, like the formula for $\pi/4$ using the Maclaurin series for $\arctan{x}$.
All series that converge absolutely also converge conditionally.
Series that converge conditionally do so when they pass the following test.

\begin{lemma}
	The series
	\begin{equation*}
		\sum_{k=0}^{\infty}{(-1)^ku_k}
	\end{equation*}
	converges if all of the following conditions are satisfied.
	\begin{enumerate}
		\item All $u_i$ are positive.
		\item There exists an integer $m$ such that for all $n > m$, $u_{n+1} \leq u_n$.
		\item The series of $u$'s pass the nth Term Test.
	\end{enumerate}
\end{lemma}

\begin{example}
	State whether the following series converges or diverges:
	\begin{equation*}
		\sum_{k=1}^{\infty}{(-1)^k\frac{1}{k}}.
	\end{equation*}
\end{example}
\begin{answer}
	\begin{enumerate}
		\item All the terms $1, 1/2, 1/3, \ldots$ are positive.
		\item All terms are less than the previous term.
		\item The terms tend to 0, passing the nth Term Test.
	\end{enumerate}

	So, by the Alternating Series Test, the series converges conditionally\footnote{You've already seen this series too. It converges to $\ln{2}$.}.
	Note that the series does not converge absolutely because it fails the P-Test.
\end{answer}

\subsubsection{Riemann Rearrangement Theorem}
\begin{theorem}[Riemann Rearrangement Theorem]
	If an infinite series is conditionally convergent, but not absolutely convergent, then its terms can be rearranged to form a divergent series or to converge to any constant.
\end{theorem}

This result may seem counter intuitive, given that we know addition is associative and commutative.
However, the result stems from the fact that we defined convergence of an infinite series at the limit of partial sums.
If we rearrange terms of the series, the partial sums and their limits can change, meaning the limit of partial sums can diverge or converge to some other value.


\begin{verbatim}
\subsection{Sine functions}
For given $x \in [0, 2\pi]$ with step size $\pi/12$, we can obtain the evaluations
of \eqref{eq:sine} at $x$ (see Table \ref{tab:sine}), and the corresponding plot
(see Figure \ref{fig:sine}).

\begin{equation}
  \label{eq:sine}
  \begin{cases}
    y_1 = \sin(x/2) \\
    y_2 = \sin(x)   \\
    y_3 = \sin(2x)
  \end{cases}
\end{equation}
\begin{table}[!hbtp]
\centering
\caption{Sine functions}
\label{tab:sine}
\begin{tabular}{ccrr}
\toprule
        $x$ & $\sin(x/2)$ &   $\sin(x)$ &  $\sin(2x)$ \\
\midrule
$0$      & $0$          & $0$  & $0$ \\
$\pi/2$  & $\sqrt{2}/2$ & $1$  & $0$ \\
$\pi$    & $1$          & $0$  & $0$ \\
$3\pi/2$ & $\sqrt{2}/2$ & $-1$ & $0$ \\
$2\pi$   & $0$          & $0$  & $0$ \\
\bottomrule
\end{tabular}
\end{table}
\begin{figure}[!hbtp]
  \centering
  \includegraphics[width=0.3\textheight]{./fig/sine.pdf}
  \caption{Sine functions}
  \label{fig:sine}
\end{figure}

\subsection{Goldbach's Conjecture}
Pursuing this type of analysis more carefully, Hardy and Littlewood in 1923 conjectured
(as part of their famous \textsl{Hardy–Littlewood prime tuple conjecture}) that for any
fixed $c \geq 2$, the number of representations of a large integer $n$ as the sum of $c$
primes $n = p_1 + \cdots + p_{c}$ with $p_1 \leq \cdots \leq p_c$ should be asymptotically
equal to
\begin{equation}
    \label{eq:hardy}
    \left( \prod_{p} \frac{p \gamma_{c,p} (n)}{(p - 1)^c}\right)
    \int_{2 \leq x_1 \leq \cdots \leq x_c: x_1 + \cdots + x_c = n}
    \frac{d x_1 \cdots d x_{c - 1}}{\ln{x_1} \cdots \ln{x_c}},
\end{equation}
where the product is over all primes $p$, and $\gamma_{c, p}(n)$ is the number of
solutions to the equation $n = q_1 + \cdots + q_c \mod p$ in modular arithmetic,
subject to the constraints $q_1, \ldots, q_c \ne 0 \mod p$. This formula
\eqref{eq:hardy} has been rigorously proven to be asymptotically valid for
$c \geq 3$ from the work of Vinogradov, but is till only a conjecture when $c = 2$.
In the latter case, the above formula simplifies to $0$ when $n$ is odd, and to
$$
2 \Pi_2 \left( \prod_{p|n; p \geq 3} \frac{p - 1}{p - 2} \right)
\int_{2}^{n} \frac{dx}{(\ln{x})^2} \approx 2 \Pi_2
\left( \prod_{p|n; p \geq 3} \frac{p - 1}{p - 2} \right) \frac{n}{(\ln{n})^2},
$$
when $n$ is even, where $\Pi_2$ is Hardy-Littlewood's twin prime constant
$$
\Pi_2 := \prod_{p \geq 3} \left( 1 - \frac{1}{(p - 1)^2} \right) = 0.6601618158\ldots
$$
This sometimes known as the \textsf{extended Goldbach conjecture}.

\emph{Reference}: \href{https://en.wikipedia.org/wiki/Goldbach's_conjecture}{Goldbach's conjecture}.
\end{verbatim}


\begin{thm}[Abstract Bayes' Rule]
	Let $\Pf$ and $\Pm$ be probability measures on the probability space $(\Omega, \F)$ with $\Pm \ll \Pf$ and Radon-Nikod\'{y}m derivate
	\begin{align}
	L = \frac{ \dx \Pm }{ \dx \Pf } .
	\end{align}
	If $\G$ is a sub-$\sigma$ algebra of $\F$ and $X$ is integrable $\G$-measurable random variable, then
	\begin{align}
	\E_{\Pm} (X| \G) = \frac{\E_{\Pf} (LX| \G)}{\E_{\Pf} (L| \G)} .
	\end{align}
\end{thm}

\begin{proof}
	First we note that
	\begin{align}
	\Pm ( \E_{\Pf} (L| \G) = 0 ) &= \int \1_{ \{ \E_{\Pf} (L| \G) = 0 \} } \dx \Pm \\
	&= \int_{ \{ \E_{\Pf} (L| \G) = 0 \} } L \dx \Pf \\
	&= \int_{ \{ \E_{\Pf} (L| \G) = 0 \} } \E_{\Pf} (L| \G) \dx \Pf \\
	&= 0
	\end{align}
	which implies that $\E_{\Pf} (L| \G) > 0$. Now we have to prove only that
	\begin{align}
	\E_{\Pm} (X| \G) \E_{\Pf} (L| \G) = \E_{\Pf} (LX| \G) .
	\end{align}
	If $G \in \G$, then $\E_{\Pm} (X| \G)$ is $\G$-measurable and thus
	\begin{align}
	\int_G \E_{\Pm} (X| \G) \E_{\Pf} (L| \G) \dx \Pf &= \int_G \E_{\Pf} (L \E_{\Pm} (X| \G) | \G) \dx \Pf  \\
	&= \int_G L \E_{\Pm} (X| \G)  \dx \Pf  \\
	&= \int_G \E_{\Pm} (X| \G)  \dx \Pm \\
	&= \int_G X \dx \Pm \\
	&= \int_G L X \dx \Pf \\
	&= \int_G \E_{\Pf} (LX | \G ) \dx \Pf .
	\end{align}
	and concludes the proof.
\end{proof}

If filtration $(\F_t)$ consists of sub-$\sigma$-algebras of $\F$, then, by Radon-Nikod\'{y}m-theorem, we have Radon-Nikod\'{y}m derivates
\begin{align}
L_t = \frac{\dx \Pm}{\dx \Pf}
\end{align}
with respect to $\sigma$-algebra $\F_t$ for all $t \geq 0$. By equation \ref{Radon-Nikodymderivateconditional},
\begin{align}
L_t = \E_{\Pf} ( L_s | \F_t )
\end{align}
for $0 \leq t < s$ implying that $(L_t)$ is a martingale with respect to the filtration $(\F_t)$. The process $(L_t)$ is called likelihood-process.

Abstract Bayes' rule and the fact $\E_{\Pf} (L_t) = 1$ implies the following theorem.

\begin{thm}
	Let $\Pf$ and $\Pm$ be probability measures with $\Pm \ll \Pf$ and Radon-Nikod\'{y}m derivate
	\begin{align}
	L = \frac{ \dx \Pm }{ \dx \Pf } .
	\end{align}
	Now the process $(X_t)$ is $\Pm$-martingale if and only if $LX_t$ is $\Pf$-martingale, where $L$ is the likelihood-process.
\end{thm}

\subsection{A consequence of Radon-Nikod\'{y}m-theorem}

If $\G$ is a sub-$\sigma$-algebra of $\F$, then Radon-Nikod\'{y}m-derivates $L_{\G}$ and $L_{\F}$ of $\Pf$ with respect to $\Pm$ need not be equal as $L_{\F}$ may not be $\G$-measurable. However, we know that for every $G \in \G$
\begin{align}
\int_G \E_{\Pf} ( L_{\F} | \G ) \dx \Pf = \int_G L_{\F} \dx \Pf = \Pm (G)
\end{align}
meaning that
\begin{align}
L_{\G} = \E_{\Pf} ( L_{\F} | \G ) .
\end{align}

Thus we have proved the following.

\begin{lemma}
	If $\Pm \ll \Pf$ and $\G$ is a sub-$\sigma$-algebra of $\F$, then
	\begin{align}
	\label{Radon-Nikodymderivateconditional}
	L_{\G} = \E_{\Pf} ( L_{\F} | \G ) ,
	\end{align}
	where $L_{\G}$ is Radon-Nikod\'{y}m-derivate of $\Pf$ with respect to $\Pm$ and filtration $\G$ and $L_{\F}$ is Radon-Nikod\'{y}m-derivate of $\Pf$ with respect to $\Pm$ and filtration $\F$.
\end{lemma}

Let $\Pm$ and $\Pf$ be equivalent measures with a filtration $(\F_t)$. W define a density process by
\begin{align}
\Lambda_t = \E_{\Pf} \left( \frac{\dx \Pm}{\dx \Pf} \ | \ \F_t \right)
\end{align}
Now
\begin{align}
\E_{\Pf} \left( \Lambda_T \ | \ \F_t \right) &= \E_{\Pf} \left( \E_{\Pf} \left( \frac{\dx \Pm}{\dx \Pf} \ | \ \F_T \right) \ | \ \F_t \right) \\
&= \E_{\Pf} \left( \frac{\dx \Pm}{\dx \Pf} \ | \ \F_t \right) \\
&= \Lambda_t
\end{align}
for all $0 \leq t \leq T$ meaning that $\Lambda_t$ is a martingale. Also
\begin{align}
\Lambda_0 = \E_{\Pf} \left( \frac{\dx \Pm}{\dx \Pf} \ | \ \F_0 \right)
\end{align}


\section*{Problem 2 Solution}

\begin{enumerate}[a)]

\item

The transmitted uncollided intensity of a neutron beam through a material with macroscopic cross section $\Sigma$ is given by
$$ I = I(0) \, e^{-\Sigma x} .$$
Solving for $\Sigma$, we find
$$ \Sigma = \frac{1}{x}\ln\left(\frac{I(0)}{I}\right) $$
Using the values provided (and considering the intensity as per the target, rather than per cm$^2$), we have
$$ \Sigma = \frac{1}{10\text{ cm}}\ln\left(\frac{2\times10^{12}\text{ neutrons/(cm}^2{\cdot}\text{s)}\left( 5\text{ cm}^2 \right)}{3.0\times10^9\text{ neutrons/s}}\right) $$
$$ \Sigma = \frac{1}{10\text{ cm}}\ln\left(3.33\times10^{3}\right) $$
$$\boxed{ \Sigma = 0.811\text{ cm}^{-1} }$$

\item

The mean free path of a particle is defined as $\lambda \equiv \frac{1}{\Sigma}$.
$$\boxed{ \lambda = 1.233\text{ cm} }$$

\item

We are told that the beam pulse is 10 $\mu$s, so when considered in conjunction with the known intensity of the neutron beam, we can find the total number of particles produced by the beam per area. We define the neutron fluence, $\Phi$, as the number of neutrons per cm$^2$, calculated as
$$ \Phi = It = (2\times10^{12}\text{ neutrons/(cm}^2\cdot\text{s)})(10^{-5}\text{ s}) = 2\times10^7\text{ neutrons/cm}^2 $$
The number of interactions, $R$, is
$$ R = \Phi \Sigma V $$
where $\Phi$ is the incident neutron fluence in $\left[\frac{\text{neutrons}}{\text{cm}^2}\right]$, $\Sigma$ is the macroscopic cross section of the shielding material in $\left[\frac{1}{\text{cm}}\right]$, and $V$ is the volume of the shield in $\left[{\text{cm}^3}\right]$.

We have already found $\Phi$, we calculated $\Sigma$ in part (a), and we can determine $V$ by multiplying the area of the beam spot by the thickness of the target.
$$ V = 5\text{ cm}^2 \times 10\text{ cm} = 50\text{ cm}^3 $$
The total number of collisions is then
$$ R = (2\times10^7\text{ cm}^{-2})(0.811\text{ cm}^{-1})(50\text{ cm}^3) $$
$$\boxed{ R = 8.11\times10^8\text{ collisions} }$$
\end{enumerate}


In this section we will study the limitations of our previous encoding of natural numbers. We will then use a positional system (the simplest of which is binary numbers) to show a much more efficient conversion process.

\subsection{Performance considerations of unary encoding}
Let us begin with an analysis of how long it takes to perform addition of two Peano numbers. Let us assume that the cost of building or decomposing terms is $O(1)$, since it can be implemented with a constant number of low-level operations such as memory allocation, runtime type checks, and similar, but it does not require recursion or iteration. Consider addition between two numbers $n + m$ with the rules:

\begin{lstlisting}
----------- r0
z + a => a

a + b => c
----------------- r1
s(a) + b => s(c)
\end{lstlisting}

We have two alternatives that clearly only depend from $n$ ($m$ does not have influx on the number of operations, since it is just returned directly as the last step): either $n=0$, or $n=n'+1$. In case of $n=0$, then rule \texttt{r0} is used, producing a result directly. This means that the number of steps $T$ performed by the system is:

$$T(n) = 1 \text{ when } n=0$$

In the case of $n=n'+1$, then rule \texttt{r1} is used. Evaluating rule \texttt{r1} requires first evaluating the expression $n'+m$\footnote{Note that this again an application of \textit{induction}, albeit less formal}, and then performing a single operation which is the addition of the \texttt{s} keyword to intermediate result \texttt{c}:

$$T(n) = T(n') + 1 = T(n-1) + 1 \text{ when } n=n'+1$$

This means that the system will perform exactly $n+1$ computation steps, therefore having a complexity of $O(n)$ where $n$ is the value of the first number. This is highly inefficient.

\subsection{Binary encoding}
Let us consider positional encoding. Specifically, let us consider binary encoding (base 2 encoding). We will define a binary number as a series of binary digits, which are either zero or one:

\begin{lstlisting}
Data [] "d0" [] Priority 0 Type Digit
Data [] "d1" [] Priority 0 Type Digit
\end{lstlisting}

The binary number itself is defined as a digit, followed by the rest of the number. We will use the comma operator to specify this separation:

\begin{lstlisting}
Data [Num] "," [Digit] Priority 10 Type Num
\end{lstlisting}

Of course numbers need to end at some point\footnote{not in real life, where we could simply assume an infinite sequence of zero's, but in a computer with limited memory this simplification is ubiquitous}, so we define a special symbol that represents the ``end of list'' number:

\begin{lstlisting}
Data [] "nil" [] Priority 0 Type Num
\end{lstlisting}

A number such as $3 = 011$ would then be expressed as \texttt{(((nil,d0),d1),d1)}. Notice that we associate the digits to the left, so that the first digit we encounter is the one of the lowest order. This is not strictly necessary, but it makes it easier to extract the digits in order of significance, and this in turns simplifies the traditional definition of addition.

\subsection{Addition}
To add two numbers, we will encode the well-known process of adding numbers with a \textit{carry}. This means that we will add, repeatedly, three digits at a time: the first two digits of the numbers that we are adding, plus the carry that comes from the previous addition of digits. We now define this sum of digits:

\begin{lstlisting}
Func [] "addDigits" [Digit Digit Digit] Priority 5 Type Expr => Num
\end{lstlisting}

We have eight instances of \texttt{addDigits}, one for each possible combination of digits (each digit has two possible values, \texttt{d0} and \texttt{d1}, thus we have $2^3$ total combinations). For each such combination, we directly return the sum of the three digits in the format of a two-digit binary number. We need no more than two digits for the result, since the maximum number we will need to return is the sum of \texttt{d1} three times:

\begin{lstlisting}
----------------------------------
addDigits d0 d0 d0 => (nil,d0),d0

----------------------------------
addDigits d0 d0 d1 => (nil,d0),d1

...

----------------------------------
addDigits d1 d1 d1 => (nil,d1),d1
\end{lstlisting}

Addition with carry performs the digit addition of the current digits of the numbers being added, plus the current carry. The two resulting digits are the lower-order unit of the result, together with the carry that will be used to add together the remaining bits of the input numbers. The function that recursively adds the digits is \texttt{addCarry}:

\begin{lstlisting}
Func [] "addCarry" [Num Num Digit] Priority 5 Type Expr => Num
\end{lstlisting}

\begin{lstlisting}
addDigits da db dr => (nil,dr',d)
addCarry a b dr' => res
----------------------------------
addCarry a,da b,db dr => res,d
\end{lstlisting}

Of course addition ends when it encounters the end of the numbers, which needs to be accompanied with a null carry:

\begin{lstlisting}
--------------------------
addCarry nil nil d0 => nil
\end{lstlisting}

We might even add an \texttt{overflow} keyword and graciously handle the case of non-null last carry as follows:

\begin{lstlisting}
--------------------------------
addCarry nil nil d1 => overflow
\end{lstlisting}

The first step of addition simply instances \texttt{addCarry} with a null initial carry:

\begin{lstlisting}
addCarry a b d0 => c
--------------------
a + b => c
\end{lstlisting}

Multiplication follows a similar scheme, and as such we will not give it here. Also, notice that we only support numbers of the same length. Supporting addition of numbers of different lengths would be relatively trivial, and as such is also not shown here.

\subsection{Performance considerations of binary encoding}
Binary encoding allows us to perform operations much faster, that is with less rules used when compared with Peano addition. Remember that Peano addition took a significant $O(n)$ amount of operations to perform, with $n$ being the value of the first operand.

Consider only the rule which performs a single step of the binary addition:

\begin{lstlisting}
addDigits da db dr => (nil,dr',d)
addCarry a b dr' => res
----------------------------------
addCarry a,da b,db dr => res,d
\end{lstlisting}

\texttt{addDigits} is just one single step, in that it directly resolves the parameters and returns the appropriate result. The only thing that matters for the purpose of the complexity of \texttt{addCarry} is the recursive call. Let us observe that the recursive call \texttt{addCarry a b dr'} uses \texttt{a} and \texttt{b} as input numbers, whereas the original addition \texttt{addCarry a,da b,db dr} used \texttt{a,da} and \texttt{b,db}, which are precisely equal to \texttt{a} and \texttt{b}, but with the additional digits \texttt{da} and \texttt{db} respectively. What is the relationship between \texttt{a} and \texttt{a,da}? Simply enough, we know that \texttt{a} is \texttt{a,da} divided by two. In general, removing the least-significant digit in a number encoded in base $b$ is equivalent to dividing the number by $b$ itself. Consider the following examples of removal of a least-significant digit:

$\begin{array}{c c c}
  110 / 2 & = & 11 \\
  111 / 2 & = & 11 \\
  1000 / 2 & = & 100
\end{array}$

This happens because a number $n$ in an arbitrary base $b$ is decomposed into:

$$d_m \times b^m \dots + d_2 \times b^2 + d_1 \times b^1 + d_0 \times b^0$$

Removing one digit decreases all powers of $b$, resulting in:

$$d_m \times b^{m-1} \dots + d_2 \times b^{2-1} + d_1 \times b^{1-1}$$

which is precisely equal to the original number divided by $b$.

When there are no more digits, it means that the number has been divided by the base ($2$ for our binary numbers) enough times to render the highest coefficient null. If the original number was $n$, we will have stopped only after $k$ steps such that $\frac{n}{2^k} = 0$. The smallest solution to this is actually well known, and is $k = \log_2 n$. This leads us to the conclusion that the complexity of this method of addition is $O(\log n)$, which is significantly faster than $O(n)$. As a way of comparison, assume that we are computing the sum $1000000 + 1000000$: the Peano addition will take about $1000000$ steps, whereas binary addition will take about $6$.


\chapter{Data}

\begin{description}
  \item[Data] is collection of data object and their attributes.
  \item[Type of Attributes] Nominal, Ordinal, Interval, Ratio.
  \item[Attribute Values Property] Distinctness, Order, Addition, Multiplication
\end{description}
\par \noindent
\marginnote{I do not focus on Types of Data Sets and Characteristics of Structured Data}One idea to determine if an interval attribute is ratio is to check if the attribute has a true or natural absolute zero point. The ratio scales are very common in physical scenario.

\section{Discrete and Continuous}

\begin{description}
  \item[Discrete Attribute] Has only a finite or countably infinite set of values
  \item[Continuous Attribute] Has real numbers as attribute values
\end{description}

\section{Data Quality}

\begin{description}
  \item[Noise] modification of original values
  \item[Outlier] considerably different than most of the other data objects
\end{description}

\section{Discretization and Binarization}

\begin{description}
  \item[Binarization] transform either a continuous attribute or a categorical attribute into one or more binary attributes
  \item[Discretisation] transform a continuous attribute into a categorical attribute
\end{description}

\section{Normalization}

\underline{Min-Max Normalization}
$$(min_A, max_A) \rightarrow (new\_min_A, new\_max_A)$$
$${v^{'}={\frac{v-min_A}{max_A-min_A}} \times (new\_max_A-new\_min_A) + new\_min_A}$$
\\
\underline{Z-score Normalization}
$$v^{'}=\frac{v-\mu_A}{\sigma_A}$$
\\
\underline{Normalisation by Decimal Scaling}
$$v^{'}=\frac{v}{10^{j}}$$
where j = smallest integer such that $Max(\mid v^{'}\mid) < 1$

\section{Proximity}

\begin{description}
\item[Similarity] Numerical measure of how alike two data objects are
\item[Dissimilarity] Numerical measure of how different two data objects are
\end{description}
\underline{Euclidean Distance}
$$dist=\sqrt{\sum_{k=1}^{n}{(p_k-q_k)^2}}$$
\\
\underline{Minkowski Distance}
$$dist={\sum_{k=1}^{n}{(\mid p_k-q_k\mid^r)}}^{1/r}$$
\begin{align*}
r = 1&: Hamming\ distance / Manhattan\ distance\\
r = 2&: Euclidean\ distance \\
r = inf&: supremum\ /\ Chebyshev\ distance, max(x - y) \\
\end{align*}
\\
\underline{Simple Matching Coefficient}
$$SMC=\frac{M_{11} + M_{00}}{M_{00} + M_{01} + M_{10} + M_{11}}$$
\\
\underline{Jaccard Coefficient}
$$J=\frac{M_{11}}{M_{01} + M_{10} + M_{11}}$$
\\
\underline{Cosine Similarity}
$$cos(p,q)=\frac{p\bullet q}{\|p\|\|q\|}$$
\underline{Tanimoto Coefficient}
$$T(p,q)=\frac{p\bullet q}{\|p\|^{2}+\|q\|^{2}-p\bullet q}$$

\section{Correlation}

$$corr(p,q)=\frac{cov(p,q)}{std(p) \times std(q)}$$
$$cov(p,q)=\frac{1}{n-1}\sum_{k=1}^{n}{(p_k-\bar{p})(q_k-\bar{q})}$$
$$std(p)=\sqrt{\frac{1}{n-1}\sum_{k=1}^{n}{(p_k-\bar{p})^{2}}}$$


\subsection{Convex Functions}
Let us first give the definition of convex sets.
\begin{definition}[Convex set]
	A set $C$ is convex, if the line segment between any two points in $C$ lies in $C$, i.e., if any $x, y\in C$ and any $\alpha$ with $0\leq \alpha \leq 1$, there holds
	\begin{equation}
	\alpha x+ (1- \alpha)y \in C.
	\end{equation}
\end{definition}

%Based on the definition above, we have the next property of convex sets.
%\begin{lemma}
%[Homework] Let $C$ be a convex set, with $x_{1}, \dots, x_{k}\in C$, and let $\alpha_{1}, \dots, \alpha_{k} \in \mathbb{R}$ satisfy $\alpha_{i} \geq 0$ and  $\sum_{i =1}^{k} \alpha_{i} = 1$, then
%	\begin{equation}
%	\sum_{i = 1}^{k}\alpha_{i}x^{i} \in C,
%	\end{equation}
%\end{lemma}

Following the definition of convex set, we define convex function as following.
\begin{definition}[Convex function]
	Let $C \subset \mathbb{R}^n$ be a convex set and $f: C \rightarrow \mathbb{R}$:
	\begin{enumerate}
		\item $f$ is called {\bf convex} if for any $x, y \in C$ and $\alpha\in [0, 1]$
		\begin{equation}
		f(\alpha x + (1- \alpha)y) \leq \alpha f(x )+ (1- \alpha)f(y).
		\end{equation}
		\item $f$ is called {\bf strictly convex} if for any $x\neq y \in C$ and $\alpha\in (0, 1)$:
		\begin{equation}
		f(\alpha x + (1- \alpha)y) < \alpha f(x ) + (1 - \alpha)f(y).
		\end{equation}
		\item A function $f$ is said to be (strictly) {\bf concave} if $-f$ is (strictly) convex.
	\end{enumerate}
\end{definition}
%With mentioned previously property of convex sets, it is easy to expend the above definition of convex function. Here we write as one lemma without proof.
%\begin{lemma}
%[Homework]Let $C$ be a convex set, with $x_{1}, \dots, x_{k}\in C$, and let $\alpha_{1}, \dots, \alpha_{k} \in \mathbb{R}$ satisfy $\alpha_{i} \geq 0$ and  $\sum_{i =1}^{k} \alpha_{i} = 1$, then
%	\begin{equation}
%	f(\sum_{i = 1}^{k}\alpha_{i}x_{i}) \leq \sum_{i = 1}^{k}\alpha_{i}f(x_{i}) .
%	\end{equation}
%\end{lemma}
Fig. \ref{convexfunc} and \ref{nonconvexfunc} are diagrams  for convex function definition.
\begin{figure}[H]
\centering                    % 大图名称
\subfigure[Convex function]{                    %第二张子图
\begin{minipage}{4cm}\centering
\includegraphics[width=1\textwidth] {6DL/figures/convexfunction.png}\label{convexfunc}
\end{minipage}   }
 \quad
 \subfigure[Nonconvex function]{                    %第二张子图
\begin{minipage}{4cm}\centering
\includegraphics[width=1\textwidth,height=0.8\textwidth] {6DL/figures/nonconvexfunction.png}\label{nonconvexfunc}
\end{minipage}   }
\caption{Convex and nonconvex functions} %
\end{figure}

\begin{lemma}
	If $f(x)$ is differentiable on $\mathbb{R}^n$, then $f(x)$ is convex if and only if
	\begin{equation}
	f( x) \ge f( y) + \nabla f( y)\cdot ( x -  y), \forall x, y \in \mathbb{R}^n.
	\end{equation}
\end{lemma}
Based on the lemma, we can first have  Fig. \ref{fig:convextangent} for convex functions, namely, the curve of a convex function is always above the tangent lines.
\begin{figure}[H]
\centering
\includegraphics[width=.5\textwidth] {6DL/figures/convextangent.png}
\caption{Tangent line of a convex function}
\label{fig:convextangent}
\end{figure}

\begin{proof}
	Let $z=\alpha x+(1-\alpha )y, 0\leq \alpha \leq 1, \forall x,y \in \mathbb{R}^n$, we have these next two Taylor expansion:
	\begin{equation}\label{key}
	\begin{aligned}
	&f(x)\geq f(z)+ \nabla f(z)(x-z),\\
	&f(y)\geq f(z)+ \nabla f(z)(y-z).
	\end{aligned}
	\end{equation}
	Then we have
	\begin{equation}
	\begin{aligned}
	\alpha f(x) + (1-\alpha) f(y)
	\geq &f(z) + \nabla f(z)[\alpha(x-z)+(1-\alpha)(y-z)]\\
	=&f(z)
	=f(\alpha x +(1-\alpha)y).
	\end{aligned}
	\end{equation}
	Thus we have
	\begin{equation}\label{key}
	\alpha f(x) + (1-\alpha) f(y) \ge f(\alpha x +(1-\alpha)y).
	\end{equation}
	This finishes the proof.

	On the other hand if	$f(x)$ is differentiable on $\mathbb{R}^n$, then
	$$
	f( x) \ge f( y) + \nabla f( y)\cdot ( x -  y), \forall x, y \in \mathbb{R}^n
	$$
	if $f(x)$ is convex.
\end{proof}

\begin{definition}[$\lambda$-strongly convex]\label{def:strongconvex}
We say that $f(x)$ is $\lambda$-strongly convex if
	\begin{equation}\label{equ:lambdastronglyconvex}
	f(x)  \ge f(y) + \nabla f(y)\cdot(x-y) + \frac{\lambda}{2}\|x-y\|^2, \quad \forall x, y \in C,
	\end{equation}
	for some $\lambda >0$.
\end{definition}
\begin{lemma}
Let $\sigma(\nabla^2f)$ be the spectrum of the operator $\nabla^2 f$. If
$$
\sigma(\nabla^2f)\subset [\lambda, L],
$$
then the condition \eqref{equ:lambdastronglyconvex} holds.
\end{lemma}

\begin{lemma}
If $f(x)$ is $\lambda$-strongly convex,
	\begin{equation}\label{strongConvIneq}
		\nabla f(x)\cdot(x - x^*) \ge f(x) - f(x^*) +\frac{\lambda}{2}\|x - x^*\|^2\ge \lambda \|x - x^*\|^2.
		\end{equation}
\end{lemma}

\begin{example}
	Consider $f(x) = \|x\|^2$, then we have
	\begin{equation}\label{key}
	\frac{\partial f}{\partial x_{i}}=2x_{i}, \nabla f = 2x \in R^n,\quad \nabla^2f=2I_{n\times n}\in\mathbb{R}^{n\times n}.
	\end{equation}
	So, we have $\lambda_{\rm min}(\nabla^2f)=2>0$ and
	\begin{equation}
	\begin{aligned}
	f(x)-f(y)-\nabla f(y)(x-y)
	=&\left \| x \right \|^2-\left \| y \right \|^2-2y(x-y)\\
	=&\left \| x \right \|^2-\left \| y \right \|^2-2xy+2\left \| y \right \|^2\\
	=&\left \| x \right \|^2-2xy+\left \| y \right \|^2\\
	=&\left \| x-y \right \|^2\\
	=&\frac{\lambda }{2}\left \| x-y \right \|^2 , \quad \lambda=2.
	\end{aligned}
	\end{equation}
	Thus, $f(x) = \|x\|^2$ is 2-strongly convex
\end{example}
\begin{lemma}
If $f(x)$ is $\lambda$-strongly convex and $\lambda(\nabla^2 f)\le L$, then
$$
\lambda\|x-y\|\le \|\nabla f(x) - \nabla f(y)\|\le L\|x-y\|.
$$
\end{lemma}
\begin{assumption}\label{ass:GD}
We make the following assumptions:
\begin{enumerate}
	\item $f(x)$ is $\lambda$-strongly convex for some $\lambda >0$ with the definition in Definition \ref{def:strongconvex}.
	\item $\nabla f$ is $L$-Lipschitz, i.e.,
	\begin{equation}\label{key}
	\|\nabla f(x) - \nabla f(y)\| \le L\|x - y\|, \forall x,y.
	\end{equation}
\end{enumerate}
\end{assumption}
\begin{example}
Actually, the loss function of the logistic regression model
	\begin{equation}\label{key}
	L(\theta) = -\log P(\theta),
	\end{equation}
	is a convex  function of $\theta$.
	Furthermore, the loss function of the regularized logistic regression model
	\begin{equation}\label{key}
	L_\lambda(\theta) = -\log P(\theta) + \lambda \|\theta\|_F^2, \lambda > 0
	\end{equation}
	  is a $\lambda'$-strongly convex function of $\theta$($\lambda'$ is related to $\lambda$).
\end{example}

We also have these following interesting properties of convex function.

\begin{proposition}[basic properties of convex function]
	\begin{enumerate}
		\item If $f(x)$, $g(x)$ are both convex, then $\alpha f(x) + \beta g(x)$ is also convex, if $\alpha, \beta \geq 0$.
		\item  Linear function is both convex and concave. Here, $f(x)$ is concave if and only if $-f(x)$ is convex.
		\item If $f(x)$ is a convex function on $\mathbb{R}^n$, then $g(y) = f(Ay+b)$ is a convex function on $\mathbb{R}^m$. Here $A \in \mathbb{R}^{m \times n}$ and $b\in \mathbb{R}^m$.
		\item  If $g(x)$ is a convex function on $\mathbb{R}^n$, and the function $f(u)$ is convex function on $\mathbb{R}$ and non-decreasing, then the composite function $f \circ g(x) = f(g(x))$ is convex.
	\end{enumerate}
\end{proposition}
The proof is straightforward and left to the interested readers.

\begin{theorem}[Jensen's inequality]
If $p_1, \cdots, p_n$ are positive numbers which sum to 1 and $f$ is a real continuous function which is convex, then
$$
f(\sum_{i=1}^n\alpha_ix_i)\le \sum_{i=1}^n \alpha_if(x_i).
$$
If $f$ is concave, then
$$
f(\sum_{i=1}^n\alpha_ix_i)\ge \sum_{i=1}^n \alpha_if(x_i).
$$
\end{theorem}