\section{Typing}

\subsection{Term}

\begin{enumerate}
    \item $\tt Type$ is a term.
    \item Variables $\tt x,y$, etc., are terms.
    \item Constants $\tt c,d$, etc., are terms.
    \item If $\tt x$ is a variable and $\tt T,U$ are terms, then $\tt\forall x:T,U$ is a term.
    \item If $\tt x$ is a variable and $\tt T,u$ are terms, then $\tt\lambda x:T.\ u$ is a term.
    \item If $\tt x$ and $\tt u$ are terms, then $\tt (t\ u)$ is a term.
    \item If $\tt x$ is a variable and $\tt t,T,u$ are terms, then $\tt let\ \ x:=t:T\ \ in\ \ u$ is a term.
\end{enumerate}

\subsection{Typing Rule}

\subsubsection{Notation}
\begin{itemize}
\item $\Gamma$ : local context.
\item $\tt u\{x/t\}$ : substitute free occurrence of variable $\tt x$ to term $\tt t$ in term $\tt u$.
\item $\mathcal{WF}(\Gamma)$ : $\Gamma$ is well-formed.
\end{itemize}

\subsubsection{Typing Rules}
\begin{equation*}
\mathcal{WF}([])[]
    \tag{\sc T-Empty}
\end{equation*}
\begin{equation*}
\frac{\Gamma\vdash {\tt T:Type\qquad \qquad x}\notin\Gamma}
    {\mathcal{WF}(\Gamma ::(\tt x:T))}
    \tag{\sc T-Ax}
\end{equation*}
\begin{equation*}
\frac{\Gamma\vdash {\tt t:T\qquad c}\notin \Gamma}
    {\mathcal{WF}(\Gamma:\tt c:=t:T)}
    \tag{\sc T-Def}
\end{equation*}
\begin{equation*}
\frac{\mathcal{WF}(\Gamma)\qquad ({\tt x:T})\in\Gamma}
    {\Gamma\vdash\tt x:T}
    \tag{\sc T-Var1}
\end{equation*}
\begin{equation*}
\frac{\mathcal{WF}(\Gamma)\qquad ({\tt x:=t:T})\in\Gamma}
    {\Gamma\vdash\tt x:T}
    \tag{\sc T-Var2}
\end{equation*}
\begin{equation*}
\frac{\mathcal{WF}(\Gamma)\qquad ({\tt c:T})\in \Gamma}
    {\Gamma\vdash\tt c:T}
    \tag{\sc T-Const1}
\end{equation*}
\begin{equation*}
\frac{\mathcal{WF}(\Gamma)\qquad ({\tt c:=t:T})\in \Gamma}
    {\Gamma\vdash\tt c:T}
    \tag{\sc T-Const2}
\end{equation*}

\begin{equation*}
\frac{\Gamma\vdash{\tt T:Type}\qquad \Gamma::({\tt x:T})\vdash\tt U:Type}
    {\Gamma\vdash\tt\forall x:T,U:Type}
    \tag{\sc T-Prod}
\end{equation*}
\begin{equation*}
\frac{\Gamma\vdash\forall{\tt x:T,U:Type}\qquad \Gamma::({\tt x:T})\vdash{\tt t:U}}
    {\Gamma\vdash\lambda{\tt x:T.\ t:\forall x:T,U}}
    \tag{\sc T-Abs}
\end{equation*}
\begin{equation*}
\frac{\Gamma\vdash\forall{\tt x:U,T}\qquad \Gamma\vdash{\tt u:U}}
    {\Gamma\vdash{\tt (t\ u):T\{x/u\}}}
    \tag{\sc T-App}
\end{equation*}
\begin{equation*}
\frac{\Gamma\vdash{\tt t:T}\qquad \Gamma::({\tt x:=t:T})\vdash{\tt u:U}}
    {\Gamma\vdash{\tt let\ \ x:=t:T\ \ in\ \ u:U\{x/t\}}}
    \tag{\sc T-Let}
\end{equation*}
\begin{equation*}
\frac{(\Gamma\vdash {\tt A_i:s_i)_{i=1..n}}\qquad (\Gamma,\tt f_1:A_1,\cdots,f_n:A_n\vdash t_i:A_i)_{i=1..n}}
    {\Gamma\vdash\tt Fix\ f_i\{f_1:A_1:=t_1\cdots f_n:A_n:=t_n\}:A_i}
    \tag{\sc T-Fix}
\end{equation*}

\subsection{Inductive Definition}

\subsubsection{Notation}
\begin{itemize}
    \item ${\tt Ind}[p](\Gamma_I:=\Gamma_C)$ : inductive definition.
    \item $\Gamma_I$ : names and types of inductive type.
    \item $\Gamma_C$ : names and types of constructors of inductive type.
    \item $p$ : the number of parameters of inductive type.
    \item $\Gamma_P$ : the context of parameters.
\end{itemize}

\subsubsection*{Example}
Here is an example:
\begin{center}
\begin{minipage}{0.6\textwidth}
\begin{minted}{coq}
Inductive list (T : Type) : Type :=
| nil : list T
| cons : T -> list T -> list T.
\end{minted}
\end{minipage}
\end{center}
$$
\tt Ind\ [1]\left([list:Type\to Type]:=
\begin{bmatrix}
\tt nil:\forall T:Type,\ list\ T\\
\tt cons:\forall T:Type,\ T\to list\ T\to list\ T
\end{bmatrix}\right)
$$
Sadly, due to our limited time and energy, our system only supports inductive definition with $|\Gamma_I|=1$,
which means inhibiting mutual inductive type like the following case:
\begin{center}
\begin{minipage}{0.6\textwidth}
\begin{minted}{coq}
Inductive tree : Type :=
    node : forest -> tree
with forest : Type :=
    | emptyf : forest
    | consf : tree -> forest -> forest.
\end{minted}
\end{minipage}
\end{center}
$$
\tt Ind\ []\left(
\begin{bmatrix}\tt tree:Type\\\tt forest:Type\end{bmatrix}
:=
\begin{bmatrix}
\tt node:forest\to tree\\
\tt emptyf:forest\\
\tt consf:tree\to forest\to forest
\end{bmatrix}\right)
$$

\subsubsection{Typing Rule}
\begin{equation*}
\frac{\mathcal{WF}(\Gamma)\qquad {\tt Ind}[p](\Gamma_I:=\Gamma_C)\in \Gamma\qquad {\tt(a:A)}\in\Gamma_i}
    {\Gamma\vdash\tt a:A}
    \tag{\sc T-Ind}
\end{equation*}
\begin{equation*}
\frac{\mathcal{WF}(\Gamma)\qquad {\tt Ind}[p](\Gamma_I:=\Gamma_C)\in \Gamma\qquad {\tt (c:C)}\in\Gamma_C}
    {\Gamma\vdash\tt c:C}
    \tag{\sc T-Constr}
\end{equation*}
\begin{equation*}
\frac{(\Gamma_P\vdash{\tt A_j:s_j'})_{j=1..k}\qquad (\Gamma_i;\Gamma_P\vdash{\tt C_i:s_{q_i}})_{i=1..n}}
    {\mathcal{WF}(\Gamma;{\tt Ind}[p](\Gamma_I:=\Gamma_C))}
    \tag{\sc T-Wf-Ind}
\end{equation*}

\subsection{Match}

\subsubsection{Notation}
The basic idea of this operator is that we have an object $\tt m$ in an inductive type $\tt I$ and we want
to prove a property which possibly depends on $\tt m$. For this, it is enough to prove the property for
$\tt m=(c_i\ u_1\ \cdots\ u_{p_i})$ for each constructor of $\tt I$.
The term for this proof will be written:
$$
\tt match\ m\ in\ I\ \_\ a\ return\ P\ with\ (c_1\ x_{11}\ \cdots\ x_{1p_1})\Rightarrow f_1\ |\ \cdots\ |\ (c_n\ x_{n1}\ \cdots\ x_{np_n})\Rightarrow
f_n\ end
$$\par
Note that the arguments of $\tt I$ corresponding to parameters must be $\tt\_$, because the result type is not
generalized to all possible values of the parameters. The other arguments of $\tt I$ (sometimes called \textit{indices}
in the literature) have to be variables ($\tt a$ above) and these variables can occur in $\tt P$. The expression after
$\tt in$ must be seen as an inductive type pattern. Notice that expansion of implicit arguments and notations apply
to this pattern. For the purpose of presenting the inference rules, we use a more compact notation:
$$
\tt case(m,(\forall ax.\ P),\lambda x_{11}\cdots x_{1p_1}.\ f_1\ |\ \cdots\ |\ \lambda x_{n1}\cdots x_{np_n}.\ f_n)
$$

\subsubsection{Type of Branch}
Before coming to the typing rule of {\tt match}, we have to deal with the type of every branch case of {\tt match}.\par
Let $\tt c$ be a term of type $\tt C$, assume $\tt C$ is a type of constructor for an inductive type $\tt I$. Let
$\tt P$ be a term that represents the property to be proved. Assume $\tt r$ is the number of parameters and
$p$ is the number of arguments.\par
Define a new type $\tt \{c:C\}^P$ which represents the type of the branch corresponding to the $\tt c:C$ constructor.
\begin{center}
\begin{tabular}{lcl}
$\tt \{c:(I\ p_1\ \cdots\ p_r\ t_1\ \cdots\ t_p)\}^P$ & $\equiv$ & $\tt (P\ t_1\ \cdots\ t_p\ c)$\\
$\tt \{c:\forall x:T,\ C\}^P$ & $\equiv$ & $\tt \forall x:T, \{(c\ x):C\}^P$
\end{tabular}
\end{center}
We write $\tt\{c\}^P$ for $\tt\{c:C\}^P$ with $\tt C$, the type of $\tt c$.

\subsubsection{Typing Rule}
\begin{equation*}
\frac{\Gamma\vdash{\tt c:(I\ q_1\ \cdots\ q_r\ t_1\ \cdots\ t_s)}\qquad
        \Gamma\vdash{\tt [P\ |\ I\ q_1\ \cdots\ q_r]}\qquad
    (\Gamma\vdash{\tt f_i:\{(c_{p_i}\ q_1\ \cdots\ q_r)\}^P})_{i=1..l}}
    {\Gamma\vdash{\tt case(c,P,f_1\ |\ \cdots\ |\ f_l):(P\ t_1\ \cdots\ t_s\ c)}}
    \tag{\sc T-Match}
\end{equation*}
$\tt [A\ |\ B]$ means the consequent of $\tt A$ is $\tt B$.


\subsection{Central limit theorem}

\subsubsection{Generalise weak law of large numbers}

\subsubsection{Characteristic function of summed IID events}

\(Z=\sum_{i=1}^nY_i\)

\(\phi_Z(t)=E[e^{itZ}]\)

\(\phi_Z(t)=E[e^{it\sum_{i=1}^nY_i}]\)

\(\phi_Z(t)=E[e^{itY}]^n\)

\(\phi_Z(t)=\phi_Y(t)^n\)

\subsubsection{Taylor series: first moments dominate with means}

\(Z=\sum_{i=1}^nY_i\)

\(Y=\dfrac{X}{n}\)

\(\phi_Z(t)=\phi_Y(t)^n\)

\(\phi_Z(t)=\phi_{\dfrac{X}{n}}(t)^n\)

\(\phi_Z(t)=\phi_X(\dfrac{t}{n})^n\)

\(\phi_X(t)=1+it\mu_X -\dfrac{(\mu_X +\sigma_X^2 )t^2}{2} +\sum_{j=3}^{\infty }\dfrac{E[X^j](it)^j}{j!}\)

\(\phi_X(\dfrac{t}{n})=1+i\dfrac{t\mu_X }{n}-\dfrac{(\mu_X +\sigma_X^2 )(\dfrac{t}{n})^2}{2} +\sum_{j=3}^{\infty }\dfrac{E[X^j](i\dfrac{t}{n})^j}{j!}\)

\(\phi_X(\dfrac{t}{n})=1+i\dfrac{t\mu_X }{n}-\dfrac{(\mu_X +\sigma_X^2 )t^2}{2n^2} +\sum_{j=3}^{\infty }\dfrac{E[X^j](i\dfrac{t}{n})^j}{j!}\)

\subsubsection{Eliminating the imaginary term}

We want \(\mu \) to be \(0\).

\(Z=\sum_{i=1}^nY_i\)

\(Y=\dfrac{X-\mu_X }{n}\)

\(\phi_Y(t)=1+it\mu_Y -\dfrac{(\mu_Y +\sigma_Y^2 )t^2}{2} +\sum_{j=3}^{\infty }\dfrac{E[Y^j](it)^j}{j!}\)

\(\mu_Y =E[\dfrac{X-\mu_X }{n}] ={\mu_X -\mu_X }{n}=0\)

\(\phi_Y(t)=1-\dfrac{\sigma_Y^2t^2}{2} +\sum_{j=3}^{\infty }\dfrac{E[Y^j](it)^j}{j!}\)

\(\sigma^2_Y =E[(\dfrac{X-\mu_X }{n})^2]\)

\(\sigma^2_Y =E[\dfrac{X^2+\mu^2_X-2X\mu_X }{n^2}]\)

\(\sigma^2_Y =\dfrac{E[X^2]+E[\mu^2_X]-E[2X\mu_X] }{n^2}]\)
\(\sigma^2_Y =\dfrac{E[X^2]-\mu^2_X}{n^2}]\)

\(\sigma^2_Y =\dfrac{\sigma^2_X}{n^2}\)

\(\phi_Y(t)=1-\dfrac{\sigma_X^2t^2}{2n^2} +\sum_{j=3}^{\infty }\dfrac{E[(\dfrac{X-\mu}{n})^j](it)^j}{j!}\)

\(\phi_Z(t)=\phi_Y(t)^n\)

\(\phi_Z(t)=[1-\dfrac{\sigma_X^2t^2}{2n^2} +\sum_{j=3}^{\infty }\dfrac{E[(\dfrac{X-\mu}{n})^j](it)^j}{j!}]^n\)

\(\phi_Z(t)=[1-\dfrac{\sigma_X^2t^2}{2n^2}]^n\)

Eliminating \(\sigma^2 \)

\(Z=\sum_{i=1}^nY_i\)

\(Y=\dfrac{X-\mu_X }{\sigma n}\)

\(\phi_Y(t)=1+it\mu_Y -\dfrac{(\mu_Y +\sigma_Y^2 )t^2}{2} +\sum_{j=3}^{\infty }\dfrac{E[Y^j](it)^j}{j!}\)

\(\mu_Y =E[\dfrac{X-\mu_X }{\sigma_X n}] ={\mu_X -\mu_X }{\sigma_X n}=0\)

\(\phi_Y(t)=1-\dfrac{\sigma_Y^2t^2}{2} +\sum_{j=3}^{\infty }\dfrac{E[Y^j](it)^j}{j!}\)

\(\sigma^2_Y =E[(\dfrac{X-\mu_X }{\sigma n})^2]\)

\(\sigma^2_Y =E[\dfrac{X^2+\mu^2_X-2X\mu_X }{\sigma^2 n^2}]\)

\(\sigma^2_Y =\dfrac{E[X^2]+\mu^2_X-2E[X]\mu_X }{\sigma^2 n^2}\)

\(\sigma^2_Y =\dfrac{E[X^2]-\mu^2_X}{\sigma^2 n^2}\)

\(\sigma^2_Y =\dfrac{\sigma^2_X}{\sigma^2 n^2}\)

\(\sigma^2_Y =\dfrac{1}{n^2}\)

\(\phi_Y(t)=1-\dfrac{t^2}{2n^2} +\sum_{j=3}^{\infty }\dfrac{E[(\dfrac{X-\mu}{\sigma n})^j](it)^j}{j!}\)

\(\phi_Z(t)=\phi_Y(t)^n\)

\(\phi_Z(t)=[1-\dfrac{t^2}{2n^2} +\sum_{j=3}^{\infty }\dfrac{E[(\dfrac{X-\mu}{\sigma n})^j](it)^j}{j!}]^n\)

\(\phi_Z(t)=[1-\dfrac{t^2}{2n^2}]^n\)

\subsubsection{Preparing for exponential expansion}

We know that

\([1+\dfrac{x}{n}]^n=e^x\)

As \(n \rightarrow \infty\).

With:

\(Z=\sum_{i=1}^nY_i\)

\(Y=\dfrac{X-\mu_X }{\sigma n}\)

We have:

\(\phi_Z(t)=[1-\dfrac{t^2}{2n^2}]^n\)

With:

\(Z=\sum_{i=1}^nY_i\)

\(Y=\dfrac{X-\mu_X }{\sigma \sqrt n}\)

We have:

\(\phi_Z(t)=[1-\dfrac{t^2}{2n}]^n\)

Which tends towards

\(\phi_Z(t)=e^{-\dfrac{1}{2}t^2}\)

\subsubsection{Rescaling}

The average of random variables, less their mean, and divided by their standard deviation multiplied by the square root of the sample size, follows a normal distribution as \(n\) increases.

What does this say about the actual distribution of sample averages?

\(Z=\sum_{i=1}^nY_i\)

\(Y_i=\dfrac{X_i-\mu_X }{\sigma_X \sqrt n}\)

\(\sum_{i=1}^nY_i\)

\(Y=\dfrac{X}{n}\)

Let's create \(Q\).

\(Q=\dfrac{Z\sigma_X }{\sqrt n}+\mu_X\)

\(Q=\dfrac{(\sum_{i=1}^nY_i)\sigma_X }{\sqrt n}+\mu_X\)

\(Q=\dfrac{(\sum_{i=1}^n(\dfrac{X_i-\mu_X }{\sigma_X \sqrt n}))\sigma_X }{\sqrt n}+\mu_X\)

\(Q=\sum_{i=1}^n(\dfrac{X_i-\mu_X }{n})+\mu_X\)

\(Q=\sum_{i=1}^n(\dfrac{X_i-\mu_X }{n}+\dfrac{\mu_X}{n})\)

\(Q=\sum_{i=1}^n(\dfrac{X_i}{n})\)

This is the sample average.

\(\phi_Q(t)=\phi_{\dfrac{Z\sigma_X }{\sqrt n}+\mu_X}(t)\)

\(\phi_Q(t)=\phi_Z(\dfrac{t\sigma_X }{\sqrt n})e^{it\mu_X}\)

\(\phi_Z(\dfrac{t\sigma_X }{\sqrt n})=e^{-\dfrac{1}{2}(\dfrac{t\sigma_X }{\sqrt n})^2}\)

\(\phi_Z(\dfrac{t\sigma_X }{\sqrt n})=e^{-\dfrac{1}{2}\dfrac{t^2\sigma^2_X }{n}}\)

\(\phi_Q(t)=e^{-\dfrac{1}{2}\dfrac{t^2\sigma^2_X }{n}}e^{it\mu_X}\)

\subsubsection{Normal distribution}

We name the normal distribution this function when \(n=1\)

\(N(\mu_X, \sigma^2_X)=e^{-\dfrac{1}{2}\dfrac{t^2\sigma^2_X }{n}}e^{it\mu_X}\)

\(N(\mu_X, \sigma^2_X)=e^{-\dfrac{1}{2}t^2\sigma^2_X }e^{it\mu_X}\)

\subsubsection{Getting the probability distribution function}

\(\phi_X(t)=e^{-\dfrac{1}{2}t^2\sigma^2_X} e^{it\mu_X}\)

\(\phi_X(t)=e^{-\dfrac{1}{2}t^2\sigma^2_X}[\cos (t\mu_X )+i\sin (t\mu_X)]\)


\chapter{Residual Neural Networks}
Recall that
\begin{equation}
f^j = \theta^j \circ g\circ f^{j-1}= \theta^j \circ g\circ
\theta^{j-1} \circ g \circ f^{j-2}
\end{equation}
\begin{equation}\label{fjj}
f^j = \theta^j \circ g\circ \theta^{j-1} \circ x_{j-1}
\end{equation}
where
\begin{equation}
  \label{eq:4}
x_{j-1}=g \circ f^{j-2}, \quad x_0=x
\end{equation}
We note \eqref{fjj} can be viewed as a nonlinear encoder for
$x_{j-1}$.  We now introduce the enhanced encoder has follows:
\begin{equation}
  \label{tildefj}
  \tilde f^j=f^{j}+\tilde \theta_{j-1}\circ x_{j-1}
\end{equation}
which is the main idea used for ResNet.

So we have the network like:
\begin{equation}
\begin{cases}
f^{2j} &= \theta^{2j} \circ g^{2j}( \tilde f^{2j-1}), \\
f^{2j-1} &= \theta^{2j-1} \circ g^{2j-1}(f^{2j-2}), \\
f_0 &= \theta^0(x).
\end{cases} \\
\end{equation}
When $\tilde \theta_{j}$ is a square ``matrix", we can just take $ \theta_{j} = I$, this is what ResNet often used.

Let us make some comments
\begin{enumerate}
\item It has been numerically observed that when $n_j=n_{j-1}$, one can take
  \begin{equation}
    \label{eq:1}
\theta_j=id.
  \end{equation}
\item In the usual CNN, it may happen that $\theta^j\approx 0$.  This
  means that the information is getting ``lost'' at the current level
  by ``improper'' (too much?) pooling or stride.  In this case,
  \eqref{tildefj} helps ``recover'' the information from the previous
  layer.   In this case
  \begin{equation}
\tilde f^j\approx x_{j-1}
  \end{equation}
\item Continuing with the above argument, it is then reasonable to
  expect that ResNet can use very large number of layers so that the
key features are still well kept from layer to layer.
\end{enumerate}

\bigskip
\bigskip
\begin{equation}
f^j = \theta^j \circ g^j(f^{j-1} + g\circ f^{j-3}) ),
\end{equation}

Take
\begin{equation}
x = g\circ f^{j-3},
\end{equation}
so $f^{j-1}$ is a function of $x$ as:
\begin{equation}
f^{j-1} = \theta^{j-1} \circ g \circ \theta^{j-1} (x).
\end{equation}

\section{ResNet model based on classical notation}
For general image classification problems, many numerical results shows that, if we just use the general CNN structure, you will get worse generallization error with deeper network. However, after 2015, Kaiming He and etc found that if they adopt the ``shortcut" structure into the general CNN they will get better result as networks become deeper and deeper in some degree and they call that kinds of CNN model as ResNet. After that, all state-of-art DL models adopt the ResNet properties to construct those deep CNN models with low generalization error.

\subsection{Shortcut connection and ResNet models}
First, we would like to talk about the so called ``shortcut connection" in general neural network. As we know, the output of $f_j$ is dependent with $f_i$ for all $ i < j$, but is just decided by $f_{j-1}$ directly and this can be shown in BP algorithm. Shortcut connection means to make the direct dependence longer like
\begin{equation}
f^j = \theta^j \circ g^j \circ h( f^{j-1}, \cdots, f^{j-k}).
\end{equation}

If we use the idea of encoder for CNN, we can take both $f^j$ with $j = 1:J$ as the nonlinear encoder of data. By the way, we can also think that $f^j$ is the nonlinear encoder of $f^i$ with $i < j$.  So, we may take $h$ as:
\begin{equation}
h(f^{j-1}, \cdots, f^{j-k}) = f^{j-1} + \sum_{s = 2}^k P_{j-s}^{j-1} f^{j-s}.
\end{equation}
or we need to add nonlinear map as:
\begin{equation}
h(f^{j-1}, \cdots, f^{j-k}) = f^{j-1} + \sum_{s = 2}^k P_{j-s}^{j-1} g^{j-s+1}(f^{j-s}).
\end{equation}

So, in fact we have many choice for change ResNet models, now we just introduce what kinds of shortcut connection in  RestNet.
\begin{equation}
f^j = \theta^j \circ g^j(f^{j-1} + P^jg^{j-2}(f^{j-3}) ),
\end{equation}
this can be shown in the next figure:
\begin{figure}[!htb]
	\center{\includegraphics[width=6cm] {figures/ShortCut.png}}
	\caption{The Architecture of ShortCut Connection with Distance 2}
	\label{LeNet-5}
\end{figure}
So, a general ResNet structure would be like:
\begin{equation}
\begin{cases}
f^{2j-1} &= \theta^{2j-1} \circ g^{2j-1}(f^{2j-2}), \\
\tilde f^{2j-1} &= f^{2j-1} + g\circ \tilde f^{2(j-1)-1}, \\
f^{2j} &= \theta^{2j} \circ g^{2j}(\tilde f^{j-1}),  \\
f_0 &= \theta^0(x).
\end{cases}
\end{equation}

\subsection{ResNet in ImageNet}
Here is a table to show the general structure for ResNet for ImageNet in 2015.
\begin{figure}[!htb]
	\center{\includegraphics[width=10cm] {figures/ResNet_ImageNet.png}}
	\caption{The Architecture of RestNet for ImageNet}
	\label{LeNet-5}
\end{figure}

The $\times 2$ block is the same to above, and the $\times 3$ block is shown next:
Here is a table to show the general structure for ResNet for ImageNet in 2015.
\begin{figure}[!htb]
	\center{\includegraphics[width=6cm] {figures/ShortCut2.png}}
	\caption{The Architecture of two kinds of shortcut connection}
	\label{LeNet-5}
\end{figure}
Or we can show it as:
\begin{equation}
f^j = \theta^j \circ g^j(f^{j-1} + P^jg^{j-3}(f^{j-4}) ),
\end{equation}
then the network would like:
\begin{equation}
\begin{cases}
f^{3j-2} &= \theta^{3j-2} \circ g^{3j-2}(f^{3(j-1)}), \\
f^{3j-1} &= \theta^{3j-2} \circ g^{3j-1}({f}^{3j-2}) \\
\tilde{f}^{3j-1} &= f^{3j-1} + g(\tilde f^{3(j-1)-1}), \\
f^{3j} &= \theta^{3j}\circ g^{3j}(\tilde f^{3j-1}) \\
f_0 &= \theta^0(x).
\end{cases}
\end{equation}

\subsection{Some properties of ResNet v.s PlainNet}
In the first paper about the ResNet, the authors proposed some properties of ResNet:
\begin{enumerate}
\item In some region, the deeper network the better results for ResNet. But, this cannot hold for plain network generally.
\begin{figure}[!htb]
	\center{\includegraphics[width=6cm] {figures/PlainNet.png}}
	\caption{The Architecture of two kinds of shortcut connection}
	\label{LeNet-5}
\end{figure}

\begin{figure}[!htb]
	\center{\includegraphics[width=10cm,height=4.5cm] {figures/PlianNetvsResNet.png}}
	\caption{The Architecture of two kinds of shortcut connection}
	\label{LeNet-5}
\end{figure}

\begin{figure}[!htb]
	\center{\includegraphics[width=12cm,height=4cm] {figures/PlianNetvsResNet-10.png}}
	\caption{The Architecture of two kinds of shortcut connection}
	\label{LeNet-5}
\end{figure}

\item Generally speaking, ResNet networks can be trained more easily than the plain network with the same depth.

\item For some case, if the pooling or convolution with stride is bigger than 1, the size of $f^{j-1}$ and $g^{j-3}(f^{j-4})$ are not the same, so we may need a "interpolation-restriction" matrix $P_{j-3}^{j-1}$ for the essential dimension is restriction but the channel dimension is restriction. And even for the dimension is the same, we can also use a transform for this operator i.e
\begin{equation}
f^j = \theta^j \circ g^j(f^{j-1} + P^jg^{j-2}(f^{j-3}) ),
\end{equation}
In the first paper about the ResNet, there are some numerical results to show that, P is only adopted when the dimension is changed.
\begin{figure}[!htb]
	\center{\includegraphics[width=6cm] {figures/ResNetProjection.png}}
	\caption{The Architecture of two kinds of shortcut connection}
	\label{LeNet-5}
\end{figure}

For the next three strategies:
\begin{enumerate}
\item Zero-padding shortcuts are used for increasing dimensions, and all shortcuts are parameter-free, i,e all $P^j$ is fix.
\item Projection shortcuts are used for increasing dimensions, and other shortcuts are identity.
\item All shortcuts are projections, i.e we need train all $P^j$.
\end{enumerate}

\section{One developed ResNet model}
For this model, which can be expressed as:
\begin{equation}\label{I_ResNet_old}
	\begin{cases}
		f^{2j-1} &= \theta^{2j-1} \circ g^{2j-1}(f^{2j-2}), \\
		\tilde f^{2j-1} &= f^{2j-1} + g\circ \tilde f^{2(j-1)-1}, \\
		f^{2j} &= \theta^{2j} \circ \tilde f^{j-1},  \\
		f_0 &= \theta^0(x).
	\end{cases}
\end{equation}
This can make sense is because of the fact that, if we take
\begin{equation}
F(x) = \theta \circ g \circ \theta(x)
\end{equation}
and noted
\begin{equation}
g^{j-2}(f^{j-3}) = x^1,
\end{equation}
then we have
\begin{equation}
x^l = P^lx^{l-1} + F^l(x^{l-1}),
\end{equation}
and for $P^{l} = I$ for almost cases.  For example, $P^l = I$ for $l = t+1: T$, which means:
\begin{equation}
x^T = x^t + \sum_{l=t}^{T} F^l(x^{l-1}).
\end{equation}
which means that information can transform for long distance especially between two pooling operations.

\newpage
\subsection{A look at some special case}
Let us take $j=9$ in \eqref{ResNet0}

\begin{equation}\label{ResNet9}
  \begin{array}{rcl}
f^0&=&\theta^0(x)\\
f^1&=&\theta^1\circ g\circ f^0(x)\\
\tilde f^1&=&f^1 \\
f^2&=&\theta^2\circ g\circ f^1(x)\\
f^3&=&\theta^3\circ g\circ f^2(x)\\
\tilde f^3&=&f^3+g\circ \tilde f^1 \\
f^4&=& \theta^4\circ g\circ \tilde f^3(x) \\
f^5&=&\theta^5\circ g\circ f^4(x)\\
\tilde f^5&=&f^5+g\circ \tilde f^3 \\
f^6&=&\theta^6\circ g\circ \tilde f^5(x)\\
f^7&=&\theta^7\circ g\circ f^6(x)\\
\tilde f^7&=&f^7+g\circ \tilde f^5 \\
 f^8&=& \theta^8\circ g\circ\tilde f^7(x) \\
 f^9&=& \theta^9 \circ g \circ f^8(x)
  \end{array}
\end{equation}

The above equation \eqref{ResNet9} shows the ResNet with 9 layers and step-2.

Now if we note
\begin{equation}
y^l = g \circ \tilde{f}^l,
\end{equation}
and
\begin{equation}
F^l(y^l) = \theta^{l+2} \circ g \circ \theta^{l+1}(y^l),
\end{equation}

So the above net work is:
\begin{equation}\label{ResNet9-1}
  \begin{array}{rcl}
f^9 &=& F^7(y^7) \\
y^7 &=& g \circ  [ F^5(y^5) + y^5 ]\\
y^5 &=& g \circ  [F^3(y^3) + y^3 ]\\
y^3 &=& g \circ  [F^1(y^1) + y^1 ]\\
y^1 &=& g \circ f^1
  \end{array}
\end{equation}
So, remove the $g$ in \eqref{ResNet9-1} seems very natural in the
above notation.

The most recent
\begin{equation}\label{ResNet9-1}
  \begin{array}{rcl}
f^9 &=& F^7(y^7) \\
y^7 &=& F^5(y^5) + y^5 \\
y^5 &=& F^3(y^3) + y^3 \\
y^3 &=& F^1(y^1) + y^1 \\
y^1 &=&  f^1
  \end{array}
\end{equation}
The most recent
\begin{equation}\label{ResNet9-new}
  \begin{array}{rcl}
f^9 &=& F^7(y^7) \\
y^7 &=& F^5(y^5) + y^5 \\
y^5 &=& F^3(y^3) + y^3 \\
y^3 &=& F^1(y^1) +  y^1 \\
y^1 &=&  f^1
  \end{array}
\end{equation}
Now adding all the formulas in \eqref{ResNet9-new}, we get
\begin{equation}\label{ResNet9-additive}
f^9 = F^7(y^7) + F^5(y^5) + F^3(y^3) +  F^1(y^1) + f^1
\end{equation}

Recall in multigrid
$$
I-B_kA_kP_k=(I-T_k)(I-B_{k-1}A_{k-1}P_{k-1})
=I-T_k(I-B_{k-1}A_{k-1}P_{k-1})-B_{k-1}A_{k-1}P_{k-1}
$$
$$
B_kA_kP_k=B_{k-1}A_{k-1}P_{k-1}
+T_k(I-B_{k-1}A_{k-1}P_{k-1})
$$
$$
B_JA_J=P_1+\sum_{k=2}^JT_k(I-B_{k-1}A_{k-1}P_{k-1})
=P_1+\sum_{k=2}^JT_k\prod_{j=1}^{k-1}(I-T_j)
$$

We note that
$$
F^1(y^1)=\theta^3\circ g\circ \theta^2\circ y^1
$$
We can take
$$
\theta^2=\check \theta^2\circ (I-\hat\theta^1)
$$
We can find
$$
\bar\theta^3\circ g\circ \bar\theta^2 = I
$$
which means
$$
\sum_{i=1}^k a_ig(w_ix+b_i)+c =x
$$
Now we make some special choices:
\begin{equation}\label{id}
\bar\theta^3\circ g\circ (\bar\theta^2\circ(\theta^2-I)) = \theta^2-I
\end{equation}
Thus
$$
y^3=\theta^2\circ y^1
$$
This recovers the linear model! Here to recover the \eqref{id} we only need the have that

\begin{equation}\label{id_cond}
\theta^3 \in \mathbb{R}^{n\times 2n} \quad \text{and} \quad \theta^2 \in \mathbb{R}^{2n \times n}.
\end{equation}

\newpage

The introduction of the original paper on ResNet \cite{HeKaiming2015} include the following statement:
\begin{quote}\it
  We can also use a square matrix $W_s$ in Eqn. (1). But we will show by experiments that the identity mapping is sufficient for addressing the degradation problem and is economical, and thus $W_s$ is only used when matching dimensions.
\end{quote}
In our notation, $W_s$ is just the $\tilde\theta^3$ in \eqref{ResNet4}.

We now compare a usual DNN as follows
\begin{equation}\label{OriginalResNet4}
  \begin{array}{rcl}
f^0&=&\theta^0(x)\\
f^1&=&\theta^1\circ g\circ f^0(x)\\
&\vdots& \\
f^9&=&\theta^9\circ g\circ f^8(x)
  \end{array}
\end{equation}
\end{enumerate}

We now study the relationship between \eqref{ResNet4} and \eqref{OriginalResNet4}.  Our question is then: Can we find weights such that the following identity holds
\begin{equation}
\label{ResNet4}
\theta^4\circ g\circ f^3=\tilde \theta^4\circ g\circ[f^3+\tilde \theta^3\circ g\circ f^1].
\end{equation}
and furthermore $\tilde \theta^3$ is non-singular?

Let us first consider the case that $g=id$, we ask
$$
\theta^4\circ f^3=\tilde \theta^4[f^3+\tilde \theta^3\circ f^1].
$$
$$
\theta^4\circ \theta^3\circ \theta^2 f^1=\tilde \theta^4[\theta^3\circ \theta^2 f^1+\tilde \theta^3\circ f^1].
$$
$$
\theta^4\circ \theta^3\circ \theta^2 =\tilde \theta^4\circ [\theta^3\circ \theta^2 +\tilde \theta^3].
$$

\newpage
\subsection{New notation}
\begin{equation}
  \label{extractor}
H_k(x)=\xi^k\circ g\circ \eta_k(x)
\end{equation}
The ResNet can then be written as
\begin{equation}  \label{RNk}
x^k=x^{k-1}+H_k(x^{k-1}), \quad k=1:J
\end{equation}
with
$$
x^0=x.
$$
We make the following observations:
\begin{enumerate}
\item \eqref{RNk} looks like nonlinear ``smoother'' or ``rougher'',
  but it is a combination of additive-multiplicative or
  BPX like.  It is like Block Gauss-Seidel with each block given by Jacobi.
\item We can make the above procedure to be multiplicative, which may
  improve efficiency.
\item Each different $k$ in \eqref{RNk} gives a new layer in the
  ResNet, but these are kind of ``fake'' layers.
\item The really new layer is only created by restriction, such as
  ``stride''
\item Different $H_k$ gives more parameters, which in a way gets
  closer to the variable convolution ... which corresponds to
  \begin{enumerate}
\item more different kernels on each pixel point
\item  more channels,
\end{enumerate}
\item The real layers should only be given when the dimension of the
  problem changes.   Namely the dimensions of different layers has to be
  different.
\item The relationship between ResNet and PlainNet is as follows:
  \begin{enumerate}
  \item PlainNet is Gauss-Seidel
\item ResNet is to convert part of the above Gauss-Seidel by a mixture
  of   Gauss-Seidal and Jacobi!
  \end{enumerate}
\end{enumerate}

\subsubsection{New terminology: ``extractor''}
We call each $H_k$ as
\begin{enumerate}
\item ``extractor''
\item smoother
\item rougher
\end{enumerate}


% !TeX root = ../main.tex
% Add the above to each chapter to make compiling the PDF easier in some editors.

\chapter{Homomorphism and Isomorphism Theorems}
In this chapter, we will discuss tools to show that two groups are isomorphic.

\begin{thm}[Homomorphism Theorem]\index{homomorphism theorem}
Let $\varphi : G \to H$ be a group homomorphism. Then, \begin{align}
    \rep{\varphi} : \Quot{G}{\ker{\varphi}} \to \im{\varphi},\quad a \ker{\varphi} \mapsto \varphi(a)
\end{align} is an isomorphism. Especially, $\Quot{G}{\ker{\varphi}} \isom \im{\varphi}$.
\end{thm} \begin{proof}
\leavevmode\begin{itemize}
    \item We will first show that $\rep{\varphi}$ is well-defined. For any $a, b \in G$, \begin{align*}
        &a \ker{\varphi} = b \ker{\varphi} \\
        \iff\quad &\inv{a} b \in \ker{\varphi} \margintag{by \cref{lem:cosets_eq}} \\
        \iff\quad &\varphi(\inv{a} b) = e_H \margintag{using the definition of the kernel \eqref{eq:kernel}} \\
        \iff\quad &\inv{\varphi(a)} \varphi(b) = e_H \margintag{using that $\varphi$ is a homomorphism \eqref{eq:homomorphism}} \\
        \iff\quad &\varphi(a) = \varphi(b) \\
        \iff\quad &\rep{\varphi}(a \ker{\varphi}) = \rep{\varphi}(b \ker{\varphi}). \margintag{using the definition of $\rep{\varphi}$}
    \end{align*} The direction ``$\Rightarrow$'' shows that $\rep{\varphi}$ is well-defined. Note that ``$\Leftarrow$'' shows that $\rep{\varphi}$ is injective.

    \item Next, we show that $\rep{\varphi}$ is a homomorphism. For any $a, b \in G$, \begin{align*}
        \rep{\varphi}(a \ker{\varphi} \cdot b \ker{\varphi}) &= \rep{\varphi}(a b \ker{\varphi}) \margintag{using the operation of the quotient group \eqref{eq:quotient_group_op}} \\[5pt]
        &= \varphi(a b) \margintag{using the definition of $\rep{\varphi}$} \\
        &= \varphi(a) \cdot \varphi(b) \margintag{using that $\varphi$ is a homomorphism \eqref{eq:homomorphism}} \\
        &= \rep{\varphi}(a \ker{\varphi}) \cdot \rep{\varphi}(b \ker{\varphi}). \margintag{using the definition of $\rep{\varphi}$}
    \end{align*}

    \item Finally, we observe that $\rep{\varphi}$ is surjective. That is, for any $a \in \im{\varphi}$, we have that $a \ker{\varphi} \in \Quot{G}{\ker{\varphi}}$. \qedhere
\end{itemize}
\end{proof}

\begin{ex}{Homomorphism theorem}{}
We have already seen in \cref{ex:homomorphisms} that for any field $K$, ${\det : \GL{n}{K} \to \woZ{K}}$ is a homomorphism with ${\ker{\det} = \SL{n}{K}}$. Thus, by the homomorphism theorem, \begin{align}
    \Quot{\GL{n}{K}}{\SL{n}{K}} \isom \woZ{K}.
\end{align}
\end{ex}

\begin{marginfigure}
    \includegraphics[width=\textwidth]{correspondence_thm.png}
    \caption{Sketch of the correspondence of subgroups $U_i$ ``between'' $G$ and $N$ and the subgroups $\Quot{U_i}{N}$ ``between'' $\Quot{G}{N}$ and $\Quot{N}{N}$.}\label{fig:correspondence_thm}
\end{marginfigure}

\begin{thm}[Correspondence Theorem]\index{correspondence theorem}
The mapping, \begin{align}
    f : \{U \subgroup G \mid N \subseteq U\} \to \{V \subgroup \Quot{G}{N}\},\quad U \mapsto \Quot{U}{N},
\end{align} is an inclusion-preserving\footnote{We say that a mapping ${f : A \to B}$ is \emph{inclusion-preserving}\index{inclusion-preserving mapping} if for any $a_1, a_2 \in A$ such that ${a_1 \subseteq a_2}$, we have ${f(a_1) \subseteq f(a_2)}$.} bijection and fore every $U \subgroup G$, \begin{align}
    U \normal G \iff \Quot{U}{N} \normal \Quot{G}{N}.
\end{align}
\end{thm} \noindent A sketch of the correspondence theorem is given in \cref{fig:correspondence_thm}.

\begin{marginfigure}
    \includegraphics[width=\textwidth]{first_isom_thm.png}
    \caption{Sketch of the first isomorphism theorem.}\label{fig:first_isom_thm}
\end{marginfigure}

\begin{thm}[First Isomorphism Theorem]\index{first isomorphism theorem}\index{isomorphism theorem}
Let $U \subgroup G$ and $N \normal G$. Then, \begin{thmlist}
    \item $U N \subgroup G$ where $U N \defeq \{x \cdot n \mid x \in U, n \in N\}$
    \item $U \cap N \normal U$
    \item $\Quot{UN}{N} \isom \Quot{U}{(U \cap N)}$
\end{thmlist}
\end{thm} \noindent A sketch of the correspondence theorem is given in \cref{fig:first_isom_thm}.

\begin{marginfigure}
    \includegraphics[width=\textwidth]{second_isom_thm.png}
    \caption{Sketch of the second isomorphism theorem..}\label{fig:second_isom_thm}
\end{marginfigure}

\begin{thm}[Second Isomorphism Theorem]\index{second isomorphism theorem}\index{isomorphism theorem}
Let $U, N \normal G$ with $N \subseteq U$. Then, $\Quot{(\Quot{G}{N})}{(\Quot{U}{N})} \isom \Quot{G}{U}$.
\end{thm}


\documentclass{article}
\usepackage{amsmath}
\usepackage[margin=1.0in]{geometry}
\usepackage{xcolor}

\begin{document}

\noindent
Does $\displaystyle \sum_{n=2}^\infty \frac{2n}{n^2+1}$
diverge, converge absolutely, or converge conditionally?

\subsection*{Solution 1}

The function $f(x)=\frac{2x}{x^2+1}$ is continuous, positive, and decreasing on $[2,\infty)$.
\begin{align*}
\int_2^\infty \frac{2x}{x^2+1}\,dx
&= \lim_{t \to \infty} \int_2^t \frac{2x}{x^2+1}\,dx\\
&= \lim_{t \to \infty} (\ln|t^2+1| - \ln |2^2+1|)\\
&= \infty
\end{align*}
Since the integral $\displaystyle \int_2^\infty \frac{2x}{x^2+1}\,dx$ diverges, the series $\displaystyle \sum_{n=2}^\infty \frac{2n}{n^2+1}$ diverges by the Integral Test.

\subsection*{Solution 2}

Since $n^2 \geq 1$, by adding $n^2$ to both sides of this inequality
\[ 2n^2 \geq n^2 + 1\]
Dividing both sides of this ineqaulity by $n(n^2+1)$, we get
\[ \frac{2n}{n^2+1} \geq \frac1n\]
Since the series $\sum \frac1{n}$ diverges by the $p$-test, the series $\sum \frac{2n}{n^2+1}$ diverges by the Direct Comparison Test.

\subsection*{Solution 3}

The series $\sum \frac1{n}$ diverges by the $p$-test. Let $a_n = \frac{2n}{n^2+1}$ and $b_n = \frac1{n}$.
\begin{align*}
\lim_{n \to \infty} \frac{a_n}{b_n}
&= \lim_{n \to \infty} \frac{2n^2}{n^2+1}\\
&= \lim_{n \to \infty} \frac{4n}{4n} \text{ by l'hopital}\\
&= \lim_{n \to \infty} \frac44 \text{ by algebra}\\
&= 1
\end{align*}
The Limit Comparison Test applies, since this limit was a positive, finite number. Therefore, the series $\sum \frac{2n}{n^2+1}$ diverges by the Limit Comparison Test.

\end{document}%%%%%%%%%%%%%%%%%

\begin{align*}
L&=\lim_{n \to \infty} \sqrt[n]{|a_n|}\\
&= \lim_{n \to \infty} \sqrt[n]{\left| \right|}\\
\end{align*}

\begin{align*}
L&=\lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|\\
&= \lim_{n \to \infty} \left| \right|\\
\end{align*}

\begin{align*}
\lim_{n \to \infty} a_n
&= \lim_{n \to \infty} \\
\end{align*}

Since $\sum |a_n| = \sum a_n$, the series $\displaystyle \sum_{n=1}^\infty AAAAAAAAAAAAAA$ converges absolutely.

Since $|r| < 1$, the series ...  converges by the Geometric Series Test.

Since $|r| \geq 1$, the series ...  diverges by the Geometric Series Test.

The function $f(x)=\frac{}{}$ is continuous, positive, and decreasing on $[1,\infty)$.

\subsection*{Solution}


\clearpage
\subsection{Pascal Expression} % (fold)
\label{sub:program-creation-pas_expression}

An \nameref{sub:expression} in Pascal is a mathematical calculation or a literal value. Each expression will have a \nameref{sub:type}, and can contain a number of mathematic operators. Table \ref{tbl:program-creation-pas operators and expresions} lists the operators that you can include in your expressions, listed in order of precedence.\footnote{Expressions follow the standard mathematic order of precedence (BODMAS).} The operators you can use depend on the kind of data that you are using within the expression.

\begin{table}[h]
  \begin{minipage}{\textwidth}
  \centering
  \begin{tabular}{|c|l|l|}
    \hline
    \textbf{Operator} & \textbf{Description} & \textbf{Example} \\
    \hline
    \texttt{ ( ) }     &   Parenthesis                 & \texttt{(1 + 1) * 2}  \\
    \texttt{mod * / div}\footnote{\texttt{div} performs an integer division, ignoring any remainder.}      &   Modulo\footnote{The remainder after division. For example 9 modulo 3 is 0, 10 modulo 3 is 1, 11 modulo 3 is 2 etc.}, Multiplication and Division & \texttt{1 / 2 * 5}  \\
     & & \texttt{1 div 2 * 5} \\
    \texttt{+ -}      &   Addition and subtraction    & \texttt{10 + 3 - 4}   \\
    \hline
  \end{tabular}
  \end{minipage}
  \caption{Pascal Operators and Example Expressions}
  \label{tbl:program-creation-pas operators and expresions}
\end{table}

\begin{table}[h]
  \begin{minipage}{\textwidth}
  \centering
  \begin{tabular}{|c|c|l|}
    \hline
    \textbf{Example Expression} & \textbf{Value} & \textbf{Type} \\
    \hline
    \texttt{ 73 }     &   73                 & \texttt{Integer}  \\
    \texttt{ 2.1 }      & 2.1   & \texttt{Single}    \\
    \texttt{ `Hello World' }      &   `Hello World'    & \texttt{String}   \\
    \texttt{ `Hello ' + `World' }      &   `Hello World'    & \texttt{String}   \\
    \texttt{ `Fred' }      &   `Fred'    & \texttt{String}   \\
    \texttt{ 3 * 2 } & 6 & \texttt{Integer} \\
    \texttt{ 1 + 3 * 2 }  & 7 & \texttt{Integer} \\
    \texttt{ (1 + 3) * 2} & 8 & \texttt{Integer} \\
    \texttt{ 7 - 3 + 1 }  & 5 & \texttt{Integer} \\
    \texttt{ 3 / 2 } & 1.5 & \texttt{Single} \\
    \texttt{ 3 div 2 } & 1 & \texttt{Integer} \\
    \texttt{ 3 mod 2 } & 1 & \texttt{Integer} \\
    \texttt{ 11 mod 3 } & 2 & \texttt{Integer} \\
    \texttt{ 3.0 / 2.0} & 1.5 & \texttt{Single} \\
    \texttt{ 3 / 2.0 } & 1.5 & \texttt{Single} \\
    \texttt{ 1 + (3 / 2.0) + 6 * 2 - 8} & 6.5 & \texttt{Single} \\
    \hline
  \end{tabular}
\end{minipage}
  \caption{Pascal Example Expressions}
  \label{tbl:program-creation-pas example expresions}
\end{table}

\mynote{
\begin{itemize}
  \item Table \ref{tbl:program-creation-pas example expresions} shows some example expressions, their values, and types.
  \item Expressions can be literal values, entered in the code.
  \item Expression can contain mathematical calculations using standard addition, subtraction, multiplication, division, and grouping.
\end{itemize}
}

% subsection c_expression (end)


\documentclass{article}
\usepackage{graphicx}
\usepackage{hyperref}
\begin{document}

\section{The Problem}

The problem originally appeared as the Puzzler on FiveThirtyEight web site on Friday, August 14, 2020.\footnote{\url{https://fivethirtyeight.com/features/are-you-hip-enough-to-be-square/}} Here is the problem:

\begin{quotation}
The Riddler Manufacturing Company makes all sorts of mathematical tools: compasses, protractors, slide rules — you name it!

Recently, there was an issue with the production of foot-long rulers. It seems that each ruler was accidentally sliced at three random points along the ruler, resulting in four pieces. Looking on the bright side, that means there are now four times as many rulers — they just happen to have different lengths.

On average, how long are the pieces that contain the 6-inch mark?
\end{quotation}

There is not a unique answer to this problem. The answer depends on the process used to determine the location of the three slices. The following sections describes an algorithm for computing the location of the slices and computes the solution for that algorithm. There is a summary at the end
with the result for each algorithm, together with some additional results for other algorithms from Python simulations.

Within these sections we observe the following conventions:
\begin{enumerate}
\item The ruler is twelve inches long. So all lengths are in inches.
\item The three points where the slices occur designated by $x$, $y$, and $z$. Unless otherwise noted, we will have $0 < x < y < z < 12$. We note that there is zero probability of any of $<$ actually being equality, so the possibility is ignored.
\item All random choices are uniformly distributed. A sentence ``Randomly choose a point between $3$ and $12$'' implies a uniform distribution where the
probability of choosing a point between $3$ and $4$ inches is $1/9$.
\end{enumerate}

\section{Choose Three Independent Points}

For this algorithm we independently choose three random points between zero and $12$ inches and then sort them in order to obtain $x$, $y$, and $z$.

We divide this into four cases:
\begin{enumerate}
\item $0 < x < y < z < 6$
\item $0 < x < y < 6 < z < 12$
\item $0 < x < 6 < y < z < 12$
\item $6 < x < y < z < 12$
\end{enumerate}

Cases 1 and 4 occur with probability $1/8$ and cases 2 and 3 occur with probability $3/8$. If we reverse the numbers on the ruler case 1 is equivalent
to case 4 and case 2 is equivalent to case 3.

For case 1 the desired length (length of the piece with the $6$ inch mark) is $12 - z$. $z$ is the maximum of three uniformly distributed random numbers
in the same interval. We know \footnote{For example,
\url{https://math.stackexchange.com/questions/460649/the-largest-of-n-random-numbers-over-a-uniform-distribution}} that the maximum is $\frac{N}{N+1}$
times the length of the interval. In this case that gives the expected value of $z = \frac{3}{4} \cdot 6  = 4.5$. So the expected length is $12 - 4.5 = 7.5$ inches.

For case 2 the desired length is $z - y$. The expected value of $z = 9$, the middle of the interval. The expected value of $y = \frac{2}{3} \cdot 6 = 4$. The
expected length is $9 - 4 = 5$ inches.

Combining the cases, remembering that cases 1 and 4 and cases 2 and 3 are equivalent, gives the expected length as $ \frac{1}{4} \cdot 7.5 + \frac {3}{4} \cdot 5 = 5 + \frac{5}{8} = 5.625$ inches.

\section{Choose $x$, then $y$, then $z$}

For this algorithm, we choose $x$ between $0$ and $12$, then $y$ between $x$ and $12$, and finally, $z$ between $y$ and $12$.

We note for this case that if $x \ge 6$, the choices for $y$ and $z$ are irrelevant and the length is $x$. If $x < 6$, we then have to look at $y$, and potentially
$z$ to compute the desired length. We will define $E$ to be the desired length, $EX(x)$ to be the expected length for  a given value of $x$ assuming $0 < x < 6$, and $EY(y)$ to be the expected length for a given value of $y$ assuming $0 < x < y < 6$.

We have
$$ E =  \frac{1}{12} \left({\int_{0}^{6} EX(x) dx} + {\int_{6}^{12} x \,dx}\right)$$

Similarly, we have
\begin{eqnarray*}
EX(x) &=& \frac{1}{12-x} \left({\int_{x}^{6} EY(y) dy} + {\int_{6}^{12} y-x \,dy}\right)\\
EY(y) &=&  \frac{1}{12-y} \left({\int_{y}^{6} 12-z \,dz} + {\int_{6}^{12} z-y \,dz}\right)
\end{eqnarray*}

Working backwards (with help from Wolfram Alpha!\footnote{\url{https://www.wolframalpha.com/input/}}):
\begin{eqnarray*}
EY(y) &=& \frac{1}{12-y} \left( \frac{1}{2} y^2 -12 y + 54 + 6(9-y) \right)\\
&=& \frac{\frac{1}{2}y^2 -18y +108}{12-y}
\end{eqnarray*}
\begin{eqnarray*}
EX(x) &=& \frac{1}{12-x} \left({\int_{x}^{6} \frac{\frac{1}{2}y^2 -18y +108}{12-y} \,dy} + {\int_{6}^{12} y-x \,dy}\right)
\end{eqnarray*}
\begin{eqnarray*}
\int_{x}^{6} \frac{\frac{1}{2}y^2 -18y +108}{12-y} \,dy &=& \frac{1}{2} \left(\frac{1}{2}x^2 + 12x + 144\,log\left(\frac{12-x}{6}\right)-90\right)\\
&&-18\left(x + 12\,log\left(\frac{12-x}{6}\right) -6\right)\\
&&+108 \, log\left(\frac{12-x}{6}\right)\\
&=&\frac{1}{4}x^2 -12x -36log\left(\frac{12-x}{6}\right) + 63\\
{\int_{6}^{12} y-x \,dy}&=&-6x +54\\
EX(x)&=&\frac{\frac{1}{4}x^2 -18x -36log\left(\frac{12-x}{6}\right) + 117}{12-x}\\
\end{eqnarray*}
\begin{eqnarray*}
E &=&  \frac{1}{12} \left({\int_{0}^{6} \frac{\frac{1}{4}x^2 -18x -36log\left(\frac{12-x}{6}\right) + 117}{12-x} dx} + {\int_{6}^{12} x \,dx}\right)
\end{eqnarray*}
\begin{eqnarray*}
\int_{0}^{6} \frac{\frac{1}{4}x^2 -18x -36log\left(\frac{12-x}{6}\right) + 117}{12-x} dx &=&\frac{1}{4}\left(144log(2) - 90\right)\\
&&-18\left(12log(2) - 6\right)\\
&&-36\left(\frac{1}{2}log^2(2)\right)\\
&&117\,log(2)\\
&=&-63\, log(2) - 18 \,log^2(2) +85.5\\
\int_{6}^{12} x \,dx &=& 54
\end{eqnarray*}
\begin{eqnarray*}
E &=& \frac{1}{12}\left(-63\, log(2) - 18 \,log^2(2) +139.5\right)\\
&=& -5.25\,log(2) - 1.5\,log^2(2)+11.625 \approx 7.265298
\end{eqnarray*}

\section{Choose $y$, then $x$ and $z$}

This algorithm is similar to the last one except we first choose $y$ between $0$ and $12$ and then choose $x$ between $0$ and $y$ and $z$
between $y$ and $12$.

We note in this case that for $y > 6$ we only have to choose $x$ and for $y < 6$ we only have to choose $z$. Further, we can see that these two
cases are symmetric: reversing the ruler transforms one into the other. So, we assume $y > 6$ and ignore the choice of $z$. Similar to the previous
we define $E$ to be the desired length and $EY(y)$ to be the expected length for a given value of y assuming $6 < y < 12$.

We have
\begin{eqnarray*}
E &=& \frac{1}{6} \int_{6}^{12} EY(y) dy\\
EY(y) &=& \frac{1}{y} \left( \int_{0}^{6} y-x \,dx + \int_{6}^{y} x \, dx \right)\\
&=& \frac{1}{y} \left( 6y - 18 + \frac{1}{2} y^2 - 18 \right)\\
&=& \frac{1}{2}y + 6 - \frac{36}{y}\\
E &=& \frac{1}{6} \int_{6}^{12}  \frac{1}{2}y + 6 - \frac{36}{y} dy\\
&=& \frac{1}{6} \left( \frac{12^2}{4} + 6 \cdot 12 - 36\,log(12) - \frac{6^2}{4} - 6 \cdot 6 + 36\,log(6)\right)\\
&=& \frac{1}{6} \left( 63 - 36 \, log(2) \right)\\
&=& 10.5 - 6 log(2) \approx 6.341117
\end{eqnarray*}

\section{Summary}

The table below summarizes the results for different algorithms. In addition to the analytical results from the preceding sections, the table includes
expected lengths from Python simulations of some additional algorithms. These results are marked with an asterisk (*). Due to time constraints,
I was unable to derive analytical results for these values. The Python program that generated these numbers can be
found at \url{https://github.com/chrisreedy/fun/blob/master/SlicedRuler/ruler.py}.

\,

\begin{tabular}{l l}
\textbf{Algorithm} & \textbf{Expected Length} \\
\hline Choose Three Independent Points & $5.625$ \\
Choose $x$, then $y$, then $z$ & $\approx 7.265298$\\
Choose $y$, then $x$ and $z$ & $\approx 6.341117$\\
Choose $x$, then $z$, then $y$ & $7.08^*$ \\
Choose Longest for next Slice & $4.47^*$ \\
Choose Slice containing 6" for next Slice & $4.35^*$
\end {tabular}

\end{document}


\paragraph{Nim}
For simplicity, we denote $a_i$ as the number of stones in the $i$-th pile, $M_i(S)$ as removing stones with the amount chosen in the set $S$ from the $i$-th pile, and $M_i=M_i[1,a_i]$. Without further explanation, it is assumed that the SG function of a game $SG=\bigoplus_{i=1}^nSG(a_i)$.

\subparagraph{Nim}
$M=\bigcup_{i=1}^nM_i$.

Normal: $SG(n)=n$.

Misere: The same, opposite if all piles are $1$'s.

\subparagraph{Nim (powers)}
Given $k$, $M=\bigcup_{i=1}^nM_i\{k^m|m\ge 0\}$.

Normal: If $k$ is odd, $SG(n)=n\%2$. Otherwise,

$$SG(n)=\biggl\{\begin{array}{lr}
2 & n\%(k+1)=k \\
n\%(k+1)\%2 & \mathrm{otherwise}\,.\end{array}$$

\subparagraph{Nim (no greater than half)}
$M=\bigcup_{i=1}^nM_i[1,\frac{a_i}{2}]$.

Normal: $SG(2n)=n,SG(2n+1)=SG(n)$.

\subparagraph{Nim (always greater than half)}
$M=\bigcup_{i=1}^nM_i[\left\lceil \frac{a_i}{2}\right\rceil, a_i]$.

Normal: $SG(0)=0,SG(n)=\left\lfloor \log_2 n\right\rfloor +1$.

\subparagraph{Nim (proper divisors)}
$M=\bigcup_{i=1}^nM_i\{x|x>1\wedge a_i\% x=0\}$.

Normal: $SG(1)=0,SG(n)=\max_x(n\%2^x=0)$.

\subparagraph{Nim (divisors)}
$M=\bigcup_{i=1}^nM_i\{x|a_i\% x=0\}$.

Normal: $SG(0)=0,SG(n)=1+\max_x(n\%2^x=0)$.

\subparagraph{Nim (fixed)}
Given a finite set $S$, $M=\bigcup_{i=1}^nM_i(S)$.

Normal: $SG_1(n)$ is eventually periodic.

Given a finite set $S$, $M=\bigcup_{i=1}^nM_i(S\cup {a_i})$.

Normal: $SG_2(n)=SG_1(n)+1$.

\subparagraph{Moore's Nim}
Given $k$, $M=\bigcup\{M_{x_1}\times M_{x_2} \dots\times M_{x_l}|l\le k \wedge\forall i(x_i<x_{i+1})\}$.

Normal: Sum all $(a_i)_2$ in base $k+1$ without carry. Lose if the result is $0$.

Misere: The same, except if all piles are $1$'s.

\subparagraph{Staircase Nim}
One can take any number of objects from $a_{i+1}$ to $a_i$($i\ge 0$).

Normal: Lose if $\bigoplus_{i=0}^{(n-1)/2}a_{2i+1}=0$.

\subparagraph{Lasker's Nim}
$M=\bigcup_{i=1}^nM_i\cup S_i$. ($S_i$: Split a pile into two non-empty piles.)

Normal: $SG(n)=\biggl\{\begin{array}{lr}
n & n\%4=1,2\\
n+1 & n\%4=3\\
n-1 & n\%4=0\,.\end{array}$

\subparagraph{Kayles}
$M=\bigcup_{i=1}^nM_i[1,2]\cup MS_i[1,2]$. ($MS_i$: Split a pile into two non-empty piles after removing stones.)

Normal: Periodic from the $72$-th item with period length $12$.

\subparagraph{Dawson's chess}
$n$ stones in a line. One can take a stone if its neighbours are not taken.

Normal: Periodic from the $52$-th item with period length $34$.

\paragraph{Ferguson game}
Two boxes with $m$ stones and $n$ stones. One can empty any one box and move any positive number of stones from another box to this box each step.

Normal: Lose if both $m$ and $n$ are odd.

\paragraph{Fibonacci game}
$n$ stones. The first player may take any positive number of stones during the first move, but not all of them. After that, each player may take any positive number of stones, but less than twice the number of stones taken during the last turn.

Normal: Win if $n$ is not a fibonacci number.

\paragraph{Wythoff's game}
Two piles of stones. Players take turns removing stones from one or both piles; when removing stones from both piles, the numbers of stones removed from each pile must be equal.

Normal: Lose if $\lfloor \frac{\sqrt{5}+1}{2}|A-B| \rfloor=\min(A,B)$

\paragraph{Mock turtles}
$n$ coins in a line. One can turn over any $1$, $2$, or $3$ coins, but the rightmost coin turned must be from head to tail.

Normal: $SG(n)=2n+[\operatorname{popcount}(n)\mathrm{\ is\ even}]$.

\paragraph{Ruler}
$n$ coins in a line. One can turn over any consecutive coins, but the rightmost coin turned must be from head to tail.

Normal: $SG(n)=\operatorname{lowbit}(n)$.

\paragraph{Hackenbush}
The game starts with the players drawing a ground line (conventionally, but not necessarily, a horizontal line at the bottom of the paper or other playing area) and several line segments such that each line segment is connected to the ground, either directly at an endpoint, or indirectly, via a chain of other segments connected by endpoints. Any number of segments may meet at a point and thus there may be multiple paths to ground.

On his turn, a player cuts (erases) any line segment of his choice. Every line segment no longer connected to the ground by any path falls (i.e., gets erased). According to the normal play convention of combinatorial game theory, the first player who is unable to move loses.

Played exclusively with vertical stacks of line segments, also referred to as bamboo stalks, the game directly becomes Nim and can be directly analyzed as such. Divergent segments, or trees, add an additional wrinkle to the game and require use of the colon principle stating that when branches come together at a vertex, one may replace the branches by a non-branching stalk of length equal to their nim sum. This principle changes the representation of the game to the more basic version of the bamboo stalks. The last possible set of graphs that can be made are convergent ones, also known as arbitrarily rooted graphs. By using the fusion principle, we can state that all vertices on any cycle may be fused together without changing the value of the graph. Therefore, any convergent graph can also be interpreted as a simple bamboo stalk graph. By combining all three types of graphs we can add complexity to the game, without ever changing the Nim sum of the game, thereby allowing the game to take the strategies of Nim.

\paragraph{Joseph cycle}
$n$ players are numbered with $0,1,2,...,n-1$. $f_{1,m}=0,f_{n,m}=(f_{n-1,m}+m)\mod n$.


\documentclass[a4paper]{article}

\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{amsmath}
\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

\usepackage{graphicx}
\usepackage{fullpage}
\title{MATH 542 Final Exam}
\author{Saket Choudhary\\skchoudh@usc.edu}

\begin{document}
\maketitle

\section*{Problem 1}
\subsection*{Problem 1a}

\begin{align*}
Y_i &= \beta_0 + \beta_1 x_{i1} + \beta_2 x_{i2} + \dots + \beta_{p-1}x_{i,p-1} + \epsilon_i\\
Y_i &= \alpha + \beta_1(x_{i1}-\bar{x}_1) + \beta_2(x_{i2} - \bar{x}_2) + \dots + \beta_{p-1}(x_{i,p-1}-\bar{x}_{p-1})\\
&= \alpha - \beta_1 \bar{x}_1 -\beta_2 \bar{x}_2 - \dots - \beta_{p-1}\bar{x}_{p-1} + \beta_1 x_{i1} + \beta_2 x_{i2} + \dots + \beta_{p-1}x_{i,p-1} + \epsilon_i\\
\implies \beta_0 &= \alpha - \beta_1 \bar{x}_1 -\beta_2 \bar{x}_2 - \dots - \beta_{p-1}\bar{x}_{p-1}\\
\implies \alpha &= \beta_0 + \beta_1 \bar{x}_1 -\beta_2 \bar{x}_2 - \dots - \beta_{p-1}\bar{x}_{p-1}
\end{align*}

\subsection*{Problem 1.2}
\begin{align*}
\bar{x_j} &= \sum_{i=1}^n \frac{x_{ij}}{n}\\
&= \frac{1}{n}\mathbf{1_n}'\mathbf{X_i}\\
Y &= \begin{pmatrix}
1 & x_{11}-\bar{x}_1 & x_{12}-\bar{x}_2 & \dots & x_{1,p-1}-\bar{x}_{p-1}\\
1 & x_{21}-\bar{x}_2 & x_{22}-\bar{x}_2 & \dots & x_{2, p-1}-\bar{x}_{p-1}\\
\vdots & \\
1 & x_{n1}-\bar{x}_1 & x_{n2}-\bar{x}_2 & \dots & x_{n,p-1}-\bar{x}_{p-1}
\end{pmatrix}\begin{pmatrix}
\alpha\\
\beta_1\\
\vdots\\
\beta_{p-1}
\end{pmatrix} + \mathbf{\epsilon}\\
&=\big(\begin{pmatrix}
1 & x_{11} & x_{12} & \dots & x_{1,p-1}\\
1 & x_{21} & x_{22} & \dots & x_{2, p-1}\\
\vdots & \\
1 & x_{n1} & x_{n2} & \dots & x_{n,p-1}
\end{pmatrix}- \begin{pmatrix}
0 & \bar{x}_1 & \bar{x}_2 & \dots & \bar{x}_{p-1}\\
0 & \bar{x}_1 & \bar{x}_2 & \dots & \bar{x}_{p-1}\\
\vdots & \\
0 & \bar{x}_1 & \bar{x}_2 & \dots & \bar{x}_{p-1}\\
\end{pmatrix} \big)\begin{pmatrix}
\alpha\\
\beta_1\\
\vdots\\
\beta_{p-1}
\end{pmatrix} + \mathbf{\epsilon}\\
&= \big(\begin{pmatrix}
\mathbf{1_n} & \mathbf{X}
\end{pmatrix} - \begin{pmatrix}
\mathbf{0_n} & \frac{1}{n}\mathbf{1_n1_n'X}
\end{pmatrix}\big)\begin{pmatrix}
\alpha\\
\beta_1\\
\vdots\\
\beta_{p-1}
\end{pmatrix} + \mathbf{\epsilon}\\
&= \begin{pmatrix}
\mathbf{1_n} & \mathbf{X}-\frac{1}{n}\mathbf{1_n1_n'X}
\end{pmatrix}\begin{pmatrix}
\alpha\\
\beta_1\\
\vdots\\
\beta_{p-1}
\end{pmatrix}+ \mathbf{\epsilon}\\
&= \begin{pmatrix}
\mathbf{1_n} & \mathbf{X_c}
\end{pmatrix}\begin{pmatrix}
\alpha\\
\beta_1\\
\vdots\\
\beta_{p-1}
\end{pmatrix}+ \mathbf{\epsilon}\\
\implies \mathbf{X_c} &= (\mathbf{I}-\frac{\mathbf{1_n1_n'}}{n})X
\end{align*}

\subsection*{Problem 1.3}

\begin{align*}
\mathbf{1_n'X_c} &= \begin{pmatrix}
1 & 1 & 1 & \dots & 1
\end{pmatrix} \begin{pmatrix}
\mathbf{I}-\frac{1}{n}\mathbf{1_n1_n'}
\end{pmatrix}\mathbf{X}\\
&= \begin{pmatrix}
1 & 1 & 1 & \dots & 1
\end{pmatrix} \begin{pmatrix}
1-\frac{1}{n} & \frac{-1}{n} & \frac{-1}{n} & \dots & \frac{-1}{n}\\
\frac{-1}{n} &1-\frac{1}{n} & -\frac{1}{n} & \dots & -\frac{1}{n}\\
\vdots\\
\frac{-1}{n} &-\frac{1}{n} & -\frac{1}{n} & \dots & 1-\frac{1}{n}\\
\end{pmatrix}\\
&= (1-\frac{1}{n}) -\frac{1}{n}*(n-1)\\
&= 0
\end{align*}

\subsection*{Problem 1.4.a}

\begin{align*}
Y_1 &= \alpha + \beta_1(x_{11}-\bar{x}_1) + \beta_2(x_{12} - \bar{x}_2) + \dots + \beta_{p-1}(x_{1,p-1}-\bar{x}_{p-1})\\
Y_2 &= \alpha + \beta_1(x_{21}-\bar{x}_1) + \beta_2(x_{22} - \bar{x}_2) + \dots + \beta_{p-1}(x_{2,p-1}-\bar{x}_{p-1})\\
\vdots \\
Y_n &= \alpha + \beta_1(x_{n1}-\bar{x}_1) + \beta_2(x_{n2} - \bar{x}_2) + \dots + \beta_{p-1}(x_{n,p-1}-\bar{x}_{p-1})\\
\hline\\
\sum_{i=1}Y_i &= n\alpha + \beta_1(\sum_{i=1}x_{i1}-n\bar{x}_1) + \beta_2(\sum_{i=1}x_{i2}-n\bar{x}_2) + \dots + \beta_{p-1}(\sum_{i=1}x_{i,p-1}-n\bar{x}_{p-1})\\
\sum_{i=1}x_{i,j}-n\bar{x}_j &= 0\\
\implies \hat{\alpha} &= \frac{\sum_{i=1}Y_i}{n}
\end{align*}
Now, we perform $Y_i-\bar{Y}$ eliminating $\alpha$ Let $Z=Y_i-\bar{Y}$m the problem then reduces to the following form: $Z = \mathbf{X_c}\beta + \epsilon$ where $\beta = \begin{pmatrix}
\beta_1 & \beta_2 & \dots & \beta_{p-1}
\end{pmatrix}$ and hence simply re-using OLS results, $\hat{\beta} =\mathbf{X_c'X_c}^{-1}\mathbf{X_c'}Y$

More rigorously:

\begin{align*}
\hat{\beta} &= (X'X)^{-1}X'Y\\
X'X\hat{\beta} &= X'Y\\
\mathbf{X'X} &= \begin{pmatrix}
\mathbf{1_n}'\\
\mathbf{X_c}'
\end{pmatrix} \begin{pmatrix}
\mathbf{1_n} & \mathbf{X_c}
\end{pmatrix}\\
&= \begin{pmatrix}
\mathbf{1_n}' \mathbf{1_n} & \mathbf{1_n'X_c}\\
\mathbf{X_c'1_n} & \mathbf{X_c'X_c}
\end{pmatrix}\\
\begin{pmatrix}
\mathbf{1_n}' \mathbf{1_n} & \mathbf{1_n'X_c}\\
\mathbf{X_c'1_n} & \mathbf{X_c'X_c}
\end{pmatrix} \begin{pmatrix}
\alpha\\
\beta_1
\end{pmatrix} &= \begin{pmatrix}
\mathbf{1_n'}\\
\mathbf{X_c'}
\end{pmatrix}\mathbf{Y}\\
\implies n\hat{\alpha} + \mathbf{1_n'X_c}\beta_1 &= \sum_{i=1}Y_i = n\bar{Y} \text{using first row}\\
\hat{\alpha} + 0 &= \bar{Y} \text{ since }\mathbf{X_c'1_n1} =0 \\
\implies \hat{\alpha} &= \bar{Y}\\
\mathbf{X_c'1_n1_n'} + \mathbf{X_c'X_c}\hat{\beta_1} &= \mathbf{X_c'Y} \text{using second row}\\
\implies \mathbf{X_c'X_c}\hat{\beta_1} &= \mathbf{X_c'Y} \text{ since }\mathbf{X_c'1_n1} =0\\
\implies \hat{\beta_1} &= \mathbf{(X_c'X_c)^{-1}X_c'Y}
\end{align*}

\subsection*{Problem 1.4.b}
Inverse of a block matrix
\begin{align*}
\begin{pmatrix}
A_{11} & A_{12}\\
A_{21} & A_{22}
\end{pmatrix}^{-1} &= \begin{pmatrix}
(A_{11}-A_{12}A_{22}^{-1}A_{21})^{-1} & -(A_{11}-A_{12}A_{22}^{-1}A_{21})^{-1}A_{12}A_{22}^{-1}\\
-(A_{22}-A_{21}A_{11}^{-1}A_{12})^{-1} & (A_{22}-A_{21}A_{11}^{-1}A_{12})
\end{pmatrix}
\end{align*}

\begin{align*}
\mathbf{X'X} &= \begin{pmatrix}
\mathbf{1_n}'\\
\mathbf{X_c}'
\end{pmatrix} \begin{pmatrix}
\mathbf{1_n} & \mathbf{X_c}
\end{pmatrix}\\
&= \begin{pmatrix}
\mathbf{1_n}' \mathbf{1_n} & \mathbf{1_n'X_c}\\
\mathbf{X_c'1_n} & \mathbf{X_c'X_c}
\end{pmatrix}\\
&= \begin{pmatrix}
\mathbf{1_n}' \mathbf{1_n} & 0\\
0 & \mathbf{X_c'X_c}
\end{pmatrix}\\
\mathbf{X'X}^{-1} &= \begin{pmatrix}
\mathbf{1_n}' \mathbf{1_n} & \mathbf{1_n'X_c}\\
\mathbf{X_c'1_n} & \mathbf{X_c'X_c}
\end{pmatrix}^{-1}\\
&= \begin{pmatrix}
n & 0\\
0 & (\mathbf{X_c'X_c})^{-1}
\end{pmatrix}\\
(X'X)^{-1}X'Y &= \begin{pmatrix}
\frac{1}{n} & 0\\
0 & (\mathbf{X_c'X_c})^{-1}
\end{pmatrix}_{(p+1) \times (p+1)}\begin{pmatrix}
\mathbf{1_n}'\\
\mathbf{X_c}'
\end{pmatrix}_{(p+1) \times n}\mathbf{Y}_{n \times 1}\\
&= \begin{pmatrix}
\frac{1}{n} & 0\\
0 & (\mathbf{X_c'X_c})^{-1}
\end{pmatrix}\begin{pmatrix}
\mathbf{1_n}'Y\\
\mathbf{X_c}'Y
\end{pmatrix}\\
&= \begin{pmatrix}
\frac{1}{n}\mathbf{1_n}'Y_{1 \times 1}\\
\mathbf{(X_c'X_c)^{-1}X_c}'Y_{p-1 \times 1}
\end{pmatrix}\\
&= \begin{pmatrix}
\bar{Y}\\
\mathbf{(X_c'X_c)^{-1}X_c}'Y
\end{pmatrix}_{p \times 1}\\
\begin{pmatrix}
\alpha\\
\beta_1\\
\beta_2\\
\vdots\\
\beta_{p-1}\end{pmatrix} &= \begin{pmatrix}
\bar{y}\\
\mathbf{(X_c'X_c)^{-1}X_c}'Y_{p-1\times 1}
\end{pmatrix}
\end{align*}

\subsection*{Problem 1.5}

Yes, column space of $X$ is identical to $\begin{pmatrix}
\mathbf{1_n} & \mathbf{X_c}
\end{pmatrix}$ since $X_c = (I-\frac{J}{n})X$

\subsection*{Problem 1.6}
\begin{align*}
\mathbf{P} &= \mathbf{X(X'X)^{-1}X'Y}\\
&= \begin{pmatrix}
\mathbf{1_n} & X_c
\end{pmatrix} \begin{pmatrix}
\frac{1}{n} & 0\\
0 & (\mathbf{X_c'X_c})^{-1}
\end{pmatrix}\begin{pmatrix}
\mathbf{1_n'}\\
\mathbf{X_c'}
\end{pmatrix}\\
\implies \mathbf{P} &= \mathbf{X_c(X_c'X_c)^{-1}X_c'} + \frac{\mathbf{1_n1_n'}}{n}\\
\end{align*}

\begin{align*}
SSE &= \mathbf{\epsilon'\epsilon}\\
&=(Y-X\hat{\beta})'(Y-X\hat{\beta})\\
&= Y'Y-\hat{\beta}'X'Y-Y'X\hat{\beta}+\hat{\beta}'X'X\hat{\beta}\\
&= Y'Y-2Y'X\hat{\beta} + \hat{\beta}'X'X\{(X'X)^{-1}X'Y\}\\
&= Y'Y-2Y'X\hat{\beta} + \hat{\beta}'IX'Y\\
&= Y'Y-2Y'X\hat{\beta} + \hat{\beta}'X'Y\\
&= Y'Y-2Y'X\hat{\beta} + Y'X\hat{\beta}'\\
&= Y'Y-Y'X\hat{\beta}\\
&= Y'Y-Y'\begin{pmatrix}
1_n & X_c
\end{pmatrix}\begin{pmatrix}
\bar{y}\\
\mathbf{(X_c'X_c)^{-1}X_c}'Y_{p-1\times 1}
\end{pmatrix}\\
&=Y'Y-Y'\begin{pmatrix}
1_n & X_c
\end{pmatrix}\begin{pmatrix}
\frac{1}{n}1_n'\mathbf{Y}\\
\mathbf{(X_c'X_c)^{-1}X_c}'Y_{p-1\times 1}
\end{pmatrix}\\
&= Y'(I-\frac{1_n1_n'}{n})Y - \mathbf{Y'X_c}\mathbf{(X_c'X_c)^{-1}X_c}'\mathbf{Y}\\
&=\sum_{i=1}^n(Y_i-\bar{Y})^2 -\mathbf{Y'X_c}\mathbf{(X_c'X_c)^{-1}X_c}'\mathbf{Y}\\
&=\sum_{i=1}^n(Y_i-\bar{Y})^2 -\mathbf{Y'P_c}\mathbf{Y}\\
\end{align*}

\subsection*{Problem 1.7}

\begin{align*}
Y_i^* &= Y_i-\bar{Y}\\
\mathbf{Y}^* &= \mathbf{Y}-\frac{1}{n}\mathbf{1_n1_n'Y}\\
\sum_{i=1}^n(Y_i-\bar{Y})^2 &= \mathbf{Y'}^{*}\mathbf{Y}^*\\
\implies SSE &= \mathbf{Y'}^{*}\mathbf{Y}^*-\mathbf{Y'^*P_c}\mathbf{Y^*}
\end{align*}

\end{document}


\documentclass{ximera}
\input{../preamble}
\title{Exercises: Probability}
%%%%%\author{Philip T. Gressman}

\begin{document}
\begin{abstract}
Various exercises relating to probability.
\end{abstract}
\maketitle

\begin{exercise}
\begin{itemize}
\item If it exists, find the value of the constant $c$ below which makes the function a PDF on the interval $[e,e^2]$. If no such constant exists, enter N/A.
\[ f(x) = c \ln x \]
\[ c = \answer{e^{-2}}. \]
\begin{hint}
You'll need to integrate by parts.
\end{hint}
\item For the same function as above, find the value of the constant $c$ which makes it a PDF on the interval $[e^{-2},e^2]$. If no such constant exists, enter N/A.
\[ c = \answer{N/A}. \]
\begin{hint}
The function $\ln x$ is negative on $[e^{-2},1)$ and positive on $(1,e^2]$. What does that imply for the possibility of it being a PDF?
\end{hint}
\end{itemize}
\end{exercise}

\begin{exercise}
Suppose $X$ is a random variable which represents the length of time your customers remain on hold before reaching a customer service agent. A reasonable model for such a random variable is to use an exponential distribution, i.e., to have a PDF given by
\[ f(x) = c e^{-cx} \]
for some positive constant $c$.  If the mean hold time is $2$ minutes, what is the standard deviation?
\[ \sigma = \answer{2}. \]
What is the probability that a customer will have to wait more than one standard deviation beyond the mean before their call is answered?
\[ P = \answer{e^{-2}} \approx 13.5\%. \]
\begin{hint}
First you need to solve for $c$.
\begin{hint}
The equation will be \[ 2 = \int c x e^{-cx} dx. \]
\begin{hint}
This gives $c = 1/2$.
\begin{hint}
The easiest formula to use here is
\[ \sigma^2 = \int_0^\infty x^2 f(x) dx - \mu^2, \]
where $\mu$ is the mean.
\end{hint}
\end{hint}
\end{hint}
\end{hint}
\end{exercise}

\begin{exercise}
Suppose $X$ is a random variable on the interval $[1,10]$ whose PDF is given by
\[ f(x) = \frac{c}{x} \]
for some positive constant $c$ (note: this is the PDF that arises in the phenomenon known as \link[Benford's Law]{https://en.wikipedia.org/wiki/Benford\%27s_law}). Compute the mean $\mu$ and the median $m$ associated to this PDF. Which is larger?
\[ \mu = \answer{\frac{9}{\ln 10}} \ \ \text{ and } m = \answer{\sqrt{10}} \]
The \wordChoice{\choice[correct]{mean}\choice{median}} is more sensitive to tails. Since this distribution has a tail extending \wordChoice{\choice{to the left/for smaller $x$}\choice[correct]{to the right/for larger $x$}}, it is expected (and in fact, true) that the mean is \wordChoice{\choice[correct]{larger}\choice{smaller}} than the median.
\begin{hint}
First you need to solve for the constant $c$.
\begin{hint}
\[ c = \frac{1}{\ln 10} \]
\end{hint}
\end{hint}
\end{exercise}

\section*{Sample Quiz Questions}
\begin{question}%%%%%[ProbabAssort001]

Find the value of \(c\) which makes the function
\[f(x) = \frac{1}{2}e^{-x} - ce^{-2x}\]
a probability density function on the interval \([0, \infty)\).
What is the value of the mean \(\mu\) of the corresponding random variable? (Hints will not be revealed until after you choose a response.)
\begin{multiplechoice}
\choice{\(\displaystyle c = 1, ~ \mu = \frac{1}{2}\)}
\choice[correct]{\(\displaystyle c = -1, ~ \mu = \frac{3}{4}\)}
\choice{\(\displaystyle c = 1, ~ \mu = 1\)}
\choice{\(\displaystyle c = -1, ~ \mu = \frac{5}{4}\)}
\choice{\(\displaystyle c = 1, ~ \mu = \frac{3}{2}\)}
\choice{\(\displaystyle c = -1, ~ \mu = \frac{7}{4}\)}
\end{multiplechoice}
\begin{feedback}
To compute the constant \(c\), we use the fact that the integral of a probability density function must equal \(1\), so
\[ \mu = \int_{0}^{\infty} \left( \frac{1}{2}e^{-x} - ce^{-2x} \right) ~ dx = 1.\]
This gives the equation \[\frac{1}{2} - \frac{1}{2}c = 1,\]
which then implies that \(c = -1\). \begin{hint} To compute the mean \(\mu\), we use the formula
\[ \mu = \int_{0}^{\infty} x \left( \frac{1}{2}e^{-x} + e^{-2x} \right) ~ dx.\]
 Calculating the integral gives \(\mu = 3/4.\) \end{hint}
\end{feedback}
\end{question}

\begin{question}%%%%%[ProbEVsinibp001]
A certain random variable \(X\) takes values in the interval \(\left[2 \pi , \frac{5}{2} \pi \right]\).
If the probability density function is given by
\[ A \sin {x} \]
 for some appropriate value of the constant \(A\),
compute the expected value \(\mu\) of \(X\).  (Hints will not be revealed until after you choose a response.)
\begin{multiplechoice}
\choice{\(\displaystyle \mu = -1 + 2 \pi\)}
\choice{\(\displaystyle \mu = 1 + \frac{3}{2} \pi\)}
\choice{\(\displaystyle \mu = 2 \pi\)}
\choice{\(\displaystyle \mu = -1 + \frac{5}{2} \pi\)}
\choice[correct]{\(\displaystyle \mu = 1 + 2 \pi\)}
\choice{\(\displaystyle \mu = \frac{5}{2} \pi\)}
\end{multiplechoice}
\begin{feedback}
The constant \(A\) will be the reciprocal of the integral
\[ \int_{2 \pi}^{\frac{5}{2} \pi} \sin {x}\, dx \]
\begin{hint}
 One can check that
\[ \begin{aligned} \int_{2 \pi}^{\frac{5}{2} \pi} \sin {x}\, dx & = 1. \end{aligned} \] \begin{hint}
To compute the expected value \(\mu\) we also need to compute the integral
\[ \int_{2 \pi}^{\frac{5}{2} \pi} x \sin {x}\, dx \]
 To compute the integral, we can use integration by parts. A reasonable strategy is to integrate  \(\sin {x}\) and differentiate  \(x\). \begin{hint}
 This gives the equality
\[ \begin{aligned} \int x \sin {x}\, dx & = - x \cos {x} - \int \left(- \cos {x}\right)\, dx \\
 & = - x \cos {x} + \sin {x}. \end{aligned} \]
Therefore
\[ \begin{aligned} \int_{2 \pi}^{\frac{5}{2} \pi} x \sin {x}\, dx & = \left. \left[- x \cos {x} + \sin {x} \right] \right|_{2 \pi}^{\frac{5}{2} \pi}\\ & = 1 - \left(- 2 \pi \right) = 1 + 2 \pi. \end{aligned} \]
Therefore the expected value is the ratio of the integrals, i.e.,
\[ \begin{aligned} \mu & = \frac{1 + 2 \pi}{1} = 1 + 2 \pi. \end{aligned} \]
\end{hint} \end{hint} \end{hint}
\end{feedback}
\end{question}

\section*{Sample Exam Questions}

\begin{question}%[2019ProbImprop]
A certain random variable $X$ has values in $(1,\infty)$ and has the property that there is some constant $C$ such that
\[ P( X > a) = C \ln \frac{a^3+1}{a^3} \]
for every $a > 1$. Compute the value of $C$ and determine whether the expected value $\mu$ of $X$ is finite or infinite. [Hint: There is enough information given to compute $C$ without calculating any integrals.]
\begin{multipleChoice}
\choice{$C = \ln 2$ and $\mu < \infty$}
\choice{$C = 1$ and $\mu < \infty$}
\choice[correct]{$C = (\ln 2)^{-1}$ and $\mu < \infty$}
\choice{$C = \ln 2$ and $\mu = \infty$}
\choice{$C = 1$ and $\mu  = \infty$}
\choice{$C = (\ln 2)^{-1}$ and $\mu = \infty$}
\end{multipleChoice}

\begin{feedback}
We know that $X$ is always greater than one, so
\[ 1 = P(X > 1) = C \ln \frac{1+1}{1}, \]
which gives $C = (\ln 2)^{-1}$.  If we let $f(x)$ denote the probability density function of $X$, then
\[ \frac{1}{\ln 2} \ln \frac{a^3+1}{a^3} = P(X > a) = \int_a^\infty f(x) dx. \]
Differentiating both sides with respect to $a$ gives
\[ \frac{1}{\ln 2} \left[ \frac{3 a^2}{a^3+1} - \frac{3}{a} \right] = - f(a) \]
so
\[ f(a) = \frac{1}{\ln 2} \left[ \frac{3}{a} - \frac{3a^2}{a^3+1} \right] = \frac{3}{a(a^3+1) \ln 2}. \]
The expected value of $X$ must equal
\[ \int_1^\infty \frac{3a}{a(a^3+1) \ln 2} da = \frac{3}{\ln 2} \int_1^\infty \frac{da}{a^3+1}. \]
This integral will be finite by direct comparison to the convergent integral $\int_1^\infty a^{-3} da$.
\end{feedback}
\end{question}

\begin{question}%%%%%[2015C.08]

The function
\[ f(x) = \begin{cases} \displaystyle \frac{k}{x^3} & 1 \leq x < \infty \\ 0 & \text{otherwise} \end{cases} \]
is a probability density function for a certain value of \(k\). For that probability density function, find the probability that \(x > 2\).
\begin{multiplechoice}
\choice{\(\displaystyle \frac{1}{2}\)}
\choice{\(\displaystyle \frac{1}{3}\)}
\choice[correct]{\(\displaystyle \frac{1}{4}\)}
\choice{\(\displaystyle \frac{2}{3}\)}
\choice{\(\displaystyle \frac{1}{5}\)}
\choice{\(\displaystyle \frac{1}{6}\)}
\end{multiplechoice}

\end{question}

\begin{question}%%%%%[2016C.07]

For a certain real number \(k\), the function
\[ f(X) = \begin{cases} \displaystyle \frac{k}{X^2+1} & \text{if } X \geq 0 \\ 0 & \text{otherwise} \end{cases} \]
is a probability density function for a continuous random variable \(X\). For this value of \(k\), find the probability that \(X > 1\).
\begin{multiplechoice}
\choice{\(0\)}
\choice{\(\displaystyle \frac{1}{3}\)}
\choice{\(\displaystyle \frac{2}{3}\)}
\choice{\(1\)}
\choice[correct]{\(\displaystyle \frac{1}{2}\)}
\choice{\(\displaystyle \frac{1}{4}\)}
\end{multiplechoice}

\end{question}

\begin{question}%%%%%[2017C.09]

Let \[ f(r) = \begin{cases} C r^2 e^{-2r/b} & r \geq 0 \\ 0 & r < 0 \end{cases}. \]
Find \(C\) so that this is a probability density function (pdf) for the random variable \(r\). Here \(b\) is a positive constant. This function is used to model the distance between the nucleus and the electron in a hydrogen atom. The constant \(b\) is called the \textit{Bohr length}. Find the mean of the pdf.
\begin{multiplechoice}
\choice{\(\displaystyle C = \frac{b^3}{4}\), mean \(\displaystyle = b\)}
\choice{\(\displaystyle C = \frac{4}{b^2}\), mean \(\displaystyle = b\)}
\choice{\(\displaystyle C = \frac{4}{b}\), mean \(\displaystyle = b^2\)}
\choice[correct]{\(\displaystyle C = \frac{4}{b^3}\), mean \(\displaystyle = \frac{3}{2}b\)}
\choice{\(\displaystyle C = \frac{4}{b^2}\), mean \(\displaystyle = \frac{3}{2}b^2\)}
\choice{\(\displaystyle C = \frac{4}{b}\), mean \(\displaystyle = \frac{3}{2}b^3\)}
\end{multiplechoice}

\end{question}

\end{document}


\chapter{Functions}

\footnote{\url{https://en.wikipedia.org/wiki/Lambda_calculus}}%
\footnote{\url{https://en.wikipedia.org/wiki/Lambda_calculus_definition}}

\index{definitions!function}%
We can think of a function as a computer.

The type \( a \to b \) is inhabited by functions that take an inhabitant of \(a\) and give an inhabitant of \(b\).

A function \(f : a \to b\) maps an \(a\) to a \(b\).

The expression \(f(x)\) means the result of applying \(f\) to \(x\).

If \(x : a\), then \( f(x) : b \).

For example, if \(f(x) = x^2 + x + 1\), then \(f(2+3) = (2+3)^2 + (2+3) + 1\) by substituting \(x\) with \(2+3\).

Don't write \(f(x)\) to mean the function \(f\).
The function is \(f\).
The expression \(f(x)\) means the result of applying \(f\) to \(x\).

\paragraph{Defining functions}

An \emph{unnamed function expression} is written like \( x \to x^2 + 1 \).

Example application: the expression \( (x \to x + 1)(5) \) reduces to \( 5 + 1 \).

\paragraph{Composing functions}

We write \(f \circ g\) to mean \( x \to f(g(x)) \).

\paragraph{Applying functions}

\paragraph{The inverse of an invertible function}

If \(f(x) = y\), then \(f^{-1}(y) = x\),
but only if \(f\) does not map anything else to \(y\).
We say that \(f^{-1}\) is the inverse of \(f\).

An invertible function is a function whose inverse is also a function.

\((f \circ g)^{-1} \equiv g^{-1} \circ f^{-1}\)

\section{Function-returning functions}

\(x \to (y \to x+y)\).

\section{Currying: one parameter is enough}

A two-input function has a type like \( (a,b) \to c \).

\index{definitions!currying}%
\emph{Currying} is the transformation from \( f(x,y) \) to \( (f'(x))(y) \) (from \(f\) to \(x \to y \to f(x,y)\)).
\emph{Uncurrying} is the inverse of currying.

We conflate \( (f'(x))(y) \) and \( f(x,y) \).

We assume that every function takes one input.

Do not confuse:
\begin{itemize}
    \item a function that takes \(n\) inputs, and
    \item a function that takes \emph{one} input that is an \(n\)-tuple.
\end{itemize}

\section{Plotting the graph of a function}

If you type \verb@sin(x)@ into Google\footnote{\url{https://www.google.com/}},
it will plot the graph of \(y = \sin(x)\) for you.

You can also type \verb@y=sin(x)@ into Wolfram Alpha\footnote{\url{https://wolframalpha.com/}}.

You can use GNU Octave.

You can use Gnuplot.

\paragraph{Thinking of an operator as a function}

An infix operator is an operator that is placed between two things.
For example, the dot in \(a \cdot b\) is an infix operator.

An operator does not have to be a symbol.
It can be a letter.
It can be a complicated notation.
For example, \(\pdv{f}{x}\).

We can treat an infix operator as another way of writing a function application.
We can think of \(a \cdot b\) as a convenient way of writing \((\cdot)(a,b)\).

\footnote{\url{https://en.wikipedia.org/wiki/Operator_(mathematics)}}%
\footnote{\url{https://en.wikipedia.org/wiki/Operator_(physics)}}

\section{Equivalence of functions, eta-reduction}
\label{sec:function-equivalence}

We write \(f \equiv g\) to mean that \(f\) gives the same result as \(g\) for all parameters.

\section{Exercises}

To evaluate something is to reduce it to normal form.

\ExerciseAnswer{Evaluate the expression \((x \to x + 1)(5)\), assuming the usual arithmetic.}{Substitute: \(5 + 1\). Reduce: \(6\).}

\ShowAnswers


\chapter{Association Rule Mining}

\begin{description}
\item[Itemset] A collection of one or more items
\item[Frequent itemset] An itemset whose support is greater than or equal to a \emph{minsup} threshold
\end{description}

$$let\ {X \Rightarrow Y}$$
\begin{align*}
Support\ Count, \sigma &=\# Trans(X \cup Y) \\
Support, s &= \frac{\# Trans(X \cup Y)}{Total\ \#\ Trans} \\
Confidence=P(Y|X)&=\frac{\# Trans(X \cup Y)}{\# Trans\ contain\ X}
\end{align*}

\noindent Goal of Association Rule Mining:

$$support \ge min\_sup$$
$$confident \ge min\_conf$$

\section{Frequent Itemset Generation}

Apriori Principle: If an itemset is frequent, then all of its subsets must also be frequent. This principle reduce the number of frequent itemset candidates
$$\forall X, Y: X \subseteq Y \Rightarrow s(X) \ge s(Y)$$
Store each candidate in a hash tree structure to count the support efficiently \\

\begin{description}
\item[Maximal Frequent itemset] none of its immediate supersets is frequent
\item [Closed itemset] none of its immediate supersets has the same support as the itemset
\end{description}

\subsection{Alternative: FP-Growth Algorithm}
Construct FP-tree from a transactional DB:
\begin{table}[h!]
\begin{tabular}{r p{12cm}}
\hline
    1: & Scan DB once, find frequent 1-itemset\\
    2: & Order frequent items in frequency descending order (L-order)\\
    3: & Process DB based on L-order\\
\hline
\end{tabular}
\end{table}

\noindent Mining frequent patterns using FP-tree:
\begin{table}[h!]
\begin{tabular}{r p{12cm}}
\hline
    1: & Construct conditional pattern base for each item in header table\\
    2: & Construct conditional FP-tree from each conditional pattern-base\\
    3: & Recursively mine conditional FP-trees and grow frequent patterns obtained so far\\
\hline
\end{tabular}
\end{table}

\section{Rule Generation}
Confidence of rules generated from the same itemset has an anti-monotone property:
$$c(ABC \rightarrow D) \ge c(AB \rightarrow CD) \ge c(A \rightarrow BCD)$$