\section{Fermi's Goldern Rule - Revision}

Next assuimg this scenario for long time $t \rightarrow \infty$ we can turn this integral into a delta distrubution as follows
\begin{equation} \label{0.1}
  \begin{aligned}
    a_{\alpha\beta}(\mb{k'},\mb{k},t') & =
    -
    \frac{i}{\hbar}
    \lim_{t \rightarrow \infty}\qty[
      \int_{-t/2}^{t/2} dt_1 \;
      e^{\frac{i}{\hbar}\qty(\varepsilon_{\beta} - \varepsilon)t_1}
      \bra{\phi_{\beta,\mb{k'}}(t')}
      V(\mb{r}) \ket{\phi_{\alpha,\mb{k}}(t')}
    ] \\
    & =
    -2\pi i \delta(\varepsilon_{\beta} - \varepsilon)
    \bra{\phi_{\beta,\mb{k'}}(t')}
    V(\mb{r}) \ket{\phi_{\alpha,\mb{k}}(t')}
    \\
    & =
    -2\pi i \delta(\varepsilon_{\beta} - \varepsilon)
    \sum_{\mb{k}}\sum_{\mb{k'}}
    \braket{\phi_{\beta,\mb{k'}}(t')}{\mb{k'}}
    \mel{\mb{k'}}{V(\mb{r})}{\mb{k}}
    \braket{\mb{k}}{\phi_{\alpha,\mb{k}}(t')}
    \\
    & =
    \sum_{k_x}\sum_{{k'}_x}
    \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} dk_y d{k'}_y \;
    \phi_{\beta}^{\dagger}(\mb{k'},t')
    \mel{\mb{k'}}{V(\mb{r})}{\mb{k}}
    \phi_{\alpha}(\mb{k},t').
  \end{aligned}
\end{equation}
Let's define the impurity potential matrix element as
\begin{equation} \label{0.2}
  V_{\mb{k}, \mb{k'}} = \mel{\mb{k'}}{V(\mb{r})}{\mb{k}}
\end{equation}
and consider this for a given $k_x$ and $k'_x$ momentum value. This information is included in the $\alpha$ and $\beta$ quantum numbers. Also we can transform the $t'$ varable into $t$ and present above as
\begin{equation} \label{0.3}
  \begin{aligned}
    a_{\alpha\beta}(t) & =
    \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} dk_y d{k'}_y \;
    V_{\mb{k}, \mb{k'}}
    \phi_{\beta}^{\dagger}(\mb{k'},t)
    \phi_{\alpha}(\mb{k},t).
  \end{aligned}
\end{equation}
Using the derivation of \textit{Floquet modes} in Eq. \eqref{3.40} we expand this as
\begin{equation} \label{0.4}
  \begin{aligned}
    a_{\alpha\beta}(t) =
    \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} dk_y d{k'}_y \;
    V_{\mb{k}, \mb{k'}} &
    {\sqrt{L_x}}
    e^{
      -ib\sin(2\omega t)
    }
    e^{
      +ik'_y  \qty[d\sin(\omega t) + y'_0]
    }
    \tilde{\chi}_{n_{\beta}}\qty(k'_y -g\cos(\omega t)) \\
    & \times
    {\sqrt{L_x}}
    e^{
      ib\sin(2\omega t)
    }
    e^{
      -ik_y  \qty[d\sin(\omega t) + y_0]
    }
    \tilde{\chi}_{n_{\alpha}}\qty(k_y -g\cos(\omega t))
  \end{aligned}
\end{equation}
and this can be simplified to
\begin{equation} \label{0.5}
  \begin{aligned}
    a_{\alpha\beta}(t) =
    L_x
    \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} dk_y d{k'}_y \;
    V_{\mb{k}, \mb{k'}} &
    e^{
      i\qty[k'_y - k_y]d\sin(\omega t)
    }
    e^{
      i\qty[k'_y y'_0 - k_y y_0]
    }
    \tilde{\chi}_{n_{\beta}}\qty(k'_y -g\cos(\omega t))
    \tilde{\chi}_{n_{\alpha}}\qty(k_y -g\cos(\omega t)).
  \end{aligned}
\end{equation}
Then by definiton of the \textit{transition probability} we can find the scattering probbaility as
\begin{equation} \label{0.6}
    \qty(A_{\alpha\beta}(t) \equiv
    a_{\alpha\beta}(t)\qty[a_{\alpha\beta}({k'}_x,k_x)]^{*}
\end{equation}

xx


\chapter{Proof of Linear Programming Formulation}
\label{LPproof}
\thispagestyle{empty}

\noindent This appendix provides the proof of Theorem \ref{lpth} that we report here for completeness:
\begin{align*}
	\underset{v}{\text{minimize}} & \; \sum_{s \in \mathcal{S}} \mu(s) v(s) \\
	\text{subject to} &  \; v(s) \geq r(s,a) + \sum_{s' \in \mathcal{S}} P(s' \mid s,a) v(s') \, , \forall a \in \mathcal{A} \, .
\end{align*}

\begin{theorem*}[Linear Programming Solution]
$v^*$ is the solution of the above linear program.
\end{theorem*}

\begin{proof}
	Let $T^*$ be the Bellman optimality operator, then the above LP can be rewritten as:
	\begin{align*}
	\underset{v}{\text{minimize}} & \; \mu^T v \\
	\text{subject to} &  \; v \geq T^* (v) \, .
\end{align*}
Using the \textit{monotonicity property} if $v \geq T^*(v)$, then $T^*(v) \geq T^*(T^*(v))$, and by applying infinite times the operator we obtain: $v \geq T^{* \infty} (v) = v^*$. Any feasible solution of the LP must satisfy $v \geq T^* (v)$, thus it must satisfy  $v \geq v^*$. Hence, assuming all entries $\mu$ are positive, $v^*$ is the optimal solution to the LP.
\end{proof}


\section{Examples}
\label{sec:examples}

\subsection{Floating Point Operations}

Principal Components Analysis is a common statistical analysis technique.  The
implementation of PCA typically involves computing the singular value
decomposition (SVD) of the input data and then projecting the data onto the
right singular vectors.  It is known that an SVD requires $6mn^2 + 20n^3$
floating point operations and that the projection onto the right singular
vectors requires an additional $2mn^2$ operations~\citep{gvl}.  Finally, as PCA
is usually performed on centered (and often scaled) data, we require an
additional $2*mn+1$ operations for centering the data.

The \thispackage package contains the example \code{pca.r} as a package demo,
which will perform a PCA on random data consisting of $10,000$ observations and
50 predictors.  The analysis uses \R's ordinary \code{prcomp()} routine, and
the performance is measured by \thispackage's \code{system.flops()}.  You can
run this demo by calling

\begin{lstlisting}[language=rr]
demo("pca", package="pbdPAPI")
\end{lstlisting}

An example output from this machine is:

\begin{Output}
      m  n  measured theoretical difference pct.error   mflops
1 10000 50 212538720   203500001    9038719   4.25274 2284.257
\end{Output}

The \thispackage package has several other demonstrations that, like this,
compare a theoretical floating point operations count against a measured count,
and then display the Mflops the computation achieved.  These demos include
an inner product calculation \code{inner.r}, a matrix-matrix product
\code{matprod.r}, and the fitting of a linear model in \code{regression.r}.

\subsection{Cache Misses}

To see the full source code described in this example, see the \code{cache_access} demo in
\thispackage.

Consider the following example, where we will fill a matrix with 1's, first by
looping over rows then columns, and then by looping over columns then rows.  For
maximum effect, we will be dropping to \CXX by way of \pkg{Rcpp}.  If you do not
have \pkg{Rcpp} installed on your system, you can still follow along (even if
you don't know \CXX), but you will not be able to recreate the timings locally.

You are probably aware that \R matrices are stored in column-major fashion.
What this means is that the data, as it is laid out in physical memory, is
easier to go from the entry with index $(i, j)$ to index $(i+1, j)$ than
it is to go to index $(i, j+1)$.

An example of accessing data poorly is the following:
\begin{lstlisting}[language=rr]
bad_cache_access <- "
  int i, j;
  const int n = INTEGER(n_)[0];
  Rcpp::NumericMatrix x(n, n);

  for (i=0; i<n; i++)
    for (j=0; j<n; j++)
      x(i, j) = 1.;

  return x;
"
\end{lstlisting}

Accessing data in this way maximizes the
amount of work the computer needs to do in searching for the data your program
needs.  Accessing your memory correctly will help minimize cache misses:

\begin{lstlisting}[language=rr]
good_cache_access <- "
  int i, j;
  const int n = INTEGER(n_)[0];
  Rcpp::NumericMatrix x(n, n);

  for (j=0; j<n; j++)
    for (i=0; i<n; i++)
      x(i, j) = 1.;

  return x;
"
\end{lstlisting}

To prove this, we can build these functions using the \pkg{inline} and
\pkg{Rcpp} packages~\citep{inline,rcpp} and then profile them with \thispackage.

\begin{lstlisting}[language=rr]
library(inline)

bad <- cxxfunction(signature(n_="integer"),
                   body=bad_cache_access, plugin="Rcpp")
good <- cxxfunction(signature(n_="integer"),
                    body=good_cache_access, plugin="Rcpp")
\end{lstlisting}

A quick check of run times shows something drastically different is happening
between the two implementations:
\begin{Output}
n <- 10000L

system.time(bad(n))
#   user  system elapsed
#  1.016   0.232   1.259

system.time(good(n))
#   user  system elapsed
#  0.201   0.155   0.357
\end{Output}

So even though we (mathematically) are doing the exact same thing to the data,
the run times differ by a factor of 3.5.  \thispackage allows us to more
thoroughly see what's happening.  We can use \code{system.cache()} to check the
L1, L2, and L3 (total) cache misses for each of these functions:

\begin{Output}
library(pbdPAPI)
n <- 10000L

system.cache(bad(n))
#$L1.total
#[1] 193580295
#
#$L2.total
#[1] 159442230
#
#$L3.total
#[1] 16895275

system.cache(good(n))
#$L1.total
#[1] 15552007
#
#$L2.total
#[1] 11580023
#
#$L3.total
#[1] 801150
\end{Output}

The L1 cache misses differ by more than an order of magnitude, 194 million to 16
million!

Perhaps a more useful measure is the \emph{cache miss ratio}, which is the
total cache misses divided by the total cache accesses (in the low-level
interface syntax, this is the return of event \code{PAPI_L2_TCM} divided by the
return of event \code{PAPI_L2_TCA}, for level 2).

Measuring this too is simple:

\begin{Output}
system.cache(bad(n), events="l2.ratio")
# L2 cache miss ratio
#            0.815597

system.cache(good(n), events="l2.ratio")
# L2 cache miss ratio
#           0.7156862

\end{Output}


\documentclass[../PHYS306Notes.tex]{subfiles}

\begin{document}
\subsection{Worksheet - Lagrange Multipliers \& Atwood Machine}
\begin{p}
Write the constraint equations $f(x,y) = const.$ for $x$ and $y$ for the simple plane pendulum and the Atwood Machine.
\end{p}
\begin{s}
The constraint equation is $f(x, y) = \sqrt{x^2 + y^2} = L$ for the simple pendulum and $f(x, y) = x + y = L$ for the Atwood machine (constrained by length of rope).
\end{s}

\begin{p}
Write down the Hamilton's principle $\delta S(x,y) = 0$ for the two constrained variables $x$ and $y$. If we try to make the action stationary, how would the deviations $\delta x$ and $\delta y$ have to be constrained?
\end{p}
\begin{s}
\begin{align*}
    \delta S + \int\left(\dpd{\LL}{x} - \dod{}{t}\dpd{\LL}{\dot{x}}\right)\delta x dt + \int \left(\dpd{\LL}{y} - \dod{}{}\dpd{\LL}{\dot{y}}\right)\delta y dt = 0
\end{align*}
But, these variations must be constrained as $x$ and $y$ are not independent of one another. We note that $\delta x, \delta y$ obey the constraints.
\end{s}

\begin{p}
For deviations that meet the conditions of the question above, write down the variations of the constraint $\delta f$. Multiply $\delta f$ by an unknown function $\lambda(t)$ and add to $\delta S$.
\end{p}
\begin{s}
\[\delta f = \dpd{f}{x}\delta x + \dpd{f}{y}\delta y = 0\]
As the displacement must leavve the constraint unchanged. Hence we may add a $0$ to the total variation of $S$ without changing anything. We therefore write:
\[\delta S = \int \left(\dpd{\LL}{x} + \lambda(t)\dpd{f}{x} - \dod{}{t}\dpd{\LL}{\dot{x}}\right)\delta x \delta t + \int\left(\dpd{\LL}{y} + \lambda(t)\dpd{f}{y} - \dod{}{t}\dpd{\LL}{\dot{y}}\right)\delta y dt = 0\]
\end{s}

\begin{p}
Can $\delta x$ and $\delta y$ be independently varied? The multiplying function of the constraint $\lambda(t)$ is so far undetermined and can be chosen as we wish. What special choice of $\lambda(t)$ makes all of the arguments $\delta x$, $\delta y$, $\delta z$... vanish? How have the Lagrange equations been modified when the dependent variables are constrained?
\end{p}
\begin{s}
Unlike the case with generalized coordinates, the variations are not independent. Consider that we can pick $\lambda (t)$ (Lagrange multiplier) to be whatever we like to make the first integral vanish. However, this then means that the second term must vanish as well, as the sum has to be zero (and if the term is to be zero for all variations, then the integrand must be zero)! In general, if we have a function of (not necessarily generalized coordinates) $\LL(x_k, \dot{x}_k, t)$ with $f_{i}(x_k, t) = 0$ ($m$ holonomic constraints), we can consider the modified Lagrangian:
\[\tilde{L}(x_k, \dot{x}_k, t) = \LL(x_k, \dot{x}_k, t) - \sum_{i=1}^m \lambda_i f_i(x_k, t)\]
And if we require that the variation vanishes $\delta S = 0$, then we get a set of generalized Lagrange equations:
\[\dpd{\LL}{x_k} + \sum_{i=1}^m \lambda_i \dpd{f_i}{x_k} = \dod{}{t}\dpd{\LL}{\dot{x}_k}\]

\end{s}

\begin{center}
    \includegraphics[scale=0.75]{Lecture-8/w8-img1.png}
\end{center}
\begin{p}
What are Lagrange's equations for $x$ and $y$ of the Atwood Machine?
\end{p}
\begin{s}
The Lagrangian is given by:
\[\LL = \frac{m_1}{2}\dot{x}^2 + \frac{m_2}{2}\dot{y}^2 + m_1gx + m_2gy\]
Hence the equations of motions are:
\[m_1\ddot{x} = m_1g + \lambda\]
\[m_2\ddot{y} = m_2g + \lambda\]
Where $\lambda$ is the Lagrange multiplier.
\end{s}

\begin{p}
Using the constraint equation for $x$ and $y$, eliminate the Lagrange multiplier and solveagain for the acceleration of $x$.
\end{p}
\begin{s}
From the constraint equation (taking two time derivatives of it) we obtain that $\ddot{x} = -\ddot{y}$. We therefore have that:
\[\ddot{x} = \frac{(m_1 - m_2)}{(m_1 + m_2)}g\]
\end{s}

\begin{p}
Now solve the equations for the Lagrange multiplier.
\end{p}
\begin{s}
Solving for $\lambda$, we have:
\[\lambda = m_1\ddot{x} - m_1g\]
We note that $m_1\ddot{x}$ is the total force of the system, so
\[m\ddot{x} = -\dpd{U}{x} - F_{T}\]
Where the first term is the conservative (gravitational) force from potential and $F_T$ is the tension/constraint force. Therefore, we have that:
\[\lambda = -F_T\]
And we have found the lagrange multiplier to be the constraint/tension force!
\end{s}

\begin{p}
Show that $\lambda \pd{f}{x}$ s the constraint force on mass $m_1$.
\end{p}
\begin{s}
Here, $\dpd{f}{x} = 1$ so $\lambda\dpd{f}{x} = \lambda = -F_T$ which is the expected result.
\end{s}
\end{document}


In this chapter, motivated by the multigrid method,
we design a unified framework for convolutional neural network
and multigrid method named MgNet.
We first propose the framework and discuss some features of the new network structure.
Then we discuss the relation between MgNet and ResNet. We will establish some new understandings
of ResNet from the point view of MgNet.

\section{MgNet: a new network structure}\label{sec:mgnet}
In this section, we introduce a new neural network structure,
named as MgNet~\cite{he2019mgnet}, motivated by the multigrid algorithm,
Algorithm \ref{alg:L-Slash0},
%and its nonlinear version in Algorithm~\ref{alg:Slash-FAS},
as discussed in the previous section.

Here we recall the most important two structures in multigrid
\begin{enumerate}
	\item iterative scheme (for linear system)
	\begin{equation}\label{eq:iterscheme}
	u^{\ell,i} = u^{\ell,i-1} + B^{\ell,i} ({f^\ell -  A^{\ell} \ast u^{\ell,i-1}}).
	\end{equation}
	\item interpolation and restriction
	\begin{equation}\label{eq:inter&rest}
	u^{\ell+1,0} = \Pi_\ell^{\ell+1} \ast_2 u^{\ell},  \quad 	f^{\ell+1} = R^{\ell+1}_\ell \ast_2 (f^\ell - A^\ell(u^{\ell})) + A^{\ell+1} \ast u^{\ell+1,0}.
	\end{equation}
\end{enumerate}

Considering the fine to coarse process of multigrid with the aforementioned
two structures in \eqref{eq:iterscheme} and \eqref{eq:inter&rest}.
We are now in a position to state the main algorithm, namely
MgNet by just putting some nonlinear activation function $\sigma$ in some places.
\begin{breakablealgorithm}
	\caption{$u^{J}={\rm MgNet}(f; J,\nu_1, \cdots, \nu_J)$}
	\label{alg:mgnet}
	\begin{algorithmic}
		\State Initialization:  $f^1 = \theta(f)$, $u^{1,0} = 0$
		%		\State Initialization $u^{1,0}$
		\For{$\ell = 1:J$}
		\For{$i = 1:\nu_\ell$}
		\State
		\begin{equation}\label{mgnet}
		u^{\ell,i} = u^{\ell,i-1} + \sigma \circ B^{\ell,i} \ast \sigma ({f^\ell -  A^{\ell} \ast u^{\ell,i-1}}).
		\end{equation}
		\EndFor
		\State Note $u^\ell = u^{\ell,\nu_\ell}$
		\begin{equation}
		\label{interpolation}
		u^{\ell+1,0} = \Pi_\ell^{\ell+1} \ast_2 u^{\ell}
		\end{equation}
		\begin{equation}
		\label{restrict-f}
		f^{\ell+1} = R^{\ell+1}_\ell \ast_2 (f^\ell - A^\ell(u^{\ell})) + A^{\ell+1} \ast u^{\ell+1,0}.
		\end{equation}
		\EndFor
	\end{algorithmic}
\end{breakablealgorithm}

\begin{theorem}\label{thm:mg-mgnet}
	If $A^\ell$, $R_\ell^{\ell+1}$ and $B^{\ell,i} = S^{\ell}$ are all linear operations as described in multigrid method
	in \S \ref{sec:mg} and all $\sigma = id$ in Algorithm \ref{alg:mgnet}.
	Then Algorithm \ref{alg:L-Slash0} is equivalent to Algorithm \ref{alg:mgnet} with any choice of $\Pi_\ell^{\ell+1}$.
\end{theorem}
\begin{proof}
	Here we replace $u^{\ell,i}$ and $f^{\ell}$ by $\tilde u^{\ell,i}$ and $\tilde f^{\ell}$ in MgNet.
	What we want to prove are
	\begin{equation}\label{eq:f-u}
	\tilde f^{\ell} =  f^{\ell} + A_\ell \tilde u^{\ell,0} \quad \text{and} \quad u^{\ell,i} = \tilde u^{\ell, i} - \tilde u^{\ell, 0},
	\end{equation}
	with $u^{\ell,i}$, $f^\ell$ in Algorithm \ref{alg:L-Slash0} and
	$\tilde u^{\ell,i}$, $\tilde f^\ell$ in Algorithm \ref{alg:mgnet} for any choice of $\Pi_\ell^{\ell+1}$.
	We prove this result by induction.
	\begin{itemize}
		\item It is easy to check that $\ell = 1$ is right by taking $\theta  = \rm{id}$.
		\item Once the above equation \eqref{eq:f-u} is right for $\ell$,
		let us prove the corresponded result for $\ell+1$.
		\begin{itemize}
			\item For $\tilde f^{\ell+1}$, as the definition in Algorithm \ref{alg:mgnet}, we have
			\begin{align*}
			\tilde f^{\ell+1} &= R_\ell^{\ell+1}(\tilde f^\ell - A^\ell \tilde u^{\ell,\nu_\ell}) + A^{\ell+1}\tilde u^{\ell+1,0}, \\
			&= R_\ell^{\ell+1}(f^\ell + A^{\ell} \tilde u^{\ell,0}- A^{\ell} \tilde u^{\ell,\nu_\ell}) + A^{\ell+1}\tilde u^{\ell+1,0} \\
			&= R_\ell^{\ell+1}(f^\ell - A^{\ell} (\tilde u^{\ell,\nu_\ell}- u^{\ell,0})) + A^{\ell+1}\tilde u^{\ell+1,0} \\
			&= R_\ell^{\ell+1}(f^\ell - A^{\ell} u^{\ell,\nu_\ell}) + A^{\ell+1}\tilde u^{\ell+1,0}, \\
			&=  f^{\ell+1} + A^{\ell+1}\tilde u^{\ell+1,0}.
			\end{align*}
			\item For $u^{\ell+1,i}$, first we have
			$$
			u^{\ell+1,0} = 0 =\tilde u^{\ell+1, 0} - \tilde u^{\ell+1, 0},
			$$
			then we prove
			\begin{equation}\label{u:i+1}
			u^{\ell+1,i} = \tilde u^{\ell+1, i} - \tilde u^{\ell+1, 0}
			\end{equation} by induction for $i$.

			We assume \eqref{u:i+1} holds for $0,1,\cdots,i-1$. Let us miner $\tilde u^{\ell+1, 0}$ in both sides of
			the smoothing process \eqref{mgnet} in Algorithm \ref{alg:mgnet}. Then we have
			\begin{align*}
			\tilde u^{\ell+1,i} - \tilde u^{\ell+1, 0} &= \tilde u^{\ell+1,i-1} - \tilde u^{\ell+1, 0} + B^{\ell+1,i} (\tilde f^{\ell+1} - A^{\ell+1} \tilde u^{\ell+1,i-1}), \\
			&= \tilde u^{\ell+1,i-1} - \tilde u^{\ell+1, 0} + B^{\ell+1,i} (f^{\ell+1} + A^{\ell+1}\tilde u^{\ell+1,0} - A^{\ell+1} \tilde u^{\ell+1,i-1} ),\\
			&= u^{\ell+1,i-1} + B^{\ell+1,i} (f^{\ell+1} - A^{\ell+1}u^{\ell+1,i-1} ).
			\end{align*}
			This is exact the smoothing process in Algorithm \ref{alg:L-Slash0} as we take $ B^{\ell+1,i} = S^{\ell+1}$.
			%			\item At last, recall that $u^{1,0} = 0$, so the output of Algorithm \ref{alg:L-Slash0}
			%			$$
			%			tilde u^{1, \nu_1} = u^{1, \nu_1},
			%			$$
			%			which is the output of Algorithm \ref{alg:L-Slash}.
		\end{itemize}
	\end{itemize}
\end{proof}

The main steps in MgNet can be understood
as solving the following  data-feature mappings in each grid $\ell$:
\begin{equation}
\label{Auf-ell}
A^\ell \ast u^\ell = f^\ell, \quad \ell=1:J,
\end{equation}
where
\begin{equation}
\label{f-ell}
f^{\ell}\in\mathbb R^{c_\ell \times m_\ell\times n_\ell},
\end{equation}
and
\begin{equation}
\label{u-ell}
u^{\ell}\in\mathbb R^{h_\ell\times m_\ell\times n_\ell},
\end{equation}
with constrain
\begin{equation}\label{key}
u \ge 0.
\end{equation}
More details about this basic assumption in image classification can be found in
~\cite{he2019constrained}.

Here, the next diagram gives a brief illustration for the above
	structure.\\
\begin{figure}[H]
	\begin{center}
		\includegraphics[width=.4\textwidth]{MgNet_3}
	\end{center}
	\caption{Structure of MgNet}
	\label{fig:mgnet}
\end{figure}

The first property of MgNet is that it recovers the multigrid methods.
Despite of the simplicity look of Algorithm \ref{alg:mgnet}, there
are rich mathematical structures and variants which we will briefly discuss below.

\subsection{Initialization: feature space channels}
Initially for $\ell=1$,  we take $m_1 = m$ and $n_1 = n$ and we may define the linear mapping
\begin{equation}
\label{eq:6}
\theta: \mathbb R^{c \times m\times n }
\mapsto \mathbb R^{c_1 \times m_1\times n_1 },
\end{equation}
to obtain $f^{1} = \theta(f)$ with $c$ %given in \eqref{data-c}
changed to the channel of the initial
data space to $c_1$.   Usually
\begin{equation}
\label{cc}
c_1\ge c.
\end{equation}
One possibility is that we choose $c_1=c$.  In this case, we choose
$\theta$ to be identity.   But in general, we may need to choose $c_1\gg
c$. One possible advantage of preprocessing the RGB ($c=3$) to
different color spaces is that we can better choose what kind of
features the CNN can detect, and under what
conditions those detections will be invariant.

One possibility of understanding and modifying this step
is to decompose the data $f$ into a number of more
specialized data
\begin{equation}
\label{decomp-f}
f=\sum_{k=1}^{c_1}\xi_kf^1_k  =\xi^Tf^1.
\end{equation}
We may use some knowledge from image processing or physics to
design a procedure to obtain the right decomposition of
\eqref{decomp-f}, or we can just train it.
Conceivably, we may view $f^{1} = \theta(f)$ as a special approximation solution of
\eqref{decomp-f} with the same sparsity pattern to $\xi$.

\subsection{Extracted units: $u^{\ell}$ and channels}
The first new feature and the main new ingredient
in the proposed neural network is the introduction
of feature variables  $u^{\ell}$ in \eqref{u-ell}, which will be known
as the extracted units.

We emphasize that the extracted-units $u^\ell$ and the data $f^\ell$ can have
different numbers of channels:\
\begin{equation}
\label{uf-channels}
u^\ell\in \mathbb{R}^{c_{u,\ell}\times m_\ell\times n_\ell  }, \quad
f^\ell\in \mathbb{R}^{c_{f,\ell} \times m_\ell\times n_\ell }
\end{equation}
One possibility is that the number of channels for both $u$ and $f$ remain
unchanged in different grids:
\begin{equation}
\label{cfl}
c_{f,\ell}=c_f, \quad \ell=1:J,
\end{equation}
and
\begin{equation}
\label{ufl}
c_{u,\ell}=c_{u}, \quad \ell=1:J.
\end{equation}
Both $c_f$ and $c_{u}$ are two super-parameters that need to be tuned,
and we may even take $c_u = c_f$.

\subsection{Poolings: $\Pi_{\ell+1}^\ell$ and $R_{\ell+1}^\ell$}
The pooling $\Pi_{\ell+1}^\ell$ in \eqref{interpolation} and
$R_{\ell+1}^\ell$ in \eqref{restrict-f} are in general different.
They can be trained in general, but they may be a priori chosen.

There are many different possibilities to choose $\Pi_{\ell+1}^\ell$.
The simplest choice of $\Pi_{\ell+1}^\ell$ is
\begin{equation}
\label{eq:8}
\Pi_{\ell+1}^\ell=0.
\end{equation}
A more sophisticated choice can be obtained by considering an
interpolation from fine grid to coarse (that, for example preserves linear function
locally).  Namely
\begin{equation}
\label{Pi}
\Pi_{\ell+1}^\ell=\bar\Pi_{\ell+1}^\ell \otimes I_{c_\ell\times c_\ell}
\end{equation}
with $\bar\Pi_{\ell+1}^\ell$ given in finite element methods.

\subsection{Data-feature mapping: $A^{\ell}$}
The second new feature of MgNet is that this data-feature mapping
only depends on the grid ${\cal T}_\ell$, and it does not depend on layers
within the same grid.  This amounts to a significant saving of the number of
parameters.  In comparison, the existing CNN, such as pre-act ResNet, can be
interpreted as a network related to the case that $A^{\ell}$ is
replaced by $A^{\ell, i}$, namely
\begin{equation}\label{u-resnet}
u^{\ell,i} = u^{\ell,i-1} + \sigma \circ B^{\ell,i} \ast \sigma (f^\ell -  A^{\ell,i} \ast u^{\ell,i-1}).
\end{equation}

The underlying convolution kernels can be different on different grids and they can
all be trained.

\subsection{Feature extractors: $\sigma \circ B^{\ell,i} \ast \sigma$}
Here we adopt the feature extractor as:
\begin{equation}
\label{extractor-ell}
\sigma\circ B^{\ell,i}\ast\sigma.
\end{equation}

Other than the level dependent extractors, the following
different strategies can be used
\begin{description}
	\item[Constant Extractors]: $B^{\ell,i}=B^{\ell}$ for   $i=1:\nu_\ell$
	\item[Scaled Extractors]:$B^{\ell,i}=\alpha_iB^{\ell}$ for   $i=1:\nu_\ell$
	\item[Variable Extractors]: $B^{\ell,i}$
\end{description}

This brief framework gives us the basic principle on designing
a CNN models for classification. All models are seen as the special
choice of data-feature mapping $A^\ell$, feature extractors $B^{\ell,i}$
and the pooling operators $\Pi_{\ell+1}^\ell$ with $R_{\ell+1}^\ell$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\endinput
Similarly, we can consider about the back-slash cycle process:
\begin{breakablealgorithm}
	\caption{$u^J={\rm MgNet1}(f; J,\nu_1, \cdots, \nu_J; \nu'_1, \cdots, \nu'_J )$}
	\label{alg:mgnet1}
	\begin{algorithmic}
		\State
		$$
		(\bar u^{1,0}, \bar u^1, f^1, \bar u^{2,0}, u^2, f^2,\cdots, \bar u^{J,0},\bar u^J, f^J) = \text{MgNet}(f; J,\nu_1, \cdots, \nu_J).
		$$
		%\State Define $\bar u^{\ell, 0 } = u^{\ell,0}$ for $\ell = 1:J$.
		\For{$\ell = J-1 : 1$}
		%		\State  and $u^{\ell,\nu_\ell'} = u^{\ell}$ then
		\begin{equation}
		%		\begin{aligned}
		u^{\ell,0} \leftarrow \bar u^{\ell} + R_{\ell}^{\ell+1} \ast_2^T (u^{\ell+1} - \bar u^{\ell+1,0}).
		%		\end{aligned}
		\end{equation}
		%				f^\ell &= f^\ell - A^\ell \bar u^{\ell,0}+ A^\ell  u^{\ell,0}.
		\For{$i = 1:\nu'_\ell$}
		\State
		\begin{equation}\label{mgnet}
		u^{\ell,i} \leftarrow u^{\ell,i-1} + (B^{\ell,i})'  ({f^\ell -  A^{\ell} \ast u^{\ell,i-1}}).
		\end{equation}
		\EndFor
		\State
		$$
		u^{\ell} \leftarrow u^{\ell,\nu_\ell'} .
		$$
		\EndFor
		\State
		$$
		(u^1,\cdots, u^{J-1}).
		$$
	\end{algorithmic}
\end{breakablealgorithm}


\section{Discussion}
\label{sect:prob-discussion}

We discuss potential applications, extensions, and connections of probabilistic property evaluation in the following.

\subsection{Probabilistic equivalence checking}
\begin{figure}[t]
    \centering
    \includegraphics{fig/build/pec-miter.pdf}
    \caption{A miter SPBN for probabilistic equivalence checking}
    \label{fig:prob-PEC}
\end{figure}
Given two SPBNs, their equivalence checking can be easily formulated under PPE and MPPE framework,
as depicted in~\cref{fig:prob-PEC}.
The property network corresponds to a miter circuit
that tests the difference of corresponding outputs of the two SPBNs,
same as in the equivalence checking of deterministic designs.
With the proposed framework,
we can analyze the average (resp. maximum) probability
that the two SPBNs are functionally different through PPE (resp. MPPE).
We refer to the equivalence checking problem as
\textit{probabilistic equivalence checking} (PEC) for the average-case analysis and
\textit{maximum probabilistic equivalence checking} (MPEC) for the worst-case analysis.
Since equivalence checking is widely encountered,
our experiments will focus on PEC and MPEC to compare the strengths and weaknesses of different solutions
that we proposed in~\cref{sect:prob-solutions}.

\subsection{Prioritized output requirement}
For some applications,
we may want to impose different criticality requirements on different output signals.
Given an SPBN $G$ over
primary inputs $X$,
internal vertices $Y$,
and auxiliary inputs $Z$,
this output-prioritized version of MPPE is naturally expressible
in terms of stochastic integer linear programming (SILP)~\cite{Schultz2003} as follows:
\begin{align*}
    \max_X \enskip \mathbb{E}[\sum_{i=1}^n w_i o_i(X,Y,Z)] \enskip s.t. \enskip \pf,
\end{align*}
where $o_i \in V_O$ is an output of $G$, $|V_O|=n$,
$w_i$ is the weight of $o_i$,
$\mathbb{E}[\cdot]$ denotes the expectation value,
and $\pf$ is a set of linear inequalities derived from the CNF formula of $G$
through the standard translation from clauses to linear constraints.
To illustrate, a clause $(x \lor \lnot y \lor z)$ in a CNF formula is transformed into
a linear inequality $(x+1-y+z)\geq 1$,
or $(x-y+z)\geq 0$ in $\pf$.
Note that the worst case formulation in~\cref{thm:prob-mppe-ssat} is a special case of
the SILP formulation where the expectation value of the miter's output is maximized.

\subsection{Connection to approximate design analysis}
Approximate design analysis assesses the deviation between
an approximate design and its exact counterpart in two scenarios:
the worse and average cases.
For the worst-case analysis, integer linear programming (ILP) can be applied
to find an input assignment to maximize the number of deviating outputs.
For the average-case analysis, model counting can be used
to compute the number of input assignments that make the two designs have different output responses.
In both cases, our PPE framework can be applied to analyze an approximate design,
which can be seen as a probabilistic design without random behavior.
For probabilistic design analysis,
if all random variables become deterministic,
then it degenerates to approximate design analysis
(from SILP to ILP under the worst-case analysis and
from weighted model counting to model counting under the average-case analysis).


\documentclass[11pt,twoside]{article}

\usepackage[headings]{fullpage}
\usepackage[utopia]{mathdesign}

\pagestyle{myheadings}
\markboth{Horn equation}{Horn equation}

\input{../../fncextra}

\begin{document}

\begin{center}
    \bf Toot your own horn
\end{center}

The use of mathematics for the design and analysis of musical instruments has a long and rich history. One of the landmarks in this area is the 1919 model known as Webster's horn equation, even though the relevant modeling and mathematics had largely been worked out by such luminaries as Daniel Bernoulli, d'Alembert, Euler, Lagrange, Green, Helmholtz, and Rayleigh. For Webster the unknown is the air pressure $u(x)$ in a long, thin tube of length $L$ with varying cross-section $A(x)$, as governed by
\begin{equation}
  \label{eq:webster}
  u'' + \frac{A'}{A} u' + \omega^2 u = 0, \qquad x\in [0,L],
\end{equation}
where $\omega$ is the time frequency in a harmonic solution, i.e., the pitch of the sound. Reasonable boundary conditions are a fixed nonzero value of $u$ at $x=0$ (the mouthpiece) and $u'(L)=0$ at the open end. In the analysis problem, $A(x)$ is taken as prescribed, whereas in design the goal is to find an $A(x)$ that optimizes some desired quality of the horn.

For some prescriptions of $A(x)$, a solution can be found in closed form. For example, if $A(x)=e^{2k x}$, then the general solution of~(\ref{eq:webster}) is
\begin{equation}
  \label{eq:solution-exp}
  u(x) = c_1 e^{s_1 x} + c_2 e^{s_2 x},
\end{equation}
where $s_1$ and $s_2$ are the roots of $s^2+2k s + \omega^2$, and $c_1$, $c_2$ are constants.

Naturally, we will focus on numerical solutions of the horn equation. The problem is a linear BVP and easy to solve for most choices of $A(x)$. The solution $u(x)$ represents a standing wave whose amplitude
\begin{equation}
  \label{eq:amplitude}
  \alpha(\omega)=\|u\|
\end{equation}
can vary considerably with the frequency $\omega$. Physically, a horn player produces more or less white noise with significant components in a wide range of frequencies. A sharp peak in $\alpha(\omega)$ indicates a resonant frequency of the horn that is likely to dominate the resulting sound.

\subsection*{Goals}

You will compute finite difference solutions of the horn equation for a few choices of $A(x)$, and look for resonances.

\subsection*{Preparation}

Read section 10.4.
% \begin{enumerate}
%     \item Derive~\eqref{eq:solution-exp}.
%     \item Let $k=3$, $L=1$, $u(0)=1$ and $u'(L)=0$. Define the amplitude using the 2-norm in~(\ref{eq:amplitude}) and make a plot of $\alpha(\omega)$ for $1\le \omega \le 16$. Computer algebra and/or numerics is \emph{strongly} recommended. How many resonant frequencies are in this range?
%\end{enumerate}

\subsection*{Procedure}

Download the template script and complete it to perform the following tasks.

\begin{enumerate}
    \item Set $L=2$, $\omega=6$, and $A(x)=e^{6x}$  in~(\ref{eq:webster}). Find the roots $s_1$ and $s_2$ in~(\ref{eq:solution-exp}). (Note that $k=3$ in that equation.) Use $u(0)=1$ and $u'(L)=0$ to solve for $c_1$ and $c_2$, and plot the solution for $0\le x \le L$.
    \item Using the same values as in step~1, apply \texttt{bvp} with $n=500$ to find a numerical solution. Plot the difference between the exact and numerical solutions at all of the finite difference nodes.
    \item Now let $A(x)=3x^2+x+1$ and $\omega=2$ (still with $L=2$). Plot the finite difference solution using $n=500$.
    \item Repeat the computation from step~3, but for 100 equally spaced values of $\omega$ between 0.2 and 3. In each case compute $\alpha(\omega)$ using the infinity norm in~(\ref{eq:amplitude}). Plot $\alpha$ as a function of $\omega$.
    \item Your amplitude graph should have two sharp spikes. Find $\omega_1$ and $\omega_2$, the horizontal coordinate of each spike. (One easy way is to use the ``data cursor'' tool in the figure's menu.) Solve the BVP for these two values of $\omega$ to get $\bfu_1$ and $\bfu_2$. Plot $\bfu_1$ and $\bfu_2$ together over $[0,L]$. You should find that $\bfu_1$ has no interior local extrema and that $\bfu_2$ has one local max or min (depending on exactly what you choose for $\omega_2$).
\end{enumerate}

% \subsection*{Extras}
% \begin{enumerate}
%   \item[E1.] Here is a creative way to check the solution even when the exact solution is unknown. Equation~(\ref{eq:webster}) can be rearranged to give
%     \begin{equation}
%       \label{eq:logAprime}
%       \log A(x) = \log A(0) - \int_0^x \frac{u''(s) + \omega^2u(s)}{u'(s)}\, ds.
%     \end{equation}
%     Use $\omega_1$ and $\bfu_1$ from the last step. Obtain the differentiation matrices and compute the vector
% \begin{verbatim}
% z = -(Dxx*u1 + omega1^2*u1) ./ (Dx*u1);
% \end{verbatim}
%     which gives the integrand of~(\ref{eq:logAprime}) at the equally spaced nodes. (The last value, $z_n$, will be useless because of the boundary condition, but this does not affect much.) For each $x_i$, perform trapezoid quadrature on $\bfz$ from element 0 to $i$ in order to approximate the integration in~(\ref{eq:logAprime}). Then finish the calculation of $A(x)$ and compare it graphically to the original $A(x)$ used to define the ODE.
% \end{enumerate}

\end{document}


\subsection{Metric compatibility}

If we have two vectors in the tangent space of a manifold with a metric tensor, we can get a scalar:

\(v^iu^jg_{ij}\)

\subsubsection{Transported metric}

If we transport two vectors along a connection, we have the metric at the new point.

\subsubsection{Metric preserving connections}

If the connection preserves the metric, then the connection is metric compatible.


\section{MgNet: a new network structure}\label{sec:mgnet}
In this section, we introduce a new neural network structure,
named as MgNet, motivated by the multigrid algorithm,
Algorithm \ref{alg:L-Slash0}, as discussed in the previous section.

%The main structure of both CNN and multigrid can be
%understood as:
%\begin{itemize}
%	\item Solving an undetermined system in grid $\ell$.
%	\item Connection of different grids by restriction and prolongation.
%\end{itemize}

First, given the data-feature equation \eqref{Auf}, we consider
its restrictions to grid $\ell$ as follows:
\begin{equation}
\label{Auf-ell}
A^\ell(u^\ell) = f^\ell, \quad \ell=1:J,
\end{equation}
where
\begin{equation}
\label{f-ell}
f^{\ell}\in\mathbb R^{m_\ell\times n_\ell\times c_\ell},
\end{equation}
and
\begin{equation}
\label{u-ell}
u^{\ell}\in\mathbb R^{m_\ell\times n_\ell\times h_\ell}.
\end{equation}
We are now in a position to state the main algorithm, namely
MgNet as:
\begin{breakablealgorithm}
	\caption{$u^J={\rm MgNet}(f; J,\nu_1, \cdots, \nu_J)$}
	\label{alg:mgnet}
	\begin{algorithmic}
		\State Initialization:  $f^1 =\theta(f)$, $u^{1,0}=0$
		%		\State Initialization $u^{1,0}$
		\For{$\ell = 1:J$}
		\For{$i = 1:\nu_\ell$}
		\State Feature extraction (smoothing):
		\begin{equation}\label{mgnet}
		u^{\ell,i} = u^{\ell,i-1} + B^{\ell,i}  ({f^\ell -  A^{\ell} (u^{\ell,i-1})}).
		\end{equation}
		\EndFor
		\State Note:
		$
		u^\ell= u^{\ell,\nu_\ell}
		$
		\State Interpolation and restriction:
		\begin{equation}
		\label{interpolation}
		u^{\ell+1,0} = \Pi_\ell^{\ell+1}u^{\ell}
		\end{equation}
		\begin{equation}
		\label{restrict-f}
		f^{\ell+1} = R^{\ell+1}_\ell(f^\ell - A^\ell(u^{\ell})) + A^{\ell+1} (u^{\ell+1,0}).
		\end{equation}
		\EndFor
	\end{algorithmic}
\end{breakablealgorithm}

The first property of MgNet is that it recovers the multigrid methods.
\begin{theorem}
	If $A^\ell$, $R_\ell^{\ell+1}$ and $B^{\ell,i} = S^{\ell}$ are all linear operations as described in multigrid method
	in \S \ref{sec:mg}.
	Then Algorithm \ref{alg:L-Slash0} is equivalent to Algorithm \ref{alg:mgnet} with any choice of $\Pi_\ell^{\ell+1}$.
\end{theorem}
\begin{proof}
	Here we replace $u^{\ell,i}$ and $f^{\ell}$ by $\tilde u^{\ell,i}$ and $\tilde f^{\ell}$ in MgNet.
	What we want to prove are
	\begin{equation}\label{eq:f-u}
	\tilde f^{\ell} =  f^{\ell} + A_\ell \tilde u^{\ell,0} \quad \text{and} \quad u^{\ell,i} = \tilde u^{\ell, i} - \tilde u^{\ell, 0},
	\end{equation}
	with $u^{\ell,i}$, $f^\ell$ in Algorithm \ref{alg:L-Slash0} and
	$\tilde u^{\ell,i}$, $\tilde f^\ell$ in Algorithm \ref{alg:mgnet} for any choice of $\Pi_\ell^{\ell+1}$.
	We prove this result by induction.
	\begin{itemize}
		\item It is easy to check that $\ell = 1$ is right by taking $\theta  = \rm{id}$.
		\item Once the above equation \eqref{eq:f-u} is right for $\ell$,
		let us prove the corresponded result for $\ell+1$.
		\begin{itemize}
			\item For $\tilde f^{\ell+1}$, as the definition in Algorithm \ref{alg:mgnet}, we have
			\begin{align*}
			\tilde f^{\ell+1} &= R_\ell^{\ell+1}(\tilde f^\ell - A^\ell \tilde u^{\ell,\nu_\ell}) + A^{\ell+1}\tilde u^{\ell+1,0}, \\
			&= R_\ell^{\ell+1}(\tilde f^\ell + A^{\ell} \tilde u^{\ell,0}- A^{\ell} \tilde u^{\ell,\nu_\ell}) + A^{\ell+1}\tilde u^{\ell+1,0} \\
			&= R_\ell^{\ell+1}(\tilde f^\ell - A^{\ell} (\tilde u^{\ell,\nu_\ell}- u^{\ell,0})) + A^{\ell+1}\tilde u^{\ell+1,0} \\
			&= R_\ell^{\ell+1}(f^\ell - A^{\ell} u^{\ell,\nu_\ell}) + A^{\ell+1}\tilde u^{\ell+1,0}, \\
			&=  f^{\ell+1} + A^{\ell+1}\tilde u^{\ell+1,0}.
			\end{align*}
			\item For $u^{\ell+1,i}$, first we have
			$$
			u^{\ell+1,0} = 0 =\tilde u^{\ell+1, 0} - \tilde u^{\ell+1, 0},
			$$
			then we prove
			\begin{equation}\label{u:i+1}
			u^{\ell+1,i} = \tilde u^{\ell+1, i} - \tilde u^{\ell+1, 0}
			\end{equation} by induction for $i$.

			We assume \eqref{u:i+1} holds for $0,1,\cdots,i-1$. Let us miner $\tilde u^{\ell+1, 0}$ in both sides of
			the smoothing process \eqref{mgnet} in Algorithm \ref{alg:mgnet}. Then we have
			\begin{align*}
			\tilde u^{\ell+1,i} - \tilde u^{\ell+1, 0} &= \tilde u^{\ell+1,i-1} - \tilde u^{\ell+1, 0} + B^{\ell+1,i} (\tilde f^{\ell+1} - A^{\ell+1} \tilde u^{\ell+1,i-1}), \\
			&= \tilde u^{\ell+1,i-1} - \tilde u^{\ell+1, 0} + B^{\ell+1,i} (f^{\ell+1} + A^{\ell+1}\tilde u^{\ell+1,0} - A^{\ell+1} \tilde u^{\ell+1,i-1} ),\\
			&= u^{\ell+1,i-1} + B^{\ell+1,i} (f^{\ell+1} - A^{\ell+1}u^{\ell+1,i-1} ).
			\end{align*}
			This is exact the smoothing process in Algorithm \ref{alg:L-Slash0} as we take $ B^{\ell+1,i} = S^{\ell+1}$.
			%			\item At last, recall that $u^{1,0} = 0$, so the output of Algorithm \ref{alg:L-Slash0}
			%			$$
			%			tilde u^{1, \nu_1} = u^{1, \nu_1},
			%			$$
			%			which is the output of Algorithm \ref{alg:L-Slash}.
		\end{itemize}
	\end{itemize}
\end{proof}

Despite of the simplicity look of Algorithm \ref{alg:mgnet}, there
are rich mathematical structures and variants which we briefly discuss below.

\subsection{Initialization: feature space channels}
Initially for $\ell=1$,  we take $m_1 = m$ and $n_1 = n$ and we may define the linear mapping
\begin{equation}
\label{eq:6}
\theta: \mathbb R^{m\times n\times c}
\mapsto \mathbb R^{m_1\times n_1\times c_1},
\end{equation}
to obtain $f^{1} = \theta(f)$ with $c$ given in \eqref{data-c} changed to the channel of the initial
data space to $c_1$.   Usually
\begin{equation}
\label{cc}
c_1\ge c.
\end{equation}
One possibility is that we choose $c_1=c$.  In this case, we choose
$\theta=$identity.   But in general, we may need to choose $c_1\gg
c$. One possible advantage of preprocessing the RGB ($c=3$) to
different color spaces is that we can better choose what kind of
features the CNN can detect, and under what
conditions those detections will be invariant.

One possibility of understanding and modifying this step
is to decompose the data $f$ into a number of more
specialized data
\begin{equation}
\label{decomp-f}
f=\sum_{k=1}^{c_1}\xi_kf^1_k  =\xi^Tf^1.
\end{equation}
We may use some knowledge from image processing or physics to
design a procedure to obtain the right decomposition of
\eqref{decomp-f}, or we can just train it.
Conceivably, we may view $f^{1} = \theta(f)$ as a special approximation solution of
\eqref{decomp-f} with the same sparsity pattern to $\xi$.

\subsection{Extracted Units: $u^{\ell}$ and channels}
The first new feature and the main new ingredient
in the proposed neural network is the introduction
of feature variables  $u^{\ell}$ in \eqref{u-ell}, which will be known
as the extracted units.

We emphasize that the extracted-units $u^\ell$ and the data $f^\ell$ can have
different numbers of channels:\
\begin{equation}
\label{uf-channels}
u^\ell\in \mathbb{R}^{m_\ell\times n_\ell \times c_{u,\ell}}, \quad
f^\ell\in \mathbb{R}^{m_\ell\times n_\ell \times c_{f,\ell} }
\end{equation}
One possibility is that the number of channels for both $u$ and $f$ remain
unchanged in different grids:
\begin{equation}
\label{cfl}
c_{f,\ell}=c_f, \quad \ell=1:J,
\end{equation}
and
\begin{equation}
\label{ufl}
c_{u,\ell}=c_{u}, \quad \ell=1:J.
\end{equation}
Both $c_f$ and $c_{u}$ are two super-parameters that need to be tuned,
and we may even take $c_u = c_f$.

\subsection{Poolings: $\Pi_{\ell+1}^\ell$ and $R_{\ell+1}^\ell$}
The pooling $\Pi_{\ell+1}^\ell$ in \eqref{restriction} and
$R_{\ell+1}^\ell$ in \eqref{restrict-f} are in general different.
They can be trained in general, but they may be a priori chosen.

There are many different possibilities to choose $\Pi_{\ell+1}^\ell$.
The simplest choice of $\Pi_{\ell+1}^\ell$ is
\begin{equation}
\label{eq:8}
\Pi_{\ell+1}^\ell=0.
\end{equation}
A more sophisticated choice can be obtained by considering an
interpolation from fine grid to coarse (that, for example preserves linear function
locally).  Namely
\begin{equation}
\label{Pi}
\Pi_{\ell+1}^\ell=\bar\Pi_{\ell+1}^\ell \otimes I_{c_\ell\times c_\ell}
\end{equation}
with $\bar\Pi_{\ell+1}^\ell$ given by~\eqref{mg-Pi}.

\subsection{Data-feature mapping: $A^{\ell}$}
The second new feature of MgNet is that this data-feature mapping
only depends on the grid ${\cal T}_\ell$, and it does not depend on layers
within the same grid.  This amounts to a significant saving of the number of
parameters.  In comparison, the existing CNN, such as ResNet, can be
interpreted as a network related to the case that $A^{\ell}$ is
replaced by $A^{\ell, i}$, namely
\begin{equation}\label{u-resnet}
u^{\ell,i} = u^{\ell,i-1} + B^{\ell,i}  (f^\ell -  A^{\ell,i} (u^{\ell,i-1}).
\end{equation}

The data-feature mapping: $A^{\ell}$ can be either linear
\eqref{linearA}, or nonlinear \eqref{nonlinearA}.  The underlying
convolution kernels can be different on different grids and they can
all be trained.

\subsection{Feature extractors: $B^{\ell,i}$}
There are some freedoms in choosing these feature extrators.
One common choice of extractors is given by \eqref{extractor}, namely
\begin{equation}
\label{extractor-ell}
B^{\ell,i}=\sigma\circ \eta^{\ell,i}\circ\sigma.
\end{equation}

Other than the level dependent extractors, the following
different strategies can be used
\begin{description}
	\item[Constant Extractors]: $B^{\ell,i}=B^{\ell}$ for   $i=1:\nu_\ell$
	\item[Scaled Extractors]:$B^{\ell,i}=\alpha_iB^{\ell}$ for   $i=1:\nu_\ell$
	\item[Variable Extractors]: $B^{\ell,i}$
\end{description}

This brief framework gives us the basic principle on designing
a CNN models for classification. All models are seen as the special
choice of data-feature mapping $A^\ell$, feature extractors $B^{\ell,i}$
and the pooling operators $\Pi_{\ell+1}^\ell$ with $R_{\ell+1}^\ell$.

\section{Some classic CNN models}\label{sec:CNNs}
In this section, we will use the notation introduced above to
give a brief description of some classic CNN models.

\subsection{LeNet-5, AlexNet and VGG}
The  LeNet-5 \cite{lecun1998gradient}, AlexNet \cite{krizhevsky2012imagenet} and VGG \cite{simonyan2014very}
can be written as:
	\begin{equation}
	\begin{cases}
	f^{1,0} &= \theta^0(f), \\
	f^{\ell,i} &= \theta^{\ell,i} \circ \sigma (f^{\ell, j-1}), \quad i = 1:\nu_\ell ~\text{and}~ \ell = 1:J,\\
	f^{\ell+1,0} &= R_\ell^{\ell+1}( f^{\ell,m+\ell}).  \\
	\end{cases}
	\end{equation}
	where $R_\ell^{\ell+1}$ can be general pooling operators and $\theta^{\ell,i}$ can be convolution with stride 1,
	or fully connected operators.
	Then the CNN model will be defined by
	\begin{equation}\label{eq:cnndefine}
	H_0(f) = f^{L,\nu_\ell}.
	\end{equation}
	In these three classic CNN models, they still need some
	extra fully connected layers in nonlinear mapping $H_0$ before the logistic regression as it contains
	a fully connected layer as in \eqref{eq:log_reg}.
	These fully connected layers are removed in ResNet to be described below.
\subsection{ResNet}
The ResNet \cite{he2016deep} can be written as
	\begin{equation}\label{ori-ResNet}
	\begin{cases}
	f^{1,0} &=R_{\rm max}\circ \sigma \circ \theta^0(f), \\
	f^{\ell,i} &= \sigma \left( f^{\ell, i-1} + \mathcal{F}^{\ell, i} (f^{\ell,i-1}) \right), \quad i = 1:\nu_\ell ~\text{and}~ \ell = 1:J ,\\
	f^{\ell+1,0} &= \sigma \left( R_\ell^{\ell+1} (f^{\ell, \nu_\ell} )+ \mathcal{F}^{\ell, 0} (f^{\ell, \nu_\ell} ) \right), \quad \ell = 1:J-1,\\
	H_0(f) &=  R_{\rm ave}(f^{L,\nu_\ell}). \\
	\end{cases}
	\end{equation}
	Here
	$$
	\mathcal{F}^{\ell,i} (f^{i-1}) = \xi^{i} \circ \sigma \circ \eta^{i} (f^{i-1}).
	$$
	Generally, $\xi^{\ell,i}$ and $\eta^{\ell,i}$ takes the form of \label{eq:conv-1} with zero padding and stride 1,
	except, $\eta^{\ell,0}$  is taken as convolution with stride 2 with the same output dimension of $R_\ell^{\ell+1}$.

\subsection{iResNet}
	The iResNet\cite{he2016identity} can be written as:
	\begin{equation}\label{eq:iResNet1}
	\begin{cases}
	f^{1,0} &=R_{\rm max}\circ \sigma \circ \theta^0(f), \\
	f^{\ell,i} &= f^{\ell, i-1} + \mathcal{F}^{\ell, i} (f^{\ell,i-1}), \quad i = 1:\nu_\ell ~\text{and}~ \ell = 1:J ,\\
	f^{\ell+1,0} &=  R_\ell^{\ell+1} (f^{\ell, \nu_\ell} )+ \mathcal{F}^{\ell, 0} (f^{\ell, \nu_\ell} ) , \quad \ell = 1:J-1,\\
	H_0(f) &=  R_{\rm ave}(f^{L,\nu_\ell}). \\
	\end{cases}
	\end{equation}
	where
	$$
	\mathcal{F}^{\ell,i} (f^{\ell,i -1}) = \xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i} \sigma (f^{\ell,i-1}).
	$$
	The only difference between ResNet and iResNet can be viewed as
	putting a $\sigma$ in different places.
	The connection of those three models are often shown with next diagrams:
	\begin{figure}[!htb]
		\begin{center}
			\includegraphics[width=.6\textwidth, height=.13\textheight]{comparison-net}
		\end{center}
		\caption{Comparison of CNN Structures}
	\end{figure}

Without loss of generality, we extract the key
feedforward steps on the same grid in different CNN models as follows.
\begin{description}
	\item[Classic CNN]
	\begin{equation}\label{eq:cCNN}
	f^{\ell,i} = \xi^i \circ \sigma (f^{\ell,i-1}) \quad \text{or} \quad f^{\ell,i} = \sigma \circ \xi^{i} (f^{\ell,i-1}) .
	\end{equation}
	\item[ResNet]
	\begin{equation}\label{eq:ResNet}
	f^{\ell,i} = \sigma( f^{\ell,i-1} + \xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i}(f^{\ell,i-1})).
	\end{equation}
	\item[iResNet]
	\begin{equation}\label{eq:iResNet}
	f^{\ell,i} = f^{\ell,i-1} + \xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i}\circ \sigma(f^{\ell,i-1}).
	\end{equation}
\end{description}

\section{Variants and generalizations of MgNet}\label{sec:relation}

%\subsection{Some properties of MgNet}
%So, this MgNet is corresponded to the multigrid methods
%with iteration in function space. A natural idea is that there is also a dual version of
%MgNet similar with the multigrid methods with iteration in dual space.
The MgNet model algorithm is one very basic and it can be generalized
in many different ways. It can also be used as a guidance to modify and
extend many existing CNN models.

The following result show how MgNet is related to he iResNet \cite{he2016identity}.
\begin{theorem}\label{thm:mgnet1}
The MgNet model Algorithm \ref{alg:mgnet},
with $A=\xi^\ell$ and $B^{\ell,i}=\sigma \circ \eta^{\ell,i}\circ\sigma$,
admits the following identities
\begin{equation}\label{dualmgnet}
f^{\ell, i} = f^{\ell, i-1} -  \xi^{\ell} \circ \sigma \circ \eta^{\ell,i}\circ \sigma (f^{\ell,i-1}), \quad i = 1:\nu_\ell, \\
\end{equation}
where
\begin{equation}
  \label{eq:5}
	f^{\ell,i} = f^{\ell} - \xi^{\ell} (u^{\ell,i}).
\end{equation}
Furthermore, \eqref{dualmgnet} represents iResNet~\cite{he2016identity}
as shown in \eqref{eq:iResNet}.
\end{theorem}

\begin{proof}
	Because of the linearity of $\xi^\ell$ and invariant within the same grid $\ell$,
	we can apply $\xi^\ell$ on both sides of \eqref{mgnet} and minus with
	$f^\ell$, thus we have
	$$
	f^{\ell} - \xi^{\ell} (u^{\ell,i})= f^{\ell} - \xi^\ell(u^{\ell,i}) -
	\xi^{\ell} \circ \sigma \circ \eta^{\ell,i}\circ \sigma (f^\ell + \xi^\ell(u^{\ell,i})).
	$$
This finish the proof with definition in \eqref{eq:5}.
\end{proof}

The above result is very simple but critically important.
In view of Theorem \ref{thm:mgnet1}, it shows how multigrid and
CNN are intimately related. Furthermore, it provides a different version
of iResNet, which can be viewed as the dual version of the original iResNet.
This relation is quit similar with the dual relation of $u$ and $f$
in multigrid method \cite{xu2017algebraic}.
\begin{lemma}\label{thm:mgnet2}
	The ResNet~\cite{he2016deep} step
 	 as in \eqref{eq:ResNet}
%	\begin{equation}\label{resnet}
%	f^{\ell,i} = \sigma( f^{\ell, i-1} - \xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i} (f^{\ell,i-1}) ).
%	\end{equation}
admits the following relation:
%(which resembles closely with \eqref{dualmgnet})
\begin{equation}\label{tilde-resnet}
\tilde f^{\ell,i} =\sigma(\tilde f^{\ell,i-1}) -
\xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i}\circ \sigma( \tilde f^{\ell,i-1}),
\end{equation}
where
\begin{equation}\label{tilde-f}
\tilde f^{\ell,i} = f^{\ell, i-1} -\xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i} (f^{\ell,i-1}).
\end{equation}
\end{lemma}
\begin{proof}
%	Now, we will establish the connection between classical ResNet and MgNet.
	First, we apply $ \xi^{\ell,i+1} \circ \sigma \circ \eta^{\ell,i+1}$
	on the both sides of \eqref{eq:ResNet} and get
	\begin{equation}\label{resnet1}
	\xi^{\ell,i+1} \circ \sigma \circ \eta^{\ell,i+1}( f^{\ell,i} ) =
	\xi^{\ell,i+1} \circ \sigma \circ \eta^{\ell,i+1}\circ \sigma( \tilde f^{\ell,i} ).
	\end{equation}
	Minus by $f^{\ell,i}$ on the both sides and recall the definition in \eqref{tilde-f}, we have
	\begin{equation*}
	\tilde f^{\ell,i+1} = f^{\ell,i} - \xi^{\ell,i+1} \circ \sigma \circ \eta^{\ell,i+1}\circ \sigma( \tilde f^{\ell,i}).
	\end{equation*}
	By the definition of $f^{\ell,i} = \sigma(\tilde f^{\ell,i})$, we finish this proof.
\end{proof}

We call the above form \eqref{tilde-resnet} as
$\sigma$-ResNet, similar to the MgNet we replace $\xi^{\ell,i}$ by $\xi^{\ell}$  and get
the next Mg-ResNet form as:
\begin{equation}\label{mg-resnet}
f^{\ell,i} =\sigma(f^{\ell,i-1}) -
\xi^{\ell} \circ \sigma \circ \eta^{\ell,i}\circ \sigma(f^{\ell,i-1}).
\end{equation}

If we take these pooling and prolongation operators
as discussed in the previous sections and focus on
the iterative forms on a certain grid $\ell$, we may
compare them all as:
\begin{table}[!htbp]
	\caption{Comparison for all iterative forms }
	\label{comparison-ALL}
	\begin{center}\scriptsize
		\resizebox{.8\textwidth}{!}{
			\begin{tabular}{|c|c|c|}
				\hline
				Primal-Dual & Model & Iterative form \\
				\hline
				\multirow{3}{*}{Feature space} & Abstract-MgNet & Solving $A^\ell(u^\ell) = f^\ell$ \\
				\cline{2-3}
				& General-MgNet & $u^{\ell,i} = u^{\ell, i-1} + B^{\ell,i} (f^\ell - A^{\ell}(u^{\ell,i-1}))$ \\
				\cline{2-3}
				& {MgNet} & $u^{\ell,i} = u^{\ell, i-1} + \sigma \circ \eta^{\ell,i}\circ \sigma (f^\ell - \xi^{\ell}(u^{\ell,i-1}))$ \\
				\hline
					\multirow{5}{*}{Data space} & iResNet & $ f^{\ell,i} = f^{\ell, i-1} -  \xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i} \circ \sigma (f^{\ell,i-1})$ \\
				\cline{2-3}
				& Mg-iResNet & $f^{\ell,i} = f^{\ell, i-1} -  \xi^{\ell} \circ \sigma \circ \eta^{\ell,i}\circ \sigma (f^{\ell,i-1})$ \\
				\cline{2-3}
				& Mg-ResNet & $f^{\ell,i} = \sigma(f^{\ell,i-1}) - \xi^{\ell} \circ \sigma \circ \eta^{\ell,i}\circ \sigma( f^{\ell,i-1})$ \\
				\cline{2-3}
				& $\sigma$-ResNet & $f^{\ell,i} = \sigma(f^{\ell,i-1}) - \xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i}\circ \sigma( f^{\ell,i-1})$ \\
				\cline{2-3}
				& ResNet & $f^{\ell,i} = \sigma(f^{\ell, i-1} -  \xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i} (f^{\ell,i-1}))$ \\
				\hline
			\end{tabular}
		}
	\end{center}
\end{table}

We can have these connections for all iterative scheme in data space:
\begin{equation}
\text{ ResNet} \xleftrightarrow{\eqref{tilde-f}} \sigma\text{-ResNet } \xleftrightarrow{\xi^{\ell,i} \leftrightarrow \xi^{\ell}} \text{Mg-ResNet}
\xleftrightarrow{\sigma(f^{\ell,i-1}) \leftrightarrow f^{\ell, i-1} } \text{Mg-iResNet} \xleftrightarrow{ \xi^{\ell} \leftrightarrow \xi^{\ell,i}} \text{iResNet}.
\end{equation}

%Because of the linearity of $\xi^{\ell,i}$, the above forms of iResNet and ResNet
%are equivalent to the previous as \eqref{dualmgnet} and \eqref{tilde-resnet}.

In this sense, these MgNet related models can be understood as
 models between iResNet and ResNet. And all these models can be
 understood as iteration in the data space as a dual relationship with
 feature space as MgNet.

The rationality of replacing  $\xi^{\ell,i}$ by layer independent $\xi^{\ell}$ may
be justified by the following theorem.
\begin{theorem}\label{thm:CNN}
On each grid $\mathcal T_\ell$,
\begin{enumerate}
	\item Any CNN model with
%	CNN and Mg-ResNet]
	\begin{equation}
	\label{CNN1}
	f^{\ell,i} =   \chi^{\ell,i} \circ \sigma (f^{\ell,i-1}),
	\end{equation}
	can be written as
	\begin{equation}\label{Res-CNN1}
	f^{\ell,i} = \sigma(f^{\ell,i-1}) - \xi^{\ell} \circ \sigma \circ \eta^{\ell,i} \circ\sigma ( f^{\ell,i-1}).
	\end{equation}
	\item Any CNN model with
%	[CNN and ResNet]
	\begin{equation}
	\label{CNN2}
	f^{\ell,i} =   \sigma\circ\chi^{\ell,i} (f^{\ell,i-1}).
	\end{equation}
	can be written as
	\begin{equation}\label{Res-CNN2}
	f^{\ell,i} = \sigma\left(f^{\ell,i-1} - \xi^{\ell} \circ \sigma \circ \eta^{\ell,i}  ( f^{\ell,i-1})\right).
	\end{equation}
\end{enumerate}

%\begin{equation}
% \chi^{\ell,i}: \mathbb{R}^{n_\ell \times n_\ell \times c_\ell}
% \mapsto \mathbb{R}^{n_\ell \times n_\ell \times c_\ell},
%\end{equation}
\end{theorem}
\begin{proof}
%	Without loss of generality, consider the classical CNN with
%	\begin{equation}
%	f^{\ell,i} =   ({\rm id}  + \tilde \eta^{\ell,i} )\circ \sigma (f^{\ell,i-1}).
%	\end{equation}
Let use prove the first case as an example,
the second case can be proven with the same process.

With similar structure in MgNet, we can take
\begin{equation}
\label{xi-cnn1}
\xi^{\ell}=  [\hat \delta_1, \cdots, \hat \delta_{{c_\ell}}],
\end{equation}
and
\begin{equation}
\label{eta-ell}
\eta^{\ell,i} = [{\rm id}_{c_\ell}, -{\rm id}_{c_\ell}] \circ (\chi^{\ell,i} - {\rm id}_{c_\ell}).
\end{equation}
Here
\begin{equation}
{\rm id}_{c_\ell}: \mathbb{R}^{n_\ell \times n_\ell \times c_\ell}
\mapsto \mathbb{R}^{n_\ell \times n_\ell \times c_\ell},
\end{equation}
is the identity map and
\begin{equation}
\hat \delta_k :  \mathbb{R}^{n_\ell \times n_\ell \times 2c_\ell}
\mapsto \mathbb{R}^{n_\ell \times n_\ell},
\end{equation}
with
\begin{equation}\label{eq:hatdelta}
\hat \delta_k([X ,Y]) = -([X]_k + [Y]_k),
\end{equation}
for any $X, Y \in \mathbb{R}^{n_\ell \times n_\ell \times c_\ell}$
and $[X,Y] \in \mathbb{R}^{n_\ell \times n_\ell \times 2c_\ell} $.

	First, we see that $\eta^{\ell,i}$ with the above
	form is a convolution from $\mathbb{R}^{n_\ell \times n_\ell \times c_\ell}$
	to  $\mathbb{R}^{n_\ell \times n_\ell \times 2c_\ell}$.
	Following the identity
	\begin{equation}
	ReLU(x) + ReLU(-x) = x,
	\end{equation}
	and the definition of $\xi^{\ell}$ i.e.
	\begin{equation}
	\xi^{\ell} = \hat \delta,
	\end{equation}
	as a special case in MgNet.
	For more details, we can give a exact form of
	$\hat \delta_k$ as in \eqref{eq:hatdelta} with
	\begin{equation}
	\hat \delta_k = [0, \cdots,0, -\delta, \cdots 0;  0, \cdots,0, -\delta, \cdots 0],  \quad k = 1:{c_\ell},
	\end{equation}
	where $\delta$ is the identity kernel during one channel.

	At last, we have
	\begin{align}
	\left[\xi^{\ell} \circ \sigma \circ [{\rm id}_{c_\ell}, -{\rm id}_{c_\ell}] (x) \right]_k &=  \left[\xi^{\ell} \circ \sigma \circ [x, -x]  \right]_k, \\
	&= \hat \delta_k ( [\sigma(x), \sigma(-x)]),  \\
	&= -\delta([\sigma(x)]_k) - \delta([\sigma(-x)]_k),\\
	&=-( \sigma([x]_k)+ \sigma(-[x]_k)) , \\
	&=  -[x]_k
	\end{align}
	Thus to say,
	\begin{equation}
	\xi^{\ell} \circ \sigma \circ [{\rm id}_{c_\ell}, -{\rm id}_{c_\ell}]  = -{\rm id}_{c_\ell}.
	\end{equation}
	Then the modified dual form of MgNet in \eqref{tilde-resnet} becomes
	\begin{align}
	f^{\ell,i} &= \sigma(f^{\ell,i-1}) - \xi^{\ell,i} \circ \sigma \circ \eta^{\ell,i} \circ\sigma ( f^{\ell,i-1}) , \\
	&=  \sigma(f^{\ell,i-1}) - \left( \xi^{\ell} \circ \sigma \circ [{\rm id}_{c_\ell}, -{\rm id}_{c_\ell}] \right)
	\circ (\chi^{\ell,i} - {\rm id}_{c_\ell})\circ \sigma(f^{\ell,i-1})\\
	&=\sigma(f^{\ell,i-1}) + (\chi^{\ell,i} -{\rm id}_{c_\ell})\circ \sigma(f^{\ell,i-1}),  \\
	&=\chi^{\ell,i} \circ  \sigma (f^{\ell,i-1}).
	\end{align}
	This covers \eqref{Res-CNN1}.
\end{proof}

\begin{remark}
Theorems~\ref{thm:CNN} shows that general CNN in
the forms of either \eqref{CNN1} or \eqref{CNN2} can be written recast
as \eqref{Res-CNN1} or \eqref{Res-CNN2} with the data-feature mapping
$A^\ell=\xi^\ell$ that is not only independent of the layers, but is
actually given a priori as in \eqref{xi-cnn1}.  In
view of Theorems~\ref{thm:mgnet1} and \ref{thm:mgnet2}, the classic
CNN models can be essentially recovered from MgNet by choosing
$\xi^\ell$ a priori as in  \eqref{xi-cnn1}.  Since
the classic CNN models have been extensively tested to be successful,
the more general MgNet with more general $\xi^\ell$ (to be trained)
are expected to be more efficient than the classic CNN models.
\end{remark}

%At last we have the next relation:
%\input{MgNet-relation.tex}

\section{Numerical experiments}\label{sec:numerics}
In this section, we present some numerical results to illustrate the
efficiency and potential of MgNet as described in Algorithm
\ref{alg:mgnet}.

\subsection{Data sets and model structure }
We choose CIFAR-10 and CIFAR-100
\cite{krizhevsky2009learning}
as two data sets for numerical tests.
Here, the CIFAR-10 dataset consists of 60000 32x32 colour
images in 10 classes, with 6000 images per class.
The CIFAR-100 dataset is just like the CIFAR-10,
except it has 100 classes containing 600 images each.
We split these two data sets with 50000 training images
and 10000 test images.

We will mainly carry out
 a comparison with study between MgNet and  ResNet \cite{he2016deep}
on these two data sets, so we choose some
similar process techniques such as there will
be a average pooling before linear regression
layers:
\begin{equation}\label{eq:ave-pooling}
R_{ave}: \mathbb{R}^{m_{J-1} \times n_{J-1} \times c_{J-1}} \mapsto \mathbb{R}^{c_{J-1}}.
\end{equation}
We use the similar ideas in MgNet to take $m_{J} = 0$, thus to choose
$$
u^{J} = \Pi_{J-1}^J u^{J-1, m_{J-1}} \in \mathbb{R}^{c_{J-1}},
$$
with
$$
\Pi_{J-1}^J  = R_{ave}.
$$
This can be true also thanks to our structure that
\begin{equation}\label{eq:c_u}
c_{u,\ell} = c_{u}, \quad 1 \le \ell \le J.
\end{equation}
Given an image $f$, similar to ResNet, we apply our MgNet as follows:
\begin{equation}\label{final-mg}
y = S \circ \theta \circ u^{J}(f),
\end{equation}
where $u^J(f)$ is the output from our MgNet as described in Algorithm
\ref{alg:mgnet},  $S$ is the soft-max mapping in \eqref{softmax} and
\begin{equation}\label{final-theta}
\theta: \mathbb{R}^{c_u} \mapsto \mathbb{R}^\kappa,
\end{equation}
represents a fully linear layer with $\kappa = 10$ for CIFAR-10 and
$\kappa = 100$ for CIFAR-100.

We will make the following choice of hyper parameters
for the MgNet:
\begin{itemize}
	\item $J$: the number of grids. As all images in CIFAR-10 or CIFAR-100
	are $32\times 32 \times 3$,  we choose $J = 4$ to be consistent with ResNet.
	\item $\nu_\ell$:  the number of smoothings in each grids. To be consistent with
	ResNet-18 or ResNet-34 we choose $\nu_\ell = 2$ or $\nu_\ell = 4$.
	\item $c_u$ and $c_f$: the number of feature and data channels.
	\item $A^\ell$: the data-feature mapping. We choose the linear case in \eqref{linearA}.
	\item $B^{\ell,i}$: the feature extractor. We choose the variable extractors as in \eqref{extractor-ell}.
	\item $R_{\ell}^{\ell+1}$: the restriction operator in \eqref{restrict-f}.
	Here we choose it as a convolution with stride $2$ which need to be trained.
	\item $\Pi_\ell^{\ell+1}$: the interpolation operator in
	\eqref{interpolation}.  Here we compare these next three
	different choices:
	\begin{enumerate}
		\item {$\Pi_0$: } $\Pi_\ell^{\ell+1} = 0$;
		\item {$\Pi_1$: }convolution with stride $2$ which need to be
		trained;
		\item {$\Pi_2$: }channel-wise interpolation as in
		\eqref{Pi}, with $\bar P_{\ell}^{\ell+1}$ as a convolution
		with one channel and stride $2$ which also need to be trained.
	\end{enumerate}
\end{itemize}

\subsection{Training algorithm}
While there are many different choices of training algorithms \cite{bottou2018optimization},
in our test, we adopt the popular
stochastic gradient descent (SGD) with mith-batch and momentum for
cross-entropy loss function.
\begin{breakablealgorithm}
	\caption{SGD with mini-batch and momentum}
	\label{alg:sgd}
	\begin{algorithmic}
\State {\bf Input}: learning rate $\eta_t$, batch size $m$, parameter Initialization $ w_0$, number of epochs $K$.
\For{Epoch $k = 1:K$} \\
\State Shuffle data and get mini-batch $B_1, \cdots, B_{\frac{N}{m}}$, choose mini-batch as: $B_{i_t}$ with
$$
i_t \equiv t \mod(\frac{N}{m}),
$$
\State Compute the gradient on $B_{i_t}$:
$$
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} h_i(w_{t}).
$$
\State Compute the momentum:
\begin{equation}
v_t = \alpha v_{t-1} - \eta_t g_t \quad (v_0 = 0).
\end{equation}
\State Update $w$:
\begin{equation}
w_{t+1} = w_t + v_t.
\end{equation}
\EndFor
\end{algorithmic}
\end{breakablealgorithm}

Here we have $h_i(w_t) = l(\classmap(f_i;w_t),y_i)$ as defined in \eqref{eq:3}, where $w_t$ notes all free parameters in MgNet and $\theta$ in \eqref{final-theta}.
We use the SGD with momentum of 0.9.
The mini-batch size is chosen as
$m=128$. The learning rate starts from 0.1 and is divided by $10$ for
every $30$ epochs, and the models are trained for up to $K=120$ epochs.
We adopt batch normalization (BN) after each convolution and before
activation, following \cite{ioffe2015batch}.  Initialization strategy
is the same with ResNet as in \cite{he2015delving}.  We
do not use weight decay and dropout.  The final Top-1 test accuracy is
shown in Table~\ref{comparison}.
\begin{table}[!htbp]
	\caption{ResNet and MgNet on CIFAR-10 and CIFAR-100.
	Our methods are named with $\nu_\ell$, ($c_u$, $c_f$), $\Pi_\ell^{\ell+1}$ by definition above.}
	\label{comparison}
	\vskip 0.15in
	\begin{center}
				%		\resizebox{.5\textwidth}{!}{
				\begin{tabular}{cccc}
					\hline
					Models & CIFAR-10 & CIFAR-100 & Params \\
					\hline
					ResNet-18 & 92.24 & 71.96 & 11.2M   \\
					ResNet-34 & 92.80 & 71.93 & 21.3M   \\
					\hline
					%				{$\nu_\ell$,  $(c_u, c_f)$, $\Pi_\ell^{\ell+1}$} & Accuracy & Accuracy & Numbers  \\
					$2, (256,256)$, $\Pi_0$ & 92.02 & 68.29 & 7.1M  \\
					$2, (256,256)$, $\Pi_1$ & 93.04 & 72.32 & 8.9M  \\
					$2, (256,512)$, $\Pi_1$ & 93.20 & 72.42 & 19.5M  \\
					$4, (256,512)$, $\Pi_2$ & 93.53& 74.26 & 17.7M  \\
					\hline
				\end{tabular}
		%		}
	\end{center}
	\vskip -0.1in
\end{table}

From the above numerical results, we find that the modified CNN models
based on MgNet structure have competitive and sometimes better
performance in comparison with standard ResNet models when applied to
both CIFAR-10 and CIFAR-100 data sets. Generally speaking, the more
channels the better performance you can achieve (see WideResNet
\cite{zagoruyko2016wide} for similar observation). Furthermore,
$\Pi_1$ and $\Pi_2$ work better than $\Pi_0$, and $\Pi_2$ can even
work better than $\Pi_1$ with fewer parameters for big enough
channel numbers.

\section{Concluding remarks}\label{sec:conclusion}
By carefully studying the connections between the traditional
multigrid method and the convolutional neural network (especially the
ResNet type) models, the MgNet established in this paper provides a
unified framework that connects both multigrid and CNN in a technical
level.  Comparing with other existing works that discuss the
connection between multigrid and CNN, MgNet goes beyond formal or
qualitative comparisons and identifies key model components that play
the same corresponding roles, from an abstract viewpoint, for these two different
methodologies.  As a result, how and why CNN models work can be
mathematically understood in a similar fashion as for multigrid method
which has a much more mature and better developed theory.  Motivated
from various known techniques from multigrid method, many variants and
improvements of CNN can then be naturally obtained.  For example, as
demonstrated from our preliminary numerical experiments, the resulting
modified CNN models quipped with fewer weights and hyper parameters
actually exhibit competitive and sometimes better performance than
standard ResNet models.

The MgNet framework opens a new door to the
mathematical understanding, analysis and improvements of deep learning
models.  The very preliminary results presented in
this paper have demonstrated the great potential of MgNet from both
theoretical and practical viewpoints.  Obviously many aspects of MgNet
should be further explored and expect to be much improved.  In fact, only very
few techniques from multigrid method have been tried in this paper and
many more in-depth techniques from multigrid require further study for
deep neural networks, especially CNN.
In particular, we believe that the MgNet framework will
lead to improved CNN that only has a small fraction of the number
of weights that are required by the current CNN. On the other hand,
the techniques in CNN can also be used to develop new generation of multigrid
and especially  algebraic multigrid methods \cite{xu2017algebraic} for solving
partial differential equations. Our ongoing works have
demonstrated great potentials for research in these directions and  many
more results will be reported in future papers.


\documentclass[12pt]{article}

\def\exercisename{DG notes}
\usepackage{mystyle}
\begin{document}
	\section*{\exercisename}

Equation to be solved:
\begin{equation*}
    \frac{\partial u}{\partial t} + \frac{\partial au}{\partial x} = 0
\end{equation*}
Multiplication by a test function $v(x)$ and integration over each element:
\begin{equation*}
	\int_{V_e} \frac{\partial u}{\partial t}\, v\, dV_e + \int_{V_e} \frac{\partial au}{\partial x}\, v\, dV_e = 0
\end{equation*}
The second term can be integrated by parts:
\begin{equation*}
	\int_{x_1}^{x_2} \frac{\partial u}{\partial t}\, v\, d{V_e}
	+ \left[ au\,v\right]_{x_1}^{x_2}
	  - \int_{x_1}^{x_2} au \frac{\partial v}{\partial x}\, d{V_e} = 0
\end{equation*}
The key idea of the method is to replace the flux $au$ on the boundary by a flux numerical $f^*$ that is evaluated from the values of $u$ in the neighbouring elements. Thanks to this numerical flux, all the equations will be coupled.
\begin{equation*}
	\int_{x_1}^{x_2} \frac{\partial u}{\partial t}\, v\, d{V_e}
	+ \left[ f^*\,v\right]_{x_1}^{x_2}
	- \int_{x_1}^{x_2} au \frac{\partial v}{\partial x}\, d{V_e} = 0
\end{equation*}
We can integrate by parts the last term:
\begin{equation*}
	\int_{x_1}^{x_2} \frac{\partial u}{\partial t}\, v\, dx  + \int_{x_1}^{x_2} \frac{\partial au}{\partial x}\, v\, dx
= \left[ (au - f^*)\,v\right]_{x_1}^{x_2}
\end{equation*}
Let $N_i(x)$ be the shape functions.
\begin{equation*}
	v(x) = \sum_i N_i(x)\, v_i
\end{equation*}
\begin{equation*}
	u(x,t) = \sum_j N_j(x)\, u_j(t)
\end{equation*}
The integral equation becomes:
\begin{equation*}
	\sum_i \sum_j
	\left[
	\int_{x_1}^{x_2} N_i(x)\,N_j(x)\, dx\, \frac{\partial u_j(t)}{\partial t}
	+
	\int_{x_1}^{x_2} N_i(x)\,\frac{\partial N_j(x)}{\partial x}\, dx\,  au_j(t)
	\right]
	\, v_i
\end{equation*}
\begin{equation*}
	= \sum_i \sum_j
	\left[
		\left( N_j(x)\,au_j(t) - f^{*}\right)\,N_i(x)
	\right]_{x_1}^{x_2}
	\, v_i
\end{equation*}
This equality should be verified for any $v_i$, thus, each coefficient of $v_i$ should vanish. This leads to a system of equations:
\begin{equation*}
 \sum_j M_{ij}\, u_j + S_{ij}\,u_j
 =
 (au_j(x_2)-f^*(x_2))\,N_i(x_2) - (au_j(x_1)-f^*(x_1))\,N_i(x_1)
\end{equation*}

\end{document}


\section{Content}

\subsection{Notation}

\begin{Defn}
  \label{defn:1}
  Some definitions, $\cos(x)$ and \(\sin(x)\) are
  \begin{eqnarray}
    \cos(x) \doteq \frac{e^{ix}+e^{-ix}}{2}  \label{eq:fefwe} \\
    \sin(x) \doteq \frac{e^{ix}-e^{-ix}}{2i} \label{eq:fefwe2}
  \end{eqnarray}
  A reference to a figure \ref{fig:a354}.
  A reference \eqref{eq:fefwe} and  \eqref{eq:fefwe2} to the above equations.
\end{Defn}

\begin{figure}[ht]\label{fig:a354}
  \begin{center}
    \includegraphics[width=0.5\textwidth]{F/sin}
    \caption{Graph of \(\sin(x)\).}
  \end{center}
\end{figure}

\lipsum[2]

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "paper"
%%% End:


\section{Imbalanced Data}

% ===
\emph{Cost Sensitive Classification}
Replace loss by: $\ell_{CS}(w;x,y) = c_y \ell(w;x,y)$

$\color{gray} \text{e.g. } \ell_{\pm} = c_\pm \ell(w;x,y)
\to c_-\cdot \hat R(w; \frac{c_+}{c_-},c_-)$

% ===
\emph{Metrics}

\begin{minipage}{.7\linewidth}
	\textbf{acc} $= \frac{\textrm{TP}+\textrm{TN}}{n}$,
	\textbf{prec} $= \frac{\textrm{TP}}{p_+}$

	\textbf{FPR} $= \frac{\textrm{FP}}{n_-}$,
	\textbf{Recall/TPR} $= \frac{\textrm{TP}}{n_+}$
\end{minipage}%
\begin{minipage}{.3\linewidth}
	\begin{tabular}{@{}l @{ }l | @{ }l}
		TP & FP & $p_+$\\
		FN & TN & $p_-$\\\hline
		$n_+$ & $n_-$ & $n$
	\end{tabular}
\end{minipage}

\textbf{Fβ score}: $F_\beta = \frac{(1+\beta)^2}{\frac{1}{\textrm{prec}} + \frac{\beta^2}{\textrm{rec}}}$,
\enskip
\textbf{ROC:} FPR vs. TPR


\section{Other Algorithms}
  \subsection{2SAT}
    Build the implication graph of the input by converting ORs $A \vee B$ to
    $!A \rightarrow B$ and $!B \rightarrow A$. This forms a bipartite graph.
    If there exists $X$ such that both $X$ and $!X$ are in the same strongly connected component,
    then there is no solution. Otherwise, iterate through the literals, arbitrarily assign
    a truth value to unassigned literals and propagate the values to its neighbors.
\begin{comment}
    \code{other/two_sat.cpp}
\end{comment}
  \subsection{DPLL Algorithm}
    A SAT solver that can solve a random 1000-variable SAT instance within a second.
    \code{other/dpll.cpp}
  \subsection{Stable Marriage}
    \ifverbose
    The Gale-Shapley algorithm for solving the stable marriage problem.
    \fi
    \code{other/stable_marriage.cpp}
\begin{comment}
  \subsection{Algorithm X}
    \ifverbose
    An implementation of Knuth's Algorithm X, using dancing links. Solves the Exact Cover problem.
    \fi
    \code{other/algorithm_x.cpp}
  \subsection{Matroid Intersection}
    Computes the maximum weight and cardinality intersection of two
    matroids, specified by implementing the required abstract methods, in
    $O(n^3(M_1+M_2))$.
    \code{other/matroid_intersection.cpp}
\end{comment}
  \subsection{Cycle-Finding}
    \ifverbose
    An implementation of Floyd's Cycle-Finding algorithm.
    \fi
    \code{other/floyds_algorithm.cpp}
  \subsection{Longest Increasing Subsequence}
    \code{other/lis.cpp}
  \subsection{Dates}
    \ifverbose
    Functions to simplify date calculations.
    \fi
    \code{other/dates.cpp}
  \subsection{Simulated Annealing}
    An example use of Simulated Annealing to find a permutation of length $n$
    that maximizes $\sum_{i=1}^{n-1}|p_i - p_{i+1}|$.
    \code{other/simulated_annealing.cpp}
  \subsection{Simplex}
    \begin{verbatim}
// Two-phase simplex algorithm for solving linear programs
// of the form
//     maximize     c^T x
//     subject to   Ax <= b
//                  x >= 0
// INPUT: A -- an m x n matrix
//        b -- an m-dimensional vector
//        c -- an n-dimensional vector
//        x -- a vector where the optimal solution will be
//             stored
// OUTPUT: value of the optimal solution (infinity if
//                   unbounded above, nan if infeasible)
// To use this code, create an LPSolver object with A, b,
// and c as arguments.  Then, call Solve(x).
    \end{verbatim}
    \code{other/simplex.cpp}
  \subsection{Fast Input Reading}
    If input or output is huge, sometimes it is beneficial to optimize the
    input reading/output writing. This can be achieved by reading all input
    in at once (using fread), and then parsing it manually. Output can also
    be stored in an output buffer and then dumped once in the end (using
    fwrite). A simpler, but still effective, way to achieve speed is to use
    the following input reading method.
    \code{tricks/fast_input.cpp}
  \subsection{128-bit Integer}
    GCC has a 128-bit integer data type named \texttt{\_\_int128}. Useful
    if doing multiplication of 64-bit integers, or something needing a
    little more than 64-bits to represent. There's also
    \texttt{\_\_float128}.
  \subsection{Bit Hacks}
    \code{tricks/snoob.cpp}


\section{Downsampling}

The analysed data was recorded at 32 kHz. For faster processing, the data was downsampled. The Nyquist frequency after downsampling needs to be sufficiently high for sharp-wave ripple detection.\footnotemark{} As ripples have a frequency of up to 250 Hz (\cref{tab:bands}), the new sampling frequency needs to be larger than 500 Hz. I chose a new sampling frequency of 1000 Hz. This allows integer subsampling of the original data, by a factor 32.

\footnotetext{The Nyquist frequency is the maximum discernible frequency in a digital signal, equal to half the sampling frequency.}

To counter aliasing, the 32 kHz data was low-pass filtered before subsampling.\footnotemark{} The cutoff frequency was chosen to be 80\% of the new Nyquist frequency (i.e. at 400 Hz for a new sampling frequency of 1000 Hz). The digital filter was applied both in forwards and backwards directions over the signal, to cancel out distortions due to filter delays that vary per frequency component. The filter type and parameters were chosen to yield both low processing time and low-distortion, high quality filtering, as was verified visually (see for example \cref{fig:downsample}).

\footnotetext{A type I Chebyshev filter was used, which is an infinite impulse response filter, with relatively steep roll-off. The filter was of order 8 and had a passband ripple of 0.05 dB. For this filter, the cutoff frequency is defined to be where the gain starts dropping below the passband ripple.}

\begin{figure}
\img[1]{downsample}
\captionn{Downsampling the data}{Blue: the recorded data at 32 kHz. Orange: the same data, after downsampling to 1000 Hz. The arrow indicates a possible neural spike, removed by the anti-aliasing filter.}
\label{fig:downsample}
\end{figure}


\section{Start with two points}
$f(s)$ is the direct connecting line between the vectors  $\vec a$ and $\vec b$. The output of $f$ is a two-vector $(\theta, \varphi)^T$
\begin{equation}\label{line}
    f(s)=(1-s)\cdot\vec{a}+s\cdot\vec{b}\qquad\forall\, s \in [0,1]
\end{equation}
\begin{equation}
    f'(s)=\vec{b}-\vec{a}
\end{equation}
K: Spherical coordinates $\rightarrow$ euclidean coordinates\\
given:
\begin{equation}\label{euklid}
    K \begin{pmatrix}\theta \\ \varphi\end{pmatrix}=
    \begin{pmatrix}
        \sin\theta\cdot\cos\varphi \\
        \sin\theta\cdot\sin\varphi \\
        \cos\theta
    \end{pmatrix}
\end{equation}
so that:
\begin{equation}
    K'=
    \begin{pmatrix}
       \cos\theta\cdot\cos\varphi && -\sin\theta\cdot\sin\varphi \\
       \cos\theta\cdot\sin\varphi && \sin\theta\cdot\cos\varphi \\
       -\sin\theta && 0
    \end{pmatrix}
\end{equation}
\subsection{Numerical solution}
\begin{equation}
    \frac{dK}{d\theta}\dbinom{\theta}{\varphi}=
    \begin{pmatrix}
        \sin\theta\cdot\cos\varphi \\
        \sin\theta\cdot\sin\varphi \\
        \cos\theta
    \end{pmatrix}
\end{equation}
See \eqref{euklid}.
\begin{equation}
    c_1=\frac{1}{2}
\end{equation}
\begin{equation}
    v_1=\underbrace{K
    \begin{pmatrix}
        \begin{matrix}
            \theta_a \\
            \varphi_a
        \end{matrix}
        +
        \begin{pmatrix}
            \theta_b-\theta_a\\
            \varphi_b-\varphi_a
        \end{pmatrix}
        \cdot c
    \end{pmatrix}
    }_{\alpha}
    -\vec{a}
\end{equation}
\begin{equation}
    c_2=\frac{c}{2}
\end{equation}
\begin{equation}
    v_2=\cdots
\end{equation}
if
\begin{equation}\label{epsilon}
    \frac{\|v_2-v_1\|}{\|v_1\|}<\varepsilon
\end{equation}
or
\begin{equation}
    \frac{\langle v_2, v_1\rangle}{\|v_2\|\cdot\|v_1\|}<\varepsilon
\end{equation}
\begin{lstlisting}[escapechar=@]
    while(@equation: \eqref{epsilon}@ > @$\varepsilon$@) {
        v1 = K(@… $\cdot$ @c) - @$\vec a$@
        v2 = K(@… $\cdot$ @c/2) - @$\vec a$@
        c = c/2;
    }
\end{lstlisting}
\begin{equation}
    \cos(\alpha)=\frac{\langle\mathrm{iter}(a,b), \mathrm{iter}(a,c)\rangle}{\|\mathrm{iter}(a,b)\|\cdot \|\mathrm{iter}(a,b)\|}
\end{equation}


\documentclass[../../main.tex]{subfiles}

\begin{document}

\subsection{Motivation}

The aim of lab is use a generalized multilinear regression model, termed the Higher-Order Partial Least Squares (HOPLS) \cite{zhao2012higher}, to predict a tensor $\underline{Y}$ from a tensor $\underline{X}$ through projecting the data onto the latent space and performing regression on the corresponding latent variables. Specifically, We have problem of recognition human body position. On video with person we need to mark the position of the main points of the body (face, shoulders, hips, elbows, lap, ankle). The experiment is carried out on JHMDB (Joint-annotated Human Motion Data Base) \cite{Jhuang:ICCV:2013}.

\subsection{Problem statement}

Given video with person and skeleton's coordinates for each frame of this video:
\[
\label{eq:example:1}
\begin{aligned}
    \mathfrak{D} = \left\{\underline{X_i}, ~\underline{Y_i}\right\}_{i=1}^{K},
\end{aligned}
\]
where ~$\underline{X_i} \in \mathbb{R}^{T \times W\times H\times C}, ~\underline{Y_i} \in \mathbb{R}^{T \times D \times I}$,\\
~$T$ - number of framres,  ~{W} - width, ~{H} - height, ~{C} - channel,\\ ~{I} - number of points in skeleton, ~{D} - picture dimension.\\
\\
In our case, $T \leqslant 40, ~W = 320, ~H = 240, ~C = 3, ~I = 15, ~D = 2.$\\
\\
The n-mode product of a tensor
$ X \in \mathbb{R}^{I_1\times ... \times I_n \times ... \times I_N}$
and matrix $A \in \mathbb{R}^{J_n\times I_n}$
is denoted by $\underline{Y} = \underline{X} \times_{i}A \in \mathbb{R}^{I_1\times ... \times I_{n-1} \times J_n \times I_{n+1} \times ... \times I_N}$
and defined as:
$$y_{i_1...i_{n-1}j_ni_{n+1}...i_N } = \sum_{i_n} x_{i_1...i_n...i_N }a_{j_ni_n}$$
$$\underline{Y} = \underline{G} \times_{1} A^{(1)} \times_{2} A^{(2)} \times_{3} ...\times_{N} A^{(N)} + \underline{E} = [[G; A^{(1)}, A^{(2)}, ..., A^{(N)}]] + \underline{E}$$

We assume $\underline{X}$ is decomposed as a sum of $rank-(1, L_2, . . . , L_N )$ Tucker blocks, while $\underline{Y}$ is decomposed as a sum of $rank-(1, K_2, . . . , K_M)$ Tucker blocks, which can be expressed as

$$\underline{X} = \sum_{r=1}^{R}\underline{G_r}\times_{1} t_r \times_{2} P^{(1)}_r \times_{3} P^{(2)}_r \times_{4} ...\times_{N} P^{(N-1)}_r + \underline{E}_r$$
$$\underline{Y} = \sum_{r=1}^{R}\underline{D_r}\times_{1} t_r \times_{2} Q^{(1)}_r \times_{3} Q^{(2)}_r \times_{4} ...\times_{N} Q^{(N-1)}_r + \underline{F}_r$$
where R is the number of latent vectors, $t_r \in \mathbb{R}^{I_1}$ is the r-th latent vector $P_r^{n} \in \mathbb{R}^{I_{n+1} \times L_{n+1}}$ and $Q_r^{m} \in \mathbb{R}^{J_{n+1} \times K_{n+1}}$ are loading matrices on mode-n and mode-m respectively, and $\underline{G_r} \in \mathbb{R}^{1 \times L_{2}\times ...\times L_{N}}$ and $\underline{D_r}\in \mathbb{R}^{1 \times K_{2}\times ...\times K_{M}}$ are core tensors.\\
\\
However the Tucker decompositions are not unique due to the permutation, rotation, and scaling issues. To alleviate this problem, additional constraints should be imposed such that the core tensors $\underline{G_r}$ and $\underline{D_r}$ are all-orthogonal, a sequence of loading matrices are column-wise orthonormal, i.e., $P^{(n)T}_rP^{(n)}_r = I$ and $Q^{(n)T}_rQ^{(n)}_r = I$, the latent vector is of length one, i.e. $\|t_r\|_F=1$.\\
\\
The main aim is to reduce Frobeniuses norm of residuals $\underline{E_r}, \underline{F_r}$. With a few propositions the following optimization problems arise

\[
\label{eq:example:2}
\begin{aligned}
   \min_{P^{(n)}, Q^{(m)}} \| [[C; P^{(1)T}, P^{(2)T}, ..., P^{(N-1)T}, Q^{(1)T}, Q^{(2)T}, ..., Q^{(M-1)T}]]\|_F^2,\\
\end{aligned}
\]
$$s.t. ~P^{(n)T}_rP^{(n)}_r = I, ~Q^{(n)T}_rQ^{(n)}_r = I$$
where $\underline{C} = COV_{\{1;1\}}(\underline{X}, \underline{Y})$.\\
Next, we can find latent vector ~$t$ from
\[
\label{eq:example:2}
\begin{aligned}
    \min_{t}\| \underline{X} - [[\underline{G};t,P^{(1)},P^{(2)}, ...,P^{(N-1)}]]\|_F^2
\end{aligned}
\]
Now we found the first latent decomposition. Next step is repiting it for $\underline{X} = \underline{E_1}, ~\underline{Y} = \underline{F_1}$.
\subsection{Problem solution}
Full algorithm is described in [1]. Now we can find solution:
$${Y}^{(new)_{(1)}} \approx T^{(new)}Q^{*T} = X^{(new)}_{(1)}WQ^{*T}$$
where ~$W$ and ~$Q^*$ have R columns, represented by
$$w_r = (P_r^{(N-1)}\otimes ...\otimes P_r^{(1)})\underline{G_{r(1)}^+}$$
$$q_r^* = \underline{D_{r(1)}}(Q_r^{(M-1)}\otimes ...\otimes Q_r^{(1)})^T.$$

\subsection{Code analysis}

The code was taken from \cite{zhao2012higher}.

\subsection{Experiment}

The experiment is carried out on JHMDB (Joint-annotated Human Motion Data Base) \cite{Jhuang:ICCV:2013}.
In our case, $T \leqslant 40, ~W = 320, ~H = 240, ~C = 3, ~I = 15, ~D = 2.$\\
The dataset has 928 videos from 21 class.
\\
Define function of similarity of $\underline{Y}^{(true)}$ and $\underline{Y}^{(predict)}$ as
$Q^2$:
$$Q^2 = 1 - \frac{\|\underline{Y}^{(true)} - \underline{Y}^{(predict)}\|_F^2}{\|\underline{Y}^{(true)}\|_F^2}$$
\\
The first experiment. We divided video into train and test part. Train has 32 frames, test has 8 frames.
\\
\begin{figure}[h!]
\centering
\includegraphics[width=1\textwidth]{figures/fig1}
\caption{Experiment 1. Dependence $Q^2$ on number of latent vectors R.}
\end{figure}
\\
The second experiment. In our dataset we have very similar videos, so we can lear our model on the first and test on the second video. It will show how robust this method.
\\
\begin{figure}[h!]
\centering
\includegraphics[width=1\textwidth]{figures/fig2}
\caption{Experiment 2. Dependence $Q^2$ on number of latent vectors R.}
\end{figure}

\end{document}


%!TEX root =  ../main.tex

\subsection{Like Powers}

\objective{Simply log expressions}

Whether we use the Triangle of Power or not, it is very powerful to recognize
that logs are simply the inverse of exponents.  Without this insight, expressions
like $\log_{10}{100} + \log_{10}{1000}$ would be very intimidating.  But with this
insight, we can paraphrase as we go in our minds: ``There is some exponent
to put on 10 and get 100.  Add to that, some exponent we put on 10 to get
1000.  Adding exponents comes from multiplying bases, so this is the same as
$\log_{10}10000$.  That is asking the question, what exponent do we put on 10
to get 10,000?  5!''

We think it is even easier in Triangles, but we might show you both styles for
the time being:

$$
\tripow{10}{}{100} + \tripow{10}{}{1000} = \tripow{10}{}{10000} = 5
$$

In other words, two logs added, is the same as one long of a multiplication.  This
works for the inverse operation of subtraction: two logs subtracted is the same
as one log of a division.  Lastly, it also works for an exponent: the log of a number
with an exponent is the same as the exponent multiplied against the log of number.

Here, Triangle notation really shines superior, because notation like $\log_2{64^2}$
is confusing.  Does it mean $\log_2{64} \cdot \log_2{64}$ or $\log_2{4096}$, which
is the difference between 36 and 12?  How much clearer is

$$
\tripow{2}{}{\tripow{64}{2}{}} vs \tripow{\tripow{2}{}{64}}{2}{}
$$

Even more consequential is the two-fold possibilities for $2^{3^4}$.  Is that $(2^3)^4$
or $2^{(3^4)}$?  That is the difference between 4096 and $2^81$, the latter of which
is a 25 digit number!  But no one can make the same mistake with

$$
\tripow{\tripow{2}{3}{}}{4}{} vs \tripow{2}{\tripow{3}{4}{}}{}
$$

\subsection{Names}
There are some logs which occur so commonly, that they base is not written.  Normally,
the word ``log'' with no base written means $\log_{10}$.  Log base $e$ is very common,
and has its own symbol $\ln$.  This acronym comes from the French, \textit{log
natural}, since $e$ is the natural number.\footnote{In French, as in Spanish, adjective
most often follow their noun, not precede it.}

Computer scientists most often used $\log_2$, which is called the binary log,
written lb.  In fact
in many disciplines, their flavor of log is the only one used, so it is assumed and the name
``log'' is written, which can fool outsiders into assuming $\log_{10}$.  For example,
Wolfram Alpha is fantastically powerful website for mathematics and other fields,
and so they use ``log'' to mean ``ln''!  We will not be so tricky, and you may assume
the solution to $\log{x}=2$ is 100.

\subsection{Change of Base}
Logs are amazing.  But given the rarity of most bases, it become necessary to ask,
Is there a way to convert them?  If we know how to re-write a log as an exponent,
and take an exponent as a multiplier, then we can:

\begin{align*}
	\log_b{a} & = c \\
	b^c &= a\\
	\ln{b^c} &= \ln{a}\\
	c \cdot \ln{b} &= \ln{a} \\
	c = \frac{\ln{a}}{\ln{b}}
\end{align*}

Notice that it does not matter what base we chose in the third line.  $\ln$ ($\log_e$)
works just as well as $\log_{\pi}$.  This rearrangement to have an arbitrary
base is called the the Change of Base Formula.

\begin{derivation}{Change of Base Formula}
$$
\tripow{b}{}{a} = \frac{\tripow{c}{}{a}}{\tripow{c}{}{b}} \quad \text{a.k.a.} \quad
\log_b{a} = \frac{\log_c{a}}{\log_c{b}}
$$
\end{derivation}

\begin{derivation}{Negative Logarithm}
$$
\tripow{b}{}{1/y} = -\tripow{b}{}{y}  = \tripow{1/b}{}{y}
\quad \text{a.k.a.} \quad
\log_b{\frac{1}{y}} = -\log_b{y} = \log_{\frac{1}{b}}{y}
$$
\end{derivation}

Students often confusing ``the sum of logs'' with ``the log of a sum''.  What can we say
about $\log_b{(a+c)}$?  On the face of it, not much.  But through some creative manipulation,
one identity can be made, which is used in Probability Theory:

\begin{align*}
	\log_b{(a+c)} 	&= + \log_b{a} - \log_b{a} + \log_b{(a+c)}\\
				&= \log_b{a} + \log_b{(a+c)} - \log_b{a}\\
				&= \log_b{a} + \log_b{\frac{a+c}{a}} \\
				&= \log_b{a} + \log_b{1 + \frac{c}{a}}\\
	\tripow{b}{}{a+c} &= \tripow{b}{}{a} + \tripow{b}{}{\left(1 + \frac{c}{a}\right)}
\end{align*}

Lastly, we have already made with the Triangle of Power:

\begin{derivation}{P-Plus Logs}
$$
\tripow{m}{}{x} \pplus \tripow{n}{}{x} = \tripow{m\cdot{}n}{}{x}
\quad \text{a.k.a.} \quad
\log_m{x} \oplus \log_n{x} = \log_{mn}{x}
$$

$$
\tripow{m}{}{x} \pminus \tripow{n}{}{x} = \tripow{m\div{}n}{}{x}
\quad \text{a.k.a.} \quad
\log_m{x} \pminus \log_n{x} = \log_{\frac{m}{n}}{x}
$$

$$
\tripow{\tripow{m}{n}{}}{}{x} = \frac{\tripow{m}{}{x}}{n}
\quad \text{a.k.a.} \quad
\log_{m^n}{x} = \frac{1}{n}\log_m{x}
$$
\end{derivation}


\section{Fundamental Theorem of Calculus}
\subsection{Integration}
Assume below that all functions under consideration are nice enough to let the integral exist.\\
Consider the following sum
$$\sum_{n=0}^{N-1}f(x_n)\Delta x$$
where $\Delta x=(b-a)/N, x_n=a+n\Delta x$.\\
The question is, how close is the finite sum above to the area under the curve of $f$, when $N$ is large?
How big is the difference between the difference between the area and the discrete area in the sum?
\begin{theorem}[Mean-vaue Theorem on Definite Integrals]
    For a continuous function $f(x)$,
    $$\int_{x_n}^{x_n+1}f(x)\,\mathrm dx=f(x_c)(x_{n+1}-x_n)$$
    for some $x_c\in(x_n,x_{n+1})$.
\end{theorem}
Expand $f(x)$ about $x\to x_n$ and evaluate it at $x_c$.
As $\Delta x\to 0$,
$$f(x_c)=f(x_n)+O(x_c-x_n)=f(x_n)+O(x_{n+1}-x_n)$$
as $|x_c-x_n|<|x_{n+1}-x_n|$.
Hence by the mean-value theorem:
\begin{align*}
    \int_{x_n}^{x_n+1}f(x)\,\mathrm dx
    &=f(x_n)(x_{n+1}-x_n)+O(x_{n+1}-x_n)(x_{n+1}-x_n)\\
    &=\Delta xf(x_n)+O(\Delta x^2)
\end{align*}
So $\epsilon=O(\Delta x^2)$.
It follows that
$$\int_a^bf(x)\,\mathrm dx=\lim_{N\to\infty}\sum_{n=0}^{N-1}f(x_n)\Delta x+\epsilon_n$$
By our bounds above, the error terms $\sum_n\epsilon_n=O(N\Delta x^2)=O((b-a)^2/N)$ vanish as $N\to\infty$.
Therefore,
$$\int_a^bf(x)\,\mathrm dx=\lim_{N\to\infty}\sum_{n=0}^{N-1}f(x_n)\Delta x$$
\begin{theorem}[Fundamental Theorem of Calculus]
    Let
    $$F(x)=\int_a^xf(t)\,\mathrm dt$$
    then $\mathrm dF/\mathrm dx=f(x)$.
\end{theorem}
\begin{proof}
    We try to evaluate the derivative of $F$,
    \begin{align*}
        \frac{\mathrm dF}{\mathrm dx}
        &=\lim_{h\to0}\frac{1}{h}\int_{x}^{x+h}f(t)\,\mathrm dt\\
        &=\lim_{h\to0}\frac{1}{h}(f(x)h+O(h^2))\\
        &=f(x)
    \end{align*}
    So $\mathrm dF/\mathrm dx=f(x)$.
\end{proof}
\begin{corollary}
    $$\frac{\mathrm d}{\mathrm dx}\int_x^bf(t)\,\mathrm dt=-f(x)$$
\end{corollary}
\begin{corollary}
    $$\int_a^{g(x)}f(t)\,\mathrm dt=f(g(x))g^\prime(x)$$
\end{corollary}
\begin{corollary}
    Let
    $$F(x)=\int f(x)\,\mathrm dx$$
    be the indefinite integral (or anti-derivative) of $f$, then
    $$\int_a^bf(t)\,\mathrm dt=F(b)-F(a)$$
\end{corollary}
\subsection{Some Integration Techniques}
The first one is integration by substitution.
\begin{example}
    $$\int\frac{1-2x}{\sqrt{x-x^2}}\,\mathrm dx=\int\frac{\mathrm du}{\sqrt u}=2\sqrt{u}+C$$
\end{example}
Trigonometric substitution
\begin{example}
    If we see something like $\sqrt{a^2-x^2}$, then we can substitute $x=a\sin\theta$.\\
    If we see something like $x^2+a^2$, we can use $x=a\tan\theta$.\\
    If we see $\sqrt{x^2-a^2}$, we can use $x=a\cosh\theta$ or $x=a\sec\theta$.\\
    If we see $\sqrt{x^2+a^2}$, we can use $x=a\sinh\theta$ or $x=a\tan\theta$.\\
    If we see $a^2-x^2$, we can use $x=a\tanh\theta$
\end{example}
And, of course, we have integration by part:
$$\int uv^\prime=uv-\int u^\prime v$$
from product rule.


\section{The \binarysearch algorithm}
\Label{sec:binarysearch}
\Label{sec:binarysearchii}

The \binarysearch algorithm is one of the four binary search
algorithms of the \cxx Standard Library \cite[\S 28.7.3.4]{cxx-17-draft}.
For our purposes we have modified
the generic implementation
to that of an array of type \valuetype.
The signature now reads:

\begin{lstlisting}[style = acsl-block]
  bool binary_search(const value_type* a, size_type n, value_type  v);
\end{lstlisting}

Again, \binarysearch requires that its input array is in increasing order.
It will return \inl{true} if there exists an index~\inl{i}
in~\inl{a} such that \inl{a[i] == v} holds.\footnote{%
   To be more precise: The \cxx Standard Library requires that
   \inl{(a[i] <= v)  && (v <= a[i])} holds.
   For our definition of \valuetype (see \S\ref{sec:frequentPattern}) this
   means that \inl{v} equals \inl{a[i]}.
}

\begin{figure}[hbt]
\centering
\includegraphics[width=0.60\textwidth]{Figures/binary_search.pdf}
\caption{\Label{fig:binarysearch}Some examples for \binarysearch}
\end{figure}

\FloatBarrier

In Figure~\ref{fig:binarysearch} we do not need to use arrows to visualize the
effects of \binarysearch.
The colors orange and grey of the sought-after values indicate whether the algorithm
returns true or false, respectively.

\subsection{Formal specification of \binarysearch and \binarysearchii}

The \acsl specification of \specref{binarysearch} is shown in the following listing.

\input{Listings/binary_search.h.tex}

Note that instead of the somewhat lengthy existential quantification
of \specref{binarysearch} we can use our previously introduced predicate
\logicref{SomeEqual} in order to achieve the following more concise
formal specification \specref{binarysearchii}.

\input{Listings/binary_search2.h.tex}

It is interesting to compare the specification of \specref{binarysearch}
with that of \specref{findii}.
Both algorithms allow to determine whether a value is contained in an array.
The fact that the \cxx Standard Library requires that \find has
\emph{linear} complexity whereas \binarysearch must have a
\emph{logarithmic} complexity can currently not be expressed with \acsl.

\subsection{Implementation of \binarysearch}

Our implementation \implref{binarysearchii} first calls \specref{lowerbound}.
Remember that if the latter returns an index \inl{0 <= i < n},
then we can be sure that \inl{v <= a[i]} holds.

\input{Listings/binary_search2.c.tex}


% based on example 7 in pythontex_gallery
% https://github.com/gpoore/pythontex/

\documentclass[12pt]{pylatex}
\usepackage{examples}

\begin{document}

\section*{A table of derivatives and anti-derivatives}

This example is based upon a nice example in the Pythontex gallery, see
\ \url{https://github.com/gpoore/pythontex/}.
It uses a tagged block to capture the {\tt\small Sympy} output for later use
in the body of the LaTeX table.

\lstset{numbers=left}

\begin{minipage}[t]{0.72\textwidth}
\begin{python}
   from sympy import *

   var('x')

   # Create a list of functions to include in the table
   funcs = [['sin(x)',r'\\'],       ['cos(x)',r'\\'],       ['tan(x)',r'\\'],
            ['asin(x)',r'\\[5pt]'], ['acos(x)',r'\\[5pt]'], ['atan(x)',r'\\[5pt]'],
            ['sinh(x)',r'\\'],      ['cosh(x)',r'\\'],      ['tanh(x)',r' ']]

   # pyBeg (CalculusTable)
   for func, eol in funcs:
       myddx = 'Derivative(' + func + ', x)'
       myint = 'Integral(' + func + ', x)'
       print(latex(eval(myddx)) + '&=' + latex(eval(myddx + '.doit()')) + r'\quad & \quad')
       print(latex(eval(myint)) + '&=' + latex(eval(myint + '.doit()')) + eol)
   # pyEnd (CalculusTable)
\end{python}
\end{minipage}
\hskip 1cm
\begin{minipage}[t]{0.28\textwidth}
\begin{latex}
   \begin{align*}
      \py {CalculusTable}
   \end{align*}
\end{latex}
\end{minipage}

\clearpage

\begin{align*}
   \py {CalculusTable}
\end{align*}

\end{document}


\section{Logistic hypothesis using logistic regression}

\subsection{Implementation}
For logistic regression the hypothesis function is a sigmoid function given by equation \ref{eq:logistic_regression}.
As before a learning rate($\alpha$) = 0.01 was used and gradient descent approach was used for parameter optimization.

\begin{equation}
\label{eq:logistic_regression}
h_{\theta}(X) = \frac{1}{e^{-\theta_0 - \theta_1 * X}}
\end{equation}

\subsection{Observation}
The final values of $\theta$s for a run are as follows:

% Theta 0 final value
\begin{equation}
\theta_0 = -3.5327129208000407
\end{equation}

% Theta 1 final value
\begin{equation}
\theta_1 = 1.3214831636264568
\end{equation}

\begin{figure}[!ht]
  \includegraphics[width=\textwidth,height=0.4\textheight,keepaspectratio]{logistic_regression_curve_0_01.png}
  \caption{Logistic regression curve}
  \label{fig:logistic_regression}
\end{figure}

The logistic curve corresponding to above $\theta$s is shown in figure \ref{fig:logistic_regression}

\subsection{Source Code}
\lstinputlisting[language=python]{task_2.py}