\nonstopmode
\documentclass[a4paper]{article}
\usepackage{natbib}
%\usepackage[usenames,dvipsnames]{xcolor}
\usepackage{amsmath}
%\usepackage{dsfont}
\usepackage{bibentry}

\input{macros}

\newtheorem{theorem}{Theorem}
%\newtheorem*{theorem*}{Theorem}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{example}[theorem]{Example}
\newtheorem{exercise}[theorem]{Exercise}

%\newcommand{\qed}{\hfill\ensuremath{\Box}}
% from llncs.cls
%\def\squareforqed{\hbox{\rlap{$\sqcap$}$\sqcup$}}
\def\squareforqed{\ensuremath{\Box}}
\def\qed{\ifmmode\squareforqed\else{\unskip\nobreak\hfil
\penalty50\hskip1em\null\nobreak\hfil\squareforqed
\parfillskip=0pt\finalhyphendemerits=0\endgraf}\fi}

\newenvironment{proof}[1][]{\noindent\ifthenelse{\equal{#1}{}}{{\it
      Proof.}}{{\it Proof #1.}}\hspace{2ex}}{\qed\bigskip}
\newenvironment{proof*}[1][]{\noindent\ifthenelse{\equal{#1}{}}{{\it
      Proof.}}{{\it Proof #1.}}\hspace{2ex}}{\bigskip}

\newcommand{\wk}{\mathord{\uparrow}}
\newcommand{\WW}{\mathbb{W}}
\newcommand{\tAr}{\mathsf{Ar}}
\newcommand{\Ar}[1]{\tAr(#1)}

\title{Normalization by Evaluation for \\ Intuitionistic Propositional Logic}
\author{Andreas Abel}
\date{July 2018}

\begin{document}
\maketitle

\nobibliography{auto-nbeSum}

\section{Intuitionistic Propositional Logic (IPL)}

\para{Formulas} and hypotheses lists (contexts).
\[
\begin{array}{lllrll}
  \Atom & \ni & P,Q & & & \mbox{atomic propositions} \\
  \Form & \ni & A,B,C,\dots
     & ::= & \top \mid A \land B \mid A \imp B
       & \mbox{negative formulas} \\
     &&& \mid & \bot \mid A \lor B \mid P
       & \mbox{positive formulas} \\
\\[1ex]
  \Cxt & \ni & \Gamma,\Delta & ::= & \cempty & \mbox{empty context}
\\ &&& \mid & \Gamma.A & \mbox{context extension}
\end{array}
\]
We write just $A$ for the singleton context $\cempty.A$.

Indices $x$ into the context $\Gamma$ of hypothesis, locating hypothesis
$A$, are written $x : \Hyp[\Gamma]A$ and defined inductively by the
following constructors:
\[
  \ru{}{\ttop : \Hyp[\Gamma.A]A}
\qquad
  \ru{x : \Hyp[\Gamma]A
    }{\pop[B] x : \Hyp[\Gamma.B]A}
\]
Derivations $t$ of the truth of a formula $A$ under assumptions
$\Gamma$ (judgement $\Gamma \der A$)
are written $t : \Tm[\Gamma]A$ and defined inductively as follows:

\paradot{Implication and hypotheses}
\begin{gather*}
  \ru{x : \Hyp[\Gamma]A}{\hyp x : \TmG A}
\qquad
  \ru{t : \Tm[\Gamma.A]B}{\impI t : \TmG{A \imp B}}
\qquad
  \ru{t : \TmG{A \imp B} \qquad
      u : \TmG A
    }{\impE t u : \TmG B}
\end{gather*}

\paradot{Truth and conjunction}
\begin{gather*}
  \ru{}{\trueI : \TmG\top}
\qquad
  \ru{t : \TmG A \qquad
      u : \TmG B
    }{\andI t u : \TmG{A \land B}
    }
\\[1ex]
  \ru{t : \TmG{A \land B}
    }{\andE1 t : \TmG A
    }
\qquad
  \ru{t : \TmG{A \land B}
    }{\andE2 t : \TmG B
    }
\end{gather*}

\paradot{Absurdity and disjunction}
\begin{gather*}
  \ru{t : \TmG A
    }{\orI1 t : \TmG {A \lor B}
    }
\qquad
  \ru{t : \TmG B
    }{\orI2 t : \TmG {A \lor B}
    }
\\[1ex]
  \ru{t : \TmG \bot
    }{\falseE t : \TmG C}
\qquad
  \ru{t : \TmG {A \lor B} \qquad
      u : \Tm[\Gamma.A]C  \qquad
      v : \Tm[\Gamma.B]C
    }{\orE t u v : \TmG C}
\end{gather*}

\section{Subformula property and normal derivations}

For every derivable judgement $\Gamma \der A$ there are infinitely
many derivations $t : \TmG A$, since arbitrary detours are allowed.
For instance to prove $\top$ we can also proceed very indirectly by
introducing a hypothesis $\top$ and then eliminating it by proof
$\trueI$:
\[\impEP{\impIP{\hyp{\ttop}}}{\trueI} : \Tm[\cempty]\top.\]
A sensible restriction is that the proof rules guarantee that only
\emph{subformulas} of $A$ and the hypotheses in $\Gamma$ are mentioned when
deriving $\Gamma \der A$.

The following indexed grammar for \emph{normal} derivations $\NfG A$ of
judgement $\Gamma \der A$ ensures the subformula property.  It is
defined mutually with a grammar $\NeG A$ of \emph{neutral} derivations.
Neutral derivations are ``straight-line'' consequences of hypotheses
(without case distinction or absurdity elimination).

\paradot{Implication and hypotheses}
\begin{gather*}
  \ru{x : \Hyp[\Gamma]A}{\hyp x : \NeG A}
\qquad
  \ru{t : \Nf[\Gamma.A]B}{\impI t : \NfG{A \imp B}}
\qquad
  \ru{t : \NeG{A \imp B} \qquad
      u : \NfG A
    }{\impE t u : \NeG B}
\end{gather*}

\paradot{Truth and conjunction}
\begin{gather*}
  \ru{}{\trueI : \NfG\top}
\qquad
  \ru{t : \NfG A \qquad
      u : \NfG B
    }{\andI t u : \NfG{A \land B}
    }
\\[1ex]
  \ru{t : \NeG{A \land B}
    }{\andE1 t : \NeG A
    }
\qquad
  \ru{t : \NeG{A \land B}
    }{\andE2 t : \NeG B
    }
\end{gather*}

\paradot{Absurdity and disjunction}
\begin{gather*}
  \ru{t : \NfG A
    }{\orI1 t : \NfG {A \lor B}
    }
\qquad
  \ru{t : \NfG B
    }{\orI2 t : \NfG {A \lor B}
    }
\\[1ex]
  \ru{t : \NeG \bot
    }{\falseE t : \NfG C}
\qquad
  \ru{t : \NeG {A \lor B} \qquad
      u : \Nf[\Gamma.A]C  \qquad
      v : \Nf[\Gamma.B]C
    }{\orE t u v : \NfG C}
\end{gather*}

\paradot{Embedding}
\[
  \ru{t : \NeG P
    }{\neu t : \NfG P}
\]

The introduction rules generate normal forms ($\tNf$s) from $\tNf$s.  The elimination
rules for negative formulas generate neutrals ($\tNe$s) from $\tNe$s, where
$\timpE$ requires a side argument in normal form.

Some care is needed with  the elimination rules for
positive formulas which are $\tfalseE$ and $\torE$.
These prove an arbitrary formula $C$, which, for the sake of the
subformula property, should not be subject to
further elimination but directly prove the goal.
Thus, in addition to restriction the elimination to neutrals
(perform case distinction only on neutrals),
we require them to produce a normal form
in case of $\torE$ from normal branches.

Any neutral derivation at atomic proposition $P$ is considered normal.
The restriction on atoms is not needed for the subformula property,
but forces derivations to be $\eta$-long.
We could drop the restriction, but even with this restriction, the
calculus is complete, meaning that for every derivation ($\tTm$) there
exists a normal derivation ($\tNf$).  This statement is called
\emph{normalization} and we will prove it constructively in the
following.

\section{Categories and Presheaves}

We define order-preserving embeddings (OPEs) $\tau : (\Delta \leq \Gamma)$ of a
context $\Gamma$ into a larger context $\Delta$ inductively by the
following constructors.
\begin{gather*}
  \ru{}{\id[\Gamma] : \Gamma \leq \Gamma}
\qquad
  \ru{\tau : \Delta \leq \Gamma
    }{\weak[A] \tau : \Delta.A \leq \Gamma}
\qquad
  \ru{\tau : \Delta \leq \Gamma
    }{\lift[A] \tau : \Delta.A \leq \Gamma.A}
\end{gather*}
For example $\weak [D] (\lift[C] (\weak[B] \id[A])) : A.B.C.D \leq
A.C$.  OPEs let us introduce extra, unused hypotheses into the
context.  Derivations of $\Gamma \der A$ can be weakened to
derivations $\Delta \der A$ for $\Delta \leq \Gamma$.

OPEs are closed under composition:  If $\tau : \Delta \leq \Gamma$ and
$\tau' : \Phi \leq \Delta$ then $\tau \comp \tau' : \Phi \leq \Delta$.
Let $\OPE$ be the category of contexts (as objects) and OPEs (as morphisms).
\begin{exercise}
  Define composition of OPEs by induction and show that $\OPE$ is
  indeed a category.
\end{exercise}

The context-indexed sets $\Hyp A$, $\Tm A$, $\Nf A$, and $\Ne A$ are
closed under weakening. \Eg{} $\tHyp$: for each $\tau : \Delta \leq \Gamma$
we have a function
\[
  \Hyp[\tau]A : \Hyp[\Gamma] A \to \Hyp[\Delta] A,
\]
\ie, a morphism in the category $\SET$ of sets and functions.
The collection $\Hyp A : (\tau : \Delta \leq \Gamma) \to
(\Hyp[\Gamma]A \to \Hyp[\Delta]A)$ of these function constitutes a
contravariant functor $\Hyp A : \OPE \to \SET$ mapping OPEs to
functions.  In other words, $\Hyp A$ is a \emph{presheaf} over $\OPE$
for each $A$, and so are $\Tm A$, $\Nf A$, and $\Ne A$.

\begin{exercise}
  Prove that $\Hyp A$, $\Tm A$, $\Nf A$, and $\Ne A$ are presheaves
  over $\OPE$ by defining the map functions and proving the functor laws!
\end{exercise}

A function $f$ between presheaves $\A$ and $\B$ is defined pointwise;
we write $f : \A \todot \B$ for the $\Cxt$-indexed collection
\[
  f_\Gamma : \A_\Gamma \to \B_\Gamma
\]
of functions (morphisms in $\SET$). Presheaves and functions between
them make a category by virtue of pointwise identity and composition.
However, typically the functions we consider are \emph{natural} in the
context index, \ie, they are \emph{natural transformations} between
presheaves, which means that they commute with OPEs.  Given $\tau :
\Delta \leq \Gamma$ and $f : \A \todot \B$, naturality means
\[
  f_\Delta \comp \A_\tau = \B_\tau \comp f_\Gamma .
\]
In words, first weakening with $\tau$ and then applying $f$ has the
same effect as applying $f$ first and perform the weakening later.
This guarantees some form of \emph{parametricity} of $f$ in the
context, in particular, $f$ cannot make decisions based on the length
of the context or its precise contents.

Presheaves over $\OPE$ and natural transformations form a category
$\PSh$, traditionally called $\widehat{\OPE}$.

\begin{exercise}
  Prove that $\PSh$ is indeed a category.
\end{exercise}

\begin{exercise}
  Prove that the following functions are morphisms in $\PSh$:
\[
\begin{array}{lllll}
  \thyp & : & \Hyp A & \todot & \Ne A \\
  \tandE1 & : & \Ne{A \land B} & \todot & \Ne A \\
  \torI1  & : & \Nf A & \todot & \Nf{A \lor B} \\
  \tfalseE & : & \Ne \bot & \todot & \Nf C \\
\end{array}
\]
  Find other examples of presheaf morphisms.
\end{exercise}

\section{NbE for the negative fragment}

We wish to define a normalization function for derivations of any
formula $A$
\[
  \tnorm^A : \Tm A \todot \Nf A .
\]
Normalization by evaluation (NbE) achieves normalization by first
evaluating terms, obtaining values and functions that compute by means
of the computation in the meta language, i.e., in $\SET$.  As a second
step, these values are reified back into syntax, yielding a normal
form.  Technically, we define a suitable presheaf $\den A$ for each
formula $A$ and two functions
\[
\begin{array}{lllll}
  \teval^A & : & \Tm A  & \todot & \den A \\
  \treify^A  & : & \den A & \todot & \Nf A \\
\end{array}
\]
which compose to $\tnorm$.

\subsection{Semantics, reflection and reification}

For now, let us focus on the negative
formulas $\top$, $A \land B$, $A \imp B$ and atoms $P$.
\[
\begin{array}{lll@{\qquad}lll}
  \den P & = & \Nf P
    & \reify[\Gamma] P t & = & t \\
  \den \top & = & \hatone
    & \reify[\Gamma] \top \_ & = & \trueI \\
  \den{A \land B} & = & \den A \hattimes \den B
    & \reify[\Gamma]{A \land B} {(a,b)} & = & \andIPP {\reify[\Gamma] A a} {\reify[\Gamma] B b} \\
\end{array}
\]
Herein, we use the unit presheaf $\hatone_\Gamma = 1$ where $1$ is the
unit set, and the pointwise product of presheaves
$(\A \hattimes \B)_\Gamma = \A_\Gamma \times \B_\Gamma$.  We note that
by these definitions, the category $\PSh$ has products, with the
pointwise terminal morphisms:
\[
\begin{array}{lll@{\qquad}lll}
   ! & : & \A \todot \hatone &
     [\_,\_] & : & (f : \C \todot \A) (g : \C \todot \B) \to (\C
                   \todot \A \hattimes \B) \\
   !_\Gamma(a) & = & ()       &
     [f,g]_\Gamma(c) & = & (f(c),g(c)) \\
\end{array}
\]

The interpretation $\den{A \imp B}$ of implication will follow the
Brouwer-Heyting-Kolmogorov (BHK) interpretation of intuitionistic
logic:
\begin{center}\itshape
  A proof of $A \imp B$ is a method turning any proof of $A$ into a
  proof of $B$.
\end{center}
However, the direct lifting of the function space $\den{A \imp
  B}_\Gamma = \den A_\Gamma \to \den B_\Gamma$ does not work.  It does
not give a presheaf because of the contravariant occurrence of
presheaf $\den A$.  Instead, we force monotonicity by quantifying over
all OPEs of $\Gamma$:
\[
  \den{A \imp B}_\Gamma = \prod_{\Delta : \Cxt} \prod_{\tau : \Delta \leq \Gamma} (\den A
  _\Delta \to \den B _\Delta)
\]
Reification is also challenging.  Let $f : \den{A \imp B}_\Gamma$ and
attempt
%$\tau : \Delta \leq \Gamma$
\[
  \reify[\Gamma]{A \imp B} f %\;\Delta\;\tau =
    = \impIP {\reify[\Gamma.A]B{(f_{\Gamma.A}\;(\weak[A]{\id[\Gamma]})}\;\mbox{?})}
\]
where hole ``?'' should be filled with an element of $\den B_{\Gamma.A}$
which intuitively should represent the new hypothesis $\ttop :
\Hyp[\Gamma.A]A$ introduced by $\timpI$.  Thus we need a natural
transformation $\Hyp A \todot \den A$ which \emph{reflects} variables
into the semantics.  In order to define it by
induction on $A$, we need to generalize it to
\[
  \treflect^A : \Ne A \todot \den A
  .
\]
Let us make sure we can define it at atoms and conjunctions:
\[
\begin{array}{lllll}
  \treflect^P & = & \tne & : & \Ne P \todot \Nf P \\
  \treflect^\top & = & !  & : & \Ne \top \todot \hatone \\
  \treflect^{A \land B} & = & \langle\, \treflect^A \comp \andE1  , \;
                             \treflect^B \comp \andE2 \rangle
    & : & \Ne{A \land B} \todot \den A \hattimes \den B \\
\end{array}
\]
Observe the eliminations $\andE i$ introduced to reflect conjunctive hypotheses!

Now, we can complete the story for implication.  Let $\tau : \Delta
\leq \Gamma$ and $a : \den A_\Delta$ and set:
\[
\begin{array}{lll}
  \treflect^{A \imp B} & : & \Ne{A \imp B} \todot \den{A \imp B} \\
  \reflect[\Gamma]{A \imp B} t\;\Delta\;\tau\;a & = &
    \reflect[\Delta]B{(\impEP {\Ne[\tau] B t} {(\reify[\Delta] A a)})}
\\[1ex]
  \treify^{A \imp B} & : & \den{A \imp B} \todot \Nf{A \imp B} \\
  \reify[\Gamma]{A \imp B} f & = &
    \impIP {\reify[\Gamma.A]B{(f_{\Gamma.A}\;(\weak[A]{\id[\Gamma]})}\;(\reflect[\Gamma.A]A{(\hyp\ttop)})}
\end{array}
\]

\begin{exercise}
  Let $A = ((P \imp Q) \land (Q \imp \top)) \imp Q$ for some atoms
  $P,Q$.  Compute $\reflect[A]A{(\hyp \ttop)}$.
\end{exercise}

\begin{exercise}
  Let $A = (P \imp Q) \imp (P \imp Q)$ for some atoms $P,Q$.
  Compute $\treify^A \comp \treflect^A$.
\end{exercise}

\begin{exercise}
  Using the constructions in this section, show that the category of
  presheaves and natural transformations is Cartesian closed.
\end{exercise}

\subsection{Evaluation}

Evaluation is turning an expression $t : \TmG A$ into a value in
$\den A$.  However, a direct definition $\teval^A : \Tm A \todot \den A$
by induction on the expression fails in case $\timpI$:
\[
\begin{array}{lcl}
   \teval^{A \imp B}_\Gamma & : & \TmG A \to \den {A \imp B} _\Gamma \\
   \teval^{A \imp B}\,(\impI t)\,_\Delta\,a & = & ? \\
\end{array}
\]
To be able to pass the argument $a$ to the function $\teval\,(\impI
t)$, we have to generalize evaluation to take an valuation of the
context into account.
\[
  \evalG{A \imp B}t  ~:~ \den \Gamma \todot \den {A \imp B}
\]
Here, $\den \Gamma$ is the presheaf obtained as the product of the
hypotheses contained in $\Gamma$:
\[
\begin{array}{lll}
  \den{\cempty} & = & \hatone \\
  \den{\Gamma.A} & = & \den\Gamma \hattimes \den A \\
\end{array}
\]
Looking up values in the environment is the following family of
natural transformations:
\[
\begin{array}{l@{~}lcl}
  \multicolumn 2 l {\tlookup^A_\Gamma} & : & \HypG A \to (\den \Gamma \todot \den A) \\
  \lookup[\Gamma.A]A \ttop    &_\Delta\,(\gamma,a) & = & a \\
  \lookup[\Gamma.B]A {\pop x} &_\Delta\,(\gamma,a) & = & \lookup[\Gamma]A x\,_\Delta\, \gamma \\
\end{array}
\]
\begin{exercise}
  Express $\tlookup$ via the projections
  $\pi_i : \A_1 \hattimes \A_2 \todot \A_i$
  out of the presheaf product.
\end{exercise}
We are ready to define evaluation:
\[
\begin{array}{lcl}
  \evalG A {\hyp x} & = & \lookupG A x \\
  \evalG {A \imp B} {\impI t}\,_\Delta\,\gamma\,_{\Delta'}\,(\tau : \Delta' \leq \Delta)\,a
    & = & \eval[\Gamma.A]B t\,_{\Delta'}\,(\den\Gamma_\tau\gamma,a) \\
  \evalG {B} {\impE t u}\,_\Delta\,\gamma & = &
    \evalG {A \imp B} t\,_\Delta\,\gamma\,_\Delta\,\id[\Delta]\,
      (\evalG A u\,_\Delta\,\gamma) \\
\end{array}
\]
\begin{exercise}
  Complete the definition of $\teval$ for conjunctions.
\end{exercise}
\begin{exercise}
  Express $\teval$ in terms of the operations of the CCC of
  presheaves!
\end{exercise}

\subsection{Normalization}

Normalization is reification after evaluation in the identity
environment:
\[
\begin{array}{lcl}
\tnorm^A & : & \Tm A \todot \Nf A \\
\norm[\Gamma] A t & = & \reify[\Gamma]A {(\eval[\Gamma] A t \,_\Gamma\, \env[\Gamma])}
\end{array}
\]
The identity environment $\env[\Gamma]$ is defined by induction on
$\Gamma$ as follows:
\[
\begin{array}{lcl}
  \env[\Gamma] & : & \den \Gamma _\Gamma \\
  \env[\cempty] & = & () \\
  \env[\Gamma.A] & = & (\den\Gamma_{\weak[A] \id[\Gamma]}(\env[\Gamma]), \;
                          \reflect[\Gamma.A] A {(\hyp \ttop)})
\end{array}
\]

\subsection{Digression: The CCC of reifiable presheaves}

Instead of defining the semantics $\den A$ of formula $A$ by induction
on $A$, we can consider a category whose objects are interpretations
of formulas, and show that this category is Cartesian closed.

An object in this category would be a quadruple consisting of:
\[
\begin{array}{lll@{\qquad}l}
  \A & : & \PSh  & \mbox{a presheaf} \\
  A  & : & \Form & \mbox{a formula} \\
  \treflect^A & : & \Ne A \todot \A
    & \mbox{a method to reflect neutrals into $\A$} \\
  \treify^A & : & \A \todot \Nf A
    & \mbox{a method to reify values of $\A$} \\
\end{array}
\]
\begin{exercise}
  Complete this definition by a suitable notion of morphism.
\end{exercise}
\begin{exercise}
  Show that this category is Cartesian closed.
\end{exercise}

%\section{Extension to the positive connectives}
\section{Disjunction and absurdity}

For sets $S_1,S_2 : \SET$ let $S_1 + S_2$ denote the disjoint union with
injections $\inj{i} : S_i \to S_1 + S_2$ and case distinction
distinction $[f_1,f_2] : S_1 + S_2 \to T$ for $f_i : S_i \to T$
such that $[f_1,f_2] \comp \inj i = f_i$.

Presheaf coproduct is given pointwise by $(\A \hatplus \B)_\Gamma = \A_\Gamma + \B_\Gamma$.
\begin{exercise}
  Show that $\hatplus$ is indeed a coproduct in $\PSh$.
\end{exercise}

\subsection{Semantics, reflection and reification}

The first idea how to model disjunction,
$\den{A \lor B}_\Gamma = \den{A}_\Gamma + \den{B}_\Gamma$, fails
because we cannot reflect disjunctive hypotheses.  \Eg, an element of
$\den{A \lor B}_{A \lor B}$ should be of the form $\inj1\,a$ or
$\inj2\,b$, but the decision on first or second injection cannot be
made here;  it depends on the hypothesis $A \lor B$.  The solution is
to extend the semantics to allow case distinction on variables (and
neutrals) before making the decisions needed for disjunctions.

Given a presheaf $\A$, we define a new presheaf $\Cover \A$
inductively by the following constructors:
\begin{gather*}
  \ru{a : \A_\Gamma}{\returnC a : \Cover\A_\Gamma}
\qquad
  \ru{t : \NeG \bot}{\falseC t : \Cover\A_\Gamma}
\\[1ex]
  \ru{t : \NeG{C \lor D} \qquad
      c : \Cover\A_{\Gamma.C} \qquad
      d : \Cover\A_{\Gamma.D}
    }{\orC t c d : \Cover\A_\Gamma}
\end{gather*}
\begin{exercise}
  Show that $\Cover\A$ is indeed a presheaf.
\end{exercise}
We can think of $c : \Cover\A_\Gamma$ as a decision tree with leaves in $\A$
and nodes labeled by terms we case on.  Nodes of type $\tfalseC$ have
no subtrees since \emph{ex falsum quod libet}, and $\torC$ nodes
casing on $t : \NeG{C \lor D}$ have
two subtrees, one having a new hypothesis $C$, and one having a new
hypothesis $D$.

We can replace all the leaves according to a function $f : \A \todot \B$,
this makes $\tCover$ a \emph{functor} in the category of presheaves.  We can
also replace all the leaves by new case trees via a natural transformation
\[
   \tjoinC ~:~ \Cover{\Cover\A} \todot \Cover\A
\]
which makes $\tCover$ a \emph{monad} in $\PSh$.
\begin{exercise}
  Define the functorial action $\Cover f$ and $\tjoinC$ and prove the
  monad laws.
\end{exercise}
Further, if the leaves of a case tree are normal forms, we can turn it into one big
normal form via:
\[
  \tpasteNf ~:~ \Cover{\Nf A} \todot \Nf A
\]
\begin{exercise}
  Define $\tpasteNf$.
\end{exercise}

\noindent
We have everything now to model disjunction:
\[
\begin{array}{lcl}
\den{A \lor B} & = & \Cover{\den A \hatplus \den B}
\\[2ex]
\treflect^{A \lor B} & : & \Ne{A \lor B} \todot \den{A \lor B} \\
\reflect[\Gamma]{A \lor B}t & = & \orC t
  {(\returnCP {\injP1 {\reflect[\Gamma.A]A{(\hyp \ttop)}}})}
  {\\ && \qquad\ (\returnCP {\injP2 {\reflect[\Gamma.B]B{(\hyp \ttop)}}})}
\\[2ex]
\treifyOr^{A \lor B} & : & \den{A} \hatplus \den{B} \todot \Nf{A \lor B} \\
\reifyOr[\Gamma]{A \lor B}(\inj1\, a) & = & \orIP1 {\reify[\Gamma]A a} \\
\reifyOr[\Gamma]{A \lor B}(\inj2\, b) & = & \orIP2 {\reify[\Gamma]B b}
\\[1.5ex]
\treify^{A \lor B} & : & \den{A \lor B} \todot \Nf{A \lor B} \\
\treify^{A \lor B} & = & \tpasteNf \comp \Cover{\treifyOr^{A \lor B}} \\
\end{array}
\]
Reification works by first reifying the leaves of the case tree using
$\treifyOr$, and then turning the whole tree into a normal form using $\tpasteNf$.
\begin{exercise}
  Compute $\reify[P \lor Q]{P \lor Q} (\reflect[P \lor Q]{P \lor Q}
  (\hyp \ttop))$.
\end{exercise}
For absurdity, we use the presheaf $\hat0_\Gamma = \{\}$ with initial
morphism $\botelim : \hat0 \todot \C$.
\[
\begin{array}{lcl}
\den{\bot} & = & \tCover\,{\hat0}
\\[2ex]
\treflect^{\bot} & : & \Ne{\bot} \todot \den{\bot} \\
\reflect[\Gamma]{\bot}t & = & \falseC t
\\[2ex]
\treify^{\bot} & : & \den{\bot} \todot \Nf{\bot} \\
\treify^{\bot} & = & \tpasteNf \comp \Cover{\botelim} \\
\end{array}
\]
To reify a value $c : \den\bot$, observe that because $\hat0$ is
empty, $c$ can only be a case tree without leaves, \ie, all branches
end in $\tfalseC$ nodes.  Thus, the mapping
$\Cover{\botelim} : \Cover{\hat0} \todot \Cover{\Nf\bot}$ is merely a
type cast.  The subsequent $\tpasteNf$ will turn the leafless tree $c$
into a normal form consisting entirely of splits $\torE$ and $\tfalseE$.

\begin{exercise}
Prove that $\treify^{A} \comp \treflect^{A}$ produces the long
$\eta$-normal form if applied to variables (\ie, hypotheses).
\end{exercise}

\subsection{Evaluation}

Evaluation poses one more challenge. Consider the term:
\[
\ru{x : \HypG {C \lor D} \qquad
    f : \Tm[\Gamma.C]{A \imp B} \qquad
    g : \Tm[\Gamma.D.A]{B} \qquad
    a : \TmG A
  }{\impEP{\orE {(\hyp x)} f {(\impI g)}} a : \TmG B}
\]
There is a redex $\impEP {\impI g} a$ separated by an $\torE$.
The function term $\orE {(\hyp x)} f {(\impI g)}$
would naturally evaluate to a case tree with functions in the
leaves.  To apply it to the evaluation of $a$, we have to apply each
leaf to that argument.  We facilitate this by implementing
\[
  \tpaste^{A \imp B} : \Cover{\den{A \imp B}} \todot \den{A \imp B}
\]
that turns a case tree of functions into a function which expects an
argument that will then be passed to all the functions in the case
tree.  In fact, we need pasting at all formulas $A$:
\[
\begin{array}{lcl}
\tpaste^A & : & \tCover\,{\den A} \todot \den A \\[1ex]
\tpaste^P & = & \tpasteNf \\
\tpaste^\bot & = & \tjoinC \\
\tpaste^{A \lor B} & = & \tjoinC \\
\tpaste^{A \land B} & = & \langle\,
  \tpaste^A \comp \Cover{\pi_1} \, , \;
  \tpaste^B \comp \Cover{\pi_2} \,\rangle \\[1ex]
\multicolumn 3 {l} {\paste[\Gamma]{A \imp B} c \;
  _\Delta \; (\tau : \Delta{\leq}\Gamma) \; (a : \den A _\Delta) \, = \,
  (\paste[\Delta]B \comp \mapC[\Delta] \varphi \comp \Cover{\den {A \imp B}_\tau})\, c
}
\end{array}
\]
In the case of implication $A \imp B$,
we need a stronger version of functoriality of $\tCover$ which
only requires a
\[
  \varphi : \prod_{\Phi} \prod_{\delta : \Phi \leq \Delta} \A_\Phi
             \to \B_\Phi
\]
to produce a $\mapC[\Delta]\varphi : \Cover\A_\Delta \to \Cover\B_\Delta$.
In our case, $\A = \den{A \imp B}$ and $\B = \den{B}$ and
\[
  \varphi\,_\Phi\;\delta\;f = f\;\id[\Phi]\;(\den A _\delta\, a)
  .
\]
Pasting allows us to complete the definition of evaluation.
\[
\begin{array}{lcl}
  \teval_\Gamma^A & : & \TmG A \to \den \Gamma \todot \den A \\[1ex]
  \evalG C {\orE t u v} _\Delta \; \gamma & = &
    \paste[\Delta]C
     {(\mapC[\Delta]{\lambda\, _\Phi\, (\tau  : \Phi \leq \Delta) \to [f,g]}\;c)} \\[1ex]
  \multicolumn 1 r {\mbox{ where } c} & = &
      \evalG{A \lor B} t\;_\Delta\;\gamma \\
  \multicolumn 1 r {f\;a} & = &
    \eval[\Gamma.A] C u \;_\Phi\;(\den\Gamma_\tau\,\gamma, a) \\
  \multicolumn 1 r {g\;b} & = &
    \eval[\Gamma.B] C v \;_\Phi\;(\den\Gamma_\tau\,\gamma, b) \\
  \\
\end{array}
\]
\begin{exercise}
  Really complete the definition of $\teval$ (all missing cases).
\end{exercise}
That's it!

\newcommand{\citeentry}[1]{\item\nocite{#1}\bibentry{#1}}

\section{Literature}

Further reading on normalization by evaluation in general:
\begin{enumerate}

\citeentry{bergerSchwichtenberg:lics91}

\citeentry{jungTiuryn:tlca93}

\citeentry{altenkirchHofmannStreicher:ctcs95}

\citeentry{abel:habil}

\end{enumerate}

% \begin{itemize}
% \item \citet{bergerSchwichtenberg:lics91}
% \item \citet{jungTiuryn:tlca93}
% \item \citet{altenkirchHofmannStreicher:ctcs95}
% \item \citet{abel:habil}
% \end{itemize}
\noindent
Literature dealing in particular with the case of disjunction /
disjoint sums:

\begin{enumerate}

\citeentry{danvy:popl96}

\citeentry{fioreSimpson:tlca99}

\citeentry{altenkirchDybjerHofmannScott:lics01}

\citeentry{balatDiCosmoFiore:popl04}

\citeentry{altenkirchUustalu:flops04}

\citeentry{barral:PhD}

\citeentry{scherer:popl17}

\end{enumerate}

% \begin{itemize}
% \item \citet{danvy:popl96}
% \item \citet{fioreSimpson:tlca99}
% \item \citet{altenkirchDybjerHofmannScott:lics01}
% \item \citet{balatDiCosmoFiore:popl04}
% \item \citet{altenkirchUustalu:flops04}

% \item \citet{barral:PhD}
% \item \citet{scherer:popl17}
% \end{itemize}

\section{Conclusion}

\paradot{Acknowledgments}  The presentation of this NbE proof and its
Agda formalization benefited greatly from discussions with Christian
Sattler.  Thanks to Jesper Cockx for listening to a prerun of the talk.

\bibliographystyle{abbrvnat}
%\bibliography{auto-nbeSum}

\end{document}


%\documentclass[handout]{beamer}
\documentclass{beamer}
\mode<article>
{
	\usepackage{amscd}
	\usepackage{times,amsmath,amsbsy,amssymb,amsfonts,epic,hyperref}
	\usepackage[pdftex]{geometry}
	\usepackage[pdftex]{graphicx}
	\usepackage{fullpage}
	\usepackage{hyperref}
}

%\mode<presentation> {
%\usetheme{Madrid}
%\usecolortheme{dolphin}
%\usecolortheme{rose}
%}
\usepackage{amsfonts,bm,mathrsfs,subfig}
%\usepackage{movie15}

\mode<presentation> {
	\setbeamertemplate{background canvas}[vertical shading][bottom=red!10,top=blue!10]
	\usetheme{Boadilla}
	\usefonttheme[onlysmall]{structurebold}
}

\usepackage{multimedia}
\usepackage{animate} %need the animate.sty file

\usepackage{pgf,pgfarrows,pgfnodes,pgfautomata,pgfheaps,pgfshade,multirow}
\usepackage{amscd}
\usepackage{amsmath,amssymb}
%\usepackage[latin1]{inputenc}
\usepackage{colortbl}
\usepackage[english]{babel}
\usepackage{times}
\usepackage[T1]{fontenc}
\usepackage{epsf}
\usepackage{epsfig}
\usepackage{graphicx}
\usepackage{bm}
\usepackage{relsize}
\usepackage{dsfont}
\usepackage{pifont}
\usepackage{comment}
\usepackage{booktabs}
\newtheorem{algorithm}{Algorithm}
\newtheorem{remark}{Remark}
\usepackage{undertilde}
\usepackage{fancybox}
\usepackage{CJK}
\usepackage{multirow}
\usepackage{tikz}
\usepackage{slashbox}

%\input{mydef}
\def\p{\partial}
\def\u{{\mathbf u}}
% \def\B{{\mathbf B}}
\def\v{{\mathbf v}}
\def\n{{\mathbf n}}
% \def\F{{\mathbf F}}
%\def\A{{\mathbf A}}
% \def\x{{\mathbf x}}
\def\DD{\displaystyle}

\newcommand*{\ucong}{\mathrel{\text{\raisebox{.25ex}{\rotatebox[origin=c]{180}{$\cong$}}}}}
\newcommand{\relu}{\mbox{{\rm ReLU}}}

\newcommand{\red}[1]{\textcolor{red}{#1}}
\newcommand{\blue}[1]{\textcolor{blue}{#1}}
\newcommand{\brown}[1]{\textcolor{brown}{#1}}
\newcommand{\green}[1]{\textcolor{green}{#1}}

\newtheorem{prop}{Proposition}
\newtheorem{thm}{Theorem}
\newtheorem{prob}{Problem}
\newtheorem{assumption}{Assumption}
\newtheorem{defn}{Definition}
\newtheorem{coro}{Corollary}
\beamertemplatenavigationsymbolsempty

\newcommand{\tabincell}[2]{\begin{tabular}{@{}#1@{}}#2\end{tabular}}

\newif\iflattersubsect

\AtBeginSection[] {
	\begin{frame}<beamer>
	\frametitle{Outline} %
	\tableofcontents[currentsection]
\end{frame}
\lattersubsectfalse
}

\AtBeginSubsection[] {
\iflattersubsect
\begin{frame}<beamer>
\frametitle{Outline} %
\tableofcontents[currentsubsection]
\end{frame}
\fi
\lattersubsecttrue
}

\begin{document}

\begin{CJK*}{UTF8}{gbsn}

\title[DNN \& FEM]
{Deep Neural Networks}

\author[Xu]{Jinchao Xu}

\institute[PSU]
{Penn State University\\[0.2cm]
	{\color{blue}\tt xu@math.psu.edu \;\; http://www.math.psu.edu/xu/}
}

\date[PSU]{Penn State \\ January 8, 2019}

\begin{frame}
\titlepage

{\footnotesize
	\centerline{Acknowledgement: }
	\centerline{{\color{blue} NSF Grants}: DMS-1522615 ~ DMS-1217142}
	\centerline{{\color{blue} DOE Grants}: DE-SC0014400 ~ DE-SC0009249}
}
\end{frame}

\begin{frame}
\frametitle{Outline}
\setcounter{tocdepth}{2}
\tableofcontents
\end{frame}

\section{Supervised learning for classification problem}
%\subsection{Deep neural networks and its relation to FEM}
\begin{frame}{A basic AI problem: classification}
\begin{itemize}
	\item Can a machine (function) tell the difference ?
\end{itemize}

\begin{figure}
	\begin{center}
		\includegraphics[width=.7\textwidth, height=.6\textheight]{figures/DL/cat-dog-1.png}
	\end{center}
\end{figure}

\end{frame}

\begin{frame}{Supervised learning $\leftrightarrow$ Function interpolation}
\begin{itemize}
\item Each image = a big vector of pixel values
\begin{itemize}
	\item $d=1280\times720\times3$ (width$\times$ height $\times$ RGB channel) $\approx$ 3M.
\end{itemize}
\pause
\item 3 different sets of points in $\mathbb{R}^d$, are they separable?
\begin{figure}
	\begin{center}
		\includegraphics[width=.3\textwidth, height=.3\textheight]{figures/DL/cat-dog-2.png}   \quad \quad  \quad \quad
		\includegraphics[width=.3\textwidth, height=.3\textheight]{figures/DL/cat-dog-3.png}
	\end{center}
\end{figure}
\pause
\item Mathematical problem: Find $f(\cdot; \Theta): \mathbb{R}^d \to \mathbb{R}^3$ such that:
$$
f(\includegraphics[width=.07\textwidth]{figures/DL/cat.png}; \Theta)
\approx \begin{pmatrix}
1\\ 0 \\ 0
\end{pmatrix}
\quad
f(\includegraphics[width=.07\textwidth]{figures/DL/dog.png}; \Theta)
\approx \begin{pmatrix}
0\\ 1 \\ 0
\end{pmatrix}
\quad
f(\includegraphics[width=.07\textwidth]{figures/DL/rabbit.png};
    \Theta) \approx
\begin{pmatrix}
0\\ 0 \\ 1
\end{pmatrix}
$$
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{How to formulate ``learning''?}
\pause
\begin{itemize}
	\item Data: $\{x_i, y_i\}_{i=1}^m$
%%	\begin{itemize}
%%		\item Image Classification: $x_i$ image, $y_i$ object class
%%		\item Speech Recognition: $x_i$ audio, $y_i$ text
%%		\item Face Recognition: $x_i$ graph with face, $y_i$ identity
%%		\item ...
%%	\end{itemize}
\pause
	\item Find $f^*$ in some function class such that $f^*(x_i) \approx y_i$.
\pause
	\item Mathematically: solve the \red{optimization problem} by parameterizing the abstract function class
	\begin{equation}\label{eq:supervise}
	\min_{\Theta} L(x,y,\Theta)
	\end{equation}
where
{\footnotesize
$$
L(x,y,\Theta)=
{\mathbb E}_{(x,y)\sim \mathcal D}[L(f(x; \Theta), y)]\approx
\frac{1}{m} \sum_{i=1}^m\|f(x_i; \Theta) - y_i\|^2
$$
}

\item Nonlinear least squares problems.
\end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Main question}

\begin{quote}\large
 What is a good function class for $f$?
\end{quote}

\end{frame}

\begin{frame}
\frametitle{What is a good function class for $f$?}
\begin{small}
\structure{Traditional machine learning models}
\begin{itemize}
\item linear regression
\item  ``nonlinear" regression,
\begin{itemize}
\item linears, polynomials, exponentials,  logarithms, trigonometric, ...
\end{itemize}
\pause
\item SVM, decision tree...
	$$
	f(x;\Theta) = \sum_{k=1}^N c_k k(x,x_k).
	$$
\end{itemize}
\pause
\structure{Traditional models in scientific  computing}
\begin{itemize}
	\item finite element methods, wavelets, spectral method ...
\end{itemize}
\end{small}
\end{frame}
\begin{frame}
{Challenges}
\begin{itemize}
	\item high dimension: image, video...
	\item big data: ImageNet, MegaFace...
	\item complex task: Go, auto-drive...
\end{itemize}
\end{frame}

\section{Deep neural networks}

\begin{frame}
  \frametitle{Linears, activation and composition}
\tiny
\pause
\begin{enumerate}
\item Start from a linear function
$$
W^0x+b^0
$$
\pause
\item Introduce an activation function $\sigma={\rm ReLU}(x)=\max(0,x)$:
	\begin{center}
		\includegraphics[width=.2\textwidth, height=.15\textheight]{figures/DL/ReLU.png}
	\end{center}
\pause
\item Compose with the activation function:
$$
x^{(1)}  = \sigma(W^0x+b^0)
$$
\pause
\item Compose with another linear function:
$$
W^1x^{(1)}+b^1
$$
\pause
\item Compose with the activation function:
$$
x^{(2)}=\sigma(W^1x^{(1)}+b^1)
$$
\pause
\item Compose with another linear function
$$
f(x;\Theta)= W^2x^{(2)}+b^2
$$
\pause
\item $\ldots$
\end{enumerate}
\end{frame}
\begin{frame}
\frametitle{Deep neural network functions}
DNN function class with super-parameters $\mathcal{N} = (n^1,\cdots,n^J)$:
\begin{equation}\label{key}
{\rm DNN}_{J}({\mathcal{N}}) := \{f:\mathbb R^d\mapsto \mathbb R ~:~ f(x; \Theta) = f^J, \Theta \in \mathbb{R}^{\mathcal{N}}\}
\end{equation}
where, with $\Theta=\Theta^J$ and $\Theta^j=(\Theta^{j-1},\theta^j)$,
\pause
\begin{itemize}
	\item Recursion: $f^j(x,\Theta^j) = (\theta^j\circ \sigma \circ
	f^{j-1})(x,\Theta^{j-1}), \quad j=1:J$
\pause
	\item Linear mapping:
	$
	\theta^j: \mathbb{R}^{n^j} \to \mathbb{R}^{n^{j+1}} \quad \text{as} \quad \theta^j(x) =  W^j x + b^j,
	$
\pause
	\item Initial: $f^0(x)=\theta^0(x).$
\pause
	\item \red{nonlinear activation}:  $\sigma: \mathbb{R} \to \mathbb{R}$.   The
	popular one is ${\rm ReLU}(u)=\max(0,u)$:
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Approximation property of ${\rm DNN}_1$}
\small
One hidden layer ${\rm DNN}_1(N)$:
$$
f=\sum_{i=1}^N a_i\sigma(w_i\cdot x+b_i)
$$
\pause
\structure{Question}
\begin{quote}
Is this function class rich enough to approximate any reasonable
(say, continuous) function?
\end{quote}
\pause
\structure{Answers:}
\begin{enumerate}
\item  NO, if $\sigma$ is a polynomial
\pause
\item  YES, if $\sigma$ is NOT a polynomial!
\end{enumerate}
\pause
\vfill
{\tiny M. Leshno, V. Lin, A. Pinkus and S. Schocken, 1993}
\end{frame}

\begin{frame}
  \frametitle{Special function class}
\structure{Special function class}
  \begin{quote}
Linears and composition with $1$  variable functions.
  \end{quote}
\pause
\structure{Approximation: any continuous function}
$$
f(x_1,\ldots,x_d)\sim \sum_{i=1}^N a_i\sigma\left(\sum_{p=1}^dw_i^px_p+b_i\right)
$$
\pause
\structure{Exact representation: any continuous function} [{\tiny
  Kolmogorov-Arnold 1957, Sprecher 1965}]
$$
f(x_1,\ldots,x_d)=\sum_{q=0}^{2d}\alert{\chi_f}\left(\sum_{p=1}^d\lambda^p\alert{\psi}(x_p+\epsilon
  q)+q\right).
$$
\pause
\structure{Disprove: Hilbert 13 problem}
\begin{quote}\tiny
	Impossibility of the solution of the general equation of the 7th degree by means of functions of only two arguments.
\end{quote}
\end{frame}

\section{ReLU DNN and Linear Finite Elements}
\begin{frame}
{What does a function in ${\rm DNN}_{\mathcal N}$ look like?}
\blue{Obviously:}
$$
{\rm DNN}_{\mathcal N}=\mbox{\rm a space of continuous piecewise linear functions!}
$$
\vskip-40pt
\pause
\begin{figure}
	\begin{center}
		%$(10, 10)$ $(20, 20)$ $ (40,40)$
		\includegraphics[width=.3\textwidth]{figures/2to10to10to1.pdf}
		\includegraphics[width=.3\textwidth]{figures/2to20to20to1.pdf}
		\includegraphics[width=.3\textwidth]{figures/2to40to40to1.pdf}
	\end{center}
\end{figure}
\vspace{-20mm}
\begin{center}
	$(10, 10)$ \qquad\qquad\qquad  $(20, 20)$ \qquad\qquad \qquad $ (40,40)$
\end{center}
\end{frame}

\begin{frame}
\frametitle{Some mathematical questions}
\begin{enumerate}
\item Connection with finite element methods
\item Approximation properties
\item What's the optimal structure?
\item Is deeper better?
\item How to reduce the number of parameters?
\end{enumerate}
\end{frame}

\begin{frame}{How is ${\rm DNN}_{\mathcal N}$ compared with (adaptive) linear FEM?}
\begin{figure}%[htbp]
\centering
\begin{minipage}[t]{0.3\textwidth}
\centering
\includegraphics[width=.7\textwidth, height=.3\textheight]{figures/dnn_grid.png}
\caption{$(40, 40)$}
\end{minipage}
\begin{minipage}[t]{0.3\textwidth}
\centering
\includegraphics[width=.7\textwidth, height=.3\textheight]{figures/fem.pdf}
\caption{Uniform Grid}
\end{minipage}
\begin{minipage}[t]{0.3\textwidth}
\centering
\includegraphics[width=.7\textwidth, height=.3\textheight]{figures/L-adaptive.pdf}
\caption{Adaptive Grid}
\end{minipage}
\end{figure}

%\pause
%Some answers:
%\begin{itemize}
%\item DNN seems to be at least as good as adaptive linear FEM!
%\pause
%\item Any linear finite element function can be written as a DNN with ReLU!
%\end{itemize}
\end{frame}

\begin{frame}{Connection of ReLU DNN and linear FEM}
\begin{theorem}[He, Li, X and Zheng 2018]
Not all linear FE functions can be represented by a
shallow neural network, namely
$$
{\rm LFE}\not\subset {\rm DNN}_1.
$$
\end{theorem}

\pause
\begin{theorem}[He, Li, X and Zheng 2018]
Given a locally convex finite element grid ${\cal
T}_h$, any linear finite element function with $N$ degrees of
freedom, it can be written as a ${\rm DNN}_J$ with
$$
J = 2+\lceil\log_2 d_h\rceil
\mbox{ and }
N_{\rm neurons}=\mathcal{O}(d_hN).
$$
where $d_h$ is the largest number of elements that share one vertex.
\end{theorem}

\end{frame}

\begin{frame}{MRDA: a sparse optimization method in Deep Learning}
\structure{Modified Regularized Dual Averaging (MRDA)}
\begin{itemize}
	\item 	Loss function with $\ell^1$ regularization term,
	\begin{equation}\label{regularized expected risk minimization}
	\min_{\Theta}~~\left\lbrace \phi(\Theta) =  \mathbf{E}_z f(\Theta,z)+\Psi(\Theta)\right\rbrace.
	\end{equation}
	\item
	$$
	\Theta_{t+1} =\mathop{\arg\min}_\Theta \left\{\frac{1}{t} \sum_{\tau=1}^{t} \left<g_{\tau},\Theta \right>+\frac{\beta_t}{t}h(\Theta)+\Psi(\Theta)\right\}.
	$$
\end{itemize}
\vspace{2.5cm}
\vfill
\tiny \structure{Ref:}  {L. Xiao 2010, X. Jia, L. Zhao, L. Zhang, J. He and J. Xu 2018}
\end{frame}

\begin{frame}\frametitle{MRDA: a sparse optimization method in Deep Learning}
%	\footnote{Jia, X., Zhao, L., Zhang, L., He, J., \& Xu, J. Modified Regularized Dual Averaging Method for Training Sparse Convolutional Neural Networks. \emph{arXiv preprint arXiv} arXiv:1807.04222, 2018}
\begin{theorem}[Jia, Zhao, Zhang, He and X 2018]\label{Convergence_Modified_RDA}
Assume there exists an optimal solution $\Theta^{\star}$ to the problem (\ref{regularized expected risk minimization}) for convex case with $\Psi(w)=\lambda\|\Theta\|_1$ that satisfies $h(\Theta^{\star})\leq D^2$ for some $D>0$, and let $\phi^{\star}=\phi(\Theta^{\star}).$ Let the sequences $\{\Theta_t\}_{t\geq 1}$ be generated by modified RDA method, and assume $\|g_t\|_{\ast}\leq G$ for some constant $G$. Then the expected cost $\mathbf{E}\phi(\bar{\Theta}_t)$ converges to $\phi^{\star}$ with rate $O(\frac{1}{\sqrt{t}})$
$$
\mathbf{E}\phi(\bar{\Theta}_t)-\phi^{\star}
= O(\frac{1}{\sqrt{t}}).
$$
\end{theorem}
\vspace{1.5cm}
\vfill
\tiny \structure{Ref:}  {L. Xiao 2010, X. Jia, L. Zhao, L. Zhang, J. He and J. Xu 2018}
\end{frame}

\begin{frame}
\frametitle{Numerical Results on CIFAR-10}
\begin{table}[H]
\centering
\caption{Different Methods on ResNet18. The models are trained for $120$ epochs. While the validation accuracies are close, proximal-SGD gives poor sparse solutions compared to MRDA, and SGD introduces no sparsity.}
\begin{tabular}{l rrr}
\hline\hline
Method & TOP 1 & TOP 5 & $\sigma$             \\ \hline
SGD    	  & 89.27 &	99.57 &	$0.00$      \\
proximal-SGD  	  & 89.80 & 99.40 & $0.03$ \\
\red{MRDA}  	  & 90.67 &	99.53 &	\red{$0.94$}     \\
[1ex]
\hline
\end{tabular}
\end{table}
\begin{equation}
\sigma = \frac{\text{number of zero parameters}}{\text{number of all parameters}}.
\end{equation}
\end{frame}

\begin{frame}
\frametitle{Numerical Results on CIFAR-10}
\begin{table}[H]
\centering
\caption{RDA on different architectures. The models are trained for $120$ epochs. The results from SGD with momentum are in the brackets. }
\begin{tabular}{l|rrrrr}
\hline\hline
Architecture & TOP 1 & TOP 5 & $\sigma$             \\ \hline
VGG16  	  & 89.01 (93.42) & 99.36 (99.79) & \red{$0.76$}  \\
ResNet18  & 90.67 (89.27) &	99.53 (99.57) &	\red{$0.94$}  \\
VGG19     & 89.50 (91.70) &	99.21 (99.40) &	\red{$0.97$} \\
\hline
\end{tabular}
\end{table}
\end{frame}

\begin{frame}
\frametitle{Numerical Results on Different Datasets}
\begin{table}[H]
\centering
\caption{RDA on ResNet18 with different datasets. The models are trained for $120$ epochs, except for CIFAR-100, which is trained for 1900 epochs. The results from SGD with momentum are in the brackets.}
\begin{tabular}{l rrr}
\hline\hline
Dataset & TOP 1 & TOP 5 & $\sigma$   \\ \hline
MNIST   & 99.63 (99.65) & 100.00 & \red{0.95}    \\
CIFAR-10 & 90.67 (89.50) & 99.53 (99.21) &	\red{0.94}  \\
CIFAR-100 & 72.29 (73.69) & 89.94 (92.43) & \red{0.56}  \\
ImageNet & 63.13 (70.58) & 84.92 (89.64) & \red{0.36} \\
\hline
\end{tabular}
\end{table}
\end{frame}

\section{CNN  and Multigrid}
\begin{frame}{Convolutional Neural Networks (CNN)}
\begin{columns}
\begin{column}{0.6\textwidth}
CNN has been successfully used in:
\begin{itemize}
\item Computer vision
\begin{itemize}
\item Classification, detection, segmentation...
\item Medical image processing,
\item Face recognition,
\end{itemize}
\item Reinforcement learning
\begin{itemize}
\item AlphaGo,
\item Automated driving,
\end{itemize}
\item Natural language processing
\begin{itemize}
\item Speech recognition,
\item  Machine translation,
\end{itemize}
\item ...
\end{itemize}
\end{column}

\begin{column}{0.4\textwidth}
\centering
\includegraphics[width=\textwidth, height= 0.4\textheight]{figures/ImageNet}  \\
\centering
\includegraphics[width=\textwidth, height= 0.4\textheight]{figures/Alphago}
\end{column}

\end{columns}

\end{frame}

\begin{frame}{Convolution operation in CNN}
\begin{tabular}{cc}
\includegraphics[width=0.45\textwidth]{figures/CNNLayer1} &
\includegraphics[width=0.45\textwidth]{figures/CNNLayer2}
\end{tabular}

\begin{equation}\label{eq:conv}
(K \ast x)_{i,j} :=\sum_{\ell=1}^c \sum_{s, t = -k}^k  K_{s+k+1,t+k+1,\ell} x_{i + s, j + t,\ell},
\end{equation}
\begin{itemize}
\item $i,j$: essential dimension
\item $\ell$: channel dimension
\end{itemize}
\end{frame}

\begin{frame}{Sub-sampling (coarsening): stride and pooling}
\begin{columns}
\begin{column}{0.5\textwidth}
\begin{center}
\includegraphics[width=.6\textwidth, height=0.4\textheight]{figures/PoolingLayer1}
\end{center}
\begin{itemize}
\item Convolution with stride
{\tiny
$$(K\ast_{s} x)_{i,j} = \sum_{\ell=1}^c\sum_{p, q = -k}^k K_{p+k+1,q+k+1}x_{is + p, js + q} .$$
}
\end{itemize}
\end{column}
\begin{column}{0.5\textwidth}
\begin{center}
\includegraphics[width=.6\textwidth, height=0.4\textheight]{figures/PoolingLayer2}
\end{center}
\begin{itemize}
\item Pooling (max-pooling)
{\tiny
$$r(x)_{i,j} = \max\{x_{(i-1)s + p,(j-1)s + q} ~:~ p,q = 1:s\}.$$
}
\end{itemize}
\end{column}
\end{columns}
\end{frame}

\begin{frame}{Fully-connected layers (DNN)}
\begin{columns}
\column{0.3\textwidth}
\includegraphics[width=\textwidth]{figures/FC}
\column{0.7\textwidth} % remember add this to the other clumn
\begin{minipage}[c][0.45\textheight][c]{\linewidth}
\begin{itemize}
\item Each neuron in the previous layer is connected with each neuron in the latter layer
\item A fully-connected layer can be represented with the following linear function
$$y=Wx+b$$
\begin{itemize}
\item $x$ --- input
\item $y$ --- output
\item $W,b$--- parameters

\end{itemize}
\item Fully-connected layer makes classification with the features extracted by the convolutional layer

\end{itemize}

\end{minipage}
\end{columns}
\end{frame}

\begin{frame}{A complete convolutional neural network example}
\begin{columns}
\column{0.25\textwidth}
\includegraphics[width=\textwidth]{figures/CNN1}
\column{0.7\textwidth} % remember add this to the other clumn
\begin{figure}
\begin{center}
\includegraphics[width=\linewidth]{figures/CNN2}
\caption{First convolutional neural network: LeNet-5 (Y. LeCun, L. Bottou, Y. Bengio, and P. Haffner, 1998)}
\end{center}
\end{figure}
\begin{itemize}
\item Convolutional layer: extract the features of the graphs
\item Fully-connected layer: make classification with the extracted features
\end{itemize}
\end{columns}
\end{frame}

\begin{frame}{Deeper and better}
\begin{figure}
\begin{center}
\includegraphics[width=0.8\textwidth]{figures/Depth}
\end{center}
\caption{Depth in ImageNet \footnote{slide from Kaiming He's presentation}}
\end{figure}
\begin{itemize}
\item  { \blue{Questions:} Why CNN works? Is deeper always better? Why?}
\item Our idea: connection with \blue{multigrid methods}!
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{{\small Convolutional Neural Networks and Multigrid
		Methods
		\footnote{\blue{J. Xu}, Deep Neural Networks and Multigrid
			Methods ({L}ecture {N}otes at {P}enn {S}tate), 2017.}
		\footnote{\blue{J. He}, \blue{J. Xu}, Residual Structure in CNN and
			Multigrid Methods \textsl{in preparation, 2018}.}
} }
\begin{columns}
	\begin{column}{0.4\linewidth}
		\begin{itemize}
			\item Connection between ResNet and MG
			{\small
				\item Hierarchy level of ``grids''
				\item PDE-based model
				\item Residual structures
				\item Smoother \emph{vs.} Convolution
				\item Restriction \emph{vs.} Pooling
				\item Adaptivity \emph{vs.} Training
				\item ...
			}
		\end{itemize}
	\end{column}
	\begin{column}{0.6\linewidth}
		\begin{figure}
			\begin{center}
				\includegraphics[width=.85\textwidth, height=.6\textheight]{figures/DL/Fig-CNN-MG.png}
			%	\caption{V-cycle \emph{vs.} ResNet}
			\end{center}
		\end{figure}
	\end{column}
\end{columns}
\end{frame}

\begin{frame}{MATH 597: Deep Learning}
\begin{description}
\item[Instructor] Professor Jinchao Xu, 314 McAllister Building, http://www.math.psu.edu/xu/
\item[Time] Tuesday and Thursday 12:05pm-1:20pm, 114 McAllister Building
\item[Office hours:] Tuesday and Thursday 3:00pm-4:00pm
\item[PREREQUISITE:]
A good knowledge of multi-variable calculus, linear
algebra and basic numerical analysis

\item[REQUIREMENT:  ]$\quad$

  \begin{itemize}
  \item Homework and programming assignments  (60\%)
\item Take-Home Midterm (20\%) Take-Home Final (20\%).
  \end{itemize}
\item[REFERENCES:]  PDF files of detailed lectures notes will be available to
the  class.
\end{description}

\end{frame}
\begin{frame}
{Topics}
\begin{itemize}
\item Elements of probability and statistics
\item Basic theory of machine learning
\item Basic structure of deep neural networks
\item Approximation properties of DNN
\item Convolutional neural networks
\item Recurrent neural networks
\item Generative Adversarial Networks
\item Stochastic gradient descent methods
\item DNN and finite element methods
\item Multigrid methods
\item MgNet
\item Python
\item MNIST
\item TensorFlow, PyTorch
\end{itemize}
\end{frame}
\end{CJK*}

\end{document}


\documentclass{article}

\title{\sc\LARGE CSCA67 Tutorial, Week 7\\
{\Large Oct. 26th-Oct. 30th, 2015}}
\date{}
\author{\sc Compiled by {\em G. Singh Cadieux}\\[1ex]
\sc Adapted from\\
A. Bretscher, \href{http://www.utsc.utoronto.ca/~bretscher/a67/lectures/implication.pdf}{\em Implication and Direct Proofs worksheet},\\
A. Bretscher, {\em Introduction to Proofs Lecture Notes} \&\\
A. Bretscher, {\em CSCA65 Lecture Notes: Formal Languages - Logic}}

\usepackage{fullpage}
\usepackage{amsmath,amssymb}
\usepackage{color}
\usepackage{multirow}
\usepackage{tikz}
\usepackage{hyperref}
\usepackage{array}

\setlength{\parindent}{0pt}

\begin{document}
\maketitle

\section{\sc Formal Logic}

A \textsc{statement} (also known as a \textsc{proposition}) is a sentence that can be evaluated to be true or false.

\subsection*{Q: {\em Which of the following are statements?}}
\begin{tabular}{p{0.5\textwidth}|p{0.5\textwidth}}
(1) $2\times 4=8$&statement (true)\\\noalign{\smallskip}
(2) $5=9$&statement (false)\\\noalign{\smallskip}
(3) It will snow this afternoon.& statement (value is unknown, but must evaluate to true or false)
\\\noalign{\smallskip}
(4) If I am happy, then I am not happy.& statement (false)\\\noalign{\smallskip}
(5) Give the definition of a statement.& not a statement\\\noalign{\smallskip}
(6) This statement is false.& not a statement (cannot be either true or false - if it is true, then it is false; if it is false, then it is true)\\\noalign{\smallskip}
(7) This statement is true.& not a statement (value cannot be determined, could be evaluated as either true or false)
\end{tabular}\\[1em]
We represent statements using symbols (which are often arabic or Greek letters).\\
For example, we can let $p$ represent the statement ``It is raining," and $q$ represent the statement ``I have an umbrella."\\[1em]
We can build more complex statements (known as ``compound statements") by combining statements using any of the following \textsc{connectives}, or \textsc{operators}.\\[1ex]
\begin{tabular}{ccclc}
Symbol&Meaning&\multicolumn{2}{l}{Example}&\multirow{6}{*}{\begin{tikzpicture}
\draw[<->,dotted] (0.1,0) -- (0.1,2.1);
\node at (0.1,2.2) {\small highest};
\node[align=center] at (0.1,-0.2) {\small lowest\\\small precedence};
\end{tikzpicture}}
\\\cline{1-4}\noalign{\addvspace{1pt}}\cline{1-4}
$\neg$&``not"&$\neg p$& It is \textit{not} raining.&\\
$\wedge$&``and"&$p\wedge q$& It is raining \textit{and} I have an umbrella.&\\
$\vee$&``or"&$p\vee q$& It is raining \textit{or} I have an umbrella.&\\
$\to$&``implies"&$p\to q$& \textit{If} it is raining, \textit{then} I have an umbrella.&\\
$\leftrightarrow$&``if and only if"&$p\leftrightarrow q$& It is raining \textit{if and only if} I have an umbrella.
\end{tabular}\\[2em]
We may also use parentheses to group statements and connectives. When parentheses are omitted, the connectives are applied according to the precedence rules (also called the ``order of operations").\\[1ex]
For example, when $\neg A\wedge B$ is parenthesized, it becomes $\neg(A)\wedge B$, since $\neg$ has higher precedence than $\wedge$. Note that this is entirely different from $\neg(A\wedge B)$.\\
Likewise, when $A\wedge B\to\neg C\vee D\wedge E$ is parenthesized, according to the precedence rules, it becomes $(A\wedge B)\to((\neg C)\vee (D\wedge E))$.\\[1ex]
When an operator is repeated, sub-expressions are usually grouped to the right. For example, when $A\to B\to C$ is parenthesized, it becomes $A\to(B\to C)$.

\subsection*{{\normalsize Given the statements\\
$p:$ ``You are in Seoul."\\
$q:$ ``You are in Kwangju."\\
$r:$ ``You are in South Korea."}\\[1ex]
Q: {\em Translate the following statement into formal logic:\\
``If you are not in South Korea, then you are not in Seoul or Kwangju."}}
We start with the most general statement: from the language ``if\ldots then\ldots", we know that the statement is an implication, with ``you are not in South Korea" as the antecedent (first half) and ``you are not in Seoul or Kwangju" as the consequent (second half).\\[1ex]
``You are not in South Korea" is the negation of $r$, and ``you are not in Seoul or Kwangju" is the negation of ``you are in Seoul or you are in Kwangju".\\[1ex]
``You are in Seoul or you are in Kwangju" is the disjunction (``or") of $p$ and $q$.\\[1ex]
Thus, in formal logic, our statement is $r\to\neg(p\vee q)$.\\[1ex]
Alternatively, because of the ambiguity of the English language, we may consider ``you are not in Seoul or Kwangju" to be the conjunction (``and") of ``you are not in Seoul" and ``you are not in Kwangju", which are, respectively, the negation of $p$ and $q$. Then our statement would be $r\to\neg p\wedge\neg q$.

\subsection*{Q: {\em Translate the following formal statement into everyday English:
$q\to(r\wedge\neg p)$}}
Here, we start with the most specific statement: $\neg p$ is the negation of ``you are in Seoul", which we express in English as ``you are not in Seoul".\\[1ex]
Then, $r\wedge\neg p$ is the conjunction (``and") of ``you are not in Seoul" and ``you are in South Korea", which we express as ``you are in South Korea and you are not in Seoul".\\[1ex]
Finally, we translate the implication to ``if\ldots then\ldots", with ``you are in Kwangju" as the antecedent (first half) and ``you are in South Korea and you are not in Seoul" as the consequent (second half).\\[1ex]
Thus, in English, our statement is ``If you are in Kwangju, then you are in South Korea and you are not in Seoul".\\[1ex]
Because of the ambiguity of the English language, there are several equivalent ways to structure this sentence. For example, we might say ``If you are in Kwangju, then you are in South Korea \textit{but} you are not in Seoul", or simply ``If you are in Kwangju, then you are in South Korea but not in Seoul".

\subsection{\sc Truth Tables}
A \textsc{truth table} is a table showing all possible truth values for a statement, depending upon the statements that make it up.\\[1ex]
When we construct a truth table for a statement $A$ containing operator(s) and constituent statements $A_1,\,A_2,\,\ldots$, we show the possible truth values of $A_1,\,A_2,\,\ldots$ on the left side of the table, and the corresponding truth values for $A$ on the right side of the table.\\
Each row of the truth table represents the truth value $A$ given the truth values of $A_1,\,A_2,\,\ldots$ in that row. \\[1ex]
The most basic truth table is the truth table for the single statement $p$, shown below. $p$ can take either a true (T) or false (F) value.\\[1ex]
\textsc{Note} that the order of the rows in a truth table is not significant, although a conventional ordering can make the truth table easier to read.
\begin{figure}[h]
\begin{minipage}{0.06\textwidth}\centering{\begin{tabular}{c}
$p$\\\hline
T\\
F
\end{tabular}}\end{minipage}
\begin{minipage}{0.12\textwidth}\centering{\begin{tabular}{c|c}
$p$&$\neg p$\\\hline
T&F\\
F&T
\end{tabular}}\end{minipage}
\begin{minipage}{0.18\textwidth}\centering{\begin{tabular}{cc|c}
$p$&$q$&$p\wedge q$\\\hline
T&T&T\\
T&F&F\\
F&T&F\\
F&F&F
\end{tabular}}\end{minipage}
\begin{minipage}{0.18\textwidth}\centering{\begin{tabular}{cc|c}
$p$&$q$&$p\vee q$\\\hline
T&T&T\\
T&F&T\\
F&T&T\\
F&F&F
\end{tabular}}\end{minipage}
\begin{minipage}{0.18\textwidth}\centering{\begin{tabular}{cc|c}
$p$&$q$&$p\to q$\\\hline
T&T&T\\
T&F&F\\
F&T&T\\
F&F&T
\end{tabular}}\end{minipage}
\begin{minipage}{0.18\textwidth}\centering{\begin{tabular}{cc|c}
$p$&$q$&$p\leftrightarrow q$\\\hline
T&T&T\\
T&F&F\\
F&T&F\\
F&F&T
\end{tabular}}\end{minipage}
\end{figure}\\
\textsc{A truth table} is particularly useful for assessing the possible truth values of a complex statement (i.e., one which contains many operators and/or constituent statements). When constructing such a truth table, we break the statement into smaller clauses and assess the truth value of each, before combining them into the original statement.\\
For example, here is a (partial) truth table for $(\neg r\vee q)\wedge p$:
\begin{center}
\begin{tabular}{ccc|c|c|c}
$r$&$q$&$p$&$\neg r$&$\neg r\vee q$&$(\neg r\vee q)\wedge p$\\\hline
T&T&T&F&T&T\\
T&T&F&F&T&F\\
T&F&T&F&F&F\\
\multicolumn{3}{c|}{\vdots}&\vdots&\vdots&\vdots
\end{tabular}
\end{center}

\subsection*{{\normalsize Given the statements\\
$p$: ``Andy is hungry."\\
$q$: ``The refrigerator is empty."\\
$r$: ``Andy is mad."}\\[1ex]
Q: {\em Construct a truth table for the following statement:\\
``If Andy is hungry and the refrigerator is empty, then Andy is mad."}}

First, we must translate this statement into formal logic. From the language ``if\ldots then\ldots", we know that the statement is an implication, with ``Andy is hungry and the refrigerator is empty" as the antecedent and ``Andy is mad" as the consequent. ``Andy is hungry and the refrigerator is empty" is the conjunction of $p$ and $q$.\\
Thus, in formal logic, our statement is $p\wedge q\to r$.\\[1ex]
To build our truth table, we begin with the smallest constituent statements, $p$, $q$, and $r$, and detemine all possible combinations of truth values for these.\\[1ex]
Then, we identify the next smallest sub-statement, based on precedence rules and parentheses - in this case, it is $p\wedge q$. We determine the truth values of this statement using the truth values of the statements that make it up - in this case, $p$ and $q$.\\[1ex]
We continue doing this until we have truth values for all sub-statements within our original statement. Finally, we use these truth values to determine the truth values of the original statement.
\begin{center}
\begin{tabular}{cccc>{\color{red}}c>{\color{red}}cc|cccc>{\color{red}}c|>{\color{red}}c|c}
$p$&$q$&$r$&\multirow{6}{*}{$\Rightarrow$}&
$p$&$q$&$r$&$p\wedge q$&\multirow{6}{*}{$\Rightarrow$}&
$p$&$q$&$r$&$p\wedge q$&$p\wedge q\to r$\\
\cline{1-3}\cline{5-8}\cline{10-14}T&T&T&&T&T&T&T&&T&T&T&T&T\\
T&T&F&&T&T&F&T&&T&T&F&T&F\\
T&F&T&&T&F&T&F&&T&F&T&F&T\\
T&F&F&&T&F&F&F&&T&F&F&F&T\\
F&T&T&&F&T&T&F&&F&T&T&F&T\\
F&T&F&&F&T&F&F&&F&T&F&F&T\\
F&F&T&&F&F&T&F&&F&F&T&F&T\\
F&F&F&&F&F&F&F&&F&F&F&F&T
\end{tabular}
\end{center}

\subsection*{{\normalsize Suppose that this statement is true, and that Andy is not mad and the refrigerator is empty.}\\
Q: {\em Is Andy hungry?}}
Each of our assumptions assigns a truth value to a statement in our truth table:
\begin{itemize}
\item ``this statement is true" $\Rightarrow$ $p\wedge q\to r$ is true
\item ``Andy is not mad" $\Rightarrow$ $r$ is false
\item ``the refrigerator is empty" $\Rightarrow$ $q$ is true
\end{itemize}
We are then being asked to find the truth value of $p$, ``Andy is hungry".\\[1ex]
We find the row of our truth table which corresponds to the truth values we know.
\begin{center}
\begin{tabular}{ccc|c|c}
$p$&$q$&$r$&\ldots&$p\wedge q\to r$\\\hline
T&T&T&\multirow{8}{*}{\ldots}&T\\
T&T&F&&F\\
T&F&T&&T\\
T&F&F&&T\\
F&T&T&&T\\
\color{red}F&\color{red}T&\color{red}F&&\color{red}T\\
F&F&T&&T\\
F&F&F&&T
\end{tabular}
\end{center}
We can see that $p$ is false in this row. Thus, Andy is \textit{not} hungry.

\subsection{\sc Logical Equivalence}
Two statements are \textsc{logically equivalent} if they have the same truth table - that is, given the same combination of truth values for their constituent statements, they both have the same truth value.\\[1ex]
For example, using truth tables, we can demonstrate that $p$ and $\neg\neg p$ are logically equivalent:
\begin{center}
\begin{tabular}{ccc|c|c}
$p$&&$p$&$\neg p$&$\neg\neg p$\\
\cline{1-1}\cline{3-5} \color{red}T&&T&F&\color{red}T\\
\color{red}F&&F&T&\color{red}F
\end{tabular}
\end{center}
Likewise, we can demonstrate that $\neg y\wedge (y\vee x)$ and $\neg y\wedge x$ are logically equivalent:
\begin{center}
\begin{tabular}{cc|c|c|c||c}
$x$&$y$&$\neg y$&$y\vee x$&$\neg y\wedge (y\vee x)$&$\neg y\wedge x$\\\hline
T&T&F&T&\color{red}F&\color{red}F\\
T&F&T&T&\color{red}T&\color{red}T\\
F&T&F&T&\color{red}F&\color{red}F\\
F&F&T&F&\color{red}F&\color{red}F\\
\end{tabular}
\end{center}

\subsection*{Q: {\em Show that $\neg(p\wedge q)$ is logically equivalent to $\neg p\vee\neg q$ using truth tables.}}
\begin{center}
\begin{tabular}{cc|c|c||c|c|c}
$p$&$q$&$p\wedge q$&$\neg(p\wedge q)$&$\neg p$&$\neg q$&$\neg p\wedge\neg q$\\\hline
T&T&T&F&F&F&F\\
T&F&F&T&F&T&T\\
F&T&F&T&T&F&T\\
F&F&F&T&T&T&T
\end{tabular}
\end{center}
$\neg(p\wedge q)$ and $\neg p\vee\neg q$ have the same truth table - thus, they are logically equivalent statements.

\subsection{\sc Using Venn Diagrams}
We can consider any statement $a$ to be the statement ``$x\in A$", where $A$ is a set and $x$ is an element.\\[1ex]
Then, if we draw a Venn diagram containing $A$, $a$ is true at every location in the diagram where an element $x$ at that location is in $A$. $a$ is false everywhere else.
\begin{center}
\begin{tikzpicture}
\draw[fill=red!50] (0,0) circle[radius=0.5in];
\draw (-.65in,-.65in) rectangle (.65in,.65in);
\node at (0,0) {A};
\node at (-.5in,.5in) {$\mathcal{U}$};
\node[pin={above right:$a$ is true}] at (.25in,.25in) {};
\end{tikzpicture}
\end{center}

\subsection*{Q: {\em Shade the region(s) where each of the following is true: $\neg a,\;\neg a\vee a,\;\neg a\wedge a$.}}
\begin{center}
\begin{tikzpicture}
\draw[fill=red!50] (-.65in,-.65in) rectangle (.65in,.65in);
\draw[fill=white] (0,0) circle[radius=0.5in];
\node at (0,0) {A};
\node at (-.5in,.5in) {$\mathcal{U}$};
\node at (0,-.8in) {$\neg(x\in A)$};
\end{tikzpicture}\qquad
\begin{tikzpicture}
\draw[fill=red!50] (-.65in,-.65in) rectangle (.65in,.65in);
\draw(0,0) circle[radius=0.5in];
\node at (0,0) {A};
\node at (-.5in,.5in) {$\mathcal{U}$};
\node at (0,-.8in) {$\neg(x\in A)\vee(x\in A)$};
\end{tikzpicture}\qquad
\begin{tikzpicture}
\draw (0,0) circle[radius=0.5in];
\draw (-.65in,-.65in) rectangle (.65in,.65in);
\node at (0,0) {A};
\node at (-.5in,.5in) {$\mathcal{U}$};
\node at (0,-.8in) {$\neg(x\in A)\wedge(x\in A)$};
\end{tikzpicture}
\end{center}
\textsc{Notice} that $\neg a\vee a$ is true for every region in the diagram. This is because, regardless of the truth value of $a$ (or of its constituent statements), $\neg a\vee a$ is always true. This type of statement is known as a \textit{tautology}.\\[1ex]
Notice also that $\neg a\wedge a$ is false for every region in the diagram. This is because, regardless of the truth value of $a$ (or of its constituent statements), $\neg a\wedge a$ is always false. This type of statement is known as a \textit{contradiction}.\\[1em]
We follow the same process to represent a statement with multiple constituent statements.\\
For example, to create a Venn diagram representing $a\wedge b$, we let $a$ be the statement ``$x\in A$" and $b$ be the statement ``$x\in B$", where $A$ and $B$ are sets and $x$ is an element. Then
\begin{center}
\begin{tikzpicture}
\draw[fill=red!50] (0,0) circle[radius=0.5in];
\draw (0.75in,0) circle[radius=0.5in];
\draw (-.65in,-.65in) rectangle (1.4in,.65in);
\node at (0,0) {A};
\node at (0.75in,0) {B};
\node at (-.5in,.5in) {$\mathcal{U}$};
\node at (0.375in,-.8in) {$x\in A$};
\end{tikzpicture}\quad
\begin{tikzpicture}
\draw[fill=red!50] (0.75in,0) circle[radius=0.5in];
\draw (0,0) circle[radius=0.5in];
\draw (-.65in,-.65in) rectangle (1.4in,.65in);
\node at (0,0) {A};
\node at (0.75in,0) {B};
\node at (-.5in,.5in) {$\mathcal{U}$};
\node at (0.375in,-.8in) {$x\in B$};
\end{tikzpicture}\quad
\begin{tikzpicture}
\draw (-.65in,-.65in) rectangle (1.4in,.65in);
\node at (-.5in,.5in) {$\mathcal{U}$};
\begin{scope}
\path[clip] (0,0) circle[radius=0.5in];
\draw[fill=red!50] (0.75in,0) circle[radius=0.5in];
\end{scope}
\draw (0,0) circle[radius=0.5in];
\draw (0.75in,0) circle[radius=0.5in];
\node at (0,0) {A};
\node at (0.75in,0) {B};
\node at (0.375in,-.8in) {$(x\in A)\wedge (x\in B)$};
\end{tikzpicture}
\end{center}

\subsection*{Q: {\em Shade the region(s) where $a\to b$ is false.}}
\begin{center}
\begin{tikzpicture}
\draw[fill=red!50] (0,0) circle[radius=0.5in];
\draw (0.75in,0) circle[radius=0.5in];
\draw (-.65in,-.65in) rectangle (1.4in,.65in);
\node at (0,0) {A};
\node at (0.75in,0) {B};
\node at (-.5in,.5in) {$\mathcal{U}$};
\node at (0.375in,-.8in) {$x\in A$};
\end{tikzpicture}\quad
\begin{tikzpicture}
\draw[fill=red!50] (0.75in,0) circle[radius=0.5in];
\draw (0,0) circle[radius=0.5in];
\draw (-.65in,-.65in) rectangle (1.4in,.65in);
\node at (0,0) {A};
\node at (0.75in,0) {B};
\node at (-.5in,.5in) {$\mathcal{U}$};
\node at (0.375in,-.8in) {$x\in B$};
\end{tikzpicture}\quad
\begin{tikzpicture}
\draw[fill=red!50] (0,0) circle[radius=0.5in];
\draw[fill=white] (0.75in,0) circle[radius=0.5in];
\draw (0,0) circle[radius=0.5in];
\draw (-.65in,-.65in) rectangle (1.4in,.65in);
\node at (0,0) {A};
\node at (0.75in,0) {B};
\node at (-.5in,.5in) {$\mathcal{U}$};
\node at (0.375in,-.8in) {$x\in A$ but $x\notin B$};
\end{tikzpicture}
\end{center}
\textsc{Notice} that this region is the region in which $a$ is true but $b$ is false. Everywhere else (that is, for every other combination of truth values for $a$ and $b$), the implication is true.

\section{\sc Additional practice problems}
{\bf Q: Which of the following are valid propositions?}
\begin{itemize}
\item $2+2=4$
\item $2+3=7$
\item If it is sunny tomorrow, I will go to the beach.
\item What is going on?
\item Stop at the red light.
\end{itemize}
{\bf Q: Which of the following propositions are true?}
\begin{itemize}
\item If $a,b,c$ are two sides and the hypotenuse of a triangle, then $a^2+b^2=c^2$.
\item If $2+2=4$ then pigs can fly.
\item If pigs can fly then pigs can get sunburned.
\item If $2+2=5$ then $2+2=4$.
\end{itemize}
{\bf Q: Use truth tables to prove the following \textit{distributive} properties:}\\[1ex]
$p\wedge(q\vee r)$ is logically equivalent to $(p\wedge q)\vee(p\wedge r)$.\\[1ex]
$p\vee(q\wedge r)$ is logically equivalent to $(p\vee q)\wedge(p\vee r)$.\\[1em]\pagebreak\\
{\bf Q: Use truth tables to prove the following \textit{associative} properties:}\\[1ex]
$p\vee(q\vee r)$ is logically equivalent to $(p\vee q)\vee r$.\\[1ex]
$p\wedge(q\wedge r)$ is logically equivalent to $(p\wedge q)\wedge r$.\\[1em]
There are 3 boxes A, B, C. Exactly one contains gold.\\
Each box has a message on top, but only one of the messages is true.\\
Box A: \textit{Gold is not in this box.}\qquad Box B: \textit{Gold is not in this box.}\qquad Box C: \textit{Gold is in box A.}\\
{\bf Q: Use implication to determine which box the gold is in.}\\[1ex]
\underline{Hint}: Start with \{$p$: Gold is in box A, $q$: Gold is in box B, $r$: Gold is in box C\},
and rewrite the messages using these propositions.\\[1em]
{\bf Q: Construct a truth table for the following compound statement:\\
$\neg(\neg(X\vee \neg Y\vee\neg Z) \vee (\neg X\wedge Y\wedge\neg Z))\vee (\neg X\vee (\neg Y\wedge Z))$\\[1em]
{\bf Q: Use a truth table to prove that the statement $((p\vee q)\wedge(\neg p))\to q$ is always true, no matter what $p$ and $q$ are.}

\end{document}


\documentclass{article} % For LaTeX2e
\usepackage{nips15submit_e,times}
\usepackage{hyperref}
\usepackage{url}
\usepackage{amsmath}
\usepackage{graphicx}
%\documentstyle[nips14submit_09,times,art10]{article} % For LaTeX 2.09

\title{Weekly Report(February.25,2019-March.1,2019)}
\author{Rui Shaopu}

% The \author macro works with any number of authors. There are two commands
% used to separate the names and addresses of multiple authors: \And and \AND.
%
% Using \And between authors leaves it to \LaTeX{} to determine where to break
% the lines. Using \AND forces a linebreak at that point. So, if \LaTeX{}
% puts 3 of 4 authors names on the first line, and the last on the second
% line, try using \AND instead of \And before the third author name.

\newcommand{\fix}{\marginpar{FIX}}
\newcommand{\new}{\marginpar{NEW}}

\begin{document}
\maketitle

\begin{abstract}
During the winter holiday, I learned several optimization algorithms first, then I also learnt a lot about classifiers and finished assignment 1 of cs231n.
\end{abstract}

\section{Learnt and Did}

Mainly I learned about classifiers and optimization algorithms. And I finished assignment 1, which gives me a big challenge, since I'm not familiar with python.

\subsection{Optimization Algorithms}
\subsubsection{Gradient Descent}
Usually we use this way to find the maximum value or the minimum value. We shall make it when the function is convex function.

Main formula:
\begin{center}
${\displaystyle J(\theta) = \frac{1}{2m}\sum_{i=1}^{m}(y_i-h_\theta(x_i))^2}$
\end{center}
partial derivation:
\begin{center}
${\displaystyle \theta'_j=\theta_j+(y_j-h_\theta(x_i))x_j^i}$
\end{center}
To get the value we want, we need to update parameters in the oppsite direction of gradient.

\subsubsection{Logistic Regression}
It is usually used to dedal with two classification problems since its unique attributes.

Main formula:
\begin{center}
${\displaystyle Y_w=\frac{1}{1+e^{-W^TX}}}$
\end{center}
\begin{center}
${\displaystyle L(W)=-\frac{1}{m}\log({\prod_{i=1}^{m}[Y_w(X_i)]^{y_i}[1-Y_w(X_i)]^{1-y_i}})}$
\end{center}
Iterative formula($\alpha$ is learning rate):
\begin{center}
${\displaystyle W_j=\frac{\alpha}{m}\sum_{i=1}^{m}(Y_w(xi)-y_i)x_{ij}}$
\end{center}

\subsubsection{Newton's method}
Newton's method drops faster than gradient descent. However, it works bad when dealing with hessian matrix.

Main formula:
\begin{center}
${\displaystyle x_{n+1}=x_n-\frac{f'(x_n)}{f"(x_n)}}$
\end{center}

Hessian matrix:
\begin{center}
${\displaystyle x_{n+1}=x_n-\frac{J_f(x_n)}{H(x_n)}}$

${\displaystyle J_f(x_n)=
\begin{bmatrix}
\frac{\partial y_1}{\partial x_1}      & \cdots & \frac{\partial y_1}{\partial x_n}      \\
\vdots & \ddots & \vdots \\
\frac{\partial y_n}{\partial x_1}      & \cdots & \frac{\partial y_n}{\partial x_n}
\end{bmatrix}
}$

${\displaystyle H(x_n)=
\begin{bmatrix}
\frac{\partial^2 f}{\partial x_1^2}   & \cdots & \frac{\partial^2 f}{\partial x_1 \partial x_n}      \\
\vdots & \ddots & \vdots \\
\frac{\partial^2 f}{\partial x_n \partial x_1}   & \cdots & \frac{\partial^2 f}{\partial x_n^2}
\end{bmatrix}
}$
\end{center}

\subsection{Classifiers}

\subsubsection{KNN}
KNN, K Nearest Neighbor, One of the most easiest classifiers.

\begin{figure}[h]
	\centering  %插入的图片居中表示
	\includegraphics[width=.4\textwidth]{5.png}
	\caption{2-dimension}  %图片的名称
	\label{fig:f5}   %标签，用作引用
\end{figure}
In the training process of this algorithm, it stores all the input and output labels of training samples. in the test process, the distance between the test sample and each training sample L1 or L2 is calculated, then the first k training sample closest to test sample are selected. Then we'll vote on the label of the k samples, the category with the largest number of votes is categorized as test samples.

KNN is very easy, however it gets a bad result with classification accuracy of 27 percent.

\subsubsection{SVM}
SVM, Support Vector Machine, is a kind of generalized linear classifier which classifies data by supervised learning. Its decision boundary is the maximum margin hyperplane for solving learning samples.

Under normal conditions, we use linear-SVM to classify two kinds of data, between which there is an obvious interval. However, if it doesn't exist, we need to find it in higher dimension. The figures below show it.

\begin{figure}[h]
	\centering  %插入的图片居中表示
	\includegraphics[width=.4\textwidth]{1.jpg}
	\caption{2-dimension}  %图片的名称
	\label{fig:f1}   %标签，用作引用
\end{figure}
\begin{figure}[h]
	\centering  %插入的图片居中表示
	\includegraphics[width=.4\textwidth]{2.jpg}
	\caption{higher dimension}  %图片的名称
	\label{fig:f2}   %标签，用作引用
\end{figure}

\subsubsection{Softmax}
In mathematics, the softmax function, also known as softargmax or normalized exponential function, is a function that takes as input a vector of K real numbers, and normalizes it into a probability distribution consisting of K probabilities.

That is, after applying softmax, each component will be in the interval(0,1), and the components will add up to 1, so that they can be interpreted as probabilities. Furthermore, the larger input components will correspond to larger probabilities. Softmax is often used in neural networks and Often acts as an activation function of a neural network

The standard (unit) softmax function ${\displaystyle \sigma : {R} ^{K}\to {R} ^{K}}$ is defined by the formula

${\displaystyle \sigma (\mathbf {z} )_{j}={\frac {e^{z_{j}}}{\sum _{k=1}^{K}e^{z_{k}}}}}$
for j = 1, …, K and ${\displaystyle {\mathbf {z}}=(z_{1},\ldots ,z_{K})\in  {R} ^{K}}$

\subsubsection{Neural Network}
Neural network is a very important part of machine learning. As I know, Convolutional neural network is the most powerful one. In assignment 1, I need to finish a two-layer neural network.

Formula:

Hyperplane: ${\displaystyle y = W X+b}$

Activation function: ${\displaystyle f_{sigmod} = \frac{1}{1+e^{z}}}$

Activation function derivation: ${\displaystyle f'_{sigmod} = f_{sigmod}(1-f_{sigmod})}$

\begin{figure}[h]
	\centering  %插入的图片居中表示
	\includegraphics[width=.4\textwidth]{4.png}
	\caption{2-dimension}  %图片的名称
	\label{fig:f3}
\end{figure}

\section{Problems}
The best problem I got is that I'm not familiar with Python libraries, such as numpy, matplotlib and so on. So was matrix operation. It wasted me a lot of time dealing with them and they still troubles me too.

\section{Plan for Next Weeks}
In the following weeks, I will keep going and finish assignment 2. I will still learn to use python and latex familiarly.

\end{document}


\documentclass[a4paper, titlepage, 11pt]{article}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{amsfonts}
\usepackage{graphicx}
\usepackage[left=3cm,right=3cm,bottom=3.5cm,top=3.5cm]{geometry}
\usepackage{fancyhdr}
\usepackage{subcaption}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage{float}
\usepackage{breqn}
\setlength{\headheight}{14.5pt}

% Theorem-like environments
	\let\oldref\ref
	\renewcommand{\ref}[1]{(\oldref{#1})}
	\newtheorem{theorem}{Theorem}[section]
	\newtheorem{cor}[theorem]{Corollary}
	\newtheorem{lemma}[theorem]{Lemma}
	\newtheorem{prop}[theorem]{Proposition}
	\newtheorem{remark}[theorem]{Remark}
	\theoremstyle{definition}
	\newtheorem*{note}{Note}
	\newtheorem{defn}[theorem]{Definition}
	\newtheorem{ex}[theorem]{Example}

	\newenvironment{solution}{\renewcommand\qedsymbol{$\square$}\begin{proof}[Solution]}{\end{proof}}

% Set header and footer
%	\pagestyle{fancy}
%	\fancyhf{}

% I like footnotes to be superscript letters rather than numbers
	\renewcommand{\thefootnote}{\alph{footnote}}
	\setlist[enumerate,1]{label={\roman*)}}

%Swap bullets for long dash in itemize
	\renewcommand\labelitemi{---}
	\renewcommand\qedsymbol{\(\blacksquare\)}

% Typesets derivatives nicely using roman d. Adapts size automatically. Stolen from https://tex.stackexchange.com/questions/135944/commath-and-ifinner/135985#135985
% \od{y}{x} is ordinary derivative, \pd{f}{x} is partial derivative. See commath documentation for others
	\usepackage{amsmath}
	\newcommand{\dif}{\mathop{}\!\mathrm{d}}
	\newcommand{\Dif}{\mathop{}\!\mathrm{D}}

	\makeatletter
	\newcommand{\spx}[1]{%
		\if\relax\detokenize{#1}\relax
		\expandafter\@gobble
		\else
		\expandafter\@firstofone
		\fi
		{^{#1}}%
	}
	\makeatother

	\newcommand\pd[3][]{\frac{\partial\spx{#1}#2}{\partial#3\spx{#1}}}
	\newcommand\tpd[3][]{\tfrac{\partial\spx{#1}#2}{\partial#3\spx{#1}}}
	\newcommand\dpd[3][]{\dfrac{\partial\spx{#1}#2}{\partial#3\spx{#1}}}

	\newcommand{\md}[6]{\frac{\partial\spx{#2}#1}{\partial#3\spx{#4}\partial#5\spx{#6}}}
	\newcommand{\tmd}[6]{\tfrac{\partial\spx{#2}#1}{\partial#3\spx{#4}\partial#5\spx{#6}}}
	\newcommand{\dmd}[6]{\dfrac{\partial\spx{#2}#1}{\partial#3\spx{#4}\partial#5\spx{#6}}}

	\newcommand{\od}[3][]{\frac{\dif\spx{#1}#2}{\dif#3\spx{#1}}}
	\newcommand{\tod}[3][]{\tfrac{\dif\spx{#1}#2}{\dif#3\spx{#1}}}
	\newcommand{\dod}[3][]{\dfrac{\dif\spx{#1}#2}{\dif#3\spx{#1}}}

	\newcommand{\genericdel}[4]{%
		\ifcase#3\relax
		\ifx#1.\else#1\fi#4\ifx#2.\else#2\fi\or
		\bigl#1#4\bigr#2\or
		\Bigl#1#4\Bigr#2\or
		\biggl#1#4\biggr#2\or
		\Biggl#1#4\Biggr#2\else
		\left#1#4\right#2\fi
	}
	\newcommand{\del}[2][-1]{\genericdel(){#1}{#2}}
	\newcommand{\set}[2][-1]{\genericdel\{\}{#1}{#2}}
	\let\cbr\set
	\newcommand{\sbr}[2][-1]{\genericdel[]{#1}{#2}}
	\let\intoo\del
	\let\intcc\sbr
	\newcommand{\intoc}[2][-1]{\genericdel(]{#1}{#2}}
	\newcommand{\intco}[2][-1]{\genericdel[){#1}{#2}}
	\newcommand{\eval}[2][-1]{\genericdel.|{#1}{#2}}
	\newcommand{\envert}[2][-1]{\genericdel||{#1}{#2}}
	\let\abs\envert
	\newcommand{\sVert}[1][0]{%
		\ifcase#1\relax
		\rvert\or\bigr|\or\Bigr|\or\biggr|\or\Biggr
		\fi
	}
	\newcommand{\enVert}[2][-1]{\genericdel\|\|{#1}{#2}}
	\let\norm\enVert
	\newcommand{\fullfunction}[5]{%
		\begin{array}{@{}r@{}r@{}c@{}l@{}}
			#1 \colon & #2 & {}\longrightarrow{} & #3 \\
			& #4 & {}\longmapsto{}     & #5
		\end{array}
	}

\title{Langevin Monte Carlo \\
  \large and the curse of dimensionality}
\author{B. Han, T.M. Hodgson, M. Holden \& M. Puza\\supervised by\\Dr Sotirios Sabanis}

% Define new commands for typesetting here
\newcommand{\R}{\mathbb{R}}
\renewcommand{\L}{\mathcal{L}}
\renewcommand{\P}{\mathbb{P}}
%\renewcommand{\epsilon}{\varepsilon}
\newcommand{\e}{\mathrm{e}}
\newcommand{\grad}{\nabla}
\newcommand{\E}{\mathbb{E}}

\begin{document}
	\maketitle
	\section*{Abstract}
	Monte Carlo methods are introduced, with a focus on Langevin-based samplers (Section \ref{sec:LMC}). The method of taming is studied and compared to Metropolised algorithms using a variety of metrics (Sections \ref{sec:MomentErrors},\ref{sec:Beyond}). Our main contribution is an extension of the paper by Brosse et al. (2018) on the tamed unadjusted Langevin algorithm, via an open source \textsc{Python} package for comparison of a variety of Langevin Monte Carlo (LMC) algorithms on a range of distributions. We have also extended the visualisation library of \cite{rogozhnikov2016hmc} to include LMC algorithms. This is presented in Section \ref{sec:Imp}, along with a discussion of the implemented metrics and pitfalls.
	\section*{Author Contribution}
	Sections \ref{sec:intro} and \ref{sec:Beyond} were written by Matthew.  Tom wrote \ref{sec:LMC}, \ref{sec:MomentErrors} and \ref{subsec:future}.  Marko wrote 5.2
	- 5.5. Bowen wrote \ref{overviewprogram} and \ref{subsec:SGLD}.\\
	The \textsc{Python} package available on GitHub was contributed to by Tom, Matthew and Marko, and is primarily the work of Marko.
	\tableofcontents
	\newpage
	\section{Introduction}\label{sec:intro}
	\input{Motivation}

	\section{Langevin Monte Carlo Algorithms}\label{sec:LMC}
	\input{algotheory}

    \section{Approximation Error}\label{sec:MomentErrors}
    \input{MomentErrors}

	\section{Beyond Moments}\label{sec:Beyond}
	\input{BeyondMoments}

	\section{Implementation} \label{sec:Imp}
	\input{WriteUp/Implementation}

    \section{Conclusion}
    \input{conclusion}

    \subsection{Stochastic Gradient Langevin Dynamics} \label{subsec:SGLD}
    \input{report}
    \section{Acknowledgements}
    The authors were supported by The Maxwell Institute Graduate School in Analysis and its Applications, a Centre for Doctoral Training funded by the UK Engineering and Physical Sciences Research Council (grant EP/L016508/01), the Scottish Funding Council, Heriot-Watt University and the University of Edinburgh

	\bibliography{langevinMC.bib}
	\bibliographystyle{plain}

\end{document}