\documentclass[notheorems]{beamer}

\usepackage[english]{babel}

\usepackage{amsmath}
\usepackage{amssymb}

% Typesetting evaluation rules (http://cristal.inria.fr/~remy/latex/)
\usepackage{mathpartir}

\usepackage[T1]{fontenc}
\usepackage{ae,aecompl}

\usepackage{listings}

\usepackage{beamerthemesplit}

\setbeamertemplate{background canvas}[vertical shading][bottom=red!10,top=blue!10]
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{headline}{}
\usetheme{Warsaw}
\useinnertheme{rectangles}

\colorlet{darkred}{red!80!black}
\colorlet{darkblue}{blue!80!black}
\colorlet{darkgreen}{green!80!black}

\lstdefinelanguage{Coq}{
  mathescape=true,
  texcl=false,
  keywords={Require, Declare},
  morekeywords={forall, exists, with, match, let, in, if, then, else},
  morekeywords={
    destruct, change, Variable, Inductive, CoInductive, Fixpoint, CoFixpoint, Definition,
    Lemma, Theorem, Proof, Axiom, Local, Save, Grammar, Syntax, Intro, Trivial,
    Qed, Intros, Symmetry, Simpl, Rewrite, Apply, Elim, Assumption, Left,
    Cut, Case, Auto, Unfold, Exact, Right},
  morekeywords={Section, Module, End},
  %emph={[1]Type, Set, nat, bool}, emphstyle={[1]\textit},
  comment=[s]{(*}{*)},
  showstringspaces=false,
}%[keywords,comments,strings]%

\lstset{
  numbers=none,
  basicstyle=\footnotesize\ttfamily,
  frame=tb,
  language=Coq,
  captionpos=b,
  xleftmargin=0em,
  xrightmargin=0em,
  aboveskip=1em,
  belowskip=1em
}

\title{Names and Numbers in Binding}

\author{Martijn Vermaat}
\institute{mvermaat@cs.vu.nl\\
http://www.cs.vu.nl/\~{}mvermaat/}
\date{Literature Study\\
March 13, 2009}

\begin{document}

\frame{\titlepage}

\frame{

  \frametitle{Names and Numbers in Binding}

  \tableofcontents

}

\section{Mechanical Reasoning about Languages}

\frame{

  \frametitle{Mechanical Reasoning}

  Shift from on-paper reasoning to mechanical reasoning:

  \begin{itemize}
    \item History of on-paper proofs and ideas
    \item Informal mechanical implementations of ideas
    \item Add a scale increase and we have a gap
  \end{itemize}

  \uncover<2->{
    \begin{block}{Goal}
      Let's make rigorous mechanical reasoning possible.
    \end{block}
  }

}

\frame{

  \frametitle{Reasoning about Languages}

  \begin{block}{Reasoning about languages}
    Often not intrinsically hard, but cumbersome in a mechanical setting.
  \end{block}

  Why?
  \uncover<2->{
    \begin{itemize}
    \item Most languages have a notion of binding
    \item Bindings and bound variables are easy on paper, hard on a computer
    \end{itemize}
  }

}

\frame{

  \frametitle{Mechanical Reasoning about Languages}

  So we want to
  \begin{quote}
    Reason about terms with bindings in tools like Coq, in a way that is close
    to the on-paper way.
  \end{quote}

  \uncover<2->{
    We need a representation for binders and variables.
  }

}

\section{Representing Bindings}

\frame{

  \frametitle{Names and Numbers in Binding}

  \tableofcontents[currentsection]

}

\frame{

  \frametitle{Representing Bindings}

  Classical problems related to binders and variables:

  \begin{itemize}
    \item $\alpha$-equivalence
    \item $\alpha$-conversion (e.g. in substitution)
  \end{itemize}

  \uncover<2->{
    Let's look at some representations.
    \begin{block}{Running example}
      Substitution in untyped $\lambda$-calculus
    \end{block}
  }

}

\subsection{Named Variables}

\frame{

  \frametitle{Names and Numbers in Binding}

  \tableofcontents[currentsubsection]

}

\frame{

  \frametitle{Traditional Representation 1/2}

  Traditional representation with named variables:
  \begin{align*}
    M ::=             &\; x
    && \text{variable} \\
    \llap{\textbar\:} &\; \lambda x.M
    && \text{abstraction} \\
    \llap{\textbar\:} &\; M \; M
    && \text{application} \\
  \end{align*}
  Abstractions introduce names.

}

\frame{

  \frametitle{Traditional Representation 2/2}

  \begin{itemize}

    \item $\alpha$-equivalent terms are routinely identified

    \item Substitution $M[N/x]$:
      \begin{align*}
        x[N/x]              &= N\\
        y[N/x]              &= y                  && \text{$x \neq y$} \\
        (\lambda y.M')[N/x] &= \lambda y.M'[N/x]  && \text{$x \neq y$ and $y$ not free in $N$} \\ % TODO: fix layout
        (M_1 \; M_2)[N/x]   &= M_1[N/x] \; M_2[N/x]
      \end{align*}

    \item Now implement this
      % alpha-quivalence is very hard
      % side-conditions are declarative
      % (\x.M)[N/x] not even defined
      % use variable convention, or a more general substitution (next slide)

  \end{itemize}

}

\frame{

  \frametitle{Simple Substitution}

  Use $\alpha$-conversion to rename bound variables and define
  substituting $N$ for $x$ in $M$ inductively$^*$ on M:

  \begin{align*}
    x[N/x]              &= \text{$N$ if $x = y$, $y$ otherwise}\\
    (\lambda y.M')[N/x] &= \lambda z.M'[z/y][N/x]  && \text{$z$ not free in $N, M'$} \\ % TODO: fix layout
    (M_1 \; M_2)[N/x]   &= M_1[N/x] \; M_2[N/x]
  \end{align*}

  % Note that M'[z/y] is not a subterm of M

  \uncover<2->{
    Already difficult enough to read, but just what we would intuitively do.
    So on paper, we can get by with some handwaving.
  }

}

\begin{frame}[fragile]

  \frametitle{Named Variables in Coq}

  Term datatype:
  \begin{lstlisting}
Inductive term : Set :=
  | Var : name -> term
  | Abs : name -> term -> term
  | App : term -> term -> term.
  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Substitution in Coq}

  Simple substitution:
  \begin{lstlisting}
Fixpoint subst (t:term) (n:name) (t':term)
  {struct t'} : term :=
  match t' with
  | Var x =>
      if eq_name x n then t else t'
  | Abs x b =>
      let z := fresh_name
                 (n :: (free_vars t) ++ (free_vars b))
      in
      Abs z (subst t n (rename x z b))
  | App f a =>
      App (subst t n f) (subst t n a)
end.
  \end{lstlisting}
  \uncover<2->{
    \emph{But this is ill-defined.}
  }

\end{frame}

\begin{frame}[fragile]

  \frametitle{Recursion on Term Size in Coq 1/3}

  Using term size as a measure:
  \begin{lstlisting}
Fixpoint size (t:term) : nat :=
  match t with
  | Var _   => 0
  | Abs x b => S (size b)
  | App f a => 1 + (size f) + (size a)
end.

Lemma size_rename : forall (n n':name) (t:term),
  size (rename n n' t) = size t.
Proof.
unfold size.
unfold rename.
induction t;
  [ case (eq_name n0 n); intro; trivial  (* Var *)
  | congruence                           (* Abs *)
  | congruence ].                        (* App *)
Qed.
  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Recursion on Term Size in Coq 2/3}

  Substitution with term size as a recursion measure:
  \begin{lstlisting}
Function subst (t:term) (n:name) (t':term)
  {measure size t'} : term :=
  match t' with
  | Var x =>
      if eq_name x n then t else t'
  | Abs x b =>
      let z := fresh_name
                 (n :: (free_vars t) ++ (free_vars b))
      in
      Abs z (subst t n (rename x z b))
  | App f a =>
      App (subst t n f) (subst t n a)
end.
(* Leaves us with 3 obligations. *)
  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Recursion on Term Size in Coq 3/3}

  Proving termination of \texttt{subst}:
  \begin{lstlisting}
Proof.
intros.
rewrite size_rename.
auto.

intros.
unfold size.
inversion f; omega.

intros.
unfold size.
inversion a; omega.
Defined.
  \end{lstlisting}
  \uncover<2->{
    We really prefer structural recursion.
  }

\end{frame}

\frame{

  \frametitle{Simultaneous Substitution}

  Stoughton suggests the structurally recursive simultaneous substitution $M \sigma$:
  \begin{align*}
    x \sigma              &= \sigma x\\
    (\lambda x.M') \sigma &= \lambda y.(M' \; \sigma[y/x])  && \text{$y$ not free in $M', \sigma$}\\
    (M_1 \; M_2) \sigma   &= M_1 \sigma \; M_2 \sigma
  \end{align*}
  where
  \begin{equation*}
    \sigma[N/y] \; x =
    \begin{cases}
      N        & \text{if $x = y$,}\\
      \sigma x & \text{otherwise}
    \end{cases}
  \end{equation*}

  \uncover<2->{
    Substituting $N$ for $x$ in $M$ is now $M \; \iota [N/x]$ with $\iota$ the identity substitution.
  }

}

\begin{frame}[fragile]

  \frametitle{Simultaneous Substitution in Coq}

  \begin{lstlisting}
Fixpoint sim_subst (l:list (term*name)) (t:term)
  {struct t} : term :=
  match t with
  | Var x =>
      apply_subst l x
  | Abs x b =>
      let z := fresh_name
                  ((free_vars_sub l) ++ (free_vars b))
      in
      Abs z (sim_subst ((Var z, x)::l) b)
  | App f a =>
      App (sim_subst l f) (sim_subst l a)
end.

Definition subst' (t:term) (n:name) (t':term)
  : term := sim_subst ((t, n) :: nil) t'.
  \end{lstlisting}

\end{frame}

\subsection{de Bruijn Indices}

\frame{

  \frametitle{Names and Numbers in Binding}

  \tableofcontents[currentsubsection]

}

\frame{

  \frametitle{de Bruijn Indices 1/2}

  Represent variable occurrences by number of binders between itself and abstraction:
  \begin{align*}
    M ::=             &\; n
    && \text{variable} \\
    \llap{\textbar\:} &\; \lambda .M
    && \text{abstraction} \\
    \llap{\textbar\:} &\; M \; M
    && \text{application} \\
  \end{align*}

  \uncover<2->{
    \begin{itemize}
      \item Harder to read
      \item $\alpha$-equivalence is term equivalence
      \item Substitution is structurally recursive
      \item No renaming (but lifting)
      \item Mechanically less tedious
    \end{itemize}
  }

}

\begin{frame}

  \frametitle{de Bruijn Indices 2/2}

  Substitution can be defined as:
  \begin{align*}
    n[N/n]             &= N\\
    m[N/n]             &= m                    && \text{$m \neq n$} \\
    (\lambda .M')[N/n] &= \lambda .M'[\uparrow \! N / n\!+\!1]\\
    (M_1 \; M_2)[N/n]  &= M_1[N/n] \; M_2[N/n]
  \end{align*}
  where $\uparrow \! M$ is $M$ with all free variables incremented.

\end{frame}

\begin{frame}[fragile]

  \frametitle{de Bruijn Indices in Coq}

  Term datatype:
  \begin{lstlisting}
Inductive term : Set :=
  | Var : nat -> term
  | Abs : term -> term
  | App : term -> term -> term.
  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Substitution in Coq}

  Substitution:
  \begin{lstlisting}
Fixpoint lift (l:nat) (t:term) {struct t} : term :=
  match t with
  | Var n   => Var (if le_lt_dec l n then (S n)
                    else n)
  | Abs u   => Abs (lift (S l) u)
  | App u v => App (lift l u) (lift l v)
end.

Fixpoint subst (t:term) (n:nat) (t':term)
  {struct t'} : term :=
  match t' with
  | Var m   => if eq_nat_dec n m then t else t'
  | Abs u   => Abs (subst (lift 0 t) (S n) u)
  | App u v => App (subst t n u) (subst t n v)
end.
  \end{lstlisting}

\end{frame}

\subsection{Locally Nameless}

\frame{

  \frametitle{Names and Numbers in Binding}

  \tableofcontents[currentsubsection]

}

\begin{frame}

  \frametitle{Locally Nameless Representation 1/2}

  Combine names and numbers -- names for free variables, de Bruijn indices for bound variables:
  \begin{align*}
    M ::=             &\; x
    && \text{free variable} \\
    \llap{\textbar\:} &\; n
    && \text{bound variable} \\
    \llap{\textbar\:} &\; \lambda .M
    && \text{abstraction} \\
    \llap{\textbar\:} &\; M \; M
    && \text{application} \\
  \end{align*}

  \uncover<2->{
    \begin{itemize}
      \item $\alpha$-equivalence is term equivalence
      \item No renaming, no lifting (but freshening)
      \item Substitution is structurally recursive
    \end{itemize}
  }

\end{frame}

\begin{frame}

  \frametitle{Locally Nameless Representation 2/2}

  Two substitution operations:
  \begin{itemize}
  \item Substitute a term for a named variable
  \item Substitute a term for a de Bruijn index\\[2em]
  \end{itemize}

  \only<2>{
  Named variable substitution:
  \begin{align*}
    x[N/x]             &= N\\
    y[N/x]             &= y                 && \text{$x \neq y$} \\
    n[N/x]             &= n\\
    (\lambda .M')[N/x] &= \lambda .M'[N/x]\\
    (M_1 \; M_2)[N/x]  &= M_1[N/x] \; M_2[N/x]
  \end{align*}
  No renaming.
  }

  \only<3->{
  de Bruijn substitution:
  \begin{align*}
    x[N/n]             &= x\\
    n[N/n]             &= N\\
    m[N/n]             &= m                    && \text{$m \neq n$} \\
    (\lambda .M')[N/n] &= \lambda .M'[N / n\!+\!1]\\
    (M_1 \; M_2)[N/n]  &= M_1[N/n] \; M_2[N/n]
  \end{align*}
  No lifting.
  }

\end{frame}

\begin{frame}[fragile]

  \frametitle{Locally Nameless Representation in Coq}

  Term datatype:
  \begin{lstlisting}
Inductive term : Set :=
  | FreeVar  : name -> term
  | BoundVar : nat -> term
  | Abs      : term -> term
  | App      : term -> term -> term.
  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{Named Variable Substitution in Coq}

  Substitute a term for a named variable:
  \begin{lstlisting}
Fixpoint subst (t:term) (x:name) (t':term)
  {struct t'} : term :=
  match t' with
  | FreeVar y  => if eq_name x y then t else t'
  | BoundVar n => t'
  | Abs b      => Abs (subst t x b)
  | App f a    => App (subst t x f) (subst t x a)
end.
  \end{lstlisting}

\end{frame}

\begin{frame}[fragile]

  \frametitle{de Bruijn Substitution in Coq}

  Substitute a term for a de Bruijn index:
  \begin{lstlisting}
Fixpoint subst (t:term) (n:nat) (t':term)
  {struct t'} : term :=
  match t' with
  | FreeVar x  => t'
  | BoundVar m => if eq_nat_dec m n then t else t'
  | Abs b      => Abs (subst t (S n) b)
  | App f a    => App (subst t n f) (subst t n a)
end.
  \end{lstlisting}

\end{frame}

\section{Implementations}

\subsection{The POPLmark Challenge}

\frame{

  \frametitle{Names and Numbers in Binding}

  \tableofcontents[currentsubsection]

}

\begin{frame}

  \frametitle{The POPLmark Challenge}

  Mechanized metatheory for the masses:
  \begin{quote}
    Your average POPL paper should include machine-checked proofs
  \end{quote}

  Set of benchmarks for measuring progress:
  \begin{itemize}
    \item Based on metatheory of System F$_\texttt{<\!:}$
    \item Binding issues are a central aspect
    \item 15 (partial) solutions
    \item Part 1a: transitivity of subtyping
  \end{itemize}

\end{frame}

\begin{frame}

  \frametitle{Syntax of System F$_\texttt{<\!:}$}

  Part 1a considers just the type language of F$_\texttt{<\!:}$.

  \begin{align*}
    T ::=             &\; X
    && \text{type variable} \\
    \llap{\textbar\:} &\; \texttt{Top}
    && \text{maximum type} \\
    \llap{\textbar\:} &\; T \rightarrow T
    && \text{type of functions} \\
    \llap{\textbar\:} &\; \forall X\texttt{\small<\!:}T.T
    && \text{universal type} \\[2em]
    \Gamma ::=        &\; \emptyset
    && \text{empty type environment} \\
    \llap{\textbar\:} &\; \Gamma, X \texttt{\small<\!:}T
    && \text{type variable binding} \\
  \end{align*}

\end{frame}

\begin{frame}

  \frametitle{Subtyping Rules of System F$_\texttt{<\!:}$}

  \begin{mathpar}
    \infer*[right=SA-Top]
           {\:}
           {\Gamma \vdash S \texttt{\large<\!:} \texttt{Top}}
    \and
    \infer*[right=SA-Refl-TVar]
           {\:}
           {\Gamma \vdash X \texttt{\large<\!:} X}
    \\
    \infer*[right=SA-Trans-TVar]
           {X \texttt{\small<\!:} U \in \Gamma
             \\ \Gamma \vdash U \texttt{\large<\!:} T}
           {\Gamma \vdash X \texttt{\large<\!:} T}
    \\
    \infer*[right=SA-Arrow]
           {\Gamma \vdash T_1 \texttt{\large<\!:} S_1
             \\ \Gamma \vdash S_2 \texttt{\large<\!:} T_2}
           {\Gamma \vdash S_1 \rightarrow S_2 \; \texttt{\large<\!:} \; T_1 \rightarrow T_2}
    \\
    \infer*[right=SA-All]
           {\Gamma \vdash T_1 \texttt{\large<\!:} S_1
             \\ \Gamma, X \texttt{\small<\!:} T_1 \vdash S_2 \texttt{\large<\!:} T_2}
           {\Gamma \vdash \forall X \texttt{\small<\!:} S_1.S_2 \; \texttt{\large<\!:} \; \forall X \texttt{\small<\!:} T_1.T_2}
  \end{mathpar}

  POPLmark 1a: transitivity of $\texttt{<\!:}$

\end{frame}

\begin{frame}

  \frametitle{Solutions in Coq}

  Named variables:
  \begin{itemize}
    \item \only<1>{Stump} \uncover<2->{{\bf Stump} (7641)}
  \end{itemize}

  de Bruijn indices:
  \begin{itemize}
    \item \only<1>{Vouillon} \uncover<2->{{\bf Vouillon} (5443)}
    \item Sallinens \uncover<2->{(unavailable)}
    \item Charg\'eraud \uncover<2->{(3727)}
  \end{itemize}

  Locally nameless:
  \begin{itemize}
    \item \only<1>{Leroy} \uncover<2->{{\bf Leroy} (1081+5414=6495)}
    \item Chlipala \uncover<2->{(2650+2400=5050)}
    \item Chargu\'eraud \uncover<2->{(803+3533=4336)}
  \end{itemize}

  Nested datatypes:
  \begin{itemize}
    \item Hirschowitz and Maggesi \uncover<2->{(2757)}
  \end{itemize}

\end{frame}

\begin{frame}

  \frametitle{Stump -- Named Variables}

  Two main techniques to avoid difficulties with named variables:
  \begin{enumerate}
    \item Free and bound variables are disjoint
      \begin{itemize}
        \item Substitution is just grafting
      \end{itemize}
    \item \uncover<2->{Use common bound variable in the bodies of the SA-All rule
      \begin{itemize}
        \item Avoid $\alpha$-equivalence issues
        \item Original rule:
          \begin{mathpar}
            \infer*[right=SA-All]
                   {\Gamma \vdash T_1 \texttt{\large<\!:} S_1
                     \\ \Gamma, X \texttt{\small<\!:} T_1 \vdash S_2 \texttt{\large<\!:} T_2}
                   {\Gamma \vdash \forall X \texttt{\small<\!:} S_1.S_2 \; \texttt{\large<\!:} \; \forall X \texttt{\small<\!:} T_1.T_2}
          \end{mathpar}
        \item \uncover<3->{Adapted rule:
          \begin{mathpar}
            \infer*[right=SA-All]
                   {\Gamma \vdash T_1 \texttt{\large<\!:} S_1
                     \\ \Gamma, X \texttt{\small<\!:} T_1 \vdash S_2[X/X_1] \; \texttt{\large<\!:} \; T_2[X/X_2]}
                   {\Gamma \vdash \forall X_1 \texttt{\small<\!:} S_1.S_2 \; \texttt{\large<\!:} \; \forall X_2 \texttt{\small<\!:} T_1.T_2}
          \end{mathpar}}}
      \end{itemize}
  \end{enumerate}

\end{frame}

\begin{frame}

  \frametitle{Vouillon -- de Bruijn Indices}

  \begin{itemize}
    \item Very clear implementation, even suggested as baseline by POPLmark team
    \item A lot of code deals with shifting (but straightforward)
    \item Proofs not by structural induction, but by induction on the size of types
    \item Narrowing and transitivity are proved separately
    \item Unfortunately no accompanying paper
  \end{itemize}

\end{frame}

\begin{frame}

  \frametitle{Leroy -- Locally Nameless Representation}

  \begin{itemize}
   \item Two substitution operations, no renaming or lifting
   \item Considering abstraction bodies, freshening is needed
   \item Proofs not by structural induction, but by induction on the size of types
   \item A lot of code deals with swaps (used for equivariance proofs)
   \item Room for improvements, some implemented by Chargu\'eraud
     \begin{itemize}
       \item Useless case in de Bruijn substitution
       \item Treat well-formed typing environments as sets
       \item Cofinite quantification of free variable in SA-All
       \item Proofs by induction on well-formdness derivation instead of size
     \end{itemize}
  \end{itemize}

\end{frame}

\subsection{Engineering Formal Metatheory}

\frame{

  \frametitle{Names and Numbers in Binding}

  \tableofcontents[currentsubsection]

}

\begin{frame}

  \frametitle{Engineering Formal Metatheory}

  Aydemir et al, POPL'08: {\em Engineering Formal Metatheory}\\[2em]

  Complete style for formalizing language metatheory:
  \begin{itemize}
    \item Building on experience from POPLmark solutions
    \item Locally nameless representation
    \item Cofinite quantification of free variables in inductive definitions of relations on terms\\[2em]
  \end{itemize}

  Implemented in this style:
  \begin{itemize}
    \item Parts of POPLmark challenge
    \item Type soundness for core ML
    \item Subject reduction for Calculus of Constructions
    \item Several small developments
  \end{itemize}

\end{frame}

\begin{frame}

  \frametitle{LNgen}

  Aydemir and Weirich, this Wednesday:
  \begin{quote}
    LNgen: Tool Support for Locally Nameless Representations\\[2em]
  \end{quote}

  Building on {\em Engineering Formal Metatheory}:
  \begin{itemize}
    \item Takes Ott-like specifications
    \item Generates locally nameless infrastructure for Coq
  \end{itemize}

\end{frame}

\section{Conclusions}

\frame{

  \frametitle{Names and Numbers in Binding}

  \tableofcontents[currentsection]

}

\begin{frame}

  \frametitle{Related Work}

  Other Representations

  \begin{block}{Nominal Representation}
    \begin{itemize}
    \item Based on nominal logic (Pitts, Gabbay)
    \item Names for variables
    \item Swapping as primitive
    \item Urban in Isabelle/HOL
    \end{itemize}
  \end{block}

  \begin{block}{Higher Order Abstract Syntax}
    \begin{itemize}
    \item Meta-variables for variables
    \item Meta-functions for functions
    \item $\alpha$-equivalence for free
    \item No renaming needed
    \item Leads to quite unusual formulations
    \end{itemize}
  \end{block}

\end{frame}

\frame{

  \frametitle{Conclusion 1/2}

  Named variables:
  \begin{itemize}
    \item Closest to paper, but too much trouble mechanically\\[1em]
  \end{itemize}

  \uncover<2->{
    de Bruijn indices:
    \begin{itemize}
    \item Good candidate for mechanical developments
    \item Involves a lot of `easy' work
    \item Not so easy to read\\[1em]
    \end{itemize}
  }

  \uncover<3->{
    Locally nameless representation:
    \begin{itemize}
    \item Improves on de Bruijn
    \item Still quite some boilerplate code
    \item Tools like LNgen might make it feasible\\[1em]
    \end{itemize}
  }

  \uncover<4->{
    Nominal approaches are promissing
  }

}

\begin{frame}

  \frametitle{Conclusion 2/2}

  Of course, it also depends on your goal:
  \begin{itemize}
    \item A language implementation might not need to be close to its concrete syntax
    \item Metatheory for many users on the other hand probably does\\[1em]
  \end{itemize}

  \ldots and on your environment:
  \begin{itemize}
    \item We focussed on Coq
    \item There are other tools
  \end{itemize}

\end{frame}

\begin{frame}

  \frametitle{Questions and Further Reading}

  Questions?\\[4em]

  \begin{block}{Literature}
    \begin{itemize}
      \item de Bruijn, 1972: {\em $\lambda$-calculus with nameless dummies}
      \item Stoughton, 1988: {\em Substitution revisited}
      \item McBride and McKinna, 2004: {\em I am not a number -- I am a free variable}
      \item Aydemir et al, 2005: {\em The POPLmark challenge}
      \item Pollack, 2006: {\em Reasoning about languages with binding}
      \item Aydemir et al, 2008: {\em Engineering formal metatheory}
    \end{itemize}
  \end{block}

\end{frame}

\end{document}


\section{Discussion} \label{sec:discuss}
\subsection{Forward Model} \label{sec:forward-model}
In the previous section, we demonstrated the accuracy of \sedflow~posteriors.
Nevertheless, a primary determining factor for the fidelity of \sedflow, or any
ML model, is the quality of the training data set and, thus, the forward model
used to construct it.
Below, we discuss the caveats and limitations of our forward model, which has
two components: the PROVABGS SPS model and noise model
(Section~\ref{sec:training}).

First, for our noise model, we assign uncertainties to noiseless photometric
fluxes based on an empirical estimate of $p(\sigma_X\given f_X)$ for each band
independently.
This is a simplicistic prescription and, as the bottom right panels of
Figure~\ref{fig:data} ($g - \sigma_r$ and $r - \sigma_r$) reveal, there are
discrepancies between the magnitude - uncertainty distributions of the training
data and observations.
Despite these discrepancies, \sedflow~provides excellent estimates of the true
posterior.
This is because we design our ANPE to include $\sigma_X$ as a conditional
variable (Section~\ref{sec:anpe_train}).
The $f_X-\sigma_X$ distribution of our training data does not impact the
accuracy of the posteriors as long as there are sufficient training data near
$\bfi{x}$ to train the NDE in that region.

A more accurate noise model will, in theory, improve the performance of
\sedflow~because the $\bfi{x}$-space of the training data will more efficiently
span the observations.
Fewer training data would be expended in regions of $\bfi{x}$-space that are
devoid of observations.
However, for our application, we do not find significantly
improved performance when we alter the noise model.
This suggests that even with our simplistic noise model, the $\bfi{x}$-space of
observations is covered sufficiently well by the training data.
We note that when we decrease $N_{\rm train}$ to below 500,000,
\sedflow~posteriors are significantly less accurate.
A more realistic forward model may reduce this $N_{\rm train}$ threshold for
accurate posteriors.
However, generating $N_{\rm train}{\sim}1,000,000$ training SEDs has a
negligible computational cost compared to MCMC SED modeling, so we do not
consider it necessary to explore this further.

Next, we consider limitations in the PROVABGS SPS model used in our forward
model.
Our SPS model uses a compact and flexible prescription for SFH and ZH that can
describe a broad range of SFHs and ZHs.
However, the prescription is derived from simulated Illustris galaxies, whose
SFHs and ZHs may be not reflect the full range of SFHs and ZHs of real
galaxies.
If certain subpopulations of observed galaxies have SFHs and ZHs that cannot be
well described by the PROVABGS prescription, they cannot be accurately modeled.
Even if the PROVABGS SFH and ZH prescriptions are sufficient, there are
limitations in our understanding of stellar evolution.

There is currently no consensus in the stellar evolution, stellar spectral
libraries, or IMF of galaxies~\citep[\emph{e.g.}][]{treu2010, vandokkum2010,
rosani2018, ge2019, sonnenfeld2019}.
The PROVABGS model uses MIST isochrones, \cite{chabrier2003} IMF, and the MILES
+ BaSeL spectral libraries.
These choices limit the range of SEDs that can be produced by the training
data.
For instance, if galaxies have significant variations in their IMF, assuming a
fixed IMF would falsely limit the range of our training data.
A more flexible SED model that includes uncertainties in SPS would broaden the
range of galaxy SEDs that can be modeled.
Data-driven approaches may also enable SED models to be more
descriptive~\citep[\emph{e.g.}][]{hogg2016, portillo2020}.
Improving  SED models, however, is beyond the scope of this work.
Our focus is on improving the Bayesian inference framework.
In that regard, the limitations of the SED model equally impacts conventional
approaches with MCMC.

We encounter the caveats above when we apply \sedflow~to the NSA catalog.
For a small fraction of NSA galaxies (588 out of 33,884), \sedflow~generates
posteriors that are outside of the prior volume.
This is because the photometry or uncertainties of these galaxies lie outside
of the support of the training data and where \sedflow~is well trained.
They either have higher photometric uncertainties, for a given magnitude, or
bluer photometric colors than the training data.
Some of these may be observational artifacts or problematic photometry.
Nevertheless, \sedflow~fails because we cannot construct training data near
them with our limited noise and SPS models.
Since this only affects a small fraction of the NSA galaxies, we flag them in
our catalog and, for completeness, infer their galaxy properties by applying
PROVABGS with MCMC sampling.
For more details, we refer readers to Appendix~\ref{sec:fail}.

To test for limitations of the forward model, we can construct additional tests
of posteriors derived from ANPE.
For instance, the $\chi^2$ of the best-fit parameter value from the
estimated posterior can be used to assess whether the best-fit model
accurately reproduces observations.
This would only require one additional model evaluation per galaxy.
One can also construct an Amortized Neural Likelihood Estimator (ANLE) using
the same training data.
Unlike the ANPE, which estimates $p(\btheta\given f_X, \sigma_X, z)$, the ANLE
would estimate $p(f_X \given \btheta, \sigma_X, z)$.
We can then further validate the posteriors by assessing whether the observed
photometry lies within the ANLE distribution.
Based on the overall high level of accuracy of \sedflow~posteriors, we do not
explore these additional tests; however, they can be used to further validate
any ANPE posteriors.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Advantages of \sedflow}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The primary advantage of \sedflow~is its computational speed.
This becomes even more pertinent if we want to add additional parameters to
address concerns about the choices in current SPS models, described above.
To relax these assumptions, SPS models would need to introduce additional
parameters that flexibly model these uncertainties~\citep{conroy2009,
conroy2010c}.
While the dimensionality of current SPS models already makes MCMC methods
computational infeasible, ANPE has been applied to higher dimensional
applications.
For instance, \cite{dax2021} constructed an accurate ANPE for a
15-dimensional model parameter and 128-dimensional conditional variable
spaces.
NDE is an actively developing field in ML and new methods are constantly
emerging~\citep[\eg][]{wu2020, dhariwal2021}.
Since ANPE can handle higher dimensionality, we can in the future include
additional parameters that model uncertainties in SPS.
This will not only improve our SED modeling, but also improve our understanding
of stellar evolution and the IMF.

In addition to enabling scalable SED modeling for the next generation galaxy
surveys, \sedflow~will also enable us to tackle other key challenges in SED
modeling.
For example, recent works have demonstrated that priors of SED models can
significantly impact the inferred galaxy properties~\citep{carnall2018,
leja2019, hahn2022}.
Even ``uniformative'' uniform priors on SED model parameters can impose
undesirable priors on derived galaxy properties such as $M_*$, SFR, SFH, or
ZH.
To avoid significant biases, galaxy studies must carefully select priors and
validate their results using multiple different choices.
With an MCMC approach, selecting a different prior means reevaluating every
posterior and repeating all the SED model evaluations in the MCMC sampling.
For an ANPE approach, the prior is set by the distribution of parameters in the
training data.
For a new prior, instead of reconstructing the training data, we can resample
it in such a way that the parameters follow the new prior.
Then, the ANPE model can be re-trained, re-validated on the test data, and
re-deployed on observations.
Each of these steps require substantially less computational resources than
generating a new set of training data or using MCMC methods.
Hence, the ANPE approach provides a way to efficiently vary the prior without
multiplying computational costs.


% !TEX root = template.tex

\begin{table*}[t]
	\centering
		\begin{tabular}{ p{7cm}p{2cm}p{2cm}p{2cm}p{2cm} }
			\hline
			Method & Accuracy & Precision & Recall & F1-Score \\
			\hline
			CNN + No centering & 77.0 & 78.0 & 75.8 & 76.8 \\
			CNN + No centering + Manual F. & 75.6 & 76.8 & 73.9 & 75.3 \\
			CNN + Centering + Manual F. & 86.2 & 86.9 & 70.2 & 77.6 \\
			\textbf{CNN + Centering + Encoder F.} & \textbf{89.2} & \textbf{88.9} &  \textbf{86.1} & \textbf{86.1} \\
			\hline
		\end{tabular}
		\caption{\label{tab:model-performance} HD
                  classification results with different features CNN
                  augmentation and data preprocessing. The tests are
                  made with our proposed architecture, linear
                  interpolation and data centering, 196 (1x16)
                  filters, max-pool (1x4), 64 fully connected neurons,
                  a dropout rate of $0.05$, l2-regularization of
                  $5e^{-5}$ and Adam learning rate of $2e^{-5}$. Here
                  we don't use OIT since we are testing our result
                  onto HD dataset, including \textit{sit} and
                  \textit{stand} labels. The test set is composed by
                  users \textit{a-b} and the reaming users form the
                  training set.}
\end{table*}

\section{Results}
\label{sec:results}

\subsection{Heterogeneity Dataset (HD)}
\label{subsec:heterogeneity-dataset}

In this section we evaluate performances between previous works and
different model architectures, and the influences of the preprocessing
techniques applied here. Similar to what was done in
\cite{ignatov2018real}, we carried out from the HD dataset some
representative users to test the model and then use the remaining ones
to train the model. In this case we selected users \textit{a} and
\textit{b} since we found that they are very representative among all
others. In fact the proposed models tend to be less precise with user
\textit{a} and more accurate with user \textit{b}. This mainly depends
on the user's style in walking, doing stairs and so on. In this way also, we
are able to compare results with other works that usually tend to
evaluate performances on unseen user.

In this training and test set settings, we evaluate the best
hyper-parameters amd results for the models, excluding OIT
preprocessing block, since this dataset was collected with a fixed orientation. \textit{sit} and \textit{stand} activities are included. The rest
of preprocessing blocks are enabled, unless otherwise specified.

\textbf{Autoencoder.} In order to search for best autoencoder model
hyper-parameters we performed a grid search. Results are reported in
Tab. \ref{tab:ae-hyperparams} where values that performs well on the
validation dataset are reported in bold. The best hyper-parameters
model used to be the one with a relatively small code size: from 24 to
36 features which is also a good thing as we do not want the
autoencoder to learn the identity, but only to keep useful
information. In this settings we get an MSE of $0.87$.

\begin{table}[t]
  \centering
  \begin{tabular}{lp{4cm}}
    \hline
    Hyper-parameter & Values \\
    \hline
    code size & \{2, 3, 4, 5, 6, 12, 18, 24, 30, \textbf{36}, 42, 48, 54, 60, 72\} \\
    batch size & \{32, \textbf{128}\} \\
    epochs & \{\textbf{150}, 200\} \\
    \hline
  \end{tabular}
  \caption{Grid-search for best hyper-parameters on autoencoder}
  \label{tab:ae-hyperparams}
\end{table}

\begin{table}[b]
	\centering
	\begin{tabular}{p{2.4cm}p{4.5cm}}
		\hline
		Hyper-parameter & Values \\
		\hline
		Distance measure & \{\textbf{euclidean}, manhattan, chebyshev, minkowski, standardized euclidean, mahalanobis\} \\
		Number of neighbors & \{3, 4, \textbf{5}, 6, 7, 8\} \\
		\hline
	\end{tabular}
	\caption{Grid-search for KNN classifier}
	\label{tab:knn-grid-search}
\end{table}

Even if the primary goal for this autoencoder is to automatically
extract features from signal for the main CNN model, we implement also
two simple classifiers in order to directly use autoencoder's feature
and check their effectiveness.

The first is a K-Nearest Neighbors (KNN) clustering algorithm, and we
perform clustering on encoder's code. The idea here is that
autoencoder should extract relevant features which may be similar
class by class. We performed fine-tuning of KNN parameters with a grid
search, looking for best values of distance measure and number of
neighbors. The Tab. \ref{tab:knn-grid-search} show the values.  We
select the euclidean distance measure and $5$ as number of neighbors.
In this case we obtained an accuracy of $76.2$\%.  From the
grid-search on KNN we surprisingly noticed that performances do not
vary significantly among different hyper-parameters: most results are
less than $5$\% far from the best. This may be an indication of the
maximum capability of this autoencoder model, so to obtain better
performances we have to add complexity to the model.

\begin{table}[b]
	\centering
	\begin{tabular}{ p{1.8cm}p{1.9cm}p{1.7cm}p{1.7cm} }
		\hline
		CNN Filters & FC2 Neurons & Accuracy & F1-Score \\
		\hline
		196 & 1024 & 84.6 & 75.3 \\
		196 & 512 & 86.1 & 75.7 \\
		\textbf{196} & \textbf{64} & \textbf{86.2} & \textbf{77.6} \\
		96 & 1024 & 82.8 & 69.9 \\
		96 & 512 & 84.4 & 73.7 \\
		96 & 64 & 89.0 & 73.0 \\
		48 & 1024 & 80.0 & 79.4 \\
		48 & 512 & 83.7 & 74.9 \\
		48 & 64 & 84.0 & 72.3 \\
		\hline
	\end{tabular}
	\caption{Hyper-parameters selection using HD results with data centering and manual features augmented CNN}
	\label{tab:model-selection}
\end{table}

The second classifier instead is based on a Feed Forward Neural
Network (FFNN). The architecture is very simple and consists on two
dense layers of $100$ neurons each with $0.1$ dropout and
\textit{ReLU} activation function, while the last is a dense layer
with a number of neurons equal to the number of classes and soft-max
activation function. The network is trained with Adam optimizer to
minimize the categorical cross-entropy loss function. In this case we
obtained an accuracy of $81.8\%$.

\textbf{CNN Network.} We fist test how number of convolutional filters and dense neurons in CL1 and FL2 will influence classification performances with data centering and manual extracted features. The results obtained are presented in Tab. \ref{tab:model-selection}. We decided to choose 196 convolutional filters and 64 dense neurons thanks to its balance between accuracy and F1-Score performances, obtaining $86.2\%$ and $77.6\%$ respectively.

To better appreciate how our preprocessing blocks affects model overall performances, we also try to disable or enable some of them. In Tab. \ref{tab:model-performance} we reported our obtained results experiments. We see that augmenting the CNN with manual extracted feature when data are not centered, lead to no significant change in performances, instead when also enabling data centering preprocessing with manual features augmented CNN the model obtain nearly $10\%$ more in accuracy and precision metrics. This prove the benefits of data centering stated previously. However we were not able to reach the same performances presented in \cite{ignatov2018real}, where the authors obtained in the same exact settings an accuracy of $97.6\%$. These empirically confirm that performances of state-of-the-art models trained with one type of sensors are worse when dealing with smartphone sensors heterogeneity. Moving on, augmenting the CNN with encoder feature increments the model performances, meaning that encoder features are more robust that manual features, as explained previously. To compare our best results in this setting with the ones presented in \cite{stisen2015smart}, we decided to perform their \textit{Leave-one-user-out cross validation} evaluation, consisting of test the model with data from one user, and train with data from all the others in a cross validation fashion and then averaging the obtained results. In this evaluation setting we obtained an average F1-score of $85.8\%$, beating their best model result of nearly $10\%$ more in F1-Score metric. This prove also that using users a and b to do our evaluation is a good compromise of the real \textit{Leave-one-user-out cross validation} evaluation performances, since we obtain nearly the same results ($90.2\%$ instead of $89.2\%$ in accuracy and $85.8\%$ instead of $86.1\%$). A confusion matrix in this latter setting is reported in Fig. \ref{fig:cnn-confusion-matrix} where we could see that the model performs nicely overall in all the considered activities, with some difficulties in distinguishing between \textit{stand} and \textit{sit} and \textit{walk} with \textit{stairs} activities.

\vspace{-0.2cm}
\begin{figure}[h]
	\centering
	\includegraphics[width=0.5\textwidth]{images/confusion_matrix.png}
	\caption{CNN confusion matrix}
	\label{fig:cnn-confusion-matrix}
\end{figure}

\vspace{-0.3cm}

\subsection{Oriented Dataset (OD)}

With this new collected dataset we want to test our models' performance
in a real use case scenario, where smartphone could be placed in
different positions and orientations. In this case we trained the
models with the entire HD as training set, and then use the OD as a
test set. It is important to remember that in this case we apply OIT
and so we condensed the \textit{sit} and \textit{stand} activities of
HD into a one class \textit{no activity} category.

\textbf{Autoencoder.}  An interesting result, as
Tab.~\ref{tab:ae-loss} confirms, is that OIT is a necessary operation
when dealing with different orientations. For example, without OIT we can see
that the autoencoder trained and tested on same data goes from $0.75$
to $10.42$ MSE: more than ten times worse. Furthermore, data from
\textit{hand} or \textit{pocket-up/down} do not inflate the loss too much. This is good
because it indicates that autoencoder is producing robust features.

\begin{table}[t]
  \centering
  \begin{tabular}{lr}
    \hline
    Scenario & Loss (MSE) \\
    \hline
    HD + OIT + OD validation & 0.75 \\
    HD + OIT + OD validation (allpos) & 0.82 \\
    HD + OD validation & 10.42 \\
    \hline
  \end{tabular}
  \caption{Autoencoder loss on different scenarios}
  \label{tab:ae-loss}
\end{table}

Tab. \ref{tab:knn-metrics} and \ref{tab:ffnn-metrics} show
respectively KNN and FFNN evaluation for the best autoencoder with
$36$ features and the two classifier with hyper-parameters selected in
Sec. \ref{subsec:heterogeneity-dataset}. From this results, we can
observe that KNN is more stable and not influenced by sensor's
position/orientation w.r.t. FFNN. Also, the two models preserve
evaluation order: on \textit{pouch} position gets best results on both
models, while the worse positions is \textit{hand+pocket}, as we
expect.

\begin{table}[h]
  \centering
  \begin{tabular}{lrrrr}
    \hline
    Positions & Accuracy & Precision & Recall & F1-score \\
    \hline
    Pouch & 80.1 & 86.4 & 80.7 & 79.0 \\
    Hand+Pocket & 79.5 & 83.7 & 79.5 & 79.6 \\
    All & 80.0 & 83.8 & 79.1 & 78.2 \\
    \hline
  \end{tabular}
  \caption{KNN evaluation onto OD between smartphone positions (pouch
    left/right/top/back, hand and pocket-up/down, all positions)}
  \label{tab:knn-metrics}
\end{table}

\vspace{-0.25cm}

\begin{table}[h]
  \centering
  \begin{tabular}{lrrrr}
    \hline
    Positions & Accuracy & Precision & Recall & F1-score \\
    \hline
    Pouch & 74.4 & 84.7 & 74.4 & 74.8 \\
    Hand+Pocket & 67.8 & 78.0 & 67.8 & 70.0 \\
    All & 69.9 & 81.3 & 69.9 & 72.4 \\
    \hline
  \end{tabular}
  \caption{FFNN evaluation onto OD between smartphone positions (pouch
    left/right/top/back, hand and pocket-up/down, all positions)}
  \label{tab:ffnn-metrics}
\end{table}

\textbf{CNN Network.} Combining autoencoder features into the CNN, we were able to reach better performances w.r.t the simpler KNN and FFNN previous presented classifiers. For the \textit{hand+pocket} new positions, we noticed a performance drop of nearly $15\%$, but given the complexity of this new unseen context we are pretty satisfied with the obtained results. As
Tab. \ref{tab:model-oit-performance} confirms, the final proposed learning strategy with a smart data preprocessing lead to good results even with new problem
settings that nearly match real use case scenarios.

\begin{table}[b]
  	\centering
	\begin{tabular}{p{1.8cm}rrrr}
	  \hline
	  Positions & Accuracy & Precision & Recall & F1-score \\
	  \hline
	  Pouch & 85.3 & 92.0 & 73.8 & 81.9 \\
	  Hand+Pocket & 70.5 & 79.5 & 63.0 & 70.2 \\
	  All & 78.0 & 84.0 & 69.0 & 75.7 \\
	  \hline
	\end{tabular}
	\caption{CNN classification comparisons onto \textit{OD} between
	  smartphone positions (pouch left/right/top/back, hand and
	  pocket-up/down and all positions).}
	\label{tab:model-oit-performance}
	\end{table}

We also try to disable OIT in the CNN model, obtaining a drop in performances of nearly $45$\% for all metrics considered. This demonstrate that OIT is an extremely useful preprocessing technique both for autoencoder and CNN models.


% !TEX root = ../00_thesis.tex

\section{Appendix B}
% \subsection*{Additional constraints for the multi-mode schedule synthesis problem}
\label{appendix:multi_mode}

In this appendix, we present how are practically implemented the constraints ensuring that
\begin{itemize}

	\item Schedules in the multi-mode case are compatible, \ie the inheritance constraints formulated in Theorem~\ref{thm:minVirtLegacy} (\cref{sec:multi_mode}) are respected

	\item Running applications can seamlessly switch between modes, \ie there is no risk of missed deadline due to a mode change, as described in \cref{sec:modeChanges}.

\end{itemize}
Finally, we conclude with a short comment on how to implement context-specific requirements in the schedule synthesis problem.

\vspace{10pt}
\fakepar{Compatible schedules}
Let us consider the schedule synthesis of mode \mode{j} where applications \app and \appB have been scheduled in higher-priority mode \mode{i}, such that
\begin{itemize}

	\item $\exists \; \app \in \legApp{j}$

	\item $\exists \; \appX \in \freeApp{j} \, , \; \appB \in \minvirtlegApp{j}{\appX}$

\end{itemize}
We denote with a superscript $^k$ a schedule value computed in mode \mode{k}, \eg $m^j.o$ is the offset of message $m$ computed in mode \mode{j}.

%legacy app: fix everything.
\app is a legacy application for mode \mode{j}. Its schedule must be reserved, which is realized simply as follows
\begin{flalign}
&\forall \; (\tau, m) \in \app.c,
&	\tau^{j}.o \; &= \; \tau^i.o
&&\\
&
&	m^i.o \; &= \; m^j.o
&&\\
&
&	m^i.d \; &= \; m^j.d
&&
\end{flalign}

%virtual legacy app: use
\appB is a virtual legacy application for application \appX. As a result, the scheduling ``space'' of the tasks of \appB must be reserved, to guarantee that the tasks of \appX will not overlap with \appB. This is achieved by adding constraints similar to those enforcing (C3), guaranteeing the validity of the task mappings (see the previous Appendix)
%
\begin{align}
\intertext{%
$	\forall\,
		\tau_\appB \in \appB.c	, \,
		\tau_\appX \in \appX.c  , \;
		\; E_\appB == E_\appX, \;
	\forall\, k_\appB \in [1..LCM/\tau_\appB.p], \; \forall\, k_\appX \in [1..LCM/\tau_\appX.p]$}
	\tau_\appB.o + \tau_\appB.e + \tau_\appB.p*k_\appB
	&\; \leq \; \tau_\appX.o + \tau_\appX.p*k_\appX \\
\texttt{or} \quad
	\tau_\appX.o + \tau_\appX.e + \tau_\appX.p*k_\appX
	&\; \leq \; \tau_\appB.o + \tau_\appB.p*k_\appB
\end{align}
%
Similarly, the \; \texttt{or} \; condition is realized using a ``big'' time constant $M$ and boolean variables.
%
\begin{align}
\tau_\appB.o + \tau_\appB.e + \tau_\appB.p*k_\appB
	&\; \leq \; \tau_\appX.o + \tau_\appX.p*k_\appX
		+ M * (1 - \lambda_{\appB,\appX}^{k_\appB,k_\appX})\\
\tau_\appX.o + \tau_\appX.e + \tau_\appX.p*k_\appX
	&\; \leq \; \tau_\appB.o + \tau_\appB.p*k_\appB
		+ M * \lambda_{\appB,\appX}^{k_\appB,k_\appX}
\end{align}

\vspace{10pt}
\fakepar{Seamless mode switch}
%message served and released in the same hyperperiod
In order to prevent the risk of missed deadline due to a mode change, we described in \cref{sec:modeChanges} a simple procedure to execute mode change requests.
\begin{enumerate}

	\item All messages must be released and served within the same hyperperiod.
	\label{rule1}

	\item The starting time \tjstart of mode \modej is set to the end of the first hyperperiod of \modei after which the execution of all applications $\app \in \Sij$ has been completed.

\end{enumerate}
%
The second rule depends on the embedded software, it is independent of the schedule synthesis.
To enforce the first rule however, additional constraints must be formulated. As discussed at the end of \cref{sec:single_mode} and illustrated in \cref{fig:openboxoptions}, the rounds schedule may be such that some messages are released in one of the mode's hyperperiod, but served only in the next hyperperiod, see~\cref{subfig:case2}.
\TODO{Now the only case presented in df(0) = -1, adapt the text}

For applications scheduled in only one mode, this is no problem. Indeed, for every mode changed, they are either starting, terminating, or not involved, which cannot lead to a deadline miss.
For the other applications, \ie scheduled in more than one mode, we enforce rule (\ref{rule1}) by setting the parameters $r_o.B_i$ in the definition of the service function $sf_i$~(see \eqref{eq:sf_def}).
For any application \app scheduled in more than one mode,
%
\begin{flalign}
&
\forall \, m_i \in \app.c,
&&r_o.B_i \; = \; 0
&&
\end{flalign}
%

\vspace{5pt}
\fakepar{Context-specific requirements}
The scheduling framework proposed in this work is very flexible thanks to the use of an ILP formulation to synthesize the schedules.
We described in the Appendices a set of constraints for this ILP formulation that solves our initial problem, as stated in \cref{sec:model}.
Then, additional constraints can be added to the formulation, to capture context-specific requirements. We give thereafter some examples of practically-relevant constraints.

\vspace{5pt}
\begin{description}

	\item[Tasks $\tau_1$ and $\tau_2$ must start simultaneously]
	\begin{equation}
	\tau_1.o \; = \;  \tau_2.o
	\end{equation}

	\item[The time difference between $\tau_1$ finishing and $\tau_2$ starting is bounded by $D$]
	\begin{equation}
	\tau_2.o - ( \, \tau_1.o + \tau_1.e \, ) \; \leq \; D
	\end{equation}

	\item[Messages $m_1$ and $m_2$ must be allocated to the same rounds]
	\begin{flalign}
	&\forall\, j\in [1..R],
	&&r_j.B_1 \; = \; r_j.B_2 \qquad
	&&
	\end{flalign}

\end{description}


\section{Tensat: Optimizing Deep Learning Computation Graphs}
\label{sec:tensat}

\newcommand{\ourname}{Tensat\xspace}
\newcommand{\tensat}{Tensat\xspace}

Deep learning frameworks and compilers
 (e.g., Tensorflow~\cite{TensorFlow}, PyTorch~\cite{NEURIPS2019_9015},
  XLA~\cite{xla}, TensorRT~\cite{TensorRT}, TVM~\cite{tvm},
  MLIR~\cite{mlir})
 have enabled diverse kinds of machine learning models to run
 efficiently on numerous compute platforms.
Neural network models in
 these frameworks are typically represented as tensor computation
 graphs.
To improve the runtime performance of a tensor graph, these
 frameworks perform various optimizations.

One of the most important optimizations is graph rewriting,
 which takes in a tensor graph $g$ and a set of semantics-preserving graph rewrites $R$,
 and by applying rewrites to $g$ seeks to find an semantically equivalent $g'$ with lower cost according to some cost model.
% which transforms a tensor graph by iteratively applying rewrite rules that substitute subgraphs with semantically equivalent ones.
%The graph rewriting optimization problem can be factored into two sub-problems: (1) discovering a comprehensive set of rewrite rules and (2) an algorithm that determines the order in which to apply the rewrite rules to optimize a given graph.
The current industry-standard approach adopted by most frameworks is to use a manually curated set of rewrite rules and rely on a heuristic strategy to determine the order in which to apply the rewrite rules.
%that often applies rewrite rules greedily.
However, this approach often leads to sub-optimal results both due to the non-comprehensive set of rewrite rules, as well as the sub-optimal graph substitution heuristic \cite{taso,metaflow}. % that considers only a small fraction of all possible graphs reachable with a given set of rewrites \cite{taso, metaflow, Fang:sampling}.

This case study aims to address the sub-optimality problem of graph rewrite strategies, while leveraging the existing rewrite rules generation technique~\cite{taso}.
Prior research has shown that searching for sequences of substitutions
\cite{taso,metaflow,Fang:sampling} outperforms heuristic approaches.
However, both heuristic and search-based solutions rely on sequential application of substitutions.
Since rewrites often depend on or enable one another,
optimization depends heavily on the order in which rewrites are applied;
the ``phase ordering'' problem strikes again.
% this classically tricky problem is known in the compilers community as the ``phase-ordering'' or ``rewrite-ordering'' problem.

\begin{table}[]
    \centering
    \begin{tabular}{ccc|cc}
    \hline
    & \multicolumn{2}{c|}{\bf Search time (s)} & \multicolumn{2}{c}{\bf Runtime speedup (\%)} \\
    & TASO & \ourname{} & TASO & \ourname{} \\
    \hline
    BERT & 13.6 & \textbf{1.4} & 8.5 & \textbf{9.2} \\
    ResNeXt-50 & 25.3 & \textbf{0.7} & 5.5 & \textbf{8.8} \\
    NasNet-A & 1226 & \textbf{10.6} & 1.9 & \textbf{7.3} \\
    NasRNN & 177.3 & \textbf{0.5} & 45.4 & \textbf{68.9} \\
    Inception-v3 & 68.6 & \textbf{5.1} & 6.3 & \textbf{10.0} \\
    SqueezeNet & 16.4 & \textbf{0.3} & 6.7 & \textbf{24.5} \\
    VGG-19 & 8.9 & \textbf{0.4} & \textbf{8.9} & \textbf{8.9} \\
    \hline
    \end{tabular}
    \caption{Comparison of optimization time and runtime speedup of the optimized computation graphs over the original graphs, TASO~\cite{taso} v.s. \ourname{}.}
    \label{table:ngraph}
\end{table}

This case study presents \ourname{}, a tensor graph superoptimization framework that employs equality saturation \cite{eqsat, eqsat-llvm, egg},
to apply all possible rewrites at once.  %comprehensively and scalably explore the search space of all equivalent graphs a given a set of rewrite rules.
\tensat splits program optimization into two phases: {\em exploration} and {\em extraction}.
The exploration phase is equality saturation using \egg as usual.
\tensat's extraction phase is totally custom;
 simple cost functions simply do not suffice for extracting efficient deep learning compute graphs.
Instead, \tensat employs an Integer Linear Programming (ILP) extraction solution,
 which requires
 a novel method to filter out invalid subgraphs from an e-graph.
% The extraction phase selects from the e-graph the equivalent program with the lowest cost according to a given cost model. The compact representation of the exponentially large search space using e-graphs enables extraction algorithms that can find the globally optimal equivalent program quickly.
%\ourname{} uses equality saturation to replace the heuristic- or sequential-search-based rewriting approaches from prior work.

% Applying equality saturation to tensor graph rewriting requires non-trivial extensions in both the exploration and extraction phases.
% We extend the exploration phase to support complex, non-local rewrite rules that are necessary to produce highly efficient tensor graphs.
% Additionally, we introduce a novel method to filter out invalid subgraphs from an e-graph,
% which enables our extraction procedure based on Integer Linear Programming (ILP) to quickly find the optimal solution.

We evaluated \ourname{} on a number of well-known machine learning models executing on a GPU.
As highlighted in \autoref{table:ngraph}, \ourname{} can synthesize optimized graphs that are up to 23\% faster in runtime than state-of-the-art \cite{taso}, while reducing the optimization time by up to 300x.
By having the e-graph compactly representing an exponential number of equivalent graphs, \ourname{} is able to cover a larger search space more efficiently than the sequential search methods.
As a result, our search approach is both extremely effective and fast enough to be used as part of a normal complation flow.
%Our approach discovers better optimized graphs than the state-of-the-art, at the same time running much faster such that it can be used as part of a normal compilation flow.
%We believe that our approach is the first tensor graph superoptimization that is both \ATTN{provably optimal} and fast enough to be used as part of a normal complation flow.

%However, doing so requires non-trivial extensions in both the saturation as well as the extraction phase: (a) extending equality saturation to support complex rewrite rules (b) formulating the extraction problem as a integer linear programming(ILP), and (c) designing pruning strategies to trade-off coverage (in terms of the number of graphs explored) for shorter time to solve the ILP problem.

%We evaluated \ourname{} on a number of well-known machine learning models (including BERT, ResNeXt-50, NasNet-A, NasRNN, and Inception-v3), comparing against TASO \cite{taso}, state-of-the-art tensor graph rewriting approach.
%up to xxx faster performing graphs in up to xxx smaller search time while exploring an exponentially larger search space.
%In addition, we have also performed a number of ablation studies to highlight the importance of the specific optimizations and extensions proposed in this paper.

\subsection{Representation}

This section describes how \ourname{} represents tensor computation graphs and rewrite rules.

\paragraph{Representing Tensor Computation Graphs}
\label{sec:language}

We use a representation based on the one in TASO~\cite{taso},
 with modifications to make it suitable for equality saturation.
\autoref{table:ops} shows the set of operators we consider.
Each operator $o_i$ corresponds to a node $n_i$ in the graph; the node represents the output tensor of the operator.
The nodes corresponding to the inputs of $o_i$ are the children nodes of $n_i$.
% Unlike TASO, we include both the input tensors and the operator parameters as the children nodes of the operator node.
% This way, all the relevant information of the tensor computation graph is represented syntactically by the nodes and the edges, without the need of additional annotations on the nodes.
Each tensor computation graph is a DAG under this representation.

The formulations in equality saturation become simpler if a graph is
single-rooted. Therefore, we combine all the final output nodes of a
graph with \emph{no-op}s to make the graph single-rooted. The no-op
nodes do not have any actual operators associated with them, and they
will not be altered during the exploration phase, so there are no side
effects.

\begin{table}[t]
    \centering
    \newcommand\act{\textsf{activation}}
    \newcommand\width{\textsf{w}}
    \newcommand\height{\textsf{h}}
    {\textbf{Types}
      \\[1mm]
      \small
    \begin{tabular}{cl|cl|cl|cl}
      $T$ & Tensor
      & $TT$ & Tensor tuple
      & $W$ & Weights tensor
      & $s_\height, s_\width$ & stride (height, width) \\
      $n$ & natural number
      & $p$  & padding
      & $a$ & activation
      & $k_\height, k_\width$ & kernel (height, width) \\
    \end{tabular}}
    \\[2em]
    \begin{tabular}{lll}
        {\bf Operator}  & {\bf Description}                    & {\bf Type signature} \\
    \hline
        \sf ewadd           & Element-wise addition                & $(T, T) \rightarrow T$ \\
        \sf ewmul           & Element-wise multiplication          & $(T, T) \rightarrow T$ \\
        \sf matmul          & Matrix multiplication                & $(a, T, T) \rightarrow T$ \\
        \sf conv $^a$       & Grouped convolution                  & $(s_\height, s_\width, p, a, T, W) \rightarrow T$ \\
        \sf relu            & Relu activation                      & $T \rightarrow T$ \\
        \sf tanh            & Tanh activation                      & $T \rightarrow T$ \\
        \sf sigmoid         & Sigmoid activation                   & $T \rightarrow T$ \\
        \sf poolmax         & Max pooling                          & $(T, k_\height, k_\width, s_\height, s_\width, p, a) \rightarrow T$ \\
        \sf poolavg         & Average pooling                      & $(T, k_\height, k_\width, s_\height, s_\width, p, a) \rightarrow T$ \\
        \sf transpose $^b$  & Transpose                            & $(T, {\sf permutation}) \rightarrow T$ \\
        \sf enlarge $^c$    & Pad a convolution kernel with zeros  & $(T, T_{\sf ref}) \rightarrow T$ \\
        \sf concat$_n$ & Concatenate along the given axis     & $(n, T, \dots, T) \rightarrow T$ \\
        \sf split $^d$      & Split a tensor into two along the axis & $(n, T) \rightarrow TT$ \\
        \sf split$_0$       & Get the first output from split      & $TT \rightarrow T$ \\
        \sf split$_1$       & Get the second output from split     & $TT \rightarrow T$ \\
        \sf merge $^e$      & Update weight to merge grouped conv  & $(W, n) \rightarrow W$ \\
        \sf reshape         & Reshape tensor                       & $(T, {\sf shape}) \rightarrow T$ \\
        \sf input           & Input tensor                         & ${\sf identifier} \rightarrow T$ \\
        \sf weight          & Weight tensor                        & ${\sf identifier} \rightarrow T$ \\
        \sf no-op            & Combine the outputs of the graph     & $(T, T) \rightarrow T$ \\
    \end{tabular}
    \\[1em]
    \caption{
        Operators supported by \ourname.
        There are four types for the nodes in our representation:
        tensor type (T), string type (S), integer type (N), and tensor tuple type (TT).
        The integer type is used to represent parameters of the
         operators, such as stride, axis, and also padding and activation modes
         (by representing different modes using different integers).
         The more complex, variable-length parameters
          (e.g. shape, axes permutation) are represented using the
          string type according to the specified formats.
          \\[1em]
          \footnotesize{
            $^a$ Same representation as TASO \cite{taso}.
                 Normal and depth-wise convolutions are special cases
                 of grouped convolutions. \\
            $^b$ Axis permutation for transpose is specified
                 using a string with format: axis$_1$\_axis$_2$\_\dots .\\
            $^c$ Pad a convolution kernel (input) with
                 zeros to make it the same size as input $T_{\sf ref}$ \\
            $^d$ Split the tensor in the given axis. The position of the split is at the place of the most recent concat. \\
            $^e$ Merge every \textit{count} number of groups in the grouped convolution. See TASO \cite{taso} for more details. \\
          }
    }\label{table:ops}
\end{table}

\paragraph{Representing Rewrite Rules}
\label{sec:rewrite}

A rewrite rule for tensor computation graph specifies that some local subgraph pattern (\textit{source pattern}) is equivalent to another subgraph pattern (\textit{target pattern}).
The input tensors to the source and target patterns are \textit{variable nodes}, which can be substituted with any concrete nodes (or e-class in equality saturation) in the current graph.
Each output tensor in the source pattern corresponds to an output tensor in the target pattern.
The two corresponding output nodes are called a pair of \textit{matched outputs}.
A rewrite rule states the equivalence between each pair of matched outputs.

We represent each source (and target) pattern using symbolic expressions (S-exprs) with variables.
Patterns with a single output is represented with an S-expr rooted on the output.
Rewrite rules with such patterns are called \textit{single-pattern rewrite rules}.
Patterns with multiple outputs are represented as a list of S-exprs rooted on each output.
Rewrite rules with multiple matched outputs are called \textit{multi-pattern rewrite rules}.
% \autoref{fig:rewrite} shows an example rewrite rule and its representation.

% \begin{figure}[t]
%     \centering
%     \includegraphics[width=\linewidth, draft=true]{figures/rewrite-example.pdf}
%     \begin{scriptsize}
%     Source: (matmul ?input$_1$ ?input$_2$), (matmul ?input$_1$ ?input$_3$) \\
%     Target: (split$_0$ (split 1 (matmul ?input$_1$ (concat$_2$ 1 ?input$_2$ ?input$_3$)))), \\
%     (split$_1$ (split 1 (matmul ?input$_1$ (concat$_2$ 1 ?input$_2$ ?input$_3$))))
%     \end{scriptsize}
%     \caption{
%     Example rewrite rule and its representation in S-expressions.
%     Identifiers starting with "?" denote variable nodes.
%     For clarity, we omit the activation mode inputs to \texttt{matmul}.
%     Arrows point from parent nodes to children nodes.
%     1 is the axis for \texttt{split} and \texttt{concat} operators.
%     % The first argument to \texttt{matmul} indicates which activation function to use.
%     }
%     \label{fig:rewrite}
% \end{figure}

\subsection{Exploration Phase}
\label{sec:saturation}

We initialize the e-graph with the original tensor computation graph.
In each iteration of the exploration phase, we search for matches of all rewrite rules in the current e-graph, and add the target patterns and equivalence relations to the e-graph.
This process continues until either the e-graph saturates or a user-specified limit (in terms of time, e-graph size, or number of iterations) is reached.
Before applying a rewrite at a found match, we perform a \textit{shape checking} to verify if the tensor shapes in the target pattern are compatible.
This is necessary since some rewrite rules requires input tensor shapes to satisfy specific preconditions, in addition to the syntactic match.
We perform shape checking in the same way as TASO \cite{taso}.

% \paragraph{Multi-Pattern Rewrite Rules}

% Multi-pattern rewrite rules are an important type of rules for tensor graph superoptimization \cite{taso}.
% However, most equality saturation toolkits only support efficient search methods to find matches for single-pattern rewrite rules \cite{egg, ematching}.
% We introduce an algorithm for applying multi-pattern rewrites, as shown in \autoref{alg:multi}. Our algorithm leverages the existing efficient search routine for single-pattern rewrites as a subroutine.
% %For clarity, we omit the handling of single-pattern rules and cycles here (those are described in \autoref{alg:cycle}).

% At the beginning of the exploration phase, we collect the set of unique S-exprs present in the source patterns of the rewrite rules after canonicalization.
% Here, if one S-expr can be transformed into another S-expr by variable renaming only, they will be mapped to the same canonicalized S-expr.
% In each iteration of the exploration phase, we use the single-pattern search subroutine to search for matches of the canonical S-exprs.
% Then for each multi-pattern rule, we take the Cartesian product of the matches found, decanonicalize the variable-to-e-class map into the original variables (using the variable renaming map stored during canonicalization), and check if the matches are compatible at the shared variables between the S-exprs (i.e., if the shared variables refer to the same e-class after the mapping).
% We apply the matches that are compatible.

% \begin{algorithm}[t]
% \small
% \caption{Applying multi-pattern rewrite rules}
% \label{alg:multi}
% \begin{algorithmic}[1]
% \Require starting e-graph $\mathcal{G}$, set of multi-pattern rewrite rules $\mathcal{R}_m$.
% \Ensure updated e-graph $\mathcal{G}$.
% \State canonicalized S-expr $e_c$ = Set(\{\})
% \For{rule $r \in \mathcal{R}_m$ }
% \For{$i = 0, \dots, |r|-1$ } \Comment{{\scriptsize $|r|$: \#S-exprs in source pattern}}
%     \State ($e$, rename\_map) = \Call{Canonical}{$r$.source[$i$]}
%     \State $e_c$.insert($e$)
%     \State $r$.map[i] = rename\_map
% \EndFor
% \EndFor

%   \For{iter = 0, \dots, MAX\_ITER}
%     \State $M$ = \Call{Search}{$\mathcal{G}, e_c$} \Comment{all matches for all patterns}
%     \For{rule $r \in \mathcal{R}_m$ }
%         \For{$i = 0, \dots, |r|-1$ }
%         \State canonical matches mc$_i$ = $M$[$r$.source[i]]
%         \State matches m$_i$ = \Call{Decanonical}{mc$_i$, $r$.map[$i$]}
%         \EndFor
%         \For{$(\sigma_0, \dots, \sigma_{|r|-1}) \in$ m$_0 \times \dots \times$ m$_{|r|-1}$}
%             \If{\Call{Compatible}{$(\sigma_0, \dots, \sigma_{|r|-1})$}}
%             \State \Call{Apply}{$\mathcal{G}, r, \sigma_0, \dots, \sigma_{|r|-1}$}
%             \EndIf
%         \EndFor
%     \EndFor
%   \EndFor
% \State \Return $\mathcal{G}$
% \end{algorithmic}
% \end{algorithm}

% In our experience, one feature of multi-pattern rules for tensor graph is that they can grow the e-graph extremely rapidly.
% Let's consider again the example rewrite rule in \autoref{fig:rewrite}.
% This rule can be matched with any two \texttt{matmul} nodes with a shared input (input$_1$).
% By applying this rule once on some match, a new \texttt{matmul} node will be created and added to the e-graph (the one on the RHS of \autoref{fig:rewrite}), which also has input$_1$ as its input.
% If the e-graph contains $N$ \texttt{matmul} nodes that has some input$_1$ at the beginning, then after iteration 1, $\mathcal{O}(N^2)$ new \texttt{matmul} nodes sharing input$_1$ will be created.
% In iteration 2, each pair in these $\mathcal{O}(N^2)$ nodes will be a match, which will create $\mathcal{O}(N^4)$ new nodes.
% Such double exponential growth can quickly explode the e-graph.

% Based on this feature, we set a separate limit $k_{\textrm{multi}}$ on the number of iterations to apply the multi-pattern rules.
% After $k_{\textrm{multi}}$ iterations, we only apply the single-pattern rules until saturation or some user-specified limit.

\subsection{Extraction Phase}

During extraction, the goal is to pick one e-node from each e-class in the e-graph to obtain an optimized graph.
The optimized graph should minimize the total cost with respect to a given cost model.
In tensor graph superoptimization, the cost model reflects the inference time taken by the graph.

\paragraph{Cost model}
We use the same cost model as TASO \cite{taso}.
Each operator has a separate and independent cost, which is the measured runtime of that operator (with the specific input sizes and parameters) on hardware.
The total cost of a graph is the sum of costs of each of its nodes.
This cost model is suitable for GPUs, since GPUs typically run one operator at a time when executing a graph.
Note that an operator can be a fused operator, consisting of multiple primitive operators, such as a fused convolution and ReLU.

\label{sec:extraction}

\paragraph{Greedy extraction}

We first experiment with a greedy extraction strategy that has been shown to be effective for certain domains~\cite{herbie, spores, egg}.
%The first extraction method we experiment with is greedy extraction.
%It has been shown to be a simple but effective method in multiple domains \cite{herbie, spores, egg}.
For each e-class, the greedy strategy computes the total cost of the subtrees rooted on each of the e-nodes, and picks the e-node with the smallest subtree cost.

Greedy extraction is not guaranteed to extract the graph with the minimum cost, even under our independent cost model.
For example, if two children of an e-node share a subgraph, greedy extraction would ignore the sharing and overestimate the cost.

\paragraph{ILP extraction}
The second approach we experiment with is formulating the extraction problem as an Integer Linear Program (ILP).

Let $i = 0, ..., N-1$ be the set of e-nodes in the e-graph.
Let $m = 0, ..., M-1$ be the set of e-classes in the e-graph.
Let $e_m$ denote the set of e-nodes within e-class $m$: $\{i | i\in e_m \}$.
Let $h_i$ denote the set of children e-classes for e-node $i$.
Let $g(i)$ denote the e-class of e-node $i$, i.e. $i\in e_{g(i)}$.
Let $m=0$ be the root e-class.
Each e-node is associated with a cost $c_i$.

We then formulate our problem as follows:
\begin{align*}
    &\textrm{Minimize: } f(x) = \sum_{i} c_i x_i
\end{align*}
Subject to:
\begin{align}
    &x_i \in \{0, 1\}, \\
    &\sum_{i\in e_0} x_i = 1, \\
    &\forall i, \forall m \in h_i, x_i \leq \sum_{j\in e_m} x_j , \\
    & \forall i, \forall m \in h_i, t_{g(i)} - t_m - \epsilon + A (1 - x_i) \geq 0, \\
    &\forall m, 0 \leq t_m \leq 1,
\end{align}

Here we introduce a binary integer variable $x_i$ for each e-node $i$; node $i$ is selected if $x_i=1$, and not selected otherwise.
Constraint (2) ensures that one node is picked in the root e-class.
Constraint (3) ensures that if a node is picked, then at least one node in each of its children e-classes needs to be picked.
We rely on the fact that at the optimal solution, each e-class can have at most one picked node (otherwise we can remove more picked nodes in this e-class to reduce the objective while still satisfying all the constraints).
Constraints (1)--(3) and the objective encode the main extraction logic.

A more subtle requirement on the extraction phase is that the extracted graph cannot contain cycles.
While the e-graph can (and likely will) contain cycles, the extracted graph is meant to map directly to an executable tensor DAG.
The extraction procedure must therefore take care to respect the acyclic invariant of DAGs.

% \autoref{fig:cycle} shows an example to illustrate how valid rewrites can produce cycles in the e-graph.
To ensure the extracted graph does not contain cycles, we introduce a real variable $t_m$ for each e-class $m$ in the ILP.
Constraint (4) ensures that the order defined by $t_m$'s is a valid topological order for the extracted graph.
Here $\epsilon < 1/M$ is a small constant for effectively encoding strict inequalities in ILP.
$A$ is a large enough constant such that $A > 1 + \epsilon$.
Constraint (5) is to limit the range for the topological order variables $t_m$'s.

We also experiment with using integer variables for $t_m$'s. In this case, $t_m$'s are constrained to take integer values between 0 to $M-1$.
Constraint (4) changes accordingly to: $\forall i, \forall m \in h_i, t_{g(i)} - t_m + A (1 - x_i) \geq 1$, where $A \geq M$.

Unlike greedy extraction, the optimal solution to the ILP is guaranteed to give a valid graph (no cycles) with the lowest cost.

% \begin{figure}
%     \centering
%     \includegraphics[width=\linewidth, draft=true]{figures/cycle.pdf}
%     \caption{Example on how a valid rewrite can introduce cycles into the e-graph. RHS is the resulting e-graph after applying the rewrite rule from \autoref{fig:rewrite} to the LHS. Dotted lines circles the e-classes. We omit the e-classes with a single node for clarity. If the node split$_1$ is picked in the right e-class, then the resulting graph will have a cycle (indicated by the red edges).}
%     \label{fig:cycle}
% \end{figure}

\paragraph{Cycle Filtering}
\label{sec:cycle}

Similar to previous work that uses ILP extraction \cite{eqsat, spores},
we find that as the size of the e-graph grows bigger, the ILP solver takes a long time and becomes the main bottleneck.
This is mainly due to the cycle constraint (4): ILP solver struggles to find a feasible solution with these constraints.
Therefore, we explore an alternative approach by filtering cycles during the exploration phase to make sure that the e-graph does not contain any cycles at the end of the exploration phase.
This way, we can get rid of the cycle constraints in the ILP.

\paragraph{Vanilla cycle filtering}
The first method is to check if applying a substitution introduces cycles to the e-graph, and discard such a substitution.
This check is run every time before applying a substitution.
Each check requires a pass over the entire e-graph.
For one iteration during the exploration phase, if we denote $N$ as the current size of the e-graph and $n_m$ as the total number of matches of the rewrite rules on the e-graph, then this vanilla cycle filtering has complexity $\mathcal{O}(n_m N)$.

\paragraph{Efficient cycle filtering}

As the number of matches $n_m$ is typically large and scales with $N$, vanilla cycle filtering can be slow.
We therefore design a novel and more efficient cycle filtering algorithm, consisting of a \textit{pre-filtering} step and a \textit{post-processing} step.
%Intuitively, in the pre-filtering step, we run one pass over the e-graph and use the information to check a large number of substitutions.
%Then we filter the remaining cycles in a post-processing steps.
\autoref{alg:cycle} shows the pseudocode for the exploration phase with efficient cycle filtering.

At the start of each iteration, we do one pass over the e-graph to record the set of descendent e-classes for each e-node (stored in a descendants map).
During the iteration, for each match of the rewrite rules, we use the pre-stored descendants map to check if  applying a rewrite introduces cycles to the e-graph; if so, we skip this match.
Line 3--9 implements the pre-filtering step.
Notice that this check is sound but not complete: a match that passes this check can still introduce cycles to the e-graph.
This is because new descendants relations introduced by the previous rewrite in this iteration are not included in the pre-stored descendants map.

\begin{algorithm}[t]
\small
\caption{Exploration phase with efficient cycle filtering}
\label{alg:cycle}
\begin{algorithmic}[1]
\Require starting e-graph $\mathcal{G}$, set of rewrite rules $\mathcal{R}$.
\Ensure updated e-graph $\mathcal{G}$, filter list $l$
  \State $l = $ $\{\}$
  \For{iter = 0, \dots, MAX\_ITER}
    \State descendants map $d$ = \Call{GetDescendants}{$\mathcal{G}, l$}
    \State matches = \Call{Search}{$\mathcal{G}, \mathcal{R}, l$}
    \For{match $\in$ matches}
      \If{\Not \Call{WillCreateCycle}{match, $d$}}
        \State \Call{Apply}{$\mathcal{G}$, match}
      \EndIf
    \EndFor
    \While{true}
      \State cycles = \Call{DfsGetCycles}{$\mathcal{G}, l$}
      \If{len(cycles) == 0}
        \State \textbf{break}
      \EndIf
      \For{cycle $\in$ cycles}
        \State \Call{ResolveCycle}{$\mathcal{G}, l$, cycle}
      \EndFor
    \EndWhile
  \EndFor
  \State \Return $\mathcal{G}, l$
\end{algorithmic}
\end{algorithm}

To resolve the cycles we missed in the pre-filtering step, we add a post-processing step at the end of each iteration (line 10-18).
We make a pass over the e-graph in DFS order and collect a set of cycles in the e-graph.
For each cycle, we choose the last node that is added to the e-graph, and add that node to a filter list.
The nodes in the filter list are considered as removed from the e-graph.
We make sure those nodes are not picked during extraction by explicitly adding constraints $\forall i \in l, x_i = 0$ to the ILP.

By constructing a descendants map once before each iteration, each of the checking in the pre-filtering step takes constant time.
The worst case complexity of the post-processing step is $\mathcal{O}(n_c N)$, where $n_c$ is the number of cycles in the e-graph.
Since $n_c$ is typically much smaller than $n_m$, this algorithm is much faster than the vanilla cycle filtering.
In practice, each DFS pass over the e-graph can find many cycles, which makes $\mathcal{O}(n_c N)$ a very conservative upper bound.

\subsection{Evaluation}

We implemented \ourname{} in Rust~\cite{rust} using \texttt{egg}~\cite{egg}.
For the extraction phase, we use SCIP \cite{scip} as the ILP solver, wrapped by Google OR-tools \cite{ortools}.

We utilize egg's \textit{e-class analysis} feature for the shape checking discussed.
An e-class analysis associates data with each e-class to support rewrites that are not purely syntactic.
We store all the relevant information of the tensors (shape, layout, split locations) in the analysis data and use these information for shape checking.

\paragraph{Experimental Setup}

We compareed \ourname{} with TASO \cite{taso} to evaluate our equality saturation based search.
We used the same set of rewrite rules as TASO for our experiments.
We evaluated on the inference graphs of 7 models:
\textbf{BERT} \cite{bert}, \textbf{ResNeXt-50} \cite{resnext50}, \textbf{NasNet-A} \cite{nasneta}, \textbf{NasRNN} \cite{nasrnn}, \textbf{Inception-v3} \cite{inceptionv3},
\textbf{VGG-19} \cite{vgg}, and \textbf{SqueezeNet} \cite{squeezenet}.
This benchmark set covers a wide range of commonly used state-of-the-art models, including both models for computer vision tasks and models for NLP tasks, both human-designed models and automatically-discovered models by neural architecture search.
We performed all experiments on a Google Cloud instance with one NVIDIA Tesla T4 GPU, a 16-core CPU, and 60 GB of memory.
% We also experimented with ResNet-50 \cite{resnet}, but find that on T4 GPU, the rewrite rules from TASO cannot provide any speedup to the graph.

For \ourname{}, our full approach uses the efficient cycle filtering algorithm (\autoref{sec:cycle}) during the exploration phase and the ILP method without the cycle constraints (\autoref{sec:extraction}) for extraction.
We set a limit on the number of nodes in the e-graph $N_{\textrm{max}}=50000$ and the number of iterations for exploration $k_{\textrm{max}}=15$.
We terminate the exploration phase when any of the limit is reached, or the e-graph is saturated.
We set a separate limit $k_{\textrm{multi}}$ on the number of iterations to apply the multi-pattern rules.
We use a default of $k_{\textrm{multi}}=1$ for the main results in \autoref{sec:speedup} and \autoref{sec:time}.
 % and study the effect of varying $k_{\textrm{multi}}$ in \autoref{sec:multi-vary}.
We set a timeout of 1 hour for the ILP solver.

For TASO's backtracking search, we use their default settings from their artifact evaluation code on the number of iterations\footnote{The number of iterations of the outer loop, see Algorithm 2 in \cite{taso} for more details} $n=100$ and the hyperparameter $\alpha=1.0$ for each benchmark.
We also test $\alpha=1.05$ as mentioned in their paper, and find that the difference is tiny (difference in speedup percentage is less than 0.1\% on average over the benchmarks).
Increasing to $n=1000$ leads to less than 1\% speedup gain with the cost of over 11x longer in optimization time on average.

\begin{figure}
  \centering
  \includegraphics[height=8cm]{tensat/all_speedup.pdf}
  \caption{
    Speedup percentage of the optimized graph with respect to the original graph, TASO v.s. \ourname{}.
    Each setting (optimizer $\times$ benchmark) is run for five times, and we plot the mean and standard error for the measurements.
  }\label{fig:speedup}
  \vspace{2em}

  \includegraphics[height=8cm]{tensat/all_optim_time.pdf}
  \caption{
    Comparison of the optimization time (log scale) between TASO and \ourname{}.
    ``TASO total'' is the total time of TASO search.
    ``TASO best'' indicates when TASO found its best result;
    achieving this time would require an oracle telling it when to stop.
  }\label{fig:overhead}
\end{figure}

\paragraph{Program Speedup}
\label{sec:speedup}

We compare the speedup percentage of the optimized graph with respect to the original graph between \ourname{} and TASO.
We use TASO's cuDNN backend to measure the runtime of the full computation graphs.
\autoref{fig:speedup} shows the results.
We can see that \ourname{} discovers better optimized graphs compared with TASO's backtracking search in most benchmarks.
\ourname{}'s optimized graphs are on average 6.6\% faster than TASO's. We see the biggest speedup of 23\% over TASO on NasRNN.
Note that for Inception-v3, \ourname{} with $k_{\textrm{multi}}=1$ gives a smaller speedup than TASO,
but increasing $k_{\textrm{multi}}$ to 2 achieves a better speedup than TASO
while still being 13.4$\times$ faster than TASO's search (see \autoref{fig:overhead}).

This improvement comes from the fact that equality saturation covers a much larger space of equivalent graphs than sequential backtracking search.
By using e-graph as a compact representation of an exponential number of equivalent graphs, \ourname{} is able to cover orders of magnitude more equivalent graphs than TASO.

\paragraph{Optimization Time}
\label{sec:time}

Another important metric is the time taken by the optimizer itself.
For \ourname{}, this is the sum of time taken by the exploration phase and the extraction phase.
For TASO, we record two times for a single backtracking search.
The first is the total time of the backtracking search with the default number of iterations ($T_{\textrm{total}}$).
The second one is the time taken to first reach the best graph found during its search ($T_{\textrm{best}}$).
$T_{\textrm{best}}$ is the best possible time for TASO's sequential backtracking search.
In practice, it is difficult (if not impossible) to achieve $T_{\textrm{best}}$ since the sequential search algorithm would have no way to know that it can stop at that point.

\autoref{fig:overhead} shows the time taken by the optimizers across benchmarks.
We can see that \ourname{} runs 9.5x to 379x faster than TASO's $T_{\textrm{total}}$, and 1.8x to 260x times faster than $T_{\textrm{best}}$.
This shows that \ourname{} can not only cover a much larger search space, but also achieve this in drastically less time.
Furthermore, \ourname{}'s optimization time is small enough that we believe our approach can be integrated into a default compilation flow instead of running the search as an additional offline autotuning process.

% \paragraph{Varying Number of Iterations for Multi-Pattern Rewrites}
% \label{sec:multi-vary}

% As we discuss in \autoref{sec:saturation}, multi-pattern rewrite rules can grow the e-graph in a extremely rapid manner.
% Here we study the effect of varying the number of iterations for multi-pattern rewrites $k_{\textrm{multi}}$.
% \autoref{fig:trend} shows the results.
% We can see the explosion of the number of nodes in the e-graph as $k_{\textrm{multi}}$ increases (due to the double exponential growth).
% For NasRNN, Inception-v3, BERT, NasNet-A, and ResNeXt-50, by increasing $k_{\textrm{multi}}$, \ourname{} discovers better graphs with larger speedups.
% But for SqueezeNet, speedup decreases with $k_{\textrm{multi}}$.
% This is due to the discrepancy between the cost model and the real graph runtime.
% As $k_{\textrm{multi}}$ increases for SqueezeNet, the cost model suggests that certain new rewrites can reduce the cost, while they in fact increase full graph runtime.
% Despite this special case where the discrepancy has an effect, \ourname{} on SqueezeNet with $k_{\textrm{multi}}=3$ still achieves a better speedup than TASO.
% By increasing $k_{\textrm{multi}}$, \ourname{} can explore a larger search space and find better optimized graphs for most benchmarks, at the cost of longer time taken by the optimizer.

% \begin{figure}
%     \centering
%     \includegraphics[width=0.29\hsize]{figures/speedup_trend.pdf}
%     \includegraphics[width=0.29\hsize]{figures/optim_trend.pdf}
%     \includegraphics[width=0.29\hsize]{figures/nodes_trend.pdf}
%     \includegraphics[width=0.09\hsize]{figures/legend_trend.pdf}
%     \caption{Effect of varying the number of iterations of multi-pattern rewrites $k_{\textrm{multi}}$.
%     For BERT, NasNet-A, NasRNN, Inception-v3, the ILP solver times out at one hour for $k_{\textrm{multi}}=3$.
%     Left: speedup of the optimized graphs (the $y$-axis is split for clarity).
%     Middle: time taken by the \ourname{}.
%     Right: final e-graph size (number of e-nodes).
%     The middle and right figures are in log scale.
%     }
%     \label{fig:trend}
% \end{figure}

% \paragraph{Ablation Study}
% \label{sec:ablation}

% In this section, we study the effect of the important design choices in our approach.

% \paragraph{Greedy v.s. ILP extraction}

% The first important design choice is the extraction method.
% \autoref{table:extraction} shows the comparison between greedy extraction and ILP extraction.
% Although greedy extraction works fine on some benchmarks (e.g. NasRNN), it fails to extract an optimized graph on others (e.g. BERT and NasNet-A).
% This is due to the nature of greedy extraction: it makes the choices on which node to pick separately and greedily, without considering the inter-dependencies between the choices.
% Consider the rewrite in \autoref{fig:rewrite} (merging two \texttt{matmul}s by \texttt{concat} and \texttt{split}) as an illustrative example.
% After applying this rewrite to the e-graph, there will be two e-classes that have multiple e-nodes: one e-class per each output.
% This rewrite can reduce the cost only if both e-classes choose the split node, since the RHS subgraph can be reused by the two outputs.
% However, greedy extraction will never pick the split nodes, since it does not know the RHS subgraph is shared between the two split nodes.

% \begin{table}[]
%     \centering
%     \begin{tabular}{cccc}
%     \hline
%         {\bf Graph Runtime (ms)} & {\bf Original} & {\bf Greedy} & {\bf ILP} \\
%     \hline
%         BERT & 1.88 & 1.88 & \textbf{1.73} \\
%         NasRNN & 1.85 & 1.15 & \textbf{1.10} \\
%         NasNet-A & 17.8 & 22.5 & \textbf{16.6} \\
%     \hline
%     \end{tabular}
%     \caption{Comparison between greedy extraction and ILP extraction, on BERT, NasRNN, and NasNet-A.
%     This table shows the runtime of the original graphs and the optimized graphs by greedy extraction and ILP extraction.
%     The exploration phase is run with $k_{\textrm{multi}} = 1$. }
%     \label{table:extraction}
% \end{table}

% \paragraph{ILP with or without cycle constraints}

% Here we study the effect of whether or not to include the cycle constraints in ILP.
% \autoref{table:cycle} presents the effect on extraction time as $k_{\textrm{multi}}$ (thus e-graph size) varies.
% With the cycle constraints, ILP solver time quickly increases with the e-graph size, and reaches timeout when $k_{\textrm{multi}}=2$.
% In our experiments, the ILP solver has not yet found a feasible solution at timeout.
% Removing the cycle constraints leads to approximately 10x--1000x speedup on ILP solving time on larger e-graphs.
% These results show that the main difficulty for the ILP solver is to satisfy the cycle constraints.
% Thus, removing the cycle constraints makes it possible for our approach to scale to larger e-graphs.

% \begin{table}[]
%     \centering
%     \begin{tabular}{ccccc}
%     \hline
%         {\bf Extraction} & \multirow{2}{*}{\bf $k_{\textrm{multi}}$} & \multicolumn{2}{c}{\bf With cycle} & {\bf Without} \\
%         {\bf time (s)} & & real & int & {\bf cycle} \\
%     \hline
%        \multirow{2}{*}{BERT} & 1 & 0.96 & 0.98 & \textbf{0.16} \\
%        & 2 & $>$3600 & $>$3600 & \textbf{510.3} \\
%        \hline
%        \multirow{2}{*}{NasRNN} & 1 & 1116 & 1137 & \textbf{0.32}  \\
%        & 2 & $>$3600 & $>$3600 & \textbf{356.7}  \\
%        \hline
%        \multirow{2}{*}{NasNet-A} & 1 & 424 & 438 & \textbf{1.81}  \\
%        & 2 & $>$3600 & $>$3600 & \textbf{75.1}  \\
%     \hline
%     \end{tabular}
%     \caption{Effect of whether or not to include cycle constraints in ILP on extraction time (in seconds), on BERT, NasRNN, and NasNet-A.
%     For the cycle constraints, we compare both using real variables and using integer variables for the topological order variables $t_m$. }
%     \label{table:cycle}
% \end{table}

% \paragraph{Efficient cycle filtering}

% To remove the cycle constraints from ILP, we need to perform cycle filtering during the exploration phase.
% Here we compare the two cycle filtering techniques introduced in \autoref{sec:cycle}.
% \autoref{table:efficient} shows the effect on the exploration phase time, as $k_{\textrm{multi}}$ varies.
% We can see that the efficient cycle filtering algorithm achieves up to 2000x speedup compared with the vanilla algorithm, making it possible to explore a larger e-graph.

% \begin{table}[]
%     \centering
%     \begin{tabular}{ccccccc}
%     \hline
%         \multirow{2}{*}{$k_{\textrm{multi}}$} & \multicolumn{2}{c}{BERT} & \multicolumn{2}{c}{NasRNN} & \multicolumn{2}{c}{NasNet-A} \\
%         & Van. & Eff. & Van. & Eff. & Van. & Eff. \\
%     \hline
%         1 & 0.18 & \textbf{0.17} & 1.30 & \textbf{0.08} & 3.76 & \textbf{1.27} \\
%         2 & 32.9 & \textbf{0.89} & 2932 & \textbf{1.47} & $>$3600 & \textbf{8.62} \\
%     \hline
%     \end{tabular}
%     \caption{Comparison between vanilla cycle filtering and efficient cycle filtering, on the exploration phase time (in seconds) for BERT, NasRNN, and NasNet-A.}
%     \label{table:efficient}
% \end{table}

%%% Local Variables:
%%% TeX-master: "../thesis"
%%% End: