\title          {\TeX\ macros for proof boxes}
\author         {Paul Taylor}

\documentclass{article}
\usepackage{boxproof}
\usepackage{a4wide}
\usepackage{daymonthyear}

\def\meta#1{\mbox{$\langle\hbox{#1}\rangle$}}
\def\macrowitharg#1#2{{\tt\string#1\bra\meta{#2}\ket}}

{\escapechar-1 \xdef\bra{\string\{}\xdef\ket{\string\}}}

\def\intro#1{{#1}{\cal I}}
\def\elim#1{{#1}{\cal E}}

\showboxbreadth 999
\showboxdepth 999
\tracingoutput 1

\let\imp\to
\def\elim#1{{{#1}{\cal E}}}
\def\intro#1{{{#1}{\cal I}}}
\def\lt{<}
\def\eqdef{=}
\def\eps{\mathrel{\epsilon}}
\def\biimplies{\leftrightarrow}
\def\flt#1{\mathrel{{#1}^\flat}}
\def\setof#1{{\left\{{#1}\right\}}}
\let\implies\to
\def\KK{{\mathsf K}}
\let\squashmuskip\relax

%=======================================================================
\begin{document}
\maketitle

\section{Introduction}
The proof
\begin{proofbox}
  \(\"1"\:\alpha\leftrightarrow\psi(x,\top)\=\\
        \:\Some\beta.\psi(x,\beta)\=\mathsf{total}\\
    \[\exists\beta\kern-1em\:\psi(x,\beta)\\
      \(\:\beta\=\\
        \:\beta=\top\=(*)\\
        \:\psi(x,\!\top)\=\mathsf{subs}\\
        \:\alpha\=\elim\leftrightarrow(\ref{1})\\
      \*\:\alpha\=\\
        \:\psi(x,\!\top)\=\elim\leftrightarrow(\ref{1})\\
        \:\beta=\top\=\mathsf{func}\\
        \:\beta\=(*)\\
      \)\:\alpha=\beta\=\intro\leftrightarrow\\
        \:\psi(x,\alpha)\=\mathsf{subs}\\
    \]  \:\psi(x,\alpha)\=\elim\exists\\
    \*  \:\psi(x,\alpha)\=\\
      \(\:\alpha\=\\
        \:\alpha=\top\=(*)\\
        \:\phi(x,\!\top)\=\mathsf{subs}\\
      \*\:\psi(x,\!\top)\=\\
        \:\alpha=\top\=\mathsf{func}\\
        \:\alpha\=(*)\\
      \)\:\alpha\leftrightarrow\psi(x,\!\top)\=\intro\leftrightarrow\\
    \)  \:\psi(x,\alpha)\leftrightarrow(\alpha\leftrightarrow\psi(x,\!\top))
          \=\intro\leftrightarrow\\
\end{proofbox}
 is produced by
\begin{verbatim}
\begin{proofbox}
  \(\"1"\:\alpha\leftrightarrow\psi(x,\top)\\
        \:\Some\beta.\psi(x,\beta)                      \=\mathsf{total}\\
    \[\exists\beta\kern-1em\:\psi(x,\beta)\\
      \(\:\beta\\
        \:\beta=\top                                    \=(*)\\
        \:\psi(x,\!\top)                                \=\mathsf{subs}\\
        \:\alpha                         \=\elim\leftrightarrow(\ref{1})\\
      \*\:\alpha\\
        \:\psi(x,\!\top)                 \=\elim\leftrightarrow(\ref{1})\\
        \:\beta=\top                                     \=\mathsf{func}\\
        \:\beta\=(*)\\
      \)\:\alpha=\beta                           \=\intro\leftrightarrow\\
        \:\psi(x,\alpha)                                 \=\mathsf{subs}\\
    \]  \:\psi(x,\alpha)                                  \=\elim\exists\\
    \*  \:\psi(x,\alpha)\\
      \(\:\alpha\\
        \:\alpha=\top                                   \=(*)\\
        \:\phi(x,\!\top)                                \=\mathsf{subs}\\
      \*\:\psi(x,\!\top)\\
        \:\alpha=\top                                   \=\mathsf{func}\\
        \:\alpha                                        \=(*)\\
      \)\:\alpha\leftrightarrow\psi(x,\!\top)   \=\intro\leftrightarrow\\
    \)  \:\psi(x,\alpha)\leftrightarrow
          (\alpha\leftrightarrow\psi(x,\!\top)) \=\intro\leftrightarrow\\
\end{proofbox}
\end{verbatim}

  Syntax as follows:
  each line is of the form
  \begin{center}
     \meta{variables}
     \meta{name}
     \verb/\:/ \meta{formula}
     \verb/\=/ \meta{reason}
     \verb/\-/ \meta{use}
     \verb/\\/
  \end{center}
  where
  \begin{itemize}
  \item \meta{variables} is something like ``$x,y$'' ---
    it's for variables declared at the beginning of $\intro\forall$- and
    $\elim\exists$-boxes.
  \item \meta{name} is a command \verb/\label{fred}/ which
    defines \verb/fred/ to be the label text, which may be used anywhere
    as \verb/\ref{fred}/ --- see {\em The \LaTeX book.}
    Local labels are also available, using \macrowitharg\lbl{name}
    or \verb/\"/\meta{name}\verb/"/; these obey the scoping rules of
    the boxes.
    You may also refer to the previous line as \verb/\ref{-}/.
  \item \meta{formula} is the proposition being asserted.
  \item \meta{reason} is \verb/\intro\land(\ref{john},\ref{mary})/
         or \verb/\elim\forall(\ref{jim})/.
  \item \meta{use} is provided for linear logic, to record the step
         which uses this one. How this accords with theory I don't yet know.
  \end{itemize}
  Note that the parts are separated by \verb/\:/, \verb/\=/ and \verb/\\/;
  these correspond to
  \begin{center}
     {\bf let } \meta{name} = \meta{expression} : \meta{type}
  \end{center}
  in a declarative language.
  The \verb/\:/, \verb/\=/ and \verb/\-/ fields are optional and may occur
  in any order. If any of them is repeated the last is taken.
  If none of them is present the \meta{variables} field is also ignored.

  Proof {\em boxes\/} are ``wrapped up'' as follows:
  \begin{itemize}
  \item the whole proof in \verb/\begin{proofbox}/...\verb/\end{proofbox}/;
  \item single-column boxes ($\intro\forall$, $\intro\imp$, $\elim\exists$),
      in \verb/\[/...\verb/\]/.
  \item multiple-column boxes are of two kinds:
    \begin{itemize}
     \item separate ($\intro\land$) boxes: \verb/\(/...\verb/\*/...\verb/\)/.
     \item stuck together ($\elim\lor$) boxes:
        \verb/\(/...\verb/\+/...\verb/\)/.
    \end{itemize}
  \end{itemize}
  You may put more than two columns in \verb/\(/...\verb/\)/ and even mix
  the \verb/\+/ and \verb/\*/ separators.

  The whole proof is enclosed in \verb/\proofbox/...\verb/\endproofbox/
  or \verb/\begin{proofbox}/...\verb/\end{proofbox}/, but the \LaTeX\
  environment form {\em must not\/} be used for nested boxes.

  If the proof occurs in paragraph mode (ie in vertical or
  unrestricted horizontal mode) then it is set as a display, using the
  full width of the page. Otherwise it uses only the required width.

  A lot of the internals are potentially configurable, but there is not
  yet a user interface suitable for doing this. This will be provided
  in the next version.

%=======================================================================
\section{Redefinable macros}
 WARNING: most of these commands will be hidden and replaced with
 optional arguments to \verb/\proofbox/ in a future version.
 Do not rely on them.

 We provide three different ways of numbering the lines of the proof:
 \begin{itemize}
 \item \verb/\runningproofline/: a global running sequence (default),

 \item \verb/\nestedproofline/: a hierarchical system with dots,

 \item \verb/\nestedproofline/: a fully hierarchical system which
 also includes the column number
 (\verb/\proof@columns/) as a letter (ASCII quote plus number).

 \end{itemize}
 \verb/\theproofline/ is the default.

 The macro \verb/\proofboxmakelabel#1/ is used to print the line label.
 We only put it in the leftmost box.
 It is printed in small non-ranging Arabic numerals
 ({\the\scriptfont1 0123456789}).
 Right-justify it in \verb/\prooflinenowidth/ if it will fit,
 otherwise let it stick out on the right, {\em i.e.}~left-justify it.

 Kill the numbers altogether with \verb/\proofboxnonumbers/.

 How to make the left column of the proof box:
 use the variables field, a space if necessary and the line label.

 How to make the middle column of the proof box:
 left justify the formula field.

 How to make the right column of the proof box:
 use the reason and use fields.

 Make the four edges of a rectangular box and the separator
 between \verb/\+/ columns.

 Use dotted lines: \verb/\dottedproofbox/.

 Leave the boxes open at the bottom: \verb/\openproofbox/

%=======================================================================
\section{Miscellaneous logical notations}

These macros are now in my \texttt{logicsym.sty}

 Print the names of the introduction and elimination rules, for example:
  \begin{quote}
     \verb/\elim\forall/ $\elim\forall$ \qquad
     \verb/\intro\land/ $\intro\land$ \qquad
  \end{quote}
 Recall that in \TeX\ the logical connectives and quantifiers are called
  \begin{center}
    \verb/\lor/ $\lor$\quad
    \verb/\land/ $\land$\quad
    \verb/\lnot/ $\lnot$\quad
    \verb/\forall/ $\forall$\quad
    \verb/\exists/ $\exists$
  \end{center}
 The following provide macros for the \verb/\implies/ {\em relation\/}
 and for the binary {\em operation\/} which yields the abstract
 \verb/\implic/ation between formulae.
 The point is that \TeX\ spaces them and breaks the lines differently:
  \begin{center}
    \verb/A\implies B/ $A\implies B$\quad{\em versus\/}\quad
    \verb/A\implic B/ $A\implic B$
  \end{center}
 There are forward and reverse, single and Double versions.

 Handle the spacing after a variable (and optionally its type)
 bound by a quantifier symbol. For example
  \begin{quote}
    \verb/\All x:X. \phi(x)/\quad prints as\quad $\All x:X.\phi(x)$\quad
     instead of\quad $\forall x:X.\phi(x)$
  \end{quote}
 We provide some commonly used forms; \verb/\iota/ ($\iota$) is Russell's
 description operator and should really be inverted.
 There are several notations for substitution.
 After writing $a[x:=b]$ throughout my book I~thought I~might change to
 $[b/x]^*a$.
 This macro reads the source in the first form and prints in the second.
 If you use it you can, like me, defer the decision about
 which notation to use until the final stages, doing
 \begin{quote}
    \verb/\renewcommand{\Subst}{\plainsubstitution}/
 \end{quote}
 if you finally decide on making substitution act on the right.
 This is already an improvement on the literal text, because it
 automatically enlarges the brackets according to the text inside.
 \verb/\Subst/ itself is (following my book)
 defined in terms of the action of a context morphism (\verb/\CtxtMor/)
 on a term. Again you can do
 \begin{quote}
    \verb/\renewcommand{\CtxtMor}{\plaincontextmorphism}/
 \end{quote}
 for something simpler.
 This macro interprets its argument as a comma-separated list
 of items in the form $x:=b$, which it switches to $b/x$.
 The simple versions.

%=======================================================================
\section{Some very easy logic exercises}

The following examples are taken from Krysia Broda's
{\em Solutions to Problems~5\/} (KB-Logic-B1-90) and took me a little
under an hour to type in.

page 1: (a)

\begin{proofbox}
   \: P\land Q \\
   \: P         \= \elim\land\\
\end{proofbox}

\begin{proofbox}
   \lbl{1}\: P \\
   \[
      \: Q \\
      \: P \= (\ref{1}) \\
   \]
   \: Q\to P \= \intro\to \\
\end{proofbox}

page 2: (c)

\begin{proofbox}
   \lbl{2}\: P\\
   \[
      \lbl{3}\: Q\\
      \: P\land Q\= \intro\land(\ref{2},\ref{3}) \\
   \]
   \: Q\to(P\land Q) \= \intro\to \\
\end{proofbox}

(g)

\begin{proofbox}
   \lbl{6}\: P\to(Q\to R)\\
   \[
      \lbl{4}\: P\to Q\\
      \[
         \lbl{5}\: P\\
         \lbl{8}\: Q \= \elim\to(\ref{4},\ref{5})\\
         \lbl{7}\: Q\to R \= \elim\to(\ref{6},\ref{5})\\
         \: R \= \elim\to(\ref{7},\ref{8})\\
      \]
      \: P\to R \= \intro\to \\
   \]
   \: (P\to Q)\to(P\to R) \= \intro\to \\
\end{proofbox}

page 3: (h)

\begin{proofbox}
   \lbl{10}\: P\to(Q\to R)\\
   \[
      \lbl{9}\: P\land Q\\
      \lbl{11}\: P \= \elim\land1(\ref{9})\\
      \lbl{12}\: Q\to R \= \elim\to(\ref{10},\ref{11})\\
      \lbl{13}\: Q \=\elim\land2(\ref{9})\\
      \: R \=\elim\to(\ref{12},\ref{13})\\
   \]
      \: P\land Q\to R \= \intro\to \\
\end{proofbox}

(i)

\begin{proofbox}
   \lbl{16}\: P\land Q\to R \\
   \[
      \lbl{14}\: P \\
      \[
         \lbl{15}\: Q \\
         \lbl{17}\: P\land Q \= \intro\land(\ref{14},\ref{15})\\
         \: R \= \elim\to(\ref{16},\ref{17})
      \]
      \: Q\to R \= \intro\to \\
   \]
   \: P\to(Q\to R) \= \intro\to \\
\end{proofbox}

(j)

\begin{proofbox}
   \lbl{18}\: P\to Q\\
   \lbl{20}\: \lnot Q\\
   \[
      \lbl{19}\: P \\
      \lbl{21}\: Q \= \elim\to(\ref{18},\ref{19})\\
      \: \bot \= \elim\lnot(\ref{20},\ref{21})\\
   \]
   \: \lnot P \= \intro\lnot \\
\end{proofbox}

page 5: (k)

\begin{proofbox}
   \lbl{22}\: \lnot P\\
   \[
      \lbl{23}\: P\\
      \[
         \: \lnot Q\\
         \: \lnot P \land P \= \intro\land(\ref{22},\ref{23})\\
         \: \bot \= \elim\lnot(\ref{22},\ref{23})\\
      \]
      \: \lnot\lnot Q \= \intro\lnot\\
      \: Q \= \lnot\lnot\\
   \]
   \: P\to Q \= \intro\to \\
\end{proofbox}

page 8: (o)

\begin{proofbox}
   \lbl{24}\: P\to Q \\
   \[
      \lbl{26}\: \lnot Q \\
      \[
         \lbl{25}\: P \\
         \lbl{27}\: Q \= \elim\to(\ref{24},\ref{25})\\
         \: \bot \= \elim\lnot(\ref{26},\ref{27})\\
      \]
      \: \lnot P \= \intro\lnot\\
   \]
   \: \lnot Q\to \lnot P \= \intro\to \\
\end{proofbox}

(p) The $\lnot\lnot$ rule is unnecessary!

\begin{proofbox}
   \lbl{28}\: P\to Q \\
   \[
      \: \lnot\lnot P\\
      \lbl{29}\: P \= \lnot\lnot\\
      \[
         \lbl{31}\: \lnot Q \\
         \lbl{30}\: Q \= \elim\to(\ref{28},\ref{29})\\
         \: Q\land\lnot Q \= \intro\land(\ref{30},\ref{31})\\
         \: \bot \= \elim\lnot\\
      \]
      \: \lnot\lnot Q \= \intro\lnot\\
   \]
   \: \lnot\lnot P\to\lnot\lnot Q \= \intro\to \\
\end{proofbox}

%=======================================================================
\section{Some more exercises}

\begin{proofbox}
   \: P\lor\lnot P \\
   \(
      \: P \\
      \: Q\to P \=\hbox{by (e)}\\
      \: (P\to Q)\lor(Q\to P) \= \intro\lor \\
   \+
      \: \lnot P\\
      \: P\to Q \=\hbox{by (k)}\\
      \: (P\to Q)\lor(P\to Q) \= \intro\lor \\
   \)
   \: (P\to Q)\lor(P\to Q) \= \elim\lor \\
\end{proofbox}

\begin{proofbox}
   \lbl{35}\: P\lor Q \\
   \[
      \lbl{30}\: (\lnot P)\land(\lnot Q) \\
      \(
         \lbl{31} \: P \\
         \lbl{32} \: \lnot P \=\elim\land1(\ref{30})\\
         \: \bot \= \elim\lnot(\ref{32},\ref{31})\\
      \+
         \lbl{33} \: Q \\
         \lbl{34} \: \lnot Q \= \elim\land2(\ref{30})\\
         \: \bot \= \elim\lnot(\ref{33},\ref{34})\\
      \)
      \: \bot \= \elim\lor(\ref{35})\\
   \]
   \: \lnot((\lnot P)\land(\lnot Q)) \= \intro\lnot \\
\end{proofbox}

\begin{proofbox}
\lbl{51} x \: \forall x'.x'<x\to p(x')\\
   \(
       \lbl{52}\: x=a \\
       \(
          \: a=c \\
               \: b<x \=\hbox{subst}(\ref{50})\\
               \: p(b) \= \elim\forall(\ref{51})\\
               \: b\neq b \= \elim\land\\
               \: \bot \= \hbox{refl}\\
       \+
          \: c<a \\
               \: c<x \=\hbox{subst}(\ref{52})\\
               \: p(c) \= \elim\forall(\ref{51})\\
               \: c\neq c \= \elim\land\\
               \: \bot \\
       \)
               \: \bot \= \elim\lor(\ref{53})\\
   \*
       \(
           \[
               \: x=b \\
                  \: a<x \\
                  \: p(a) \= \elim\forall\\
                  \: a\neq a\\
                  \: \bot \\
           \] \: x\neq b \= \intro\lnot \\
       \*
           \[
              \: x=c \\
                  \: b<x \\
                  \: p(b) \= \elim\forall\\
                  \: b\neq b\\
                  \: \bot
           \] \: x\neq c \= \intro\lnot \\
       \)
   \)
   \: x\neq a \land x\neq b \land x\neq c \= \intro\land\\
   \: p(x) \= \hbox{def} \\
\end{proofbox}

\begin{proofbox}
   \: a<b \\
   \lbl{40}\= b<c \\
   \: a<c \lor (a=c \lor c<a) \\
   \(
       \lbl{43}\: a=c \lor c<a \\
       \: p(x)\equiv(x\neq a\land (x\neq b \land x\neq c))\=\hbox{def}\\
       \[
          \lbl{41} x \: \forall x'.x'<x\to p(x')\\
          \(
              \lbl{42}\: x=a \\
            \(
               \: a=c \\
               \: b<x \=\hbox{subst}(\ref{40})\\
               \: p(b) \= \elim\forall(\ref{41})\\
               \: b\neq b \= \elim\land\\
               \: \bot \= \hbox{refl}\\
            \+
               \: c<a \\
               \: c<x \=\hbox{subst}(\ref{42})\\
               \: p(c) \= \elim\forall(\ref{41})\\
               \: c\neq c \= \elim\land\\
               \: \bot \\
            \)
            \: \bot \= \elim\lor(\ref{43})\\
         \*
            \(
               \[
                  \: x=b \\
                  \: a<x \\
                  \: p(a) \= \elim\forall\\
                  \: a\neq a\\
                  \: \bot \\
               \]
               \: x\neq b \= \intro\lnot \\
            \*
               \[
                  \: x=c \\
                  \: b<x \\
                  \: p(b) \= \elim\forall\\
                  \: b\neq b\\
                  \: \bot \\
               \]
               \: x\neq c \= \intro\lnot \\
            \)
       \)
       \: x\neq a \land x\neq b \land x\neq c \= \intro\land\\
       \: p(x) \= \hbox{def}\\
    \]
    \: \forall x.(\forall x'.x'<x\to p(x'))\to p(x) \= \intro\forall\\
   \: p(a) \= \elim\forall\\
   \: a\neq a \= \elim\land\\
   \: \bot\\
   \: a<c \= \elim\bot \\
\+
   \: a<c \\
\)
\: a<c \= \elim\lor\\
\end{proofbox}

%=======================================================================
\section{Krysia Broda's dragons exercise}

\def\happy#1{{{\mathsf{happy}}(#1)}}
\def\child#1#2{{{\mathsf{child}}(#1,#2)}}
\def\fly#1{{{\mathsf{fly}}(#1)}}
\def\dragon#1{{{\mathsf{dragon}}(#1)}}
\def\green#1{{{\mathsf{green}}(#1)}}
\def\parent#1#2{{{\mathsf{parent}}(#1,#2)}}

\def\all{\forall}
\def\some{\exists}
\def\imp{\Rightarrow}
\def\impby{\Leftarrow}

\advance\proofboxmargin-3em

\begin{proofbox}
\lbl{happy}\:\all x.\happy x\impby[\all y.\child y x\imp\fly y]
        \land\dragon x\\
\lbl{fly}\:\all x.\green x \land \dragon x \imp \fly x\\
\lbl{green}\:\all x.[\some y.\parent y x \land \green y]\imp\green x\\
\lbl{dragon}\:\all z.\all x.\child x z \land \dragon z \imp \dragon x\\
\lbl{parent}\:\all x.\all y.\parent x y \impby \child y x\\
\[\all x_0\lbl{Dragon}\:\dragon{x_0}\\
        \lbl{Green}\:\green{x_0}\\
        \[\all y_0\lbl{child}\:\child {y_0}{x_0}\\
                \(      \lbl{Parent}\:\parent{x_0}{y_0}
                                \=\elim\all(\ref{parent})\\
                        \:\parent{x_0}{y_0} \land\green{x_0}
                                \=\intro\land(\ref{Green})\\
\lbl{greenish}  \:\some z.\parent z {y_0} \land \green z
                                \=\intro\some(z:=x_0)\\
                        \:\green{y_0}\=\elim\all(\ref{green},x:=y_0)\\
                \*
                        \:\dragon{y_0}\=\elim\all(\ref{dragon},x:=y_0,
                                z:=x_0,\ref{Dragon})\\
                \)\lbl{greendragon}\:\green{y_0}\land\dragon{y_0}
                                \=\intro\land\\
                \:\fly{y_0}\=\elim\all(\ref{fly},x:=y_0,\ref{greendragon})\\
        \]\lbl{taught}\:\all y.\child y{x_0}\imp\fly y\=\intro\all\\
        \:\happy x\=\elim\all(\ref{happy},\ref{taught},\ref{Dragon})\\
\]\:\all x.\happy x \impby \green x \impby \dragon x\=\intro\all\\
\end{proofbox}
\noindent
Where the previous deduction is a premise of a rule, the reference is omitted.
Where the substitution is of the same letter (possibly with a subscript) it
is omitted, except for the $\alpha$-conversion (change of bound variable name)
in line~\ref{greenish}.

%=======================================================================
\section{Proof boxes from my book}

$$\proofboxformulawidth=18em\relax
\begin{proofbox}
\[\forall x:\lbl{ind box hyp}\:\forall x'.x'\prec x\imp\phi(x')
   \=\mbox{induction hypothesis}\\
\:\vdots\\
\lbl{ind box pred}\:u\prec x\=\mbox{various terms $u$}\\
\:\phi(u)\=\elim\forall(\ref{ind box hyp},\ref{ind box pred})\\
\:\vdots\\
\:\phi(x)\=\mbox{the property}\\
\]\:\forall x.(\forall x'.x'\prec x\imp\phi(x'))\imp\phi(x)\=\intro\forall\\
\:\forall x.\phi(x)\=\mbox{$\prec$-induction for $\phi$}\\
\end{proofbox}$$
$$\begin{proofbox}
\lbl{ind eg 0}\:\phi(0)\=z\\
\lbl{ind eg s}\:\forall n.\phi(n)\imp\phi(n+1)\=s\\
\lbl{ind eg 1-}\:\phi(0)\imp\phi(1)\=\elim\forall(\ref{ind eg s})\\
\lbl{ind eg 1}\:\phi(1)\=\elim\imp(\ref{ind eg 1-},\ref{ind eg 0})\\
\lbl{ind eg 2-}\:\phi(1)\imp\phi(2)\=\elim\forall(\ref{ind eg s})\\
\lbl{ind eg 2}\:\phi(2)\=\elim\imp(\ref{ind eg 2-},\ref{ind eg 1})\\
\lbl{ind eg 3-}\:\phi(2)\imp\phi(3)\=\elim\forall(\ref{ind eg s})\\
\lbl{ind eg 3}\:\phi(3)\=\elim\imp(\ref{ind eg 3-},\ref{ind eg 2})\\
\end{proofbox}$$
$$\begin{proofbox}
\lbl{mrwf1}\:\forall x.(\forall x'.x'\lt x\imp\phi(x'))\imp\phi(x)
\=\mbox{hypothesis}\\
\lbl{mrwf4}\:\psi(y)\eqdef\forall x.(fx=y)\imp\phi(x)\=\mbox{definition}\\
\[\forall y:\lbl{mrwf2}\:\forall y'.y'\prec y\imp\psi(y') \\
   \[\forall x:\lbl{mrwf3}\:fx=y\\
      \[\forall x':\:x'\lt x\\
          \:fx'\prec y\=\hbox{monotonicity}\\
          \:\psi(fx')\=\elim\forall(\ref{mrwf2})\\
          \:\phi(x')\=\elim\forall(\hbox{def }\ref{mrwf4},\ref{mrwf3})\\
      \]
      \:\forall x'.x'\lt x\imp\phi(x')\=\intro\forall\\
      \:\phi(x)\=\elim\forall(\ref{mrwf1})\\
   \]
   \:\forall x.(fx=y)\imp\phi(x)\=\intro\forall\\
  \:\psi(y)\=\hbox{def}(\ref{mrwf4})\\
\]
\:\forall y.(\forall y'.y'\prec y\imp\psi(y'))\imp\psi(y)\=\intro\forall\\
\:\forall y.\psi(y)\=(Y,{\prec})\hbox{-induction}\\
\end{proofbox}$$
$$\begin{proofbox}
\lbl{qwf1}\:\forall y.[\forall y'.y'\eps y\imp\phi(y')]\imp\phi(y)\\
\[\lbl{qwf2}\forall x:\:\forall x'.x'\prec x\imp\phi(fx')\\
\[\forall y':\:y'\eps fx\\
\lbl{qwf3}\:\exists x'.x'\prec x\land y'=fx'
\=\hbox{surj on pred}\\
\[\exists x':\:x'\prec x\\
\:y'=fx'\\
\:\phi(fx')\=\elim\forall(\ref{qwf2})\\
\:\phi(y')\=\hbox{substitution}\\
\]\:\phi(y')\=\elim\exists(\ref{qwf3})\\
\]\:\forall y'.y'\eps fx\imp\phi(y')\=\intro\forall\\
\:\phi(fx)\=\elim\forall(\ref{qwf1},y:=fx)\\
\]\:\forall x.[\forall x'.x'\prec x\imp\phi(fx')]\imp\phi(fx)\=\intro\forall\\
\end{proofbox}$$
$$\begin{proofbox}
\lbl{trwf1}\:\forall x_2,x.[x_2\ll x \biimplies
x_2\prec x \lor \exists x_1. x_2\ll x_1\prec x]\\
\lbl{trwf2}\:\forall x.[\forall x'.x'\prec x\imp\phi(x')]\imp\phi(x)
\=\mbox{hypothesis}\\
\lbl{trwf3}\:\psi(x)\eqdef\forall x_2.x_2\ll x\imp\phi(x_2)
\=\mbox{definition}\\
\lbl{trwf4}\:\forall x.\psi(x)\imp\phi(x)
\=\intro\forall(\elim\forall(\ref{trwf2},\ref{trwf3}))\\
\(\:\phi(x)\\
\:\forall x_2.x_2\ll x\imp\phi(x_2)\\
\[\forall x_1:\:x_2\ll x_1\\
\:x_2\ll x\\
\:\phi(x_2)\\
\:\forall x_2.x_2\ll x\imp\phi(x_2)\\
\:\psi(x_1)\\
\]\:\forall x_1.x_1\prec x\imp\psi(x_1)\\
\+\lbl{trwf5}\forall x:\:\forall x_1.x_1\prec x\imp\psi(x_1)\\
\[\forall x_2:\:x_2\ll x\\
\lbl{trwf7}\:x_2\prec x_1\lor\exists x_1.x_2\ll x_1\prec x
\=\elim\forall(\ref{trwf1})\\
\(\:x_2\prec x_1\\
\:\psi(x_2)\=\elim\forall(\ref{trwf5})\\
\:\phi(x_2)\=\elim\forall(\ref{trwf4})\\
\+\exists x_1:\:x_2\ll x_1\prec x\\
\:\psi(x_1)\=\elim\forall(\ref{trwf5})\\
\:\phi(x_2)\=\elim\forall(\ref{trwf3},x:=x_1)\\
\)\:\phi(x_2)\=\elim{\exists{\lor}}(\ref{trwf7})\\
\]\:\forall x_2.x_2\ll x\imp\phi(x_2)\=\intro\forall\\
\:\psi(x)\=\hbox{def}(\ref{trwf3})\\
\)\:\forall x.[\forall x_1.x_1\prec x\imp\psi(x_1)]\biimplies\psi(x)
\=\intro{\forall{\biimplies}}\hfill\\
\end{proofbox}$$
$$\begin{proofbox}
\:\forall U.[\forall V.V\flt\prec U\imp\phi(V)]\biimplies\phi(U)\\
\:\phi(\emptyset)\\
\[\forall x,U:\:\phi(U)\\
\lbl{fltindxhyp}\:
        [\forall y.y\prec x\imp[\forall V.\phi(V)\imp\phi(V\cup\setof y)]]\\
\[\forall V_0:\:V_0\flt\prec U\\
\:\phi(V_0)\=\mbox{premise}\\
\:\emptyset\flt\prec\setof x\imp\phi(V_0)\\
\[\forall y,V_1:\:V_1\flt\prec x\\
\lbl{fltindV0V1}\:\phi(V_0\cup V_1)\\
\lbl{fltindy<x}\:y\prec x\\
\:\phi(V_0\cup V_1\cup\setof y)
        \=\elim\forall(\ref{fltindxhyp},\ref{fltindy<x},V:=V_0\cup V_1,
        \ref{fltindV0V1})\\
\]
\:\forall y,V_1.V_1\flt\prec x\land y\prec x\land\phi(V_0\cup V_1)
        \imp\phi(V_0\cup V_1\cup\setof y)\=\intro\forall\\
\:\forall V_1.V_1\flt\prec\setof x\imp\phi(V_0\cup V_1)
        \=\mbox{$\KK$-induction}\\
\]
\:\forall V_0,V_1.V_0\flt\prec U\land V_1\flt\prec\setof x
        \imp\phi(V_0\cup V_1)\=\intro\forall\\
\:\forall V.V\flt\prec U\cup\setof x\imp\phi(V)\=\mbox{Lemma}\\
\:\phi(U\cup\setof x)\=\mbox{premise}\\
\]
\:\forall x.[\forall y.y\prec x\imp[\forall V.\phi(V)\imp\phi(V\cup\setof y)]]
        \imp[\forall U.\phi(U)\imp\phi(U\cup\setof x)]\=\intro\forall\\
\:\forall x.[\forall U.\phi(U)\imp\phi(U\cup\setof x)]
        \=\mbox{$\prec$-induction}\\
\:\forall U.\phi(U)\=\mbox{$\KK$-induction}\\
\end{proofbox}$$
$$\proofboxnonumbers\proofboxformulawidth=4em
\begin{array}{cc}
\begin{proofbox}\[\qquad\:\phi\=\qquad\\\:\vdots\\\:\psi\\\]
\:\phi\imp\psi\=\intro\imp\end{proofbox}&
\begin{proofbox}\[\forall x':\\\:\vdots\=\qquad\\\:\phi(x')\\\]
\:\forall x.\phi(x)\=\intro\forall\end{proofbox}\\
\begin{proofbox}\qquad\:\phi\lor\psi\\
\(\qquad\:\phi\=\qquad\\\:\vdots\\\:\alpha\\
\+\:\psi\=\qquad\\\:\vdots\\\:\alpha\\\)
\:\alpha\=\elim\lor\end{proofbox}&
\begin{proofbox}\:\exists x.\phi(x)\\
\[\exists x':\:\phi(x')\=\qquad\\\:\vdots\\\:\alpha\\\]
\:\alpha\=\elim\exists\end{proofbox}
\end{array}$$
$$\proofboxformulawidth=30em \begin{proofbox}
\(\:\alpha\biimplies\psi(x,\top)\\
\:\exists\beta.\psi(x,\beta)\=\mathsf{total}\\
\[\exists\beta:\:\psi(x,\beta)\\
\(\:\beta\\
\:\beta=\top\\
\:\psi(x,\top)\=\mathsf{subs}\\
\:\alpha\=\elim\biimplies\\
\*\:\alpha\\
\:\psi(x,\top)\=\elim\biimplies\\
\:\beta=\top\=\mathsf{func}\\
\:\beta\\
\)\:\alpha=\beta\=\intro\biimplies\\
\:\psi(x,\alpha)\=\mathsf{subs}\\
\]\:\psi(x,\alpha)\=\elim\exists\\
\*
\:\psi(x,\alpha)\\
\(\:\alpha\\
\:\alpha=\top\\
\:\phi(x,\top)\=\mathsf{subs}\\
\*\:\psi(x,\top)\\
\:\alpha=\top\=\mathsf{func}\\
\:\alpha\\
\)
\:\alpha\biimplies\psi(x,\top)\=\intro\biimplies\\
\)\:\psi(x,\alpha)\biimplies(\alpha\biimplies\psi(x,\top))\=\intro\biimplies\\
\end{proofbox}$$
$$\proofboxformulawidth=7em \proofboxmargin=2em
\begin{proofbox}
\[\lbl{pnn0h}\:\alpha\\
\lbl{pnn0r}\:\alpha\imp\alpha\\
\:\alpha\=\qquad\quad(\ref{pnn0h})\\
\]\:\alpha\imp(\alpha\imp\alpha)\imp\alpha\=\intro\forall\\
\proofboxnonumbers\:(0)\\
\end{proofbox}
\hskip 0pt minus3em\relax
\begin{proofbox}
\[\lbl{pnn1h}\:\alpha\\
\lbl{pnn1r}\:\alpha\imp\alpha\\
\:\alpha\=\elim\imp(\ref{pnn1r},\ref{pnn1h})\\
\]\:\alpha\imp(\alpha\imp\alpha)\imp\alpha\=\intro\forall\\
\proofboxnonumbers\:(1)\\
\end{proofbox}
\hskip 0pt minus3em\relax
\begin{proofbox}
\[\lbl{pnn2h}\:\alpha\\
\lbl{pnn2r}\:\alpha\imp\alpha\\
\lbl{pnn21}\:\alpha\=\elim\imp(\ref{pnn2r},\ref{pnn2h})\\
\:\alpha\=\elim\imp(\ref{pnn2r},\ref{pnn21})\\
\]\:\alpha\imp(\alpha\imp\alpha)\imp\alpha\=\intro\forall\\
\proofboxnonumbers\:(2)\\
\end{proofbox}
%  \hskip 0pt minus3em\relax
%  \begin{proofbox}
%  \[\lbl{pnn3h}\:\alpha\\
%  \lbl{pnn3r}\:\alpha\imp\alpha\\
%  \lbl{pnn31}\:\alpha\=\elim\imp(\ref{pnn3r},\ref{pnn3h})\\
%  \lbl{pnn32}\:\alpha\=\elim\imp(\ref{pnn3r},\ref{pnn31})\\
%  \:\alpha\=\elim\imp(\ref{pnn1r},\ref{pnn32})\\
%  \]\:\alpha\imp(\alpha\imp\alpha)\imp\alpha\=\intro\forall\\
%  \proofboxnonumbers\:(3)\\
%\end{proofbox}%Overfull \hbox (17.42128pt too wide) detected
$$
$$\vbox{\proofboxformulawidth=12em \relax
\begin{proofbox}
\lbl{pthora1}\:\phi\land\psi\\
\[\lbl{pthora2}\forall\alpha:\:\phi\implies(\psi\implies\alpha)\\
\lbl{pthora3}\:\phi\=\elim{{\land}1}(\ref{pthora1})\\
\lbl{pthora4}\:\psi\implies\alpha
\=\elim\implies(\ref{pthora2},\ref{pthora3})\\
\lbl{pthora5}\:\psi\=\elim{{\land}2}(\ref{pthora1})\\
\lbl{pthora6}\:\alpha
\=\elim\implies(\ref{pthora4},\ref{pthora5})\\
\]\:\forall\alpha.(\phi\implies(\psi\implies\alpha))\implies\alpha
\=\intro{\forall{\implies}}\\
\end{proofbox}}
\kern6em
%
% Underfull \hbox (badness 10000) in paragraph
% Overfull \hbox (282.68869pt too wide) detected
%
\vbox{\baselineskip=\proofboxbaselineskip\relax
$p_1:\phi\times\psi$\\
$p_2:\phi\to(\psi\to\alpha)$\\
$p_3=\pi_1(p_1):\phi$\\
$p_4=p_2(p_3):\psi\to\alpha$\\
$p_5=\pi_2(p_1):\psi$\\
$p_6=p_4(p_5):\alpha$\\
$p_7=\lambda\alpha.\lambda p_2.p_6
%  =\lambda\alpha\lambda p_2.p_2(\pi_1(p_1))(\pi_2(p_1))
%  :\Pi\alpha.(\phi\to(\psi\to\alpha))\to\alpha
$\\}$$

%=======================================================================
\section{From my JSL paper}

%--------------------------------------------------------------------------
%                 save the proof box so that we can find out its height
\setbox9=\vbox{%
\proofboxbaselineskip=1.42em
%
%  parameters for version 1:
   \proofboxmargin=80pt
   \proofboxformulawidth=\hsize
   \advance\proofboxformulawidth-2\proofboxmargin
\iffalse
   \def\isX{:X}%
   \def\isPfX{:\finpower(X)}%
\else
   \def\isX{}\def\isPfX{}%
\fi
%
\begin{proofbox}%%*** WARNING: the names of the labels dont match their numbers
\lbl{fltwf1}\:\All U.[\All V.V\flt\prec U\Implies\phi(V)]
\biimplies\phi(U)\\
%\lbl{fltwf2}\:\All V.V\flt\prec\emptyset\Implies\bot
%   \=\mbox{def}({\flt\prec})\\
\lbl{fltwf3}\:\phi(\emptyset)
    \=\elim\forall(\ref{fltwf1},\mbox{def}({\flt\prec}))\\ %\ref{fltwf2})\\
\lbl{fltwf4}\:\psi(x)\eqdef\forall U.\phi(U)\Implies\phi(U\cup\setof x)\\
\[\lbl{fltwf5}\lbl{fltindxhyp}\forall x\isX
   \:\forall y.y\prec x\Implies\psi(y)\\
%\:[\forall y.y\prec x\Implies[\All V.\phi(V)\Implies\phi(V\cup\setof y)]]\\
\[\lbl{fltwf8}\forall V_0\isPfX
\lbl{fltwf10}\:\phi(V_0)\\
\lbl{fltwf9}\:\theta(W)\eqdef W\flt\prec\setof x\Implies\phi(V_0\cup W)\\
\lbl{fltwf11}\:\theta(\emptyset)
   \=\mbox{def}(\ref{fltwf9},\ref{fltwf10})\\
\[\forall W\isPfX
   \:\theta(W)\\
  \forall y\isX\lbl{fltwf13}\: W\flt\prec\setof x\Implies\phi(V_0\cup W)
   \=\mbox{def}(\ref{fltwf9})\\
\[\lbl{fltwf14}\:W\cup\setof y\flt\prec\setof x
   \equiv W\flt\prec\setof x\land y\prec x\\
%\lbl{fltwf15}\:W\flt\prec\setof x\=\ref{fltwf14},\hbox{def}({\flt\prec})\\
\lbl{fltwf16}\:\phi(V_0\cup W)\=\elim\Implies(\ref{fltwf13},\ref{fltwf14})\\
%\lbl{fltwf17}\:y\prec x\=\ref{fltwf14},\hbox{def}({\flt\prec})\\
\lbl{fltwf18}\:\psi(y)\equiv\forall V.\phi(V)\Implies\phi(V\cup\setof y)
\=\elim\forall(\ref{fltwf5},\ref{fltwf14}),\mbox{def}(\ref{fltwf4})\\
\:\phi(V_0\cup W\cup\setof y)\=\elim\forall(\ref{fltwf18},\ref{fltwf16})\\
\]
\:W\cup\setof y\flt\prec\setof x\Implies\phi(V_0\cup W\cup\setof y)
   \=\intro\Implies\\
\:\theta(W\cup\setof y)\=\mbox{def}(\ref{fltwf9})\\
\]
\:\theta(\emptyset)\land
     \forall y.\All W.\big[\theta(W)\Implies\theta(W\cup\setof y)\big]
     \=\intro\land(\ref{fltwf11},\intro\forall)\\
\:\forall W.\theta(W)\=\mbox{$\KK$-induction for }\theta\\
\:\All W.W\flt\prec\setof x\Implies\phi(V_0\cup W)\=\mbox{def}(\ref{fltwf9})\\
\]
%\:\All V_0.V_0\flt\prec U\Implies\All W.\theta(W)\=\intro\forall\\
\lbl{fltwf24}\:\squashmuskip
\All V_0.\phi(V_0)\Implies
      \big(\All W.W\flt\prec\setof x\Implies\phi(V_0\cup W)\big)
        \=\intro\forall\\    %\mbox{Currying, def(\ref{fltwf9})}\\
\[\lbl{fltwf6}\forall U\isPfX
   \:\phi(U)\\
%   \:V_0\flt\prec U \=\elim\forall(\ref{fltwf1},\ref{fltwf6},\ref{fltwf8})\\
\lbl{fltwfa}\:U=\emptyset\lor U\neq\emptyset
   \=\hbox{Proposition~\ref{free semil}}\\
\lbl{fltwfb}\:
   U=\emptyset\Implies\All W.W\flt\prec(U\cup\setof x)\Implies\phi(W)
   \=\elim\forall(\ref{fltwf24},V_0=\emptyset,\ref{fltwf3})\\
\[\lbl{fltwfe}\forall V_0\isPfX\:V_0\flt\prec U\\
\lbl{fltwff}\forall W\isPfX\:W\flt\prec\setof x\\
\label{fltwfg}\:\phi(V_0)
   \=\elim{\forall{\Impliedby}}(\ref{fltwf1},\ref{fltwf6},\ref{fltwfe})\\
\lbl{fltwfh}\:\phi(V_0\cup W)
   \=\elim\forall(\ref{fltwf24},\ref{fltwfg},\ref{fltwff})\\
\]\:\All V_0,W.V_0\flt\prec U\land W\flt\prec\setof x\Implies\phi(V_0\cup W)
  \=\intro\forall\\
\lbl{fltwfk}\:
   U\neq\emptyset\Implies\forall V.V\flt\prec(U\cup\setof x)\Implies\phi(V)
  \=\hbox{Lemma \ref{flt power distr}}\\
\lbl{fltwf27}\:\forall V.V\flt\prec(U\cup\setof x)\Implies\phi(V)
        \=\elim\lor(\ref{fltwfa},\ref{fltwfb},\ref{fltwfk})\\
\:\phi(U\cup\setof x)\=\elim\forall(\ref{fltwf1},\ref{fltwf27})\\
\]\lbl{fltwf29}
\:\forall U.\phi(U)\Implies\phi(U\cup\setof x)\equiv\psi(x)
        \=\intro\forall,\hbox{def}(\ref{fltwf4})\\
\]
\:\forall x.[\All y.y\prec x\Implies\psi(y)]\Implies\psi(x)\=\intro\forall\\
\lbl{fltwf32}\:\forall x.\psi(x)\=\mbox{$\prec$-induction for }\psi\\
\:\phi(\emptyset)\land\All x.\All U.
        \big[\phi(U)\Implies\phi(U\cup\setof x)\big]
        \=\intro\land(\ref{fltwf3},\mbox{def}(\ref{fltwf4},\ref{fltwf32}))\\
\:\All U.\phi(U)\=\mbox{$\KK$-induction for }\phi      \\%\;\hfill\qedsymbol\\
\end{proofbox}%
}% end of \setbox9=\vbox

\box9

%=======================================================================

\nocite{*}
\bibliographystyle{plain}
%\bibliography{boxuser}

\begin{thebibliography}{1}

\bibitem{VickersSJ:reasprog}
K.~Broda, S.~Eisenbach, H.~Khoshnevisan, and Steven Vickers.
\newblock {\em Reasoned Programming}.
\newblock International Series in Computer Science. Prentice Hall, 1994.

\bibitem{TaylorP:intso}
Paul Taylor.
\newblock Intuitionistic sets and ordinals.
\newblock {\em Journal of Symbolic Logic}, 61:705--744, 1996.

\bibitem{TaylorP:prafm}
Paul Taylor.
\newblock {\em Practical Foundations of Mathematics}.
\newblock Number~59 in Cambridge Studies in Advanced Mathematics. Cambridge
  University Press, 1999.

\end{thebibliography}

%=======================================================================

\end{document}


\documentclass[11pt]{article}
\usepackage{xcolor}
\usepackage{bbm}
\usepackage{amsmath}
\usepackage{hyperref}
\DeclareMathOperator*{\argmin}{arg\,min}
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    filecolor=magenta,
    urlcolor=violet,
}
\newcommand{\numpy}{{\tt numpy}}    % tt font for numpy

\topmargin -.5in
\textheight 9in
\oddsidemargin -.25in
\evensidemargin -.25in
\textwidth 7in

\begin{document}

% ========== Edit your name here
\title{SAILON Evaluation Metrics: Rigorously Defined}
\author{Sam Showalter}
\maketitle

\medskip
\section{Experimental Setup}

The SAILON evaluation system is abstractly defined as follows. \textit{A priori}, a number of trials $T$ is defined, where each trial $t \in T$ contains a single batch $b$ of $n$ samples. The total number of samples across the entire evaluation is denoted $Tn = N$.

Each sample $\mathbf{x}$ is a video of one of $k$ possible actions, or an action unseen during training time as denoted the $k + 1$ novel class. These samples are independently fed through a network and novelty detection system, considered jointly in this setup as $f_\theta(.)$. In this setting, the output for any sample $\mathbf{x_i}$ fed through $f_\theta(.)$ is a scalar label $y_{\theta i} \in \{0,1, ... k, k+1\}$ where $k + 1$ represents a novel input sample.

In addition, there is a third metric set \textit{a-priori} on the trials of the evaluation. At an unspecified trial $t_a \in T$, the presence of novel instances begins and will remain for all subsequent trials. Before trial $t_a$, no novel samples are present, though the model $f_\theta(.)$ can at any time predict $y_{\theta i} = k + 1$.

The evaluation is run sequentially in a batch-wise fashion over trials. Accordingly, $\mathbf{x}_{ti}$ represents the $i^{th}$ sample of the $t^{th}$ batch, and $y_{ti}$ represents the corresponding label. In addition, a baseline model $f_\phi(.)$, whose predictions are denoted $\hat{y}_{\phi ti} \in \{0, 1, ... , k\}$, cannot predict novelty and is often referred to as the \textbf{baseline model}. All evaluation metrics are determined based on the relationship of predicted labels $\hat{y}_{\theta ti}$ to the ground-truth labels $y_{ti}$ of the evaluation and to the baseline predictions $\hat{y}_{\phi ti}$.

Implicitly, if a given trial $t$ at least one sample in its batch $b_t$ was predicted to be novel ($\hat{y}_{\theta ti} = 1$), the trial itself is tagged with binary label 1 for its batch $\hat{l}_t = 1$, signifying the agent believes novelty has started. After the first trial possessing novelty occurs, all future trials will be given a novelty label = 1 regardless of the predictions in its batch. The ground truth labels across trials is denoted $l_t$

\section{Trial-based Evaluation Metrics: Detection Performance}%
\label{sec:trial_based_evaluation_metrics}

First, trial-based evaluation metrics are evaluated, with an optimal performance defined as predicting novelty for the first time in batch $b_{ta}$ belonging to trial $t_a \in T$. Failure modes in terms of false positives and false negatives are defined as follows, where $t$ represents the $t^{th}$ trial and $\hat{l}_t$ represents the binary label defining if any samples in batch $b_t$ were classified as novel.

\begin{equation}
    \begin{aligned}
      TP &= \hat{l}_t = l_t \\
      FP &= \hat{l}_t > l_t \\
      FN &= \hat{l}_t < l_t
    \end{aligned}
\end{equation}

These metrics are then utilized to generate slightly more sophisticated evaluation metrics across trials, including:

\begin{equation}
    \begin{aligned}
      \text{Correctly Detected Trials (CDT)} &= \sum^{T}_{t = 1} \mathbbm{1}( (FP_{t} = 1) \ \& \ (TP_t >= 1)) \\
      \text{Trial Failures} &= (t_a - \argmin_t(l_t = 1)) \\
      \text{False Positive Trial \%} &= \frac{FP}{T} \\
      \text{False Negative Trial \%} &= \frac{FN}{T}
    \end{aligned}
\end{equation}

In summary, the primary goal of this family of evaluation metrics is to minimize the number of false positives. This is far more important to PAR than the other metrics. Trial Failures exclusive looks at how many trials too early you declared novelty.

\section{Sample-based Evaluation Metrics: Reaction Performance}%
\label{sec:sample_based_evaluation_metrics_reaction_performance}

The next family of evaluation metrics concerns sample-based evaluation metrics. These metrics conducted over batches (unless specified otherwise) and then averaged. The evaluation metrics are defined below.

\begin{equation}
    \begin{aligned}
      FP &= (\hat{y}_{\theta ti} = k+1) \ \& \ (y_{ti} \neq k+1)) \\
      FN &= (\hat{y}_{\theta ti} \neq k+1) \ \& \ (y_{ti} = k+1))
    \end{aligned}
\end{equation}

These can be computed in any way, but in particular there are several metrics that compare these FP, FN values in the pre- and post-novelty regime. The formulas are the same for the baseline model, but $\theta$ is replaced by $\phi$.

\begin{equation}
    \begin{aligned}
      \text{Accuracy} &= \frac{1}{T} \sum^{T}_{t=1} \frac{1}{n} \sum^{n}_{i=1} \mathbbm{1}(\hat{y}_{\theta ti} = y_{ti}) \\
      \text{Baseline Accuracy} &= \frac{1}{T} \sum^{T}_{t=1} \frac{1}{n} \sum^{n}_{i=1} \mathbbm{1}(\hat{y}_{\phi ti} = y_{ti}) \\
    \end{aligned}
\end{equation}

Per trial asymptotic performance is defined as the ``measure in the change in accuracy over post-novelty rounds''. No notation or formulae of any kind accompany this statement. However, some additional metrics include:

\begin{equation}
    \begin{aligned}
      \text{Novelty Response Performance (NRP)} &= \frac{( \text{Accuracy} | t \geq t_a)}{( \text{Accuracy} | t < t_a)}  \\
      \text{Overall Performance Task Improvement (OPTI)} &= \frac{( \text{Accuracy} | t \geq t_a)}{( \text{Baseline Accuracy} | t \geq t_a)} \\
      \text{Asymptotic Performance Task Improvement (APTI)} &= \frac{( \text{Accuracy per Trial} | t \geq t_a)}{( \text{Baseline Accuracy per trial} | t \geq t_a)} \\
      \text{Area Under Activity Monitor Operating Characteristic (AUAMOC)} &= \text{AUROC of TP and FP}
    \end{aligned}
\end{equation}

In words, \textbf{NRP} is the ratio of the average batch accuracy in batches after novelty over the average batch accuracy in batches before novelty - we want this as high as possible. \textbf{OPTI} is the ratio of post-novelty accuracy in our model over post-novelty accuracy of the baseline. We also want this to be as high as possible. \textbf{APTI} is a more granular \textbf{OPTI} that makes use of per-trial differences. \textbf{AUAMOC} is a simple AUROC curve plotted for global TP and FP, \textbf{threshold based on anomaly score}

\section{Oustanding items}%
\label{sec:oustanding_items}

\begin{enumerate}
  \item Top-k classification with anomaly score not a probability - how do they intend to do this?
\end{enumerate}

\end{document}


\chapter{Background}
\label{ch:02-background}

We can make human language manageable to computers by using a sort of different methods and techniques. This is the bread and butter of any NLP researcher. Here, following the NLP community, we will focus only in the techniques derived from the machine learning field. In this chapter, we will present some abstract machine learning models follow by some applications in NLP.

\section{Machine Learning}

\textit{Machine learning} is the branch of computer science that deals with programs that can improve with experience (i. e., learn). Machine learning is divided into three main subareas: \textit{supervised learning}, \textit{reinforcement learning} and \textit{unsupervised learning}.

We will work with supervised learning only. In this setting, we assume that the process of interest is defined by an unknown function $g:X\rightarrow Y$. We try to approximate $g$ by changing the parameters of a function $f$ through an optimization process. The optimization is done by defining an \textit{error function} that evaluates how well $f$ approximates $g$ in the part of $g$ that we have access: the training data $D = \{(\vect{x}^{(1)}, \vect{y}^{(1)}), \dots ,(\vect{x}^{(N)}, \vect{y}^{(N)})\}$ (where $g(\vect{x}^{(i)})=\vect{y}^{(i)}$).

$f$ can be a function from any family of models. One family that is having a lot of success for language tasks is the \textit{neural network} family.

\section{Neural Networks}

A neural network is a non-linear function $f(\vect{x}; \vect{\theta})$. It is defined by a collection of parameters $\vect{\theta}$ and a collection of non-linear transformations. It is usual to represent $f$ as a compositions of functions:

\begin{align}
f(\vect{x}; \vect{\theta}) &= f^{(2)}(f^{(1)}(\vect{x}; \vect{W}_1, \vect{b}_1); \vect{W}_2, \vect{b}_2)\\
&= softmax(\vect{W}_2 (\sigma(\vect{W}_1\vect{x} + \vect{b}_1)) + \vect{b}_2)
\end{align}

The output of these intermediary functions are referred as \textit{layers}. So in the example above, $\vect{x}$ (the output of the identity function) is the \textit{input layer}, $f^{(1)}(\vect{x}; \vect{W}_1, \vect{b}_1)$ is the \textit{hidden layer} and $f^{(2)}(f^{(1)}(\vect{x}; \vect{W}_1, \vect{b}_1); \vect{W}_2, \vect{b}_2)$ is the \textit{output layer}. Since each layer is a vector, we normally speak about the \textit{dimension} of a layer. For historical reasons we also say that each entry on a layer is a \textit{node} or a \textit{neuron}.  Models with a large number of hidden layers are called \textit{deep models}, for this reason the name \textit{deep learning} is used.

\par A neural network is a function approximator: it can approximate any Borel measurable function from one finite dimensional space to another with any desired nonzero amount of error. This theoretical result is know as the \textit{universal approximation theorem} \cite{Cybenko}. Without entering in the theoretical concepts, it suffice to note that the family of Borel mensurable functions include all continuous functions on a closed and bounded subset of $\mathbb{R}^n$.

Different deep learning architectures are used in NLP: \textit{convolutional architectures} have a good performance in tasks were it is required to find a linguistic indicator regardless of its position (e.g., document classification, short-text categorization, sentiment classification, etc); high quality word embeddings can be achieved with models that are a kind of \textit{feedforward neural network} \cite{Mikolov23}. But for a variety of works in natural language we want to capture regularities and similarities in a text structure. That is way \textit{recurrent} and \textit{recursive} models have been widely used in the field. Here we are focused on generative models and since recurrent models have been producing very strong results for language modeling \cite{goldberg15}, we will concentrate on them.

\section{Recurrent Neural Network}
\label{sec:RNN}

\textit{Recurrent neural network} (RNN) is a family of neural network specialized in sequential data $\vect{x}^{(1)}, \dots, \vect{x}^{(\tau)}$. As a neural network, a RNN is a parametrized function that we use to approximate one hidden function from the data. What makes RNN unique is a recurrent definition of one of its hidden layer:

\begin{equation}
\vect{h}^{(t)} = g(\vect{h}^{(t-1)}, \vect{x}^{(t)}; \vect{\theta})
\end{equation}

$\vect{h}^{(t)}$ is called \textit{state}, \textit{hidden state}, or \textit{cell}.

\par This recurrent equation can be unfolded for a finite number of steps $\tau$. For example, when $\tau =3$:

\begin{align}
\vect{h}^{(3)}& = g(\vect{h}^{(2)}, \vect{x}^{(3)}; \vect{\theta})\\
 & = g(g(\vect{h}^{(1)}, \vect{x}^{(2)}; \vect{\theta}), \vect{x}^{(3)}; \vect{\theta})\\
 & = g(g(g(\vect{h}^{(0)}, \vect{x}^{(1)}; \vect{\theta}), \vect{x}^{(2)}; \vect{\theta}), \vect{x}^{(3)}; \vect{\theta})\\
\end{align}

Using a concrete example consider the following classification model define by the equations:

\begin{equation}
f(\vect{x}^{(t)}, \vect{h}^{(t-1)}; \vect{V}, \vect{W}, \vect{U}, \vect{c}, \vect{b}) = \vect{\hat{y}}^{(t)}
\end{equation}
 \vspace{0.2cm}
\begin{equation}
\vect{\hat{y}}^{(t)} = softmax(\vect{V} \vect{h}^{(t)} + \vect{c})
\end{equation}
\vspace{0.2cm}
 \begin{equation}
\vect{h}^{(t)} = g(\vect{h}^{(t-1)}, \vect{x}^{(t)}; \vect{W},\vect{U}, \vect{b})
\end{equation}
\vspace{0.2cm}
\begin{equation}
\vect{h}^{(t)} = \sigma(\vect{W} \vect{h}^{(t-1)} + \vect{U} \vect{x}^{(t)} + \vect{b})
\end{equation}

This kind of model can create an output $\vect{\hat{y}}^{(t)}$ at each time $t$, or the model can produce a single output $\vect{\hat{y}}$ after processing an entire input sequence. This choice depends on the learning problem that is being modeled.

With the model's prediction at hand, we can use a loss function (like cross entropy for the classification problem) and apply the back-propagation algorithm to optimize the model. These models look complex, but it quite straightforward to compute the gradients \cite[p.~374]{DeepLearningbook}.

Although this kind of deep learning model is very useful, it presents a severe flaw. When computing the gradients there is a lot of repeated matrix multiplication using the recurrent weight matrix (in the example above, the matrix $\vect{W}$). Depending on some configurations of this matrix \textit{the gradients may vanish or explode exponentially with respect to the number of time steps}.

Thus, handing long-term dependencies became a problem when using RNNs. Different solutions were proposed, the most effective results came from some modifications of this model. We will present the two most famous modifications: \textit{the gated recurrent unit} and \textit{the long short-term memory}.

\section{Gated Recurrent Unit}
\label{sec:GRU}

To capture long-term dependencies on a RNN  the authors of the paper \cite{ChungGCB14}  proposed a new architecture called \textit{gated recurrent unit} (GRU). This model was constructed to make each hidden state  $\vect{h}^{(t)}$ to adaptively capture dependencies of different time steps. It work as follows, at each step $t$ one candidate for hidden state is formed:

\begin{equation}
\vect{\widetilde{h}}^{(t)} = tahn(\vect{W} (\vect{h}^{(t-1)} \odot  \vect{r}^{(t)}) + \vect{U} \vect{x}^{(t)} + \vect{b})
\end{equation}

where $\vect{r}^{(t)}$ is a vector with values in $[0, 1]$ called a \textit{reset gate}, i.e.,  a vector that at each entry outputs the probability of reseting the  corresponding entry in the previous hidden state $\vect{h}^{(t-1)}$. Together with $\vect{r}^{(t)}$ we define an \textit{update gate}, $\vect{u}^{(t)}$. It is also a vector with values in $[0, 1]$. Intuitively we can say that this vector decides how much on each dimension we will use the candidate update. Both $\vect{r}^{(t)}$ and $\vect{u}^{(t)}$ are defined by $\vect{h}^{(t-1)}$ and $\vect{x}^{(t)}$; they also have specific parameters:

\begin{equation}
\vect{r}^{(t)} = \sigma(\vect{W}_{r} \vect{h}^{(t-1)} + \vect{U}_{r} \vect{x}^{(t)} + \vect{b}_{r})
\end{equation}

\begin{equation}
\vect{u}^{(t)} = \sigma(\vect{W}_{u} \vect{h}^{(t-1)} + \vect{U}_{u} \vect{x}^{(t)} + \vect{b}_{u})
\end{equation}

At the end the new hidden state $\vect{h}^{(t)}$ is defined by the recurrence:

\begin{equation}
\vect{h}^{(t)} = \vect{u}^{(t)} \odot \vect{\widetilde{h}}^{(t)} + (1 - \vect{u}^{(t)}) \odot \vect{h}^{(t-1)}
\end{equation}

Note that the new hidden state combines the candidate hidden state $\vect{\widetilde{h}}^{(t)}$ with the past hidden state $\vect{h}^{(t-1)}$ using both $\vect{r}^{(t)}$ and $\vect{u}^{(t)}$ to adaptively copy and forget information.

\section{Long Short-Term Memory}
\label{sec:LSTM}

\textit{Long short-term memory} (LSTM) is one of the most applied versions of the RNN family of models. Historically it was developed before the GRU model, but conceptually we can think in the LSTM as an expansion of the model presented in the last session. Because of notation differences they can look different. LSTM is also based on parametrized gates; in this case three: the \textit{forget gate}, $\vect{f}^{(t)}$, the \textit{input gate}, $\vect{i}^{(t)}$, and the \textit{output gate}, $\vect{o}^{(t)}$. The gates are defined only by $\vect{h}^{(t-1)}$ and $\vect{x}^{(t)}$ with specific parameters:

\begin{equation}
\vect{f}^{(t)} = \sigma(\vect{W}_{f} \vect{h}^{(t-1)} + \vect{U}_{f} \vect{x}^{(t)} + \vect{b}_{f})
\end{equation}

\begin{equation}
\vect{i}^{(t)} = \sigma(\vect{W}_{i} \vect{h}^{(t-1)} + \vect{U}_{i} \vect{x}^{(t)} + \vect{b}_{i})
\end{equation}

\begin{equation}
\vect{o}^{(t)} = \sigma(\vect{W}_{o} \vect{h}^{(t-1)} + \vect{U}_{o} \vect{x}^{(t)} + \vect{b}_{o})
\end{equation}

Intuitively $\vect{f}^{(t)}$ should control how much informative will be discarded, $\vect{i}^{(t)}$ controls how much information will be updated, and $\vect{o}^{(t)}$ controls how munch each component should be outputted. A candidate cell, $\tilde{\vect{c}}^{(t)}$ is formed:

\begin{equation}
\tilde{\vect{c}}^{(t)} = tahn(\vect{W} \vect{h}^{(t-1)} + \vect{U} \vect{x}^{(t)} + \vect{b})
\end{equation}

And a new cell $\vect{c}^{(t)}$ is formed by forgetting some information of the previous cell $\tilde{\vect{c}}^{(t-1)}$ and by adding new values from $\tilde{\vect{c}}^{(t)}$ (scaled by the input gate)

\begin{equation}
\vect{c}^{(t)} = \vect{f}^{(t)} \odot \vect{c}^{(t-1)} + \vect{i}^{(t)} \odot \tilde{\vect{c}}^{(t)}
\end{equation}

The new hidden state, $\vect{h}^{(t)}$, is formed by filtering $\vect{c}^{(t)}$:

\begin{equation}
\vect{h}^{(t)} = \vect{o}^{(t)} \odot tanh(\vect{c}^{(t)})
\end{equation}

Until now we have presented general deep learning theory, now we will focus on the specificities of these models applied to natural language problems.

\section{Language model}

We call \textit{language model} a probability distribution over sequences of tokens in a natural language.

\[
P(x_1,x_2,x_3,x_4) = p
\]

This model is used for different NLP tasks such as speech recognition, machine translation, text auto-completion, spell correction, question answering, summarization and many others.

The classical approach to a language model was to use the chain rule of probability and a Markovian assumption, i.e., for a specific $n$ we assume that:

\begin{equation}
P(x_1, \dots, x_T) = \prod_{t=1}^{T} P(x_t \vert x_1, \dots, x_{t-1}) = \prod_{t=1}^{T} P(x_{t} \vert x_{t - (n+1)}, \dots, x_{t-1})
\end{equation}

This gave raise to models based on $n$-gram statistics. The choice of $n$ yields different models; for example, the
\textit{unigram} language model ($n=1$) is defined as:
\begin{equation}
P_{uni}(x_1, x_2, x_3, x_4) = P(x_1)P(x_2)P(x_3)P(x_4)
\end{equation}

where $P(x_i) = count(x_i)$ and $count$ is a function that counts tokens occurrence in a corpus.\\

Similarly the \textit{bigram} language model ($n=2$) is defined as:
\begin{equation}
P_{bi}(x_1,x_2,x_3,x_4) = P(x_1)P(x_2\vert x_1)P(x_3\vert x_2)P(x_4\vert x_3)
\end{equation}
where
\begin{equation}
P(x_i\vert x_j) = \frac{count(x_i, x_j)}{count(x_j)}
\end{equation}

With these basic statistics we can already define useful language models. It is observed that higher $n$-grams yields better performance. This comes with a price though, higher $n$-grams requires great amounts of memory \cite{Heafield}. For this motive $n$-grams based language models that are trained on large corpora uses at most $5$-grams.

Since \cite{Mikolov11} the landscape has change, instead of using one approach that is specific for the language domain, we can use a general model for sequential data prediction, a RNN. The RNN's ability to deal with unrestricted size sequence input permits to abandon the $n$-gram model's restricted context assumption.

To understand the language model task as a machine learning task, some details should be clear.

First, the learning task is to estimate the probability distribution

\begin{equation}
\label{languagedistri}
P(x_{n} = \text{word}_{j^{*}} | x_{1}, \dots ,x_{n-1})
\end{equation}

for any $(n-1)$-sequence of words $x_{1}, \dots ,x_{n-1}$.

Second, the function $f$ being used to approximate \ref{languagedistri} is trained on a corpus in the following way: each word $x_t$ of the corpus will be used as input to $f$ (in the form of an one-hot vector), and the immediate subsequent word, say $x_{t+1}$, will be used as a target. The training is done by minimizing the cross entropy loss between the model's output and the probability distribution given by the target.

One example is in order. Suppose our corpus $\corpus$ is compose only by the lines below:

\begin{quote}
Yes, here we go again, give you more, nothing lesser\\
Back on the mic is the anti-depressor\\
Ad-Rock, the pressure, yes, we need this\\
The best is yet to come, and yes, believe this\\
\end{quote}

From this corpus we can construct a vocabulary list $\Vocab$ as follows: after preprocessing the text we create a list of the most frequent words (in this case we can take all words) with the size $V$ (here $V=27$). Hence, we can treat each word in the text either by an index referring to the word position on $\Vocab$ or as a one-hot vector that codifies this index (e.g., "the" would be identified with $0$, "yes" with $1$, "we" with $2$, and so on).

Then, the dataset is the collection of words
\[
D = \{(<eos>, \text{Yes}), (\text{Yes}, \text{here}), (\text{here}, \text{we}),\dots,(\text{believe}, \text{this}), (\text{this}, <eos>)\}
\]
where $<eos>$ is the "end of sentence" token (also a member of $\Vocab$).

A simple recurrent language model $f(\vect{x}^{(t)}, \vect{\theta})$ is defined by the following equations:

\begin{equation}
\vect{e}^{(t)} = \vect{E}\vect{x}^{(t)}
\end{equation}
\vspace{0.2cm}
\begin{equation}
\vect{h}^{(t)} = \sigma(\vect{W}\vect{h}^{(t-1)}+ \vect{U}\vect{e}^{(t)}+ \vect{b})
\end{equation}
\vspace{0.2cm}
\begin{equation}
f(\vect{x}^{(t)}, \vect{\theta}) = \vect{\hat{y}}^{(t)} = softmax(\vect{V}\vect{h}^{(t)} + \vect{c})
\end{equation}

where $\vect{E} \in \mathbb{R}^{d,V}$ is the matrix of word embeddings, $\vect{x}^{(t)} \in \mathbb{R}^{V}$ is one-hot word vector at time step $t$, $\vect{y}^{(t)} \in \mathbb{R}^{V}$ is the ground truth at time step $t$ (also a one-hot word vector) and $d$ is the size of the word embeddings.

For each word $x_t$ let $j_t$ be the index of the subsequent word, so at each time $t$ the point-wise loss is:

\begin{align}
\label{lossCE}
L^{(t)}(\vect{\theta}) &= CrossEntropy(\vect{y}^{(t)},\vect{\hat{y}}^{(t)})\\
    &= - \log({\vect{\hat{y}}^{(t)}}_{j_t})\\
        &= - \log P(x_{t+1} = \text{word}_{j_t}|x_{1}, \dots, x_{t})\\
        &= - \log P(x_{t+1}|x_{1}, \dots, x_{t})
\end{align}

The loss $L$ is the mean of all point-wise losses

\begin{equation}
L(\vect{\theta})=\frac{1}{T}\sum_{t=1}^{T}L^{(t)}(\vect{\theta})
\end{equation}

With the loss function defined, we apply some optimization algorithm like \textit{stochastic gradient descent} to choose the optimal parameters for the language model:

\begin{equation}
\vect{\theta}^{*} = \argmin_{\vect{\theta}} L(\vect{\theta})
\end{equation}

Because of the historical connections with information theory, the \textit{perplexity} ($PP$) metric is often used to evaluate a language model. This metric can be thought as the weighted average branching factor of a language.

Given $\corpus = x_1, x_2, \dots, x_T$, we define the perplexity of $\corpus$ ($PP(\corpus)$) as:

\begin{align}
PP(\corpus) &= P(x_1, x_2, \dots, x_T)^{-\frac{1}{T}}\\
      &= \sqrt[T]{\frac{1}{P(x_1, x_2, \dots, x_T)}}\\
      &= \sqrt[T]{\prod_{i=1}^{T}\frac{1}{P(x_i \vert x_1,\dots, x_{i-1})}}
\end{align}

Using \ref{lossCE} we can relate cross entropy loss and perplexity:

\begin{align}
        L(\vect{\theta}) &=\frac{1}{T} \sum_{t=1}^{T} L^{(t)}(\vect{\theta})\\
          &=\frac{1}{T} \sum_{t=1}^{T} - \log P(x_{t+1}|x_{1}, \dots, x_{t})\\
          &=\frac{1}{T} \sum_{t=1}^{T} \log ((\frac{1}{P(x_{t+1}|x_{1}, \dots, x_{t})})\\
          &= \log\left( \sqrt[T]{\prod_{i=1}^{T}\frac{1}{P(x_i \vert x_1,\dots, x_{i-1})}} \right)\\
          &= \log(PP(\corpus))
\end{align}

Hence,

\begin{equation}
2^{L(\vect{\theta})} = PP(\corpus)
\end{equation}

Thus, by finding the parameters that minimize the cross entropy error we are also minimizing the perplexity of the language model.

\section{Sequence-to-Sequence}
\label{sec:Seq2seq}

There is one powerful application of RNN based language model. The authors from \cite{Sustskever} used two RNNs to create an end-to-end translation model that is now know as \textit{the encoder-decoder} or \textit{the sequence-to-sequence} (seq2seq) architecture. This architecture is define as follows: let $\vect{x}^{(1)}, \dots, \vect{x}^{(n)}$ be a source sentence in the one-hot representation,  let $\vect{y}^{(1)}, \dots, \vect{y}^{(m)}$ be the target sentence also in the one-hot format. $f_{enc}$ (the \textit{encoder}) is a RNN with the sole purpose of creating a vector representation of input language's sequences. $f_{dec}$ (the \textit{decoder}) is a language model for the target language. These models are trained together mapping source sentences to target sentences.

For example, suppose the training pair $(\vect{x}^{(1)}, \dots, \vect{x}^{(n)}, \vect{y}^{(1)}, \dots, \vect{y}^{(m)})$ is ("Nas tardes de fazenda há muito azul demais", "In the farm’s afternoons there is too much blue"). Here we want the model to translate one Portuguese sentence to an English one. We first encode the Portuguese sentence in the vector $\vect{s}$, i.e.,

\begin{equation}
\vect{s} = f_{enc}(\vect{x}^{(n)}, \vect{h}^{(n-1)})
\end{equation}

Then using the control English sentence as a target, at each time $t$ we compute the cross entropy error between the decoder prediction $f_{dec}(\vect{y}^{(t)}, \vect{\tilde{h}}^{(t-1)})$ and the target $\vect{y}^{(t+1)}$ ($\vect{y}^{(0)}$ is the $<eos>$ token). The decoder uses the vector representation of the source sentence $\vect{s}$ as an initial hidden state (i.e., $\vect{\tilde{h}}^{(0)} = \vect{s}$). The goal of this model is to to approximate the probability distribution over the tokens from the target language given the sentence of the source language, i.e.,

\begin{equation}
\vect{\tilde{h}}^{(t)} = f_{dec}(\vect{y}^{(t)}, \vect{\tilde{h}}^{(t-1)})
\end{equation}

\begin{equation}
\label{decpred}
p(y_t | y_1, \dots, y_{t-1}, x_1, \dots, x_{n}) = softmax(\vect{W}_{s}  \vect{\tilde{h}}^{(t)} + \vect{b}_s)
\end{equation}

One limitation of this architecture is that the source sentence, in some cases, has more features than the decoder embedding $\vect{s}$ can properly summarize. To address that some attention mechanisms were introduced.

\section{Attention}
\label{sec:Attention}

The attention-based models are models built on top of the seq2seq architecture. The encoding part continues the same as before, but now at each time $t$ a context vector $\vect{c}^{(t)}$ is defined to capture relevant source-side information to help the prediction of the current target word $\vect{y}^{(t)}$. Once $\vect{c}^{(t)}$ is constructed the attention hidden state is defined as:

\begin{equation}
\vect{\tilde{h}}^{(t)} = tahn(\vect{W}_c[\vect{c}^{(t)};\vect{h}^{(t)}])
\end{equation}

With the attention hidden state defined, the model's prediction is the same as the one defined in \ref{decpred}.

The core of this technique is the definition of $\vect{c}^{(t)}$. There are different strategies available, here we will focus only on one: \textit{global attention}.

Let $\vect{a} \in \mathbb{R}^{m,n}$. We will use this matrix as an alignment matrix, i.e., at the end of the training $\vect{a}_{ts}$ should reflect the probability of the source representation $\vect{h}^{(s)}$ be relevant for the output $\hat{y}^{(t)}$. We define $\vect{a}_{ts}$ as

\begin{equation}
\vect{a}_{ts} = \frac{exp(score(\vect{\tilde{h}}_t,\vect{h}_s))}{\sum_j exp(score(\vect{\tilde{h}}_t,\vect{h}_j))}
\end{equation}

Where $score$ is a content-based function that can have different implementations:

\begin{equation}
score(\vect{\tilde{h}}_t,\vect{h}_s) = \begin{cases}
\vect{\tilde{h}}_t ^{\top}\vect{h}_s\\
\vect{\tilde{h}}_t ^{\top}\vect{W}_a \vect{h}_s\\
\vect{v}_a ^{\top}tahn(\vect{W}_a[\vect{\tilde{h}}_t;\vect{h}_s])\\
\end{cases}
\end{equation}

At the end, a global context vector $\vect{c}^{(t)}$ is computed as the weighted average, according to $\vect{a}_t$ over all source states:

\begin{equation}
\vect{c}^{(t)} = \sum_{s} \vect{a}_{ts}\vect{h}^{(s)}
\end{equation}


\section{Learning Motion Primitive using Kinesthetic Demonstrations}
\label{sec:Learning motion primitive}
The main idea of the work is to infer the relevant set of features,
which describe the demonstrated motion primitive.
Based on the learnt relevant features, during the execution phase
the robot tries to predict the goal of the motion primitive, starting
from a new initial condition.

The demonstrations consist of recording two  states of the world
called as the start state and the end state.
So each demonstrations have 2 snapshots of the world in the initial state and the final state.
Based on these snapshots we try to infer the intent of the demonstrations.

The motion primitive in our representation consist of pre-conditions and the post-conditions.
The execution of each motion primitive depends on the features used to explain it.
Each motion primitive is defined by a set of features which are a subset of the complete feature space,
which are also relevant for the robot to execute the respective motion primitive.

\subsection{Features}
Features are the basic building block, which are used to describe a motion primitive.
A Feature $f$ can be defined as any quantitative parameter of the world $W$.
For example the pose of the robot, color of the box, distance between box and robot,
displacement of the tooltip in time.

Features are broadly classified into :
\begin{enumerate}
    \item features describing object properties $f(O)$
    \begin{enumerate}
        \item features describing robot properties $f(O_r)$ (eg : pose of the robot)
        \item features describing environment properties $f(O_e)$(eg: color of the box)
    \end{enumerate}
    \item features describing relation between object properties $g(O_1, O_2)$
    \begin{enumerate}
        \item relation between  different objects $g(O_1, O_2)$ (eg : distance between box and robot)
        \item relation between same object in time $g(O_{1i}, O_{1f}) $(eg : displacement of tooltip)
    \end{enumerate}
\end{enumerate}

Thus the collection of featues can be defined as
\begin{equation}
    \begin{aligned}
    F &= f(O) + g(O_1 , O_2) \\
      &= (f(O_r) + f(O_e))  +  (g(O_1, O_2) + g(O_{1i}, O_{1f}))
    \end{aligned}
\end{equation}

As explained in section \ref{sec:Proposed motion primitive}
the proposed motion primitive is defined using pre and post conditions of the features.
\begin{equation}
    \begin{aligned}
    & \text{Motion primitive} := (f_s, f_e ) \\
    & where  \nonumber \\
    &    f_s = \text{features in start of demonstrations} \nonumber\\
    &    f_e = \text{features in end of demonstrations} \nonumber
    \end{aligned}
\end{equation}

\subsection{Modelling Motion Primitive}

The learning problem can be considered as a supervised learning problem, where
the model has to learn from the set of demonstrations to predict the output
$f_e$ when a new input $f_s$ is provided as explained in figure \ref{model}.
\begin{figure}[htp]
\centering
\includegraphics[scale=0.40]{images/model.png}
\caption{Model of motion primitive}
\label{model}
\end{figure}

When the number of training set is limited,learning on
 all the features of the world is not a feasible solutions.
 The approach taken here is to find the relevant features on which the
learning can be made most effectively.
The prespective adopted in finding the relevant features is that in multiple
demonstrations of the same action, some features will share similar values
in all the demonstrations. Since we are demonstrating a single action
there has to be consistency in some features of the action.
We need to identify these consistent features and these become the relevant
features. The consistency increases the relevance of the feature to
sucessfully predict from new unseen start states.

We will consider a bivariate probability density function $\phi_i$  by
considering 2 variables, the value of a feature in start of demonstration and
its value at the end of the demonstration.

Let $s_i$ be the $i^{th}$ feature of $f_s$,
and $e_i$ be the $i^{th}$ feature of $f_e$, \\
The bivariate distribution is given by :
\begin{equation}
    \phi_i(s_i , r_i) = \eta_i I_i ( \lfloor s_i \rfloor, \lfloor e_i \rfloor)
\end{equation}
where operator $\lfloor .  \rfloor$ is  a quantization operator, that returns the bin unit in
the histogram $I_i$, that corresponds to the feature.

The bi-variate distribution is an appropriate criteria to comment on the
relevance of the feature. Using the distribution we can conclude on the
convergence of the feature. A feature which has converged, the final values
will lie on a straight line in the distribution. If all the final values lie on
a straight line it ensures that whatever maybe the initial value the end values
always remain constant. The constant value in all the demonstration concludes
that the corresponding feature is relevant with respect to the intent of the
motion primitive.
This is explained in the figure \ref{fig:feature distribution}. Higher the
degree of convergence, higher the relevance of the feature for the motion
primitive.
\begin{figure}
    \centering
    \begin{subfigure}[b]{0.4\textwidth}
        \includegraphics[scale=0.5]{images/arm_joint_1JoinPDF.png}
        \caption{Distribution of the feature describing arm joint 1.}
        \label{sub fig 1}
    \end{subfigure}
    \begin{subfigure}[b]{0.4\textwidth}
        \includegraphics[scale=0.5]{images/d_linear_m0_gripperJoinPDF.png}
        \caption{Distribution of feature describing distance of object 0 and tooltip. }
        \label{sub fig 2}
    \end{subfigure}
    \caption[Bivariate distribution of features]{An example of bivariate
        distributions of two features for the \textit{move} arm relative to object motion
        primitive.The x-axis is the end values of demonstration while the
        y-axis is the start value of the demonstration. The dots are the
        feature values across 12 demonstrations.
        Based on the distribution in \ref{sub fig 1}, we can say that
        there is no certaininty in the end values. This
        results in lower relvance to the motion primitive . Based on the
        distribution in \ref{sub fig 2},we can say that the end states are
        more concentrated.This ensures that for any value in the start state
        we are sure that the end values always remain same.
        This results in a higher relevance with respect to the motion
    primitive.}\label{fig:feature distribution}
\end{figure}

To calculate this relevance in the distribution, we use two different
measuring  methods. 1) Entropy 2) Conditional entropy.
We compute the
entropy $H_i$ and conditional entropy $CH_i$ of the bi-variate distribution $\phi_i(s_i, r_i)$.

Entropy of a discrete random variable is given by,
\begin{equation}
    H(X) = - \sum P(X) \log P(X)
\end{equation}

For a pair of discrete random variables X and Y, which are co related
conditional entropy of X given Y $h(X|Y)$ is given by :
\begin{equation}
    H(X|Y) =  - \sum _{k = -K}^{K} p_X(x|y) p_Y(y|x) \log \frac{p_Y(y|x)}{p_X(x|y) p_Y(y|x)}
\end{equation}

Based on the entropy or conditional entropy we define the model relevance of a motion primitive.
\begin{equation}
    p(f | \theta ) = \prod_i e^{-E_i}
\end{equation}
where $\theta$ is the set of all $\phi$, and $E_i$ can be $H_i$ or $CH_i$

\subsection{Expert knowledge base}
In our work we create the knowledge base based on expert knowledge of the tasks being performed.
An individual subset of feature space $F$ is called as a template $t$. $t_i \subset F $ .
For example the template $t_1$ contains all the features describing the joint angles of the robot, template $t_2$ contains features which describes
distance of tooltip to manipulated object.

A collection of the templates is called as a knowledge base $K$.

$K = t_1 \and t_2 \ldots t_n $

\begin{figure}[htp]
\centering
\includegraphics[scale=0.50]{images/feature_space.png}
\caption[Feature space venn diagram]{Relation between feature space F, knowledge base K and the templates t}
\label{}
\end{figure}
Thus for each template $t_i$ we have created a low dimension subset of the feature space $F$.
Our aim is not to find minimum set of features but to find relevant set of features which can describe an action.

The novelty in our approach is the representation of the expert knowledge base.
The knowledge base has been structured using the effect metrics.
The templates in the knowledge base are organised on the basis of the effect metrics.
The structured nature of the knowledge base helps in determining the relevant features with
less number of demonstrations.
\subsubsection{Effect Metrics}
Effects are defined as changes to the robot-world relationship and/or to positions, orientations
and states of external objects and robot \cite{alissandrakis_action_2006}

In this work we only consider changes to the robot-world relationship and changes to positions, orientations and states of the robot. Based on the orientation and position of the
robot in environment 2 types of effect metrics can be used, \textit{position} and \textit{orientation}

\begin{figure}[htp]
\centering
\includegraphics[scale=0.70]{images/position_effect_metrics.png}
\caption[Position effect metrics]{Position effect metrics.
 To measure the change in the relation of change in positions between
robot and environment objects
\textit{relative displacement, absolute position, relative position and mirror position}
 effect metrics can be used. First row shows the demonstration and
 their resulting effects. The second rows represents the
 corresponding object (in different workspace ) how it needs to be
 moved (from dashed to solid outline) by an imitator to match the
corresponding effects according to the metric. The grey triangle
 are superimposed to show the relative position of the objects are
the same in final state. \cite{alissandrakis_action_2006} }
\label{position effect metrics}
\end{figure}
\begin{figure}[htp]
\centering
\includegraphics[scale=0.70]{images/angular_effect_metrics.png}
\caption[Orientation effect metrics]{Orientation effect metrics.  To measure the change in the relation of change in orientation between robot and environment objects \textit{relative rotation, absolute orientation, relative orientation and mirror rotation} effect metrics can be used.First row shows the demonstration and their resulting effects. The second rows represents the corresponding object (in different workspace) how it needs to be moved. (from dashed to solid outline) by an imitator to match the corresponding effects according to the metric. The guide lines are superimposed to show the relative orientation of the objects are the same in final state. \cite{ alissandrakis_action_2006}}
\label{orientation effect metrics}
\end{figure}

The different position metrics are
\begin{itemize}
	\item relative displacement
	\item absolute position
	\item relative position
	\item mirror position
\end{itemize}

the different orientation metrics are
\begin{itemize}
	\item relative rotation
	\item absolute orientation
	\item relative orientation
	\item mirror rotation
\end{itemize}

Depending on the effect metric the same demonstration can be interpreted as different.
The example in figure \ref{effect metrics} explains this.
\begin{figure}[htp]
\centering
\includegraphics[scale=0.8]{images/effect_position.png}
\caption[Effect metrics in imitation]{The figure illustrates three examples
 of the imitation based on a single demonstration. The left box is the
 demonstrated action. The right box is the possible final state based on
 the position metrics used. The grey triangles are superimposed to show
 the relative distance is maintained \cite{alissandrakis_action_2006}}
\label{effect metrics}
\end{figure}

Based on the effect metrics the knowledge base is organized in different templates which describe an action.
The templates developed in this work are categorized based on the effect metrics.

\subsection{Selecting template from the knowledge base}
For computing the relevance, we use a 3 stage procedure:
\begin{enumerate}
    \item Comparing mean of start value and end value of the feature. $\rightarrow$ To determine that the feature has changed.
            (This was used because features which were not changed in demonstration also have low entropy. So to remove these features this condition
            check was introduced.)
    \item Comparing the standard deviation of start value and end value of the featue $\rightarrow$ To determing the feature shows convergence in information.
            (This was used because there were features which changed and had low entropy but they were divergent in nature. So to remove the effect of
            these features this condition check was introduced as explained in figure \ref{fig:box plot})
    \item Calculating the entropy/conditional entropy of the final value of features whose values have changed.
\end{enumerate}

\begin{algorithm}[H]
 \KwData{\\ $f_s$ : start value of the features \\
         $f_e$ : end value of the features\\
        K : knowledge base }
 \KwResult{\\ relevance $L$ of each Template }
 R=0\;
 \For{each template $t$ in K}{
    \For{each feature $f$ in $t$ }{
  read start values $s$ of feature $f$ from $f_s$\;
  read end values $e$ of featue $f$ from $f_e$\;
  \If{mean(s) equals mean(e)}{
       $ R += 1$\;
   }
   \If{standard deviation(s) is less or equal to standard deviation(e)}{
       $ R += 1$\;
   }
    $R += \text{entropy(e)} or\text{ conditional entropy}(e|s)$\;
  }
 }
 \caption{Algorithm for computing relevance of the templates in knowledge base}
\end{algorithm}

\begin{figure}
    \centering
    \begin{subfigure}[b]{0.3\textwidth}
        \includegraphics[scale=0.25]{images/boxplot_same_mean.png}
        \caption{}
        \label{sub box 1}
    \end{subfigure}
    \begin{subfigure}[b]{0.3\textwidth}
        \includegraphics[scale=0.25]{images/boxplot_noconvergence.png}
        \caption{}
        \label{sub box 2}
    \end{subfigure}
    \begin{subfigure}[b]{0.3\textwidth}
        \includegraphics[scale=0.25]{images/boxplot_converged.png}
        \caption{}
        \label{sub box 3}
    \end{subfigure}
    \caption[Box plot of features]{Box plots of features. Each figure has 2 box plots.
Plot I is of the start value of the feature while plot F is of the end value
of the feature. Figure \ref{sub box 1} is box plot of a featue whose mean values
have not changed for both start and end values of the feature .
Figure \ref{sub box 2} is the box plot of a feature whose mean value
has changed but the final values has not converged.
Figure \ref{sub box 3} is the box plot of a feature whose mean value
has changed and the final values have converged.} \label{fig:box plot}
\end{figure}

For each $t_i \in K $, we compute a score $\beta_i$ that combines model fitting and the number of
features in the template.
\begin{equation}
    \beta_i = -2 \log (p (f | \theta_i)) - \alpha_i L_i
\end{equation}
where $\theta_i$ are the distributions related to the features of $t_i$
and $L_i$ is the number of features in the template.
The first term of equation  computes relevance and the second term, weighted by $\alpha$, encourages the usage of templates
consisting of a large number of features $L_i$ .

The most relevant template $T^*$ is selected as :
\begin{equation}
    T^* = \operatornamewithlimits{argmin}_i (\beta_i)
\end{equation}