% ******************************* Thesis Appendix B ********************************

\chapter{Supporting information: Chapter 5}

\graphicspath{{Appendix4/Figures/}}

\section{Supplementary figures and tables}
\newpage
\begin{figure}[H]
\flushleft{\textbf{Figure S5.1.} Positive pairwise species correlations to the environment derived from single-predictor LVMs for the full dataset. Connecting lines between species nodes denote positive mean posterior correlations with credible intervals excluding zero. Line colour and thickness indicates the strength of the positive correlation where darker and thicker lines are closer to 1. Species node labels combine the first letters of the genus and specific epiphet.}
\includegraphics[width=1.0\linewidth]{networkplots-positive-unshrink}\\
\end{figure}

\begin{figure}[H]
\flushleft{\textbf{Figure S5.2.} Abundance of each species (excl. \textit{Empodisma minus}) through time. Trend lines obtained with thin-plate splines in a GAMM framework (see main text for model fitting). Shaded region represents 95\% confidence intervals.}
\includegraphics[width=1.0\linewidth]{timeplots}\\
\end{figure}

\begin{table}[H]
  \centering
  \flushleft\textbf{Table S5.1.} Negative and positive pairwise environmental and residual correlations in each of the four plot clusters.
    \begin{tabular}{lllll}
    \toprule
          & \multicolumn{2}{l}{No. sig. env. correlations} & \multicolumn{2}{l}{No. sig. res. correlations} \\
    \midrule
          & -ve   & +ve   & -ve   & +ve \\
    deep soil-moisture & 6,0,1,0 & 12,2,4,13 & 8,11,7,0 & 10,14,30,41 \\
    shallow soil-moisture & 2,0,4,0 & 3,3,2,0 & 10,10,5,0 & 17,9,16,42 \\
    pH    & 0,0,3,0 & 1,1,11,0 & 7,10,6,0 & 11,13,23,41 \\
    total iron & 2,0,2,0 & 2,1,3,1 & 10,11,7,0 & 6,8,17,41 \\
    phosphorous & 4,0,9,3 & 3,5,7,3 & 12,11,4,0 & 7,10,26,34 \\
    organic carbon & 1,0,6,0 & 0,0,10,16 & 8,10,6,0 & 6,12,27,39 \\
    exchangeable aluminium & 2,0,2,6 & 4,8,2,12 & 10,16,5,0 & 15,20,25,41 \\
    exchangeable magnesium & 3,0,5,1 & 3,1,9,8 & 10,11,6,0 & 11,13,28,41 \\
    cation exchange capacity & 1,0,5,1 & 2,3,6,0 & 8,13,5,0 & 14,10,21,43 \\
    \bottomrule
    \end{tabular}%
  \label{tab:addlabel}%
\end{table}%

\includepdf[pages={-}, offset=0.4cm 0cm,  addtotoc= {1,section,1,LVM Primer,hlabel}]{Appendix4/LVM-primer.pdf}

\subsection{fitLVM-auxiliaryfunctions.R}

\begin{verbatim}
##########################################
## Auxilary functions for fitting LVMs  ##
##########################################

## Calculate the correlation between spp due to shared/diverging env responses
extract.env.cor <- function(fit.mod, X, y) {
	n.species <- ncol(y)
	fit.mcmcBase <- fit.mod$BUGSoutput
	mcmc.runs <- mcmc(fit.mcmcBase$sims.matrix, start = 1,
                    thin = fit.mcmcBase$n.thin)
	rm(fit.mcmcBase)

	all.shared.env.mat <- matrix(0,nrow(mcmc.runs),n.species^2)

	## For each MCMC sample, find the correlation between spp responses
	for(t in 1:nrow(mcmc.runs)) {
		cw.spp.coef <- matrix(mcmc.runs[t,grep("spp.coef\\[",colnames(mcmc.runs))],
                          n.species,byrow=F)
		eta.mat <- X%*%t(cw.spp.coef) ## On linear predictor scale
		all.shared.env.mat[t,] <- as.vector(cor(eta.mat))
		}

	## Average/Median over the MCMC samples
	env.mat.mean <- sig.env.mat.mean <- matrix(apply(all.shared.env.mat,2,mean),
                                             n.species,byrow=F)
	env.mat.median <- sig.env.mat.median <- matrix(apply(all.shared.env.mat,2,median),
                                                 n.species,byrow=F)

	get.cor.intervals <- HPDinterval(as.mcmc(all.shared.env.mat), prob = 0.95)
	id.sign.cors <- which(get.cor.intervals[,1] > 0 | get.cor.intervals[,2] < 0)
	sig.env.mat.mean[-id.sign.cors] <- 0
	sig.env.mat.median[-id.sign.cors] <- 0

	return(list(envcor.mean = env.mat.mean,
              envcor.median = env.mat.median,
              sig.envcor.mean = sig.env.mat.mean,
              sig.envcor.median = sig.env.mat.median))
	}

## Produce the residual correlation
extract.residual.cor <- function(fit.mod, X, y) {
	fit.mcmcBase <- fit.mod$BUGSoutput
	mcmc.runs <- mcmc(fit.mcmcBase$sims.matrix,
                    start = 1,
                    thin = fit.mcmcBase$n.thin)
	rm(fit.mcmcBase)

	n.species <- ncol(y)
	n.sites <- nrow(y)
	Tau.arr <- matrix(NA,nrow(mcmc.runs),n.species^2)
	for(t in 1:nrow(mcmc.runs)) {
		lvs <- matrix(mcmc.runs[t,grep("LV",colnames(mcmc.runs))],
                  n.sites,byrow=F)
		lv.coefs <- matrix(mcmc.runs[t,grep("lv.coef",colnames(mcmc.runs))],
                       n.species,byrow=F)
		Tau.arr[t,] <- as.vector(cor(lvs%*%t(lv.coefs))) }

	## Average/Median over the MCMC samples
	Tau.mat.mean <- sig.Tau.mat.mean <- matrix(apply(Tau.arr,2,mean),
                                             n.species,byrow=F)
	Tau.mat.median <- sig.Tau.mat.median <- matrix(apply(Tau.arr,2,median),
                                                 n.species,byrow=F)

	get.cor.intervals <- HPDinterval(as.mcmc(Tau.arr), prob = 0.95)
	id.sign.cors <- which(get.cor.intervals[,1] > 0 | get.cor.intervals[,2] < 0)
	sig.Tau.mat.mean[-id.sign.cors] <- 0
	sig.Tau.mat.median[-id.sign.cors] <- 0

	return(list(rescor.mean = Tau.mat.mean,
              rescor.median = Tau.mat.median,
              sig.rescor.mean = sig.Tau.mat.mean,
              sig.rescor.median = sig.Tau.mat.median))
	}

## Produce mean and median point estimates from fit
extract.params <- function(fit.mod) {
	fit.mcmcBase <- fit.mod$BUGSoutput
	mcmc.runs <- mcmc(fit.mcmcBase$sims.matrix, start = 1,
                    thin = fit.mcmcBase$n.thin)
	rm(fit.mcmcBase)
	n.species <-  length(grep("spp.int",colnames(mcmc.runs)))

	all.spp.coef <- mcmc.runs[,grep("spp.coef",colnames(mcmc.runs))]
	spp.coef.mean <- matrix(apply(all.spp.coef,2,mean),n.species,byrow=F)
	spp.coef.median <- matrix(apply(all.spp.coef,2,median),n.species,byrow=F)

	all.spp.int <- mcmc.runs[,grep("spp.int",colnames(mcmc.runs))]
	spp.int.mean <- apply(all.spp.int,2,mean)
  spp.int.median <- apply(all.spp.int,2,median)

	all.spp.phi <- mcmc.runs[,grep("spp.phi",colnames(mcmc.runs))]
	spp.phi.mean <- apply(all.spp.phi,2,mean)
  spp.phi.median <- apply(all.spp.phi,2,median)

	all.mu.beta <- mcmc.runs[,grep("mu.beta",colnames(mcmc.runs))]
	mu.beta.mean <- apply(all.mu.beta,2,mean)
  mu.beta.median <- apply(all.mu.beta,2,median)

	all.tau.beta <- mcmc.runs[,grep("tau.beta",colnames(mcmc.runs))]
	tau.beta.mean <- apply(all.tau.beta,2,mean)
  tau.beta.median <- apply(all.tau.beta,2,median)

	return(list(spp.coef.mean = spp.coef.mean, spp.coef.median = spp.coef.median,
              spp.int.mean = spp.int.mean, spp.int.median = spp.int.median,
              spp.phi.mean = spp.phi.mean, spp.phi.median = spp.phi.median,
              mu.beta.mean = mu.beta.mean, mu.beta.median = mu.beta.median,
              tau.beta.mean = tau.beta.mean, tau.beta.median = tau.beta.median))
	}
\end{verbatim}


%!TEX root = ../copatterns-thesis.tex
\chapter{Related Work}
\label{cha:related-work}

% \section{Copatterns}
% \label{sec:related_work_copatterns}
% \todo{Relate to 'Unnesting of Copatterns' by Setzer et al.}

\section{Size-change Termination}
The termination checker in Idris uses the size-change
principle\,\citep{BradyIdrisImpl13}. Where the productivity checking procedure
presented in Chapter~\ref{cha:infer-guard-recurs} is essentially a type check
with intermediate term inference, the size-change principle is purely based on
syntactic analysis. The size-change principle for termination was first proposed
for a strict first-order functional language (without loop constructs) by Lee,
Jones, and Ben-Amram\,\citep{LeeJones01SizeChange}. The principle essentially
states that if infinitely many recursive calls to a function would lead to
infinite decrease in some parameter value, then the function must be
terminating, since any value of an inductive type must have finite size. This
last condition is of particular importance, as the size-change principle cannot
in general recognize functions as being terminating if they have parameters that
do not exhibit a well-founded order. Lee, Jones, and Ben-Amram present two
realizations of the principle, one using automata and one using a call graph. In
the graph formulation, termination is approximated by identifying any recursive
calls (both direct and indirect) through cycles in the call graph, and then
constructing a ``size-change graph'' for each of these. The size-change graphs
are then used to discover whether infinite descent in some parameter value is
present. One of the limitations of this approach is that parameter values must
decrease monotonically: Values cannot at any point become structurally larger,
even though the total change in size in a call chain would ultimately lead a to
a decreasing value. Two examples of size-change terminating functions are shown
in Figure~\ref{fig:sizechange_plus_map}. In both examples, the recursive call
happens on structurally smaller input.

\begin{figure}
\begin{lstlisting}[mathescape,title=\idrisBlock]
plus : Nat $\to$ Nat $\to$ Nat
plus Z      m = m
plus (S n$'$) m = plus n$'$ m

map : (a $\to$ b) $\to$ Vect n a $\to$ Vect n b
map f []        = []
map f (x :: xs) = f x :: map f xs
\end{lstlisting}
\caption{Two size-change terminating functions.}
\label{fig:sizechange_plus_map}
\end{figure}

Since its first-order formulation, the principle has been proven to be applicable to more expressive cases. Jones and Bohr\,\citep{Jones04Untyped} showed that size-change termination can be applied to the untyped lambda calculus using abstract interpretation\,\citep{Jones:1995}. A set of safe size-change graphs for a program is generated by defining evaluation rules without an environment component, thus overapproximating the number of possible values for any given variable. Given these rules, a corresponding overapproximated set of safe size-change graphs is generated for further termination analysis. Note that due to the undecidability of the halting problem, it is impossible in general to compute the exact set of safe size-change graphs for a given function call.

Following the work on the untyped lambda calculus, Sereni and Jones generalized the size-change principle to handle a higher-order functional language with user-defined data types and general recursion\,\citep{Sereni05terminationanalysis,Sereni06Phd}. Here, a termination criterion is presented which works for arbitrary control-flow graphs, and in turn is able to give an approximation of termination for both strict and lazy functional programs. A key point in this work is how different approaches to control-flow and call graph construction may influence the preciseness of the termination analysis.

All of the previously mentioned implementations of the size-change principle can
only approximate termination for programs involving data which exhibits some
well-founded order. Nevertheless, Avery\,\citep{Avery06} presented a formulation
in which it is possible to detect size-change termination for non-well-founded
data types --- in particular, this formulation is shown to work for a language
with a non-well-founded integer type. Instead of identifying infinite descent using a well-founded partial order on parameter, as proposed by Lee at al., Avery's analysis is based on a decrease in invariants which are found to hold for each program point. The idea is that if the value of some invariant (which can involve arbitrarily many values) can be shown to decrease on every passage of a program point, then the program terminates. A simple example of a size-change terminating program in Avery's implementation is given in Figure~\ref{fig:avery_example}. The invariant for the inner loop is \texttt{n - j + i}, while the one for the outer loop is \texttt{n - i}. These cannot decrease indefinitely, and therefore the program is size-change terminating.

\begin{figure}
\begin{lstlisting}
for (i = 0; i <= n; i++) {
  for (j = 0; j-i <= n; j++);
}
return;
\end{lstlisting}
\caption{An example program involving integers written in a subset of C, which is size-change terminating in Avery's formulation.}
\label{fig:avery_example}
\end{figure}

While size-change termination for non-well-founded data could be a step towards
using a size-change approach for an approximation of productivity, it is unclear
which invariants one would have to infer for corecursive programs. In our
solution, we do not directly establish invariants, but instead encode constraints
in the types of guarded recursive programs using the guardedness type
constructor ($\laterkappa$) and quantification over clocks. Together with a
user-supplied modality (either causal or non-causal) we can then establish
whether a guarded recursive version of a user-written complies with these constraints.

In more recent work, Hyvernat\,\citep{Hyvernat13} has proposed a formulation of the size-change principle for functional languages which to a certain degree solves the problem of non-monotonic decrease in parameter values. The motivation behind this work is to incorporate size-change termination into the PML language\,\cite{PMLLanguage}. Non-monotonic decrease is detected by tracking the size of a parameter throughout the entire control-flow graph, instead of merely recording whether each call in isolation leads to a decrease in some value.

The termination criterion most similar to the size-change principle predates the original article by Lee, Jones, and Ben-Amram\,\citep{LeeJones01SizeChange}. This criterion was developed by Abel for the \texttt{foetus} termination checker\,\citep{Abel98foetus}, and forms the basis of the totality checker implemented in Agda\,\citep{Norell:thesis}. Analogous to the size-change principle, Abel identifies recursive calls in a call graph and performs a termination analysis by tracking changes in parameter sizes. This initial presentation of \texttt{foetus} makes no mention of productivity for corecursive programs, although such an extension has since been presented by Altenkirch and Danielsson\,\citep{AltenkirchNAD10}.

\section{Syntactic Guardedness}
\label{sec:synt-guard}
% Telford and Turner nævnes mere
% Hyvernat flyttes herned

% Due to the duality between inductive and coinductive types, it may be compelling to imagine a productivity analysis which, dual to the size-change principle, works by identifying structurally larger values. Such a dual notion of ``size-change productivity'' is exactly the idea behind the syntactic guardedness checkers found in Idris, Agda, and Coq\,\citep{Coq:manual}. Where values become structurally smaller by pattern matching, they become structurally larger by constructor application. Therefore, the \emph{guardedness principle} states that a coinductive definition is guarded if all corecursive calls appear directly under a coinductive constructor. This has the implication that the productivity of corecursive functions can be detected by a purely syntactic check.

The syntactic guardedness principle was first proposed by Coquand\,\citep{Coquand94} as a tool for constructive reasoning about infinite objects. Coquand argues that previous coinductive proof methods rely on impredicative definitions, which he deems to be ``unsatisfactory'' for constructive reasoning. Based on Milner's work on process calculi\,\citep{Milner:1989}, Coquand defines an infinite object to be productive if a (not necessarily well-founded) computation tree can be associated with it, through an analogy between infinite proof objects and processes. This observation is then synthesized into a ``guarded induction principle''. In practice, the guarded induction principle can be used for constructive proofs through a syntactic check, which verifies that all corecursive references appear directly under a coinductive constructor.

In continuation of Coquand's efforts, Gim\'{e}nez\,\citep{Gimenez95} formalized an extension of the Calculus of Constructions\,\citep{Coquand:1988}, in which a modification of Coquand's original formulation of the guarded induction principle is provided. In particular, the guarded induction principle is modified such that it can be applied to types with second-order quantification.

The guarded induction principle described by Coquand and Gim\'{e}nez was
intended to be used within a proof system (e.g. Coq) for easier reasoning about
infinite objects. Confirming Coquand's own observation, Telford and
Turner\,\citep{Telford98ensuringthe} argue that the guarded induction principle
is too conservative in a programming setting. In their system of
\emph{Elementary Strong Functional Programming}
(ESFP)\,\citep{Telford97ensuringstreams,Telford98ensuringthe,Telford:jucs_6_4:ensuring_termination_in_esfp},
Telford and Turner therefore extend practical use of the guarded induction
principle to detect a wider range of function as productive. They achieve this
by considering guardedness more abstractly over a domain of ``guardedness
levels'', such that productive corecursion is bounded by a given
``depth''. Intuitively, their system works by detecting that a program produces
more elements that it consumes, i.e. that recursive references are subject to
more introduction forms (constructor applications) than elimination forms (case
analysis). Since any program can be higher-order, the guardedness level of a
function is calculated by a ``guardedness function'',  where the guardedness
levels of both bound and free variables are taken into account. As an advanced
example, Telford and Turner show that the program generating the Hamming numbers
(a problem first discussed by Dijkstra\,\citep{Dijkstra:1997}) is accepted as a
productive program by their system. Just as Telford and Turner, one of our goals
with the implementation of an inference system for guarded recursion was to
overcome the restrictive nature of the syntactic guardedness principle. Where
the approach taken by Telford and Turner is essentially an extension of the
underlying idea of syntactic guardedness, our approach provides productivity
checking by guarded recursion through term inference and type
checking. Consequently, our productivity analysis is bounded by the type
constraints for which we can infer guarded recursive programs, and not by the
syntactic structure of the program written by the user.

In the same work, Hyvernat\,\citep{Hyvernat13} discusses an approach similar to
the idea behind the work of Telford and Turner. He proposes that productivity
can be detected by counting the number of abstractions and applications in a
program, respectively. Given that one does not reduce under abstractions, a
function can then be considered productive if it has more abstractions than
applications, since this necessarily means that the program will not diverge.

Another approach to coping with the conservative nature of the original
syntactic guardedness principle is to consider alternative programming styles, making productive definitions easier to write. Following the incorporation of the guardedness condition into the Agda totality checker as described by Altenkirch and Danielsson (they attribute their approach to Abel)\,\citep{AltenkirchNAD10}, Danielsson\,\citep{Danielsson10beatingthe} described a method for working around the guardedness condition whenever it rejects a productive program. When a corecursive reference appears under a call to a function which is not a constructor, he designs an embedded domain-specific language in which said function is implemented as a constructor, making the guarded induction principle applicable. He is then able to define an interpreter for this language, such that the original program is ultimately accepted as productive. None of these steps happen automatically, but must be done manually. Although useful, Danielsson argues that efficiency is a concern, and that the best solution might be to entirely move away from using guardedness for productivity. Sized types have since been implemented in Agda.

%\section{Guarded Recursion}
%Nakano
%neelk
%Atkey and McBride
%Flere papers fra Aarhus + Mogel

% As presented in Section~\ref{sec:guarded-recursion}, guarded recursive type
% systems can be used to define necessarily productive programs. The general
% notion underlying guarded recursion was introduced by
% Nakano\,\citep{Nakano:2000}, although he does not himself use the term ``guarded
% recursion''. He introduces a modal typing system with recursive types,
% $\lambda\!\bullet\!\mu$ (called ``lambda-later-mu''), in which it is possible to encode provably non-diverging programs. This is achieved partly by giving a modal type to the Y-combinator (see Figure~\ref{fig:nakano_Y}), and partly by defining types such that recursive occurrences are only available in the context of a modality $\bullet$, e.g. infinite binary trees as a type $\mu X.A\times\bullet X\times\bullet X$. Nakano suggests that his system could be related to temporal logic, where the $\bullet$ modality reprensents notions of ``future time''. Additionally, he notes that the system is not intended to be used as a type system for programming languages, but serves to widen the range of programs to which the idea of proofs-as-programs can be applied.

% \begin{figure}
% \[
% \vdash \lambda f. (\lambda x. f (x x)) (\lambda x. f (x x)) : (\bullet X \to X) \to X
% \]
% \caption{The Y-combinator as derived in Nakano's $\lambda\!\bullet\!\mu$ system\,\citep{Nakano:2000}.}
% \label{fig:nakano_Y}
% \end{figure}

% Inspired by Nakano's approach, Atkey and McBride\,\citep{Atkey:2013} present a similar typing discipline based on the simply typed lambda calculus, paving the way for a more practical use of guarded recursion. They introduce applicative programming\,\citep{Mcbride:2008} over modalities, which coupled with Nakano's fixed point combinator makes programming with modalities more accessible, and provides a more compositional system for productivity checking. Their main contribution is the introduction of clock variables into a type system with guarded recursion, such that it is possible to distinguish between values that are in the process of construction and values that are fully constructed. Fully constructed values can be identified by universal quantification over clock variables, enabling the user to extract values from the modal world of guarded recursion.

% Following the work by Atkey and McBride, M\o gelberg\,\citep{Mogelberg:2014} extends the idea of a type system with guarded recursion and clock variables to a dependently typed setting. Based on work by Birkedal and M\o gelberg\,\citep{BirkedalL:sgdtuniverse-conf} and Birkedal, M\o gelberg, Schwinghammer, and St\o vring\,\citep{BirkedalL:sgdt-journal}, he models a dependently typed lambda calculus with guarded recursion and clocks using the topos of trees model. This results in an extensional type theory for reasoning about guarded recursive types. Although no intensional formulation of the model is given, it is expected that such a formulation exists.

% % In extensional type theory, definitional and propositional equality is the same
% Typing systems with modalities on types as introduced by Nakano has also been applied in the field of functional reactive programming. Krishnaswami\,\citep{Krishnaswami13} presents a typing system where both time leaks (depending on past values for arbitrary intervals of time) and space leaks (non-permanent memory leaks from capturing too much history) are prevented by construction. This is accomplished by typing all values with \emph{temporal recursive types}, using modalities on types to indicate the time at which a value becomes available. In the context of a global clock, programs are then evaluated according to two operational semantics: one giving the semantics for a program at the current clock tick, and one for advancing the global clock. Whenever the global clock is advanced, all values which can no longer be referenced due to time constraints are discarded from the environment, thus eliminating space leaks. Accompanying this mechanism are rules which make references to unavailable data impossible. Time leaks are prevented by unfolding all values at the moment they become available. Since Krishnaswami's approach enforces causality (that the first \emph{n} outputs only depend on the first \emph{n} input values), it can essentially be used to encode guarded recursion, at least in the simply typed formulation by Atkey and McBride. But where Atkey and McBride emphasize that clock variables localize the encoding of productivity, Krishnaswami introduces a global clock to which all programs must adhere.

\section{Sized Types}
An alternative type-based approach to totality checking is sized types. Where the intuitive abstraction of guarded recursion is time, sized types attach sizes to values at the type level. The motivation behind sized types is to express totality proofs using induction on sizes, such that the structure of the term in question can be disregarded. With size-change termination, termination proofs are constructed by ensuring that all recursive arguments happen on structurally smaller terms. Analogously, totality proofs using sized types are constructed by ensuring that all values are defined in terms of data with a structurally smaller \emph{size}.  Hence, the term ``size'' implies no specific term structure on the data involved in the totality proof.

Sized types for totality checking were first proposed by Hughes, Pareto, and Sabry\,\citep{Hughes96} for reactive systems, where each data type introduced into a program is associated with a family of sized types indicating the bounds of a value of that type. A similar idea was developed by Amadio and Coupet-Grimal\,\citep{Amadio98}, where guard conditions are introduced into the type system to ensure the productivity of coinductive data, following the work of Coquand\,\citep{Coquand94} and Gim\'{e}nez\,\citep{Gimenez95}.

Eduarde Gim\'{e}nez also presented a system for typing recursive definitions in an extension of the Calculus of Constructions using sized types\,\citep{Gimenez98structuralrecursive}. A notable result of this work is that any well-typed term in the proposed extension is normalizing with respect to lazy evaluation, substantially widening the domain of functions to which type-based termination is applicable. In the wake of this extension, Abel\,\citep{Abel99terminationchecking} wrote a quite accessible paper on using sized types for totality checking, showing that bidirectional type checking\,\citep{Pierce00} is suitable for a system with sized types. In addition, Abel described how infinite streams defined as coinductive types can be encoded in a language as functions on natural numbers, and that the productivity of a stream can be understood in terms of its \emph{definedness}, meaning the number of times it can safely be unfolded.

The notion of definedness is also an important part of Abel and Pientka's work
on applying sized types to a system with copatterns\,\citep{Abel13Wellfounded},
although here it is defined more precisely as the \emph{depth} of a value of
coinductive type. By directly using the notion of depth in the type system, they
show that totality proofs for both coinductive definitions with copatterns and
inductive definitions (as well as mixed inductive-coinductive definitions) can
be constructed by well-founded induction on sizes within the type system. The
method is shown to work for System F\textsubscript{$\omega$}, and has later been
implemented in Agda along with copatterns.

From the perspective of automation, sized types are similar to
guarded recursion. Presently, there does not seem to exist a procedure for
inferring all sizes for a given definition, so these must be given manually by
ther user. The same holds for guarded recursion. While we have no knowledge of
the existence of a
general procedure for inferring all kinds of guarded recursive definitions, the
system presented in Chapter~\ref{cha:infer-guard-recurs} paves the way for
further investigation into this area.

%structurally smaller

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../copatterns-thesis"
%%% End:


% $Id$ %
\subsection{Nim}
\screenshot{plugins/images/ss-nim}{Nim}{img:Nim}
Rules of Nim: There are 21 matches. Two players (you and the \dap{})
alternately pick a certain number of matches and the one who takes the
last match loses.  You can take up to twice as many matches as the
\dap{} selected, and vice versa.

\begin{btnmap}
\ButtonLeft \opt{HAVEREMOTEKEYMAP}{& } & Decrease the number of matches \\
\ButtonRight \opt{HAVEREMOTEKEYMAP}{& } & Increase the number of matches \\
\ButtonPlay \opt{HAVEREMOTEKEYMAP}{& } & Remove the number of matches you have selected \\
\ButtonStop \opt{HAVEREMOTEKEYMAP}{& } & Exit the game \\
\end{btnmap}


\chapter{Adding Memory to the Agents}
\section{Overview}
In the previous chapters, we have assumed that the agents interact in fully observable problems, which are modeled as MDPs. This means that the observation that the agent receives from the environment is sufficient to fully understand its state. Thus:

\begin{equation}
    s_{t} = o_{t}
\end{equation}

However, this is not always true. In robotics problems, robots observe their environment through sensors, and, commonly, the data obtained from them gives partial information about the robot's state. So, a significant set of real-world problems is better described by Partially Observable Markov Decision Processes (POMDPs).

There are different reasons why a problem could be partially observed. One of them is when a state is described by time-dependent phenomena, and the current observations only get partial information about them. For instance, an agent may need to estimate the velocity of a flying drone using instantaneous RGB images that capture its position. Unless observations from different time steps are combined, the agent is not going to be able to make the estimation of the drone's velocity. Other examples of time-dependent phenomena are temporary occlusions or corrupted communication systems between the sensors and the agent.

All of these scenarios are not covered by the variations of D-COACH proposed so far. In this chapter, we aim to add memory to the policies trained with D-COACH, in order to have a system capable of solving sequential decision-making problems with partially observed $\text{time-dependent}$ states. The idea is to validate this approach through simulations, establishing a baseline for further research in memory-based deep interactive learning.
\newpage

\section{Methodology}
There are two well-known approaches for adding memory to agents in sequential decision-making problems when using DNNs as function approximators:

\begin{enumerate}
    \item \textbf{Observation stacking \cite{atari}}: This approach consists in stacking a fixed number of past observations to the current one, and using this stack as the input of the policy.
    \item \textbf{Recurrent models \cite{hausknecht2015deep}:} This approach uses policies that incorporate RNN layers into its neural network architecture. Given that these models have an internal state, they can store information from the past (i.e. they have memory), and use it in future inferences.
\end{enumerate}

One of the main issues of observation stacking is that the memory of these models is determined by the number of stacked observations. Problems that need to remember events in medium or large sequences would require larger stacks. In high-dimensional state problems, the size of the input can increase considerably as the number of stacked observations increments, generating high overheads. In contrast, RNN-based models have the ability to model information for an arbitrarily long amount of time \cite{lample2017playing}. Also, they have no input-related overheads because when these models are evaluated they use as input one observation at a time. In RNNs, the memory-related overhead is determined by the size of their hidden states and the length of the sequences used when updating the weights of the models, so the latter is only effective when training.

From an engineering point of view, using RNNs in products for real-world applications with long or medium time dependencies could lower their costs in comparison to using observation stacking. The input overhead of approaches based on stacked observations would be too high in these scenarios; thus, more powerful (expensive) computers would be needed.

Given the more practical usage of recurrent models and their capability of representing arbitrarily long sequences, in this chapter we use RNN-based policies (with LSTM layers) to test the viability of D-COACH for solving problems in POMDP settings.

\subsection{Learning to Remember}
Even though RNNs are networks with the capability of storing information from past observations, they have to learn to do this. Commonly, in DRL approaches this is implicitly learned when backpropagating the error that aims to maximize the expected return. This gives the intuition that when using D-COACH, backpropagating the correction error through the recurrent layers should be enough for learning well-performing policies. Nevertheless, preliminary tests showed that shaping recurrent models with corrective feedback made the agents to rapidly overfit to the first set of corrections, loosing the capability of learning interesting behaviors and, as a consequence, failing to solve the tasks.

To overcome this problem, we propose to have DNNs separated into two parts: (1) transition model and (2) policy. The transition model is in charge of learning the dynamics of the environment in a supervised manner using samples collected by the agent while the policy part is shaped using corrective feedback.

In MDPs, a transition model is capable of predicting the next state of the environment as a function of the current state and the taken action $M(s_{t},a_{t}) = s_{t+1}$ (as in Equation \ref{eq:model}). In contrast, in POMDPs the agent does not have direct access to its state, so, instead, the transition model can be trained to predict the next observation $o_{t+1}$. To achieve this, the neural network architecture must include recurrent layers (LSTMs in this case). LSTMs can represent in their hidden state $h_{t}$ information from past observations, which is crucial to predict the next observation when the environment is partially observed. Thus, the objective of the first part of the DNN is to learn $M(o_{t},a_{t}, h_{t-1}) = \widetilde o_{t+1}$, which, as a consequence, learns to embed past observations in $h_{t}$.

The second part of the DNN, the policy, takes as input the concatenation of the hidden state of the transition model computed in the last time step $h_{t-1}$ with the current observation of the environment $o_{t}$ and uses this information as if it were the state. D-COACH is used to update the weights of this part as it is done in fully-observable low-dimensional state problems with D-COACH OFF. In this chapter, we are assuming that a representation of past observations combined with the current observation is a good approximation of $s_{t}$:

\begin{equation}
s_{t}\approx (h_{t-1}, o_{t})
\end{equation}

We call this approach Memoryful ONline state representation learning D-COACH (D-COACH MON). A summary of this approach is presented in Figure \ref{fig:mb_dcoach}.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.4\linewidth]{imagenes/cap4/model_based_dcoach.pdf}
    \caption{D-COACH MON general structure.}
    \label{fig:mb_dcoach}
\end{figure}

\newpage

\subsection{Low-dimensional State with Memory}
\label{sec:ld_memory}
As it has been done previously in this work, we first study the low-dimensional state case. The objective is to learn the dynamics of the environment online i.e. as the policy is shaped interactively.

If we look back to the approach taken in Chapter 3, the idea was similar. In that case the objective was to learn a low-dimensional representation of a high-dimensional input online. The strategy was to share the encoder layers of an autoencoder between the policy and the autoencoder, and to update them using both the cost of the policy and the one of the autoencoder. In this case, the hidden state of the LSTM could be interpreted as a representation of past observations. So, this recurrent layers could be shared between the policy and the model, updating them using both costs. A simplified version of this approach is shown in Figure \ref{fig:ld_model_rip}.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.6\linewidth]{imagenes/cap4/ld_model_rip.pdf}
    \caption{Low-dimensional observations general neural network architecture for POMDPs (option 1).}
    \label{fig:ld_model_rip}
\end{figure}

The shortcoming presented in the approach shown in Figure \ref{fig:ld_model_rip} is that in this case the same problem that we found in the preliminary tests appears. The error of the policy does not work well when updating the weights of recurrent layers using the D-COACH strategy, even when using the auxiliary cost of the model.

Alternatively, the approach that was finally taken was to update the model and the policy separately and simultaneously. If we go back to Figure \ref{fig:mb_dcoach}, this would mean that the \textbf{Transition Model} box is updated with transitions collected by the agent, and, separately, the \textbf{Policy} box is updated with corrective feedback. The proposed transition model architecture (\textbf{Transition Model} box in Figure \ref{fig:mb_dcoach}) consists of LSTM and FNN layers, as shown in Figure \ref{fig:ld_model_win}. The policy architecture (\textbf{Policy} box) is simply a composition of FNN layers, as it is done in the other variations of D-COACH.

\newpage

\begin{figure}[H]
    \centering
    \includegraphics[width=0.5\linewidth]{imagenes/cap4/ld_model.pdf}
    \caption{Low-dimensional observations general neural network architecture for POMDPs (option 2).}
    \label{fig:ld_model_win}
\end{figure}

Figure \ref{fig:ld_model_win} presents the general structure of the neural network architecture that is proposed for problems with low-dimensional observations. A detailed version of the network architecture is provided in Figure \ref{fig:detailed_ld}.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.8\linewidth]{imagenes/cap4/ld_model_det.pdf}
    \caption{D-COACH MON low-dimensional observations neural network architecture.}
    \label{fig:detailed_ld}
\end{figure}

As mentioned before, the transition model and the policy are trained as two separate networks. These networks depend on each other. The policy uses as part of its input $h_{t-1}$, which is a vector that the transition model outputs. Similarly, the transition model uses the last taken action as part of its input, which is a vector generated by the policy. When an update of the architecture is done, the transition model is first updated, and, consecutively, the policy is updated. A summary of this process is presented in Figure \ref{fig:ld_mon_train}.

\begin{figure}[h]
\centering
\subfloat[][Transition model update step.]{\includegraphics[width=0.49\linewidth]{imagenes/cap4/ld_mon1.pdf}}
\subfloat[][Policy update step.]{\includegraphics[width=0.45\linewidth]{imagenes/cap4/ld_mon2.pdf}}
\caption[Update step of D-COACH MON neural network for low-dimensional observations.]{Update step of D-COACH MON neural network for low-dimensional observations. First (a), the transition model is updated in a supervised manner using as input a batch with sequences of observations and actions where the labels correspond to $o_{t+1}$. Consecutively (b), the policy is updated using human corrective feedback using as input $o_{t}$ and $h_{t-1}$.}
\label{fig:ld_mon_train}
\end{figure}

\subsection{High-dimensional State with Memory}
In the high-dimensional case, agents need to learn two representations: (1) spatial and (2) temporal. Spatial representation are understood as a low-dimensional representation of high-dimensional data, which is what it has been covered in the former chapters using autoencoders. In contrast, temporal representations refer to encapsulating previous observations in a memory, which is what it has been done in Section \ref{sec:ld_memory} using LSTMs.

In preliminary tests, we found that the most effective way of doing this under the setting of D-COACH is to treat the autoencoder and the recurrent layers as one architecture to represents the transition function model. A model with the capability of learning from high-dimensional states, as it can be seen in Figure \ref{fig:rnn_hd}.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.5\linewidth]{imagenes/cap4/hd_model.pdf}
    \caption{High-dimensional observations general neural network architecture for POMDPs.}
    \label{fig:rnn_hd}
\end{figure}

Instead of using separate costs for the model and the autoencoder, the autoencoding cost for reconstructing $o_{t+1}$ is used. So, Equation \ref{eq:ae} instead of being $L(x_{t},\widetilde x_{t})$ it would be  $L(x_{t},\widetilde x_{t+1})$. By doing this, the hidden state of the LSTM represents both the high-dimensional input and past observations, given that the autoencoder tries to reconstruct and predict $o_{t+1}$. As it was done for the low-dimensional observation case, Figure \ref{fig:detailed_hd} introduces the detailed neural network architecture for the high-dimensional observation case.

\begin{figure}[h]
    \centering
    \includegraphics[width=\linewidth]{imagenes/cap4/hd_model_det.pdf}
    \caption{D-COACH MON high-dimensional observations neural network architecture.}
    \label{fig:detailed_hd}
\end{figure}

In the same way as in the low-dimensional observation case, the transition model and the policy are trained separately and simultaneously. The only difference is that in this case, when the policy is updated it uses the encoding layers to generate an encoded low-dimensional representation of the observation $e(o_{t})$ that the policy uses as input, but only the policy layers are updated. This is also different from what is done in D-COACH OFF, where the encoding layers are also updated when training the policy. Figures \ref{fig:hd_mon_train1} and \ref{fig:hd_mon_train2} summarize a D-COACH MON update step for high-dimensional observations.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.93\linewidth]{imagenes/cap4/hd_mon1.pdf}
    \caption{Transition model update step.}
    \label{fig:hd_mon_train1}
\end{figure}

\begin{figure}[h]
    \centering
    \vspace{1.5cm}
    \includegraphics[width=0.93\linewidth]{imagenes/cap4/hd_mon2.pdf}
    \caption{Policy update step.}
    \label{fig:hd_mon_train2}
    \vspace{1.5cm}
\end{figure}

\subsection{The Algorithm}

In Algorithm \ref{algorithm:DeepCOACH-M}, the pseudocode of D-COACH MON is presented. The hidden state of the LSTM is denoted as  $h^{\mathrm{LSTM}}$ and the human corrective feedback as $h$.

In every time step, the agent executes an action based on its last observation and in the current hidden state of the LSTM (line 5). This hidden state is updated using its previous value and the most recent observation and action (line 7). In this occasion, a buffer in charge of storing the samples of the transition function model ($\mathcal{D}$) is incorporated. In both buffers, $\mathcal{B}$ and $\mathcal{D}$, sequences with length $\tau$ are stored (lines 8 and 18), which is necessary for training the LSTM. This is done following the \emph{bootstrapped random updates} \cite{hausknecht2015deep} strategy. As in the former versions of D-COACH, $\mathcal{B}$ is used for updating the policy (lines 16 and 22) by replaying past corrections. In contrast, $\mathcal{D}$ replays past transitions of the environment in order to update the transition function model (lines 17 and 24). The transition model is updated every time feedback is given (lines 15 and 17), in the same way as it is done with the policy in all of the variations of D-COACH, including this one (lines 14 and 16).

\newpage

\begin{algorithm}[H]
\caption{D-COACH MON: Memoryful Online State Representation Learning}\label{algorithm:DeepCOACH-M}
\begin{algorithmic}[1]
\State \textbf{Require:} error magnitude $e$, policy buffer update interval $b$, policy buffer sampling size $N$, policy buffer min. size $k$, policy buffer max. size $K$, model buffer update interval $d$, transition function model buffer sampling size $M$, transition function model buffer min. size $l$, transition function model buffer max. size $L$, training sequence length $\tau$.
\State \textbf{Init:} $\mathcal{B} = []$, $\mathcal{D} = []$
\For{t = 1,2,...}{}
\State \textbf{observe} observation $o_{t}$
\State \textbf{execute} action $a_{t}=\pi(o_{t}, h^{\mathrm{LSTM}}_{t-1})$
\State \textbf{feedback} human corrective advice $h_{t}$
\State \textbf{compute} $h^{\mathrm{LSTM}}_{t}$ from $M(o_{t}, a_{t},h^{\mathrm{LSTM}}_{t-1})$
\State \textbf{append} $(o_{t-1},...,o_{t-\tau},a_{t-1},...,a_{t-\tau},o_{t})$ to $\mathcal{D}$
\If{length($\mathcal{D}$) $> L$ }
\State $\mathcal{D} = \mathcal{D}[2:L+1]$
\EndIf
\If{$h_{t}$ is not \textbf{0}}
\State $\mathit{error}_{t} = h_{t}\cdot e$
\State $y_{label(t)} = a_{t} + \mathit{error}_{t}$
\State \textbf{update} $\pi$ using SGD with $(o_{t}, h^{\mathrm{LSTM}}_{t-1}, y_{\mathit{label}(t)})$
\State \textbf{update} $M$ using SGD with $(o_{t-1},...,o_{t-\tau},a_{t-1},...,a_{t-\tau},o_{t})$
\State \textbf{update} $\pi$ using SGD with a mini-batch of sequences sampled from $\mathcal{B}$
\State \textbf{update} $M$ using SGD with a mini-batch of sequences sampled from $\mathcal{D}$
\State \textbf{append} $(o_{t},...,o_{t-\tau},a_{t-1},...,a_{t-\tau}, y_{\mathit{label}(t)})$ to $\mathcal{B}$
\If{length($\mathcal{B}$) $> K$ }
\State $\mathcal{B} = \mathcal{B}[2:K+1]$
\EndIf
\EndIf
\If{mod(t, b) is 0 and length($\mathcal{B}$) $\geq$ $k$}
\State \textbf{update} $\pi$ using SGD with a mini-batch of sequences sampled from $\mathcal{B}$
\EndIf
\If{mod(t, d) is 0 and length($\mathcal{D}$) $\geq$ $l$}
\State \textbf{update} $M$ using SGD with a mini-batch of sequences sampled from $\mathcal{D}$
\EndIf
\EndFor
\end{algorithmic}
\end{algorithm}

\section{Experiments and Results}
Simulated teachers were used in three different problems for validating D-COACH MON. The main idea behind these experiments is to compare D-COACH MON with D-COACH ON in partially observable problems.

\begin{enumerate}
    \item \textbf{Partially Observed Cart-Pole:} A partially observed low-dimensional observation scenario. This is the same environment used in Chapter 3 with a modification in the observation space of the agent. In the standard fully-observable Cart-Pole the state has four dimensions, which consists of the position $x$ and velocity $\dot x$ of the cart and the angle $\theta$ and angular velocity $\dot \theta$ of the pole, such that $s=(x, \dot x, \theta, \dot \theta)$. In this case, we take out the derivatives present in the state, such that $s=(x, \theta)$.

    \item \textbf{Partially Observed Cart-Pole from Pixels:} The Cart-Pole environment is modified to use as input raw pixels from an image. The size of the Cart-Pole with respect to the background is incremented to make it easier to learn a low-dimensional representation of the input. The image dimensions are $(600 \times 400 \times 3)$, which is then cropped to $(400 \times 400 \times 3)$ and finally downsampled to $(32 \times 32 \times 1)$ (see Figure \ref{fig:cartpoles}).

    \begin{figure}[H]
    \centering
    \hspace{0.4cm}
    \subfloat[][Original Cart-Pole environment.]
    {
    \setlength{\fboxsep}{0pt}%
    \setlength{\fboxrule}{1pt}%
    {\fbox{\includegraphics[width=0.35\linewidth]{imagenes/cap4/cartpole_LD.png}}}
    }
    \hspace{1.3cm}
    \captionsetup{justification=centering}
    {
    \setlength{\fboxsep}{0pt}%
    \setlength{\fboxrule}{1pt}%
    \subfloat[][Cart-Pole from pixels \\ $(600 \times 400 \times 3)$.]{\fbox{\includegraphics[width=0.35\linewidth]{imagenes/cap4/cartpole_HD.png}}}
    }
    \newline
    {
    \setlength{\fboxsep}{0pt}%
    \setlength{\fboxrule}{1pt}%
    \subfloat[][Cart-Pole from pixels neural network input \\ $(32 \times 32 \times 1)$. ]{\fbox{\includegraphics[width=0.25\linewidth]{imagenes/cap4/cartpole_hd2.png}}}
    }
    \caption{Cart-Pole environment comparison.}
    \label{fig:cartpoles}
    \end{figure}

    \item \textbf{Partially Observed Car Racing:} A partially observed high-dimensional state scenario. The Car Racing problem presented in Chapter 3 is modified such that the indicators at the bottom of the image are no longer available (see Figure \ref{fig:no_inds_car_racing}).
\end{enumerate}

\begin{figure}[h]
    \centering
    \includegraphics[width=0.7\linewidth]{imagenes/cap4/car_racing_no_inds.PNG}
    \caption[Partially observed Car Racing.]{Partially observed Car Racing: indicators at the bottom of the observation are removed.}
    \label{fig:no_inds_car_racing}
\end{figure}

All the results that present averaged data in the form of a curve have confidence intervals that represent the $60^{th}$ percentile of the data. For problems with low-dimensional observations, the neural network architecture presented in Figure \ref{fig:detailed_ld} was used, and the hyper-parameters values are shown in Table \ref{table:mon_ld_hyp}. For the high-dimensional observation case, the neural network architecture introduced in \ref{fig:detailed_ld} was used, and the hyper-parameters shown in Tables \ref{table:mon_hypers1}, \ref{table:mon_hypers2} and \ref{table:mon_hypers3} were used. The rows in the tables with no values were not used i.e. replace FC with identity. In the same way as in Chapter 3, the hyper-parameters of the neural networks used in these experiments were tuned with preliminary experiments.

\begin{table}[]
\centering
\caption[Low-dimensional observation D-COACH MON neural network hyper-parameters.]{D-COACH MON low-dimensional observation neural network hyper-parameters.}
\label{table:mon_ld_hyp}
\begin{tabular}{lcc}
\textbf{Layer}                                              & \multicolumn{1}{l}{\textbf{Activation}} & \multicolumn{1}{l}{\textbf{$N^{\circ}$ neurons}}                \\ \hline \hline
FC1                                                         & ---                                     & ---                                                    \\ \hline
FC2                                                         & ---                                     & ---                                                    \\ \hline
FC3                                                         & ReLU                                    & 300                                                    \\ \hline
\begin{tabular}[c]{@{}l@{}}FC4\\ (size obs.)\end{tabular}   & tanh                                    & Part. Cart-Pole: 2                                     \\ \hline
FC5                                                         & ---                                     & ---                                                    \\ \hline
FC6                                                         & ReLU                                    & 150                                                    \\ \hline
FC7                                                         & ReLU                                    & 64                                                     \\ \hline
FC8                                                         & ReLU                                    & 64                                                     \\ \hline
\begin{tabular}[c]{@{}l@{}}FC9\\ (size action)\end{tabular} & tanh                                    & Part. Cart-Pole: 1                                     \\ \hline
R1                                                          & LSTM                                    & \begin{tabular}[c]{@{}c@{}}150\\ (size h)\end{tabular} \\ \hline
\end{tabular}
\end{table}

\begin{table}[H]
\vspace{1cm}
\parbox{\linewidth}{
\centering
\caption[High-dimensional observation D-COACH MON convolutional and deconvolutional layers hyper-parameters]{D-COACH MON high-dimensional observations convolutional and deconvolutional layers hyper-parameters. Autoencoder latent space size: $8\times8\times4$ (Car Racing) and $4\times4\times4$ (HD Cart-Pole).}
\label{table:mon_hypers1}
\begin{tabular}{lcccc}
\textbf{Layer} & \multicolumn{1}{l}{\textbf{Activation}} & \multicolumn{1}{l}{\textbf{Filters}} & \multicolumn{1}{l}{\textbf{Filter size}} & \multicolumn{1}{l}{\textbf{Stride}} \\ \hline \hline
C1             & ReLU                                    & 16                                   & $3\times3$                                    & 2                                   \\ \hline
C2             & ReLU                                    & 8                                    & $3\times3$                                    & 2                                   \\ \hline
C3             & ReLU                                    & 4                                    & $3\times3$                                    & 2                                   \\ \hline
DC1            & ReLU                                    & 8                                    & $3\times3$                                    & 2                                   \\ \hline
DC2            & ReLU                                    & 16                                   & $3\times3$                                    & 2                                   \\ \hline
DC3            & ReLU                                    & 1                                    & $3\times3$                                    & 2                                   \\ \hline
\vspace{0.2cm}
\end{tabular}}
\end{table}

\begin{table}[H]
\centering
\caption{High-dimensional observation D-COACH MON observation spaces sizes.}
\label{table:mon_hypers3}
\begin{tabular}{lc}
\textbf{Problem}     & \textbf{Input size} \\ \hline \hline
Car Racing     & $64\times64\times1$                   \\ \hline
HD Cart-Pole    & $32\times32\times1$  \\ \hline
\end{tabular}
\vspace{1cm}
\end{table}

\begin{table}[H]
\centering
\caption{High-dimensional observation D-COACH MON fully-connected and LSTM layers hyper-parameters.}
\label{table:mon_hypers2}
\begin{tabular}{lcc}
\textbf{Layer}                                              & \multicolumn{1}{l}{\textbf{Activation}} & \multicolumn{1}{l}{\textbf{$N^{\circ}$ neurons}}                                        \\ \hline
FC1                                                         & ---                                     & ---                                                                                \\ \hline
FC2                                                         & ---                                     & \begin{tabular}[c]{@{}c@{}}Car Racing: 256\\ HD Cart-Pole: 64\end{tabular} \\ \hline \hline
FC3                                                         & ReLU                                    & 300                                                                                \\ \hline
FC4                                                         & ReLU                                    & \begin{tabular}[c]{@{}c@{}}Car Racing: 256\\ HD Cart-Pole: 64\end{tabular} \\ \hline
FC5                                                         & ReLU                                    & 150                                                                                \\ \hline
FC6                                                         & ReLU                                    & 150                                                                                \\ \hline
FC7                                                         & ReLU                                    & 64                                                                                \\ \hline
FC8                                                         & ReLU                                    & 64                                                                                \\ \hline
\begin{tabular}[c]{@{}l@{}}FC9\\ (action size)\end{tabular} & tanh                                    & \begin{tabular}[c]{@{}c@{}}Car Racing: 3\\ HD Cart-Pole: 1\end{tabular}     \\ \hline
R1                                                          & LSTM                                    & \begin{tabular}[c]{@{}c@{}}150\\ (size h)\end{tabular}                                  \\ \hline
\end{tabular}
\end{table}

\subsection{Validation Low-Dimensional State}

D-COACH MON is tested in the partially observed cart-pole because it is a way of validating if the proposed methodology works in a simple setting. Figure \ref{fig:ld_cartpole_model} shows the learning curves of D-COACH MON and D-COACH ON. The same strategy used in Chapter 3 for training policies with a simulated teacher is employed in this section.

\begin{figure}[H]
    \centering
    \includegraphics[width=0.7\linewidth]{imagenes/cap3/cartpole_LD_model.pdf}
    \caption[Partially observed Cart-Pole results for simulated teacher comparing D-COACH ON and D-COACH MON.]{Partially observed cart-pole results for simulated teacher comparing D-COACH ON and D-COACH MON.  Buffer: $K = 200$; $k=5$; $b = 10$; $N = 50$; $L=1\mathrm{e}5$; $l=5$; $d=1$; $M=50$; $\tau=8$; $P_{h}$: $\alpha = 0.35$; $\tau = 0.0003$; $\emph{e}=1$. Simulated teacher network learning rate: $0.005$; Transition model learning rate: $0.0005$.}
    \label{fig:ld_cartpole_model}
\end{figure}

D-COACH ON is not able to learn a well performing policy in this case. This was expected, since the velocities of both the cart and the pole are essential for making good decisions in a problem of these characteristics. Figure \ref{fig:cp_ex} shows an example of two scenarios wherein at time step 2 the observation would be the same in the partially observed cart-pole (if we only focus on the angle of the pole). In this example the angular velocity of the pole changes direction between scenarios, but an agent without memory is not able to tell the difference. As a consequence, in $t=2$, it would make the same decision in two opposite scenarios.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.65\linewidth]{imagenes/cap4/cartpole_ex.pdf}
    \caption{Cart-pole partial observability example.}
    \label{fig:cp_ex}
    \vspace{-0.2cm}
\end{figure}

In contrast, if the agent makes decisions based on previous observations it can understand that these two scenarios are different and take better decisions. This is shown in the blue curve of Figure \ref{fig:ld_cartpole_model}, where the agent is able to learn a well performing policy in about 12 minutes.

\subsection{Validation High-Dimensional State}
In this section, we validate D-COACH MON in two different problems (once again, the simulated teacher training strategy introduced in Chapter 3 is used). Cart-pole is a problem that was not originally designed to be solved using as input raw pixels of an image, so we do not expect to obtain a perfect performance (500 score average). The main goal of testing D-COACH with this problem is to observe if the proposed model (autoencoder + LSTM) is able to embed both the past and high-dimensional inputs adequately.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.7\linewidth]{imagenes/cap3/cartpole_HD_model.pdf}
    \vspace{-0.2cm}
    \caption[Partially observed Cart-Pole from pixels results for simulated teacher comparing D-COACH ON and D-COACH MON.]{Partially observed cart-pole from pixels results for simulated teacher comparing D-COACH ON and D-COACH MON.  Buffer: $K = 1000$; $k=20$; $b = 1$; $N = 20$; $L=1\mathrm{e}5$; $l=20$; $d=1$; $M=50$; $\tau=4$; $P_{h}$: $\alpha = 0.35$; $\tau = 0.00003$; $\emph{e}=1$; $m=50$; $\epsilon=0.0011$. Simulated teacher network learning rate: $0.0003$; Transition model learning rate: $0.0005$; Autoencoder learning rate: $0.0003$.}
    \label{fig:cp_hd}
\end{figure}

Figure \ref{fig:cp_hd} shows a slower learning speed and a less successful convergence of D-COACH MON if we compare its performance with the one obtained in the low-dimensional observation partially observed Cart-Pole (see Figure \ref{fig:ld_cartpole_model}). This is expected, since in this case the agent must also learn to extract features from the image. Also, D-COACH ON is not able to learn a well performing policy for this problem. So, even though D-COACH MON is not able to achieve the score of 500, a considerable improvement is made over its memoryless version, showing the importance of using recurrence in this type of problems.

Finally, D-COACH MON was validated in the partially observed Car Racing problem. Figure \ref{fig:po_cr} shows that D-COACH ON (orange curve) is able to learn a policy that achieves an average return of $\sim700$ after $\sim15$ minutes of training, which is an acceptable score for this problem. Nevertheless, D-COACH MON (blue curve) shows that it is possible to obtain an even better performance if memory is added to the neural network, achieving an average return of $\sim800$ after $\sim15$ minutes of training (problem is considered to be solved with a score of 900), which is similar to the performance of $\text{D-COACH}$ ON in the fully observable Car Racing (see Figure \ref{fig:simulatedteachers}). The difference between both curves shows that D-COACH MON is a more robust and powerful approach than $\text{D-COACH}$ ON, capable of enhancing the performance of the agent in problems where $\text{D-COACH}$ ON could appear to be sufficient.

\begin{figure}[H]
    \centering
    \includegraphics[width=0.7\linewidth]{imagenes/cap3/car_racing_lstm.pdf}
    \caption[Partially observed Car Racing results for simulated teacher comparing D-COACH ON and D-COACH MON.]{Partially observed Car Racing results for simulated teacher comparing D-COACH ON and D-COACH MON.  Buffer: $K = 2000$; $k=20$; $b=10$; $N = 8$; $L=1\mathrm{e}5$; $l=20$; $d=1$; $M=8$; $\tau=10$; $P_{h}$: $\alpha = 0.9$; $\tau = 0.000015$; $\emph{e}=(1, 1, 1)$; $m=50$; $\epsilon=0.0011$. Simulated teacher network learning rate: $0.003$; Transition model learning rate: $0.0005$; Autoencoder learning rate: $0.003$.}
    \label{fig:po_cr}
\end{figure}

\section{Discussion}
In this chapter we introduced a variation of D-COACH with memory, D-COACH MON. The main objective was to extend D-COACH in order for it to work in POMDPs where the state is described by time-dependent phenomena. Low-dimensional and high-dimensional problems were covered, introducing a different model architecture for each case.

D-COACH MON offers a framework that learns both the policy and the model online. This makes it easy for humans to interact in the learning process of the agents, given that no human effort is needed in extra learning steps as in the offline state representation learning version of D-COACH (Chapter 3). The experiments show that the proposed method is effective for enhancing the performance of agents in POMDPs. As discussed previously, memory-based approaches may be key to solve problems in real-world scenarios, where is common to face problems with partial observability.

As it is stated in the overview of this chapter, RNNs have an extra overhead when training, because sequences must be used for updating the weights of the recurrent layers. Given that D-COACH is an IML approach, updates are done in real-time while the teacher is providing feedback. We found that the overhead produced when updating the recurrent layers is too high for running this approach in CPU; thus, a GPU was used. This is a disadvantage with respect to the previously proposed variations of D-COACH, given that those models are able to run in real-time using CPUs.


\chapter{Clock Drift}\label{chap:6}
\minitoc

In the previous section, we introduced a timed automata model that
describes a high level representation of systems execution.
However, this type of model assumes that components clocks are perfectly synchronous which is
hardly the case in practice.
Effectively, clocks are able to measure time up to a certain precision and will be likely
to drift since they are implemented based on oscillators that are not perfect: the oscillator
frequency is not constant, it changes depending on environmental conditions such as
temperature, humidity and aging.
Figure~\ref{fig:drift_exp} illustrates an example of two clocks $x$ and $y$ having different rate
with respect to an implicit perfect reference time. Figure (a) shows the case
where clock $x$ evolves steadily faster than clock $y$. Whereas, in Figure (b)
clock $y$ is initially faster than clock $x$, then the trend is inverted after some time.

\begin{figure}[ht]
\begin{minipage}[b]{0.5\linewidth}
\centering
\includegraphics[scale=.55]{Figures/drift1}\\
  \vspace*{5mm} \small{(a)}
\vspace{4ex}
\end{minipage}%%
\begin{minipage}[b]{0.5\linewidth}
\centering
\includegraphics[scale=.55]{Figures/drift2}\\
  \vspace*{5mm} \small{(b)}
\vspace{4ex}
\end{minipage}
\caption{Example of Clocks with Different Rates}
\label{fig:drift_exp}
\end{figure}

In this chapter, we present a distributed timed automata model where clocks advance at
different rates and we study the resulting effect on the system behavior.

\section{Distributed Timed Systems with Independent Clock Rates}

\subsection{Expressing Clock Constraints Using Local Clock}

When building distributed real-time systems, a common practice is to use
local clocks as time references as explained in Chapter~\ref{chap:3}.
These clocks measure the absolute time elapsed since the system startup and are never reset.
This approach reduces the effort of keeping track of the actual time progress in components
and enable to have a common time scale.

The idea consists in mapping each clock of a component to a (unique) local clock.
Thus, the value of component clocks are obtained by simply shifting their corespondent local
clock by a constant amount of time as soon as the clocks are not reset.
Effectively, for each clock $x$ of a component, we introduce a real variable $\rho_x$ that
stores the absolute time of its last reset (with respect to its local clock), that is, if $x$
is mapped to a local clock $g$, then each time $x$ is reset, $\rho_x$ is update to the current
value of g.
Notice that the value of $x$ can be found by the equality $x=g-\rho_x$.
As a result, any timing constraints $c$ of a component $B_i$ can be expressed using a
local clock $g$ as follows:
\begin{equation}
  \label{eq:glob}
  c = \bigwedge_{x_i\in\X_i} l\triangleleft x_i\triangleright u
    =\bigwedge_{x_i\in\X_i} l+\rho_{x_i}\triangleleft g\triangleright u+\rho_{x_i}
\end{equation}

where $\triangleleft\in\{<,\le\}$ and $\triangleright\in\{>,\ge\}$.
Notice any timing constraints of Definition~\ref{eq:cc} can be written on the form of
inequalities.

\subsection{Distributed Timed System}

Let $S=(\Loc,\loc_0,\X,\D,\gamma,\E_{\gamma},\{f_e\}_{e\in\gamma},\I)$ be a timed system of $n$
components synchronizing through the interaction set $\gamma$.
For an interaction $\alpha$, we denote by $\clock{\alpha}$ the set of clocks appearing in its
timing constraints, that is, $\clock{\alpha}=\{x\in\X|\forall (\loc,\alpha,g,r,\loc')\in
\E_{\gamma}, x \ \circlled{$\in$} \ g \}$, with \circlled{$\in$} denoting the presence of
$x$ in the guard $g$.

Given an interaction partition $\{\gamma_k\}_{k=1}^m$, we put $\clock{\gamma_k}$ to denote
the set of clocks appearing in the timing constraints of interactions of $\gamma_k$, that is,
$\clock{\gamma_k}=\{\cup_{\alpha\in\gamma_k}\clock{\alpha}\}$.
We formalize the independent evolution of clocks by defining an \emph{ownership map} that assigns
a set of clocks to a unique local clock based on interaction partitioning .
Particularly, we assign to each class of interaction a unique local clock.
We require additionally that clock of each components is mapped only to a unique local clock.
This avoid timing inconsistency and ensure that each clock of a component is evaluated using
a unique local
clock. This constraints immediately the interaction partitioning as follows:
\begin{equation}\label{eq:tcf}
  \bigwedge_{\substack{i,j\in\{0,\cdots,m\}\\i\neq j}}\clock{\gamma_i}\cap\clock{\gamma_j}=
  \emptyset
\end{equation}

\begin{definition}[Distributed Timed System]\label{def:drft}
Given a timed system $S=(\Loc,\loc_0,\X,\D,\gamma,\E,$\\$\{f_e\}_{e\in\gamma},\I)$
and an interaction partitioning satisfying the constraint~\ref{eq:tcf}.
We define the corresponding distributed timed system with independent local clock rates
as the tuple
  $S^{dt}=(\Loc,\loc_0,\X^{dt},\D^{dt},\gamma,\E^{dt},\{f_e^{dt}\}_{e\in\gamma},\I^{dt},\pi)$
such that:
\begin{itemize}
  \item $\X^{dt}$ is the set of local clocks (a unique clock per class of interaction)
  \item $\pi:\X\lto\X^{dt}$ is a many to one mapping between clocks $\X$ of $S$ and local clocks
    $\X^{dt}$
  \item $\D^{dt}=\D\cup\{\cup_{x\in\X}\rho_x|\rho_x\in\realpoz\}$ is the set of data with
    $\rho_x$ being real valued variables storing absolute reset times
  \item $\E^{dt}$ is such that for every $(\loc,\alpha,g,r,\loc')\in\E$, $\alpha\in\gamma$, we
    have the corresponding transition $(\loc,\alpha,g^{dt},\emptyset,\loc')\in\E^{dt}$ where:
  \begin{itemize}
    \item $g^{dt}$ is the guard $g$ expressed using the local clock $x^{dt}\in\X^{dt}$ where
      $\forall x_i,x_j\in\clock{\alpha},i\neq j,\pi(x_i)=\pi(x_j)=x^{dt}$
    \item $f_e^{dt}=f_e\cup\{\rho_x:=x^{dt}|\forall x\in r\}$ is the transfer function updating
      reset variables in addition to data variables of $\D$
  \end{itemize}
  \item $\I^{dt}$ is the set of location invariants expressed using local clocks
\end{itemize}
\end{definition}

Notice that clocks appearing in location invariants appear necessarily in at least an
interaction since we assume timed system with well formed guard.

\begin{property}[Semantics]
  The semantics of a distributed timed system $S^{dt}=(\Loc,\loc_0,\X^{dt},\D^{dt},$\\$\gamma,
  \E^{dt},\{f_e^{dt}\}_{e\in\gamma},\I^{dt},\pi)$ is defined by the timed transition system
  $\ttsb{dt}=(\Q^{dt},q_0^{dt},\gamma,\lto_{\gamma})$ where:
  \begin{itemize}
    \item $\Q^{dt}=\Loc\times\V(\X_{dt}\cup\tcal{D}_{dt})\times\Delta$ where $\Loc$ is
      the set of global locations, $\V(\X_{dt}\cup\tcal{D}_{dt})$ is the set of clock and
      data valuations, and $\Delta=\real$ is the set offsets of local clocks in
      $\X_{dt}$ with respect to an implicit (perfect) reference clock
    \item $q_0^{dt}=(\loc_0,0,0)$ is the initial state.
    \item $\lto_{\gamma}\subseteq\Q^{dt}\times(\gamma\cup\realpos)\times\Q^{dt}$ is the set of
      labeled transitions defined by the rules:
    \begin{itemize}
      \item $(\loc,\val,\delta)\transit{\alpha}_{\gamma}(\loc',\val[r],\delta)$ for
        $\alpha\in\gamma$, if
      $(\loc,\alpha,g,r,\loc')\in\E^{dt}\wedge \val\models g$
    \item $(\loc,\val,\delta)\transit{d}_{\gamma}(\loc,\val',\delta')$ for $d\in\realpos$,
      such that $\val'=\val+d-\delta+\delta'\wedge \val'\models\I^{dt}(\loc)\wedge\val'\ge\val$
    \end{itemize}
  \end{itemize}
\end{property}

The above semantics models drifting clocks by introducing an offset variable $\delta$ that
stores for each clock in $x^{dt}$ its actual drift value with respect to an implicit
reference time. The common practice when clocks are subject to drift is to
\emph{regularly resynchronize} the clocks using different methods, such as bit-stuffing
or any other clock synchronization scheme.
Consequently, we consider a more realistic model where clocks can drift up
to a certain value $\epsilon$ with respect to a reference clock. This value is generally
computed based on three parameters: \emph{(i)} the post-synchronization gap, \emph{(ii)}
the longest gap between synchronizations, and \emph{(iii)} the clock precision parameter
usually given by the constructor.
This induces that $\delta\in[-\epsilon,\epsilon]^{|\X^{dt}|}$.

\subsection{Properties}

\begin{property}\label{pr:rd}
  $\forall x,y\in\X^{dt},x\neq y,\delta=(\cdots,\delta_x,\cdots,\delta_y,\cdots)$
  where $|\delta|=|\X^{dt}|$, we have $\delta_x-\delta_y\in[-2\epsilon,2\epsilon]$
\end{property}

Property~\ref{pr:rd} states that the relative drift between local partitions
clock is bounded by $2\epsilon$. This results from the fact that all local
clocks are kept within $\epsilon$ of a reference clock.

In order to attest the correctness of the distributed semantics, we compare
its corresponding time transition system ($\ttsb{dt}$) and the timed transition system
of the standard semantics ($\ttsb{g}$).

Let $R$ be the relation:
\begin{align*}
  R=\{(q^{dt},q)\in\Q_{dt}\times\Q|q^{dt}=(\loc^{dt},\val^{dt},\delta), q=(\loc,\val) \\
  \text{ such that: }
  \begin{cases}
  \loc^{dt}=\loc,\\
  \forall x\in\X,\val(x)=\valdt(\pi(x))-\rho_x-\delta_x
\end{cases}
\end{align*}

The relation $R$ relates states of the distributed semantics with
states of the standard semantics having the same location configuration,
and whose clock valuations expressed on local clocks are $\delta\in[-\epsilon,\epsilon]$ close.
We call such states $\epsilon-$similar.

\begin{lemma}\label{lem:equiv}
  For $\epsilon=0$, we have $\ttsb{dt}\sim_{R}\ttsb{g}$
\end{lemma}

Lemma~\ref{lem:equiv} describes the fact that a given system and its corresponding
distributed model are bisimilar when $\Delta=0$, that is, when clocks advance at the same
rate (perfect clocks).

\begin{property}\label{pr:enabled1}
  Let $(\qdt,q)\in \RS$ such that $\qdt$ satisfies $\enabled{\alpha}$, then
  $q$ satisfies \\$\enabledbackwardb{\alpha}{0}{\epsilon}\vee\enabled{\alpha}
  \vee\enabledforwardb{\alpha}{0}{\epsilon}$.
\end{property}

Property~\ref{pr:enabled1} expresses that for states $(\qdt,q)\in\RS$ if it exists
an interaction enabled at $\qdt$ then this interaction is either enabled at $q$,
will be enabled after a time progress of $\epsilon$ or is up to $\epsilon$ after
the deadline of $\alpha$. This property flows directly from the $\epsilon-$similarity
of states $(\qdt,q)\in\RS$, the form of interactions timing constraints
(conjunction of intervals)
and the fact that clocks involved in the same interaction advance at the same rate.
It points out that any execution from
state $\qdt$ might not be always possible from state $q$ or any of its time successor.

\begin{lemma}\label{lm:enabled1}
  Let $(\qdt,q)\in\RS$ such that $\qdt$ satisfies $\enabled{\alpha}\wedge
  \enabledbackwardb{\alpha}{\epsilon}{\epsilon}$, then
  $q$ satisfies $\enabledbackwardb{\alpha}{0}{\epsilon}$.
\end{lemma}
Lemma~\ref{lm:enabled1} can be deduced straightforwardly from property~\ref{pr:enabled1}.
It expresses that for states $(\qdt,q)\in\RS$, if it exists an interaction $\alpha\in\gamma$
such that $\alpha$ is enabled at $\qdt$ and that clock valuations are up to $\epsilon$
before the deadline of this interaction then, $\alpha$ can be executed from $q$ or any of its
$\epsilon-$time successors (by doing a time progress up to $\epsilon$).

The usual notion of simulation as defined in~\ref{def:sim} is too precises.
It requires that each trace in one system
can be matched~\emph{exactly} by a trace in the other system, that is,
two states can be distinguished even for an infinitesimally
small mismatch between timings ($\epsilon\neq0$).
Thus, we rely on the following quantitative variant of simulation~\cite{drift:esim}.

\begin{definition}[$\epsilon-$simulation]\label{def:esim}
  Given two TTS, $\TTSc{1}$ and $\TTSc{2}$, a relation $R\subseteq\Q_1\times\Q_2$ is a:
  \begin{itemize}
    \item Strong timed $\epsilon-$simulation, if for any $(q_1,q_2)\in R$, $\sigma\in\sum$,
      $d,\epsilon\in\realpoz$
    \begin{itemize}
      \item $q_1\transit{\sigma}\q_1'$ implies $q_2\transit{\sigma}q_2'$ for some $q_2'\in\Q_2$
        with $(q_1',q_2')\in R$
      \item $q_1\transit{d}\q_1'$ implies $q_2\transit{d'}q_2'$ for some $q_2'\in\Q_2$ and
        $d'\in\realpoz$ with $|d'-d|\le\epsilon$ and $(q_1',q_2')\in R$
    \end{itemize}
    \item Timed action $\epsilon-$simulation, if for any $(q_1,q_2)\in R$, $\sigma\in\sum$,
      $d,\epsilon\in\realpoz$
    \begin{itemize}
      \item $q_1\transit{d,\sigma}\q_1'$ implies $q_2\transit{d',\sigma}q_2'$ for some $q_2'
        \in\Q_2$ and $d'\in\realpoz$ with $|d'-d|\le\epsilon$ and $(q_1',q_2')\in R$
    \end{itemize}
  \end{itemize}
  If there exists a strong timed (resp. timed action) $\epsilon-$simulation
  between $\ttsb{1}$ and $\ttsb{2}$ w.r.t $R$, then we write
  $\ttsb{1}\sqsubseteq^{\epsilon}_{R}\ttsb{2}$ (resp. $\tts{1}\sqsubseteq^{\epsilon*}_{R}
  \ttsb{2})$
\end{definition}

This approach characterizes
the degree of closeness between timed systems: it generalizes the (boolean)
notions of timed simulation
(yes or no) to metrics over timed system. Formally, for a positive real number
$\epsilon$, a state $q_1$ is
told $\epsilon-$similar to another state $q_2$ if there is a time-abstract simulation
that can relates both states in
the sense that the difference between the delays of time-step transitions is at most $\epsilon$.

Although this definition of simulation is less
restrictive, Property~\ref{pr:enabled1} gives the intuition that for some states
$(\qdt,q)\in\RS$ there may be interactions that can be executed from $\qdt$ but not from
$q$, which make the $\epsilon-$simulation impossible.

\section{Robust Distributed Semantics}
As explained in previous sections, the distributed semantics may
exhibit new behavior with respect to the standard semantics.
In this section, we identify the problematical states of the
distributed semantics and provide sufficient conditions
that will guarantee timed action $\epsilon-$simulation.

\begin{definition}[Potentially Bad States]\label{def:pbss}
  We denote by $\Upsilon$ the set of potentially bad states of the
  distributed semantics characterized as follows:
  \[\Upsilon=\{q=(\loc,\val)|\exists\alpha\in\gamma,\val\models g_{\alpha}\wedge
  \val+\epsilon\not\models g_{\alpha} \} \]
\end{definition}

The intuition behind this characterization results form Property~\ref{pr:enabled1}.
These states may yield possible execution of interactions in the distributed semantics that
are not possible in the related states of the standard semantics (states that are
$\epsilon-$similar).

\begin{proposition}\label{prop:esim}
  Let $\ttsb{dt}$ be the timed transition system of the distributed semantics.
  We have:
  \[\Upsilon\cap\reach^*(\ttsb{dt})=\emptyset\implies\ttsb{dt}\sqsubseteq^{\epsilon*}_{\RS}
  \ttsb{g}\]
  where $\reach^*(\ttsb{dt})$ is the projection of states of the distributed semantics
  on state variables of the standard semantics, that is, by considering only locations, clocks,
  and data variables.
\end{proposition}

\begin{proof}[Proof of Proposition~\ref{prop:esim}]

  In order to prove proposition~\ref{prop:esim}, we need to show that if for any
  $(\qdt,q)\in\RS$, $\sigma\in\gamma$ and $d\in\realpoz$:
      \begin{displaymath}
        \qdt\transit{d,\sigma}\qdtb \implies \exists q'\in\Q^g, q\transit{d',\sigma}q',
        \text{ with }
        d'\in\realpoz, \ |d'-d|\le\epsilon \text{ and } (\qdtb,q')\in\RS
      \end{displaymath}

      Let $(\qdt,q)\in\RS$ such that $\Upsilon\cap\reach^*(\ttsb{dt})=\emptyset$ and
      $\exists\sigma\in\gamma,\exists d\in\realpoz,$
  $\qdt\transit{d,\sigma}\qdtb$. We distinguish two cases:
  \paragraph{\textbf{Case 1: $d > 0$}\\}
  We have $\qdt \transit{d,\sigma}\qdtb$. This means:
    \[
    \begin{cases}{}
      \valdtb=(\valdt+d-\delta+\delta')[r^{dt},f_{\sigma}] \\
      \valdt +d-\delta+\delta'\models\I^{dt}(\loc)\wedge g_{\sigma}^{dt} \\
      \valdt +d-\delta+\delta'+\epsilon\models\I^{dt}(\loc)\wedge g_{\sigma}^{dt}&
      \qdt\notin\Upsilon
    \end{cases}\]
  where $[r^{dt},f_{\sigma}]$ means after applying reset and transfer function.
  When expressed on original clocks the above becomes:
    \[\begin{cases}{}
      \valdtb=(\val+\rho+d+\delta')[r^{dt},f_{\sigma}]\\
      \val+d+\delta'\models\I(\loc)\wedge g_{\sigma} \\
      \val+d+\delta'+\epsilon\models\I(\loc)\wedge g_{\sigma}
    \end{cases}\]
  We also have $\delta'\in[-\epsilon,\epsilon]$. Consequently we have:

  \begin{enumerate}
    \item $\delta'\in[0,\epsilon]$, then:\\
      \[\begin{cases}{}
        \valdtb=\val+d+\delta'+(\rho)[r^{dt},f_{\sigma}]\\
      \val+d+\epsilon\models\I(\loc)\wedge g_{\sigma} \label{1}
    \end{cases}\]
    By putting $d'=d$, and  since $r^{dt}=\emptyset$ (Definition~\ref{def:drft}) and
      $f_{\sigma}$ applies only on $\rho$ we obtain:
      \[\begin{cases}
        \valdtb=\val+d'+\delta'+(\rho)[r^{dt}] \\
      \val+d'+\epsilon\models\I(\loc)\wedge g_{\sigma} \label{1}
      \end{cases}\]%%%%%
      Consequently, we can conclude that $q\transit{d',\sigma}q'$ such $(\qdtb,q')\in\RS$
    \item $\delta'\in[-\epsilon,0]$, then:\\
      \[\begin{cases}{}
        \valdtb=\val+d+\delta'+(\rho)[r^{dt},f_{\sigma}]\\
      \val+d-\epsilon+\epsilon\models\I(\loc)\wedge g_{\sigma} \label{1}
    \end{cases}\]
    By putting $d'=d$, and  since $r^{dt}=\emptyset$ (Definition~\ref{def:drft}) and
      $f_{\sigma}$ applies only on $\rho$ we obtain:
      \[\begin{cases}
        \valdtb=\val+d'+\delta'-\epsilon+(\rho)[r^{dt}] \\
      \val+d'\models\I(\loc)\wedge g_{\sigma} \label{1}
      \end{cases}\]%%%%%
      Consequently, we can conclude that $q\transit{d',\sigma}q'$ such $(\qdtb,q')\in\RS$
  \end{enumerate}

  \paragraph{\textbf{Case 2: $d = 0$}\\}
   Since $\Upsilon\cap\reach^*(\ttsb{dt})=\emptyset$, using the same methodology of case 1,
   we can conclude that $\exists q'\in\Q^g, q\transit{d',\sigma}q',\text{ with }
        d'\in\realpoz, \ |d'-d|\le\epsilon \text{ and } (\qdtb,q')\in\RS$.
\end{proof}

Proposition~\ref{prop:esim} highlights an interesting property on the reachable states of the
distributed semantics. It states that if all the reachable states of the
distributed semantics are not potential bad states, i.e, $\Upsilon\cap\reach^*(\ttsb{dt})=
\emptyset $, then there exists a timed action $\epsilon-$simulation between the timed transition
systems of the standard semantics and the distributed semantics.

Consequently, the idea is to restrict the distributed semantics in order to
avoid such states.
This can be achieved by shrinking the upper bound of every transition timing constraints, if it
exists, by $\epsilon$.
Let $\ttsb{\dte}$ be the timed transition system after shrinking every existing upper bound timing
constraints by an amount of $\epsilon$. Then, we have the following lemma:
\begin{lemma}
  $\ttsb{\dte}\subseteq\ttsb{dt}$
\end{lemma}
The above lemma derives straightforwardly from the fact that shrinking does not introduce new
behavior. Conversely, it just restrains the set of states from which interactions can be
executed.

\begin{lemma}\label{lem:shrink}
  $\Upsilon\cap\reach^*(\ttsb{\dte})=\emptyset$.
\end{lemma}
The above lemma is a direct consequence of the shrinking operation.
In fact, since clocks of the distributed semantics an those of the standard semantics can have
at most $\epsilon$ difference in values, by shrinking upper bound timing constraints we avoid
all possible execution of interactions for clock values that are $\epsilon$ before its due date.
Consequently, this ensures that $\Upsilon\cap\reach*(\ttsb{\dte})=\emptyset$ and allows for any
related state of the standard semantics to execute the same interaction
directly or after doing a timed step.
%This can be effectively achieved by restricting the progress of time based to
%$\epsilon$ before the due date of interactions.
%Preventing the progress of time $\epsilon$ time units
%before any interaction deadline will prevent any execution not possible
%in the corresponding state of the standard semantics.
%Formally, this restriction can be written as follows:
%\begin{equation}\label{eq:res}
% (\loc,\val,\delta)\transit{d}_{\gamma}(\loc,\val',\delta') \text{ for } d\in\realpos,
%      \text{ such that }\val'=\val+d-\delta+\delta'\wedge \val'\models\I(\loc)\wedge
%      (\loc,\val',\delta')\notin\Upsilon
%\end{equation}

%The above restriction of time progress helps to avoid reaching the potential bad states
%characterized by Definition~\ref{def:pbss}. However, this restriction does not guarantee
%the non-reachability of such states. In fact, interaction executions may as well lead
%to state of $\Upsilon$. Consequently, to prove that any reachable state of the distributed
%semantics is not a potentially bad state, we need to ensure that any \emph{arrival state},
%that is, state reachable immediately after the execution of an interaction is not a potentially
%bad state.
The following lemma is a direct consequence of Proposition~\ref{prop:esim} and Lemma~\ref{lem:shrink}.
\begin{lemma}
  $\ttsb{\dte}\sqsubseteq^{\epsilon*}_{\RS}\ttsb{g}$
\end{lemma}

\begin{definition}[Robustness]\label{def:rob}
  For a given timed system S, we say that S is robust to clock drifts iff $\reach(\ttsb{\dte})$
  is deadlock free.
\end{definition}

Restricting the distributed semantics through shrinking allows to avoid the reachability of some states
(the potential bad states). This enforces the timed action $\epsilon-$simulation between states
of the standard semantics and those of the restricted distributed semantics.
However, since shrinking reduces the set of reachable states, the timed action $\epsilon-$simulation
alone is not enough. We must also ensure that shrinking does not introduce any deadlocks.
The two aforementioned point together form our definition of robustness.

\section{Discussion}

In this chapter, we revisited the robustness concept usually used to check whether a given
system still satisfies the specification (a set of properties) when subject to different
perturbations such as clock drift. We first modeled the behavior of a timed system with
independent clock rates and subject to a resynchronization scheme. Thereafter, we used
a variant of simulation that relates states with a timing difference up to a certain $\epsilon$,
and provide a characterization of potential bad states that may invalidate this variant of
simulation. Then, we suggested a strategy based on shrinking upper bound timing constraints
in order to avoid such states which allows to attest the timed action $\epsilon-$simulation.

Sankur et al.~\cite{ocan} studied the robustness problem in timed automata against guards
shrinking. They provided a method for deciding whether shrinking all timing constraints (guards)
of a timed automaton, by possibly different amounts, results in a timed automaton that
preserves some time-abstract behaviors, and is not blocking. The latter represents their
robustness notion.
It was proven that if the automaton is shrinkable, then several properties established on the
initial automaton are preserved in the implementation (when clocks are not perfect).
Otherwise, then the conclusion is that the model is vulnerable to the slightest variations in the
measure of time and should thus be considered as non-robust, and the design should be reviewed.
From a theoretical aspect, the proposed analysis was formulated as a parameter synthesis
problem. \texttt{Shrinktech}~\cite{ocan2} is a tool that implements the simulation-shrinkability
algorithm presented in~\cite{ocan}. It is compatible with the Kronos~\cite{kronos} model checker,
that can minimize the region graph of a timed automaton~\cite{tl} needed by the tool.
The tool has been used to attest the shrinkability of several case studies, such as
the Philips Audio Retransmission protocol, Fischer's Mutual
Exclusion protocol (up to 4 agents), and some other asynchronous circuit models.
These experiments showed that the tool is capable of treating shrinkability of timed automata
with thousands of edges (w.r.t graphs with more than sixty thousand transitions).
The bottleneck of the tool was also identified as being the size of the full bisimilarity graph,
which is often costly to compute, and that may require long processing time for shrinkability analysis.
Moreover, the non-shrinkability of most models was mainly due equality constraints, or to the fact
that some models was designed at a high level of abstraction where imprecisions were
not taken into account.


    \foldertitle{dbase}{Basic Database Management}{dbase/Contents}

	\paragraph{Loading and saving
databases}\label{loading-and-saving-databases}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \href{dbase/dbload}{\texttt{dbload}} - Create database by loading CSV
  file.
\item
  \href{dbase/dbsave}{\texttt{dbsave}} - Save database as CSV file.
\item
  \href{dbase/xls2csv}{\texttt{xls2csv}} - Convert XLS file to CSV file.
\end{itemize}

\paragraph{Getting information about
databases}\label{getting-information-about-databases}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \href{dbase/dbnames}{\texttt{dbnames}} - List of database entries
  filtered by name and/or class.
\item
  \href{dbase/dbprintuserdata}{\texttt{dbprintuserdata}} - Print names
  of database tseries along with specified fields of their userdata.
\item
  \href{dbase/dbrange}{\texttt{dbrange}} - Find a range that encompasses
  the ranges of the listed tseries objects.
\item
  \href{dbase/dbsearchuserdata}{\texttt{dbsearchuserdata}} - Search
  database to find tseries by matching the content of their userdata
  fields.
\item
  \href{dbase/dbuserdatalov}{\texttt{dbuserdatalov}} - List of values
  found in a specified user data field in tseries objects.
\end{itemize}

\paragraph{Converting databases}\label{converting-databases}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \href{dbase/array2db}{\texttt{array2db}} - Convert numeric array to
  database.
\item
  \href{dbase/db2array}{\texttt{db2array}} - Convert tseries database
  entries to numeric array.
\item
  \href{dbase/db2tseries}{\texttt{db2tseries}} - Combine tseries
  database entries in one multivariate tseries object.
\end{itemize}

\paragraph{Batch processing}\label{batch-processing}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \href{dbase/dbbatch}{\texttt{dbbatch}} - Run a batch job to create new
  database fields.
\item
  \href{dbase/dbclip}{\texttt{dbclip}} - Clip all tseries entries in
  database down to specified date range.
\item
  \href{dbase/dbcol}{\texttt{dbcol}} - Retrieve the specified column or
  columns from database entries.
\item
  \href{dbase/dbcomment}{\texttt{dbcomment}} - Create model-based
  comments for database tseries entries.
\item
  \href{dbase/dbfun}{\texttt{dbfun}} - Apply a function to database
  fields.
\item
  \href{dbase/dbplot}{\texttt{dbplot}} - Plot from database.
\item
  \href{dbase/dbredate}{\texttt{dbredate}} - Redate all tseries objects
  in a database.
\end{itemize}

\paragraph{Combining and splitting
databases}\label{combining-and-splitting-databases}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \href{dbase/dbextend}{\texttt{dbextend}} - Combine tseries
  observations from two or more databases.
\item
  \href{dbase/dbmerge}{\texttt{dbmerge}} - Merge two or more databases.
\item
  \href{dbase/dbminuscontrol}{\texttt{dbminuscontrol}} - Create
  simulation-minus-control database.
\item
  \href{dbase/dbsplit}{\texttt{dbsplit}} - Split database into mutliple
  databases.
\end{itemize}

\paragraph{Overloaded operators for
databases}\label{overloaded-operators-for-databases}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \href{dbase/dbminus}{\texttt{-}} - Remove entries from a database.
\item
  \href{dbase/dbmtimes}{\texttt{*}} - Keep only the database entries
  that are on the list.
\item
  \href{dbase/dbplus}{\texttt{+}} - Merge entries from two databases
  together.
\end{itemize}

\paragraph{Getting on-line help on database
functions}\label{getting-on-line-help-on-database-functions}

\begin{verbatim}
help dbase
help dbase/function_name
\end{verbatim}