\begin{figure*}
%\vspace{-1em}
\centering
%\includegraphics[width=0.99\linewidth, trim={10cm 0 0 0},clip]{../yellowfin_iclr2018/manuscript_for_revision/experiment_results/resnet/mom_dynamic_3_annotated.pdf}
\includegraphics[width=0.99\linewidth]{../yellowfin_iclr2018/manuscript_for_revision/experiment_results/resnet/mom_dynamic_3_annotated.pdf}
	\vspace{-0.5em}
	\caption{
	When running \tuner, total momentum $\hat{\mu}_t$ equals algorithmic value in synchronous settings (left); $\hat{\mu}_t$ is greater than algorithmic value on 16 asynchronous workers (middle).
	\Asynctuner automatically lowers algorithmic momentum and brings total momentum to match the target value (right).
%Red dots are measured $\hat{\mu}_t$ at every step with red line as its running average.
	Red dots are total momentum estimates, $\hat{\mu}_T$, at each iteration.
The solid red line is a running average of $\hat{\mu}_T$.
%	When running \tuner, total momentum $\hat{\mu}_t$ is greater than algorithmic value on 16 asynchronous workers (left).
%	\Asynctuner automatically lowers algorithmic momentum and matches total momentum to the target value (right).
%%Red dots are measured $\hat{\mu}_t$ at every step with red line as its running average.
%	Red dots are total momentum estimates, $\hat{\mu}_T$, at each iteration.
%    The solid red line is a running average of $\hat{\mu}_T$.
	}
	\label{fig:we-can-measure}
%\vspace{-0.35em}
\end{figure*}

\section{\Asynctuner}
\label{sec:async_tuner}

Asynchrony is a parallelization technique that avoids synchronization barriers \citep{recht2011hogwild}.
In this section, we propose a {\em closed momentum loop} variant of \tuner to accelerate convergence in asynchronous training.
%To handle the momentum dynamics of asynchronous parallelism, we propose a {\em closed momentum loop} variant of \tuner.
After some preliminaries, we show the mechanism of the extension:
it measures the dynamics on a running system and controls momentum with a negative feedback loop.
\paragraph{Preliminaries}
%Asynchrony is a popular parallelization technique \citep{recht2011hogwild} that avoids synchronization barriers.
When training on $M$ asynchronous workers, staleness (the number of model updates between a worker's read and write operations) is on average $\tau=M-1$,
i.e., the gradient in the SGD update is delayed by $\tau$ iterations as $\nabla f_{S_{t - \tau}}(x_{t - \tau} )$.
Asynchrony yields faster steps, but can
increase the number of iterations to achieve the same solution,
a tradeoff between hardware and statistical
efficiency~\citep{DBLP:journals/pvldb/ZhangR14}.
\citet{mitliagkas2016asynchrony} interpret asynchrony as added momentum dynamics.
Experiments in \citet{hadjis2016omnivore} support this finding, and demonstrate that reducing algorithmic momentum can compensate for asynchrony-induced momentum
and significantly reduce the number of iterations for convergence.
Motivated by that result, we use the model
in~\eqref{equ:exp_async_update_app}, where the total momentum, $\mu_T$, includes both asynchrony-induced and algorithmic  momentum, $\mu$, in~\eqref{eqn:momentum_gd}.
\begin{equation}
	\mathbb{E}[ x_{t+1} - x_t ]
	= \mu_T \mathbb{E}[x_t - x_{t-1}] - \alpha \mathbb{E}\nabla f(x_{t})
\label{equ:exp_async_update_app}
\end{equation}
We will use this expression to design an estimator for the value of total momentum, $\hat{\mu}_T$.
This estimator is a basic building block of \asynctuner, that {\em removes the need to manually compensate for the effects of asynchrony}.

\paragraph{Measuring the momentum dynamics}
\Asynctuner estimates total momentum $\mu_{T}$ on a running system and uses a negative feedback loop to adjust algorithmic momentum accordingly.
Equation~\eqref{equ:exp_async_update} gives an estimate of $\hat{\mu}_T$ on a system with staleness $\tau$, based on \eqref{equ:exp_async_update}.
\begin{align}
\hat{\mu}_T
					= \mathop{\mathsf{median}}\left(
							\frac{x_{t - \tau} - x_{t - \tau-1} + \alpha \nabla_{S_{t-\tau -1}} f(x_{t - \tau - 1} )}
							{x_{t - \tau-1} - x_{t - \tau-2}}
					\right)
\label{eqn:momentum_measurement}
\end{align}
We use $\tau$-stale model values to match the staleness of the gradient,  and perform all operations in an elementwise fashion.
This way we get a total momentum measurement from each variable;
the median combines them into a more robust estimate.

\paragraph{Closing the asynchrony loop}
Given a reliable measurement of $\mu_{T}$,
we can use it to adjust the value of algorithmic momentum so that the total momentum matches the \emph{target momentum} as decided by \tuner in Algorithm~\ref{alg:basic-algo}.
\Asynctuner in Algorithm~\ref{alg:async-algo} %(in Appendix~\ref{sec:async_yf})
uses a simple negative feedback loop to achieve the adjustment.
%Figure~\ref{fig:we-can-measure} demonstrates that under asynchrony the measured total momentum is strictly higher than the algorithmic momentum (middle plot), as expected from theory;
%closing the feedback loop (right plot) leads to total momentum matching the target momentum.
%Closing the loop, as we will see, improves performance significantly.
%Note for asynchronous-parallel training, as the estimates and parameter tuning is unstable in the beginning when there are only a small number of iterations, we use initial learning $\frac{1}{\tau + 1}$ instead of $1.0$ to prevent overflow in the beginning.

%\begin{algorithm}[H]
%	\caption{\Asynctuner}
%	\begin{algorithmic}[1]
%%	\State Input: $\mu\gets0$, $\alpha \gets \frac{1}{\tau + 1}$, $\gamma\gets0.01, \tau$ (staleness)
%	\State Input: $\mu\gets0$, $\alpha \gets 0.0001$, $\gamma\gets0.01, \tau$ (staleness)
%	\For { $t\gets1$ to $T$}
%	\State $x_t\!\gets\!x_{t - 1} + \mu (x_{t - 1} - x_{t - 2} ) - \alpha \nabla_{S_t} f(x_{t - \tau - 1} )$
%	\State $\mu^*,\alpha \gets \Call{\tuner}{\nabla_{S_t} f(x_{t - \tau - 1} ), \beta}$ %(get momentum from the dynamic range)
%	\State $\hat{\mu_T}
%					\gets \mathop{\mathsf{median}}\left(
%							\frac{x_{t - \tau} - x_{t - \tau-1} + \alpha \nabla_{S_{t-\tau-1}} f(x_{t - \tau - 1} )}
%							{x_{t - \tau-1} - x_{t - \tau-2}}
%					\right)$ \Comment{Measuring total momentum}
%	\State $\mu \leftarrow \mu + \gamma \cdot (\mu^* - \hat{\mu_T})$ \Comment{Closing the loop}
%	\EndFor
%\end{algorithmic}
%\label{alg:async-algo}
%\end{algorithm}

%In Section~\ref{sec:async_tuner}, we briefly discuss the mechanism of our designed \Asynctuner in asynchronous-parallel setting. In this appendix, we expand the details in total momentum estimator, $\hat{\mu_T}$, and present the full \Asynctuner in Algorithm~\ref{alg:async-algo} with extensive discussion.
%\paragraph{Measuring the momentum dynamics}
%Remember, we use the formula in~\eqref{equ:exp_async_update_app} to model the momentum dynamics in asynchronous-parallel systems
%\Asynctuner estimates total momentum $\mu_{T}$ on a running system and uses a negative feedback loop to adjust algorithmic momentum accordingly.
%\begin{equation}
%	\mathbb{E}[ x_{t+1} - x_t ]
%	= \mu_T \mathbb{E}[x_t - x_{t-1}] - \alpha \mathbb{E}\nabla f(x_{t})
%\label{equ:exp_async_update_app}
%\end{equation}
%Equation~\eqref{eqn:momentum_measurement_app} gives an estimate of $\hat{\mu_T}$ on a system with staleness $\tau$, based on \eqref{equ:exp_async_update_app}.
%\begin{align}
%\hat{\mu_T}
%					= \mathop{\mathsf{median}}\left(
%							\frac{x_{t - \tau} - x_{t - \tau-1} + \alpha \nabla_{S_{t-\tau -1}} f(x_{t - \tau - 1} )}
%							{x_{t - \tau-1} - x_{t - \tau-2}}
%					\right)
%\label{eqn:momentum_measurement_app}
%\end{align}
%We use $\tau$-stale model values to match the staleness of the gradient,  and perform all operations in an elementwise fashion.
%This way we get a total momentum measurement from each variable;
%the median combines them into a more robust estimate.
%
%%\label{subsec:closed_loop_YF}
%%\begin{figure}
%%\centering
%%\includegraphics[width=0.95\linewidth]{experiment_results/resnet/mom_dynamic_3_annotated.pdf}
%%	\caption{
%%	Momentum dynamics on CIFAR100 ResNet.
%%	Running \tuner, total momentum is equal to algorithmic momentum in a synchronous setting (left). Total momentum is greater than algorithmic momentum on 16 asynchronous workers, due to asynchrony-induced momentum (middle).
%%	Using the momentum feedback mechanism of \asynctuner, lowers algorithmic momentum and brings total momentum to match the target value on 16 asynchronous workers (right).
%%	Red dots are individual total momentum estimates, $\hat{\mu}_T$, at each iteration.
%%The solid red line is a running average of those estimates.
%%	}
%%	\label{fig:we-can-measure}
%%\end{figure}
%
%\paragraph{Closing the asynchrony loop}
%Given a reliable measurement of $\mu_{T}$,
%we can use it to adjust the value of algorithmic momentum so that the total momentum matches the \emph{target momentum} as decided by \tuner in Algorithm~\ref{alg:basic-algo}.
%\Asynctuner in Algorithm~\ref{alg:async-algo} %(in Appendix~\ref{sec:async_yf})
%uses a simple negative feedback loop to achieve the adjustment.
%%Figure~\ref{fig:we-can-measure} demonstrates that under asynchrony the measured total momentum is strictly higher than the algorithmic momentum (middle plot), as expected from theory;
%%closing the feedback loop (right plot) leads to total momentum matching the target momentum.
%%Closing the loop, as we will see, improves performance significantly.
%%%Note for asynchronous-parallel training, as the estimates and parameter tuning is unstable in the beginning when there are only a small number of iterations, we use initial learning $\frac{1}{\tau + 1}$ instead of $1.0$ to prevent overflow in the beginning.
%%
%%%\begin{algorithm}[H]
%%%	\caption{\Asynctuner}
%%%	\begin{algorithmic}[1]
%%%%	\State Input: $\mu\gets0$, $\alpha \gets \frac{1}{\tau + 1}$, $\gamma\gets0.01, \tau$ (staleness)
%%%	\State Input: $\mu\gets0$, $\alpha \gets 0.0001$, $\gamma\gets0.01, \tau$ (staleness)
%%%	\For { $t\gets1$ to $T$}
%%%	\State $x_t\!\gets\!x_{t - 1} + \mu (x_{t - 1} - x_{t - 2} ) - \alpha \nabla_{S_t} f(x_{t - \tau - 1} )$
%%%	\State $\mu^*,\alpha \gets \Call{\tuner}{\nabla_{S_t} f(x_{t - \tau - 1} ), \beta}$ %(get momentum from the dynamic range)
%%%	\State $\hat{\mu_T}
%%%					\gets \mathop{\mathsf{median}}\left(
%%%							\frac{x_{t - \tau} - x_{t - \tau-1} + \alpha \nabla_{S_{t-\tau-1}} f(x_{t - \tau - 1} )}
%%%							{x_{t - \tau-1} - x_{t - \tau-2}}
%%%					\right)$ \Comment{Measuring total momentum}
%%%	\State $\mu \leftarrow \mu + \gamma \cdot (\mu^* - \hat{\mu_T})$ \Comment{Closing the loop}
%%%	\EndFor
%%%\end{algorithmic}
%%%\label{alg:async-algo}
%%%\end{algorithm}
%%
%
%
%

\begin{algorithm}[h]
	\caption{\Asynctuner}
	\begin{algorithmic}[1]
%	\State Input: $\mu\gets0$, $\alpha \gets \frac{1}{\tau + 1}$, $\gamma\gets0.01, \tau$ (staleness)
	\State Input: $\mu\gets0$, $\alpha \gets 0.0001$, $\gamma\gets0.01, \tau$ (staleness)
	\For { $t\gets1$ to $T$}
	\State $x_t\!\gets\!x_{t - 1} + \mu (x_{t - 1} - x_{t - 2} ) - \alpha \nabla_{S_t} f(x_{t - \tau - 1} )$
	\State $\mu^*,\alpha \gets \Call{\tuner}{\nabla_{S_t} f(x_{t - \tau - 1} ), \beta}$ %(get momentum from the dynamic range)
	\State $\hat{\mu_T}
					\gets \mathop{\mathsf{median}}\left(
							\frac{x_{t - \tau} - x_{t - \tau-1} + \alpha \nabla_{S_{t-\tau-1}} f(x_{t - \tau - 1} )}
							{x_{t - \tau-1} - x_{t - \tau-2}}
					\right)$ \Comment{Measuring total momentum}
	\State $\mu \leftarrow \mu + \gamma \cdot (\mu^* - \hat{\mu_T})$ \Comment{Closing the loop}
	\EndFor
\end{algorithmic}
\label{alg:async-algo}
\end{algorithm}

%%%%%%%%%%%%%%%%% latest backup version %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%Asynchrony is a parallelization technique that avoids synchronization barriers \citep{recht2011hogwild}.
%%In this section, we propose a {\em closed momentum loop} variant of \tuner to accelerate convergence in asynchronous training.
%%To handle the momentum dynamics of asynchronous parallelism, we propose a {\em closed momentum loop} variant of \tuner.
%%After some preliminaries, we show the mechanism of the extension:
%%it measures the dynamics on a running system and controls momentum with a negative feedback loop.
%%\paragraph{Preliminaries}
%%Asynchrony is a popular parallelization technique \citep{recht2011hogwild} that avoids synchronization barriers.
%%When training on $M$ asynchronous workers, staleness (the number of model updates between a worker's read and write operations) is on average $\tau=M-1$,
%%i.e., the gradient in the SGD update is delayed by $\tau$ iterations as $\nabla f_{S_{t - \tau}}(x_{t - \tau} )$.
%%It yields faster steps, but can
%%increase the number of iterations needed,
%%a tradeoff between hardware and statistical
%%efficiency~\citep{DBLP:journals/pvldb/ZhangR14}.
%It yields better hardware efficiency, i.e. faster steps, but can
%increase the number of iterations to a given metric, i.e. statistical efficiency, as a tradeoff~\citep{DBLP:journals/pvldb/ZhangR14}.
%%a tradeoff between hardware and statistical
%%efficiency~\citep{DBLP:journals/pvldb/ZhangR14}.
%%In this section, we propose a {\em closed momentum loop} variant of \tuner to reduce the number of iterations it needs to converge in asynchronous training.
%%In this section, we propose a {\em closed momentum loop} variant of \tuner to reduce the number of iterations for convergence in asynchronous training.
%%\paragraph{\Asynctuner}
%\citet{mitliagkas2016asynchrony} interpret asynchrony as added momentum dynamics.
%%It is empirically supported in \citet{hadjis2016omnivore} that manually reducing algorithmic momentum can compensate for asynchrony-induced momentum
%%and significantly reduce the number of iterations to converge.
%We design \asynctuner, a variant of \tuner to automatically control algorithmic momentum, compensate for asynchrony and accelerate convergence.
%We use the formula in~\eqref{equ:exp_async_update} to model the dynamics in the system, where the total momentum, $\mu_T$, includes both asynchrony-induced and algorithmic  momentum, $\mu$, in~\eqref{eqn:momentum_gd}.
%\begin{equation}
%	\mathbb{E}[ x_{t+1} - x_t ]
%	= \mu_T \mathbb{E}[x_t - x_{t-1}] - \alpha \mathbb{E}\nabla f(x_{t})
%\label{equ:exp_async_update}
%\end{equation}
%We first use~\eqref{equ:exp_async_update} to design an robust estimator $\hat{\mu}_T$ for the value of total momentum at every iteration.
%%This estimator is a basic building block of \asynctuner, that {\em removes the need to manually compensate for the effects of asynchrony}.
%Then we use a simple negative feedback control loop to adjust the value of algorithmic momentum so that $\hat{\mu}_T$ matches the \emph{target momentum} decided by \tuner in Algorithm~\ref{alg:basic-algo}.
%%We refer to Appendix~\ref{sec:async_app} for details on estimator $\hat{\mu}_T$ and \Asynctuner in Algorithm~\ref{alg:async-algo}.
%%\Asynctuner in Algorithm~\ref{alg:async-algo} (in Appendix~\ref{sec:async_app}) %(in Appendix~\ref{sec:async_yf})
%%uses a simple negative feedback loop to achieve the adjustment.
%In Figure~\ref{fig:we-can-measure},
%we demonstrate momentum dynamics in an asynchronous training system.
%As directly using the target value as algorithmic momentum, \tuner (middle) presents total momentum $\hat{\mu}_T$ strictly larger than the target momentum, due to asynchrony-induced momentum. \Asynctuner (right) automatically brings down algorithmic momentum, match measured total momentum $\hat{\mu}_T$ to target value and, as we will see, speeds up convergence comparing to \tuner. We refer to Appendix~\ref{sec:async_app} for details on estimator $\hat{\mu}_T$ and \Asynctuner in Algorithm~\ref{alg:async-algo}.
%%
%%so that visually demonstrates the mechanism of \Asynctuner in handling the momentum dynamics under asynchrony. In asynchronous-parallel setting, the measured total momentum is strictly higher than the algorithmic momentum (middle plot), as expected from theory.
%%Closing the feedback loop (right plot) leads to total momentum matching the target momentum and, as we will see, improves performance significantly.
%
%%\begin{figure*}
%%%\vspace{-2.5em}
%%\centering
%%\includegraphics[width=\linewidth]{experiment_results/resnet/mom_dynamic_3_annotated.pdf}
%%	\caption{
%%	When running \tuner, total momentum $\hat{\mu}_t$ equals algorithmic value in synchronous settings (left); $\hat{\mu}_t$ is greater than algorithmic value on 16 asynchronous workers (middle).
%%	\Asynctuner automatically lowers algorithmic momentum and brings total momentum to match the target value (right).
%%Red dots are measured $\hat{\mu}_t$ at every step with red line as its running average.
%%%	Red dots are total momentum estimates, $\hat{\mu}_T$, at each iteration.
%%%The solid red line is a running average of $\hat{\mu}_T$.
%%	}
%%	\vspace{-0.25em}
%%	\label{fig:we-can-measure}
%%\end{figure*}

%%%%%%%%%%%%%%%%%%%%%% latest backup versions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\begin{figure}
%%\centering
%%\includegraphics[width=0.95\linewidth]{experiment_results/resnet/mom_dynamic_3_annotated.pdf}
%%	\caption{
%%%	Momentum dynamics on CIFAR100 ResNet.
%%	Running \tuner on a ResNet, total momentum equals algorithmic value in a synchronous setting (left). Total momentum is greater than algorithmic value on 16 asynchronous workers, due to asynchrony-induced momentum (middle).
%%	\asynctuner automatically lowers algorithmic momentum and brings total momentum to match the target value (right).
%%	Red dots are total momentum estimates, $\hat{\mu}_T$, at each iteration.
%%The solid red line is a running average of $\hat{\mu}_T$.
%%	}
%%	\label{fig:we-can-measure}
%%\end{figure}
%Asynchrony is a parallelization technique that avoids synchronization barriers \citep{recht2011hogwild}.
%%In this section, we propose a {\em closed momentum loop} variant of \tuner to accelerate convergence in asynchronous training.
%%To handle the momentum dynamics of asynchronous parallelism, we propose a {\em closed momentum loop} variant of \tuner.
%%After some preliminaries, we show the mechanism of the extension:
%%it measures the dynamics on a running system and controls momentum with a negative feedback loop.
%%\paragraph{Preliminaries}
%%Asynchrony is a popular parallelization technique \citep{recht2011hogwild} that avoids synchronization barriers.
%%When training on $M$ asynchronous workers, staleness (the number of model updates between a worker's read and write operations) is on average $\tau=M-1$,
%%i.e., the gradient in the SGD update is delayed by $\tau$ iterations as $\nabla f_{S_{t - \tau}}(x_{t - \tau} )$.
%%It yields faster steps, but can
%%increase the number of iterations needed,
%%a tradeoff between hardware and statistical
%%efficiency~\citep{DBLP:journals/pvldb/ZhangR14}.
%It yields better hardware efficiency, i.e. faster steps, but can
%increase the number of iterations to a given metric, i.e. statistical efficiency, as a tradeoff~\citep{DBLP:journals/pvldb/ZhangR14}.
%%a tradeoff between hardware and statistical
%%efficiency~\citep{DBLP:journals/pvldb/ZhangR14}.
%%In this section, we propose a {\em closed momentum loop} variant of \tuner to reduce the number of iterations it needs to converge in asynchronous training.
%%In this section, we propose a {\em closed momentum loop} variant of \tuner to reduce the number of iterations for convergence in asynchronous training.
%%\paragraph{\Asynctuner}
%\citet{mitliagkas2016asynchrony} interpret asynchrony as added momentum dynamics.
%%It is empirically supported in \citet{hadjis2016omnivore} that manually reducing algorithmic momentum can compensate for asynchrony-induced momentum
%%and significantly reduce the number of iterations to converge.
%We design a {\em closed momentum loop} variant of \tuner to control algorithmic momentum, compensate for asynchrony and accelerate convergence.
%We use the formula in~\eqref{equ:exp_async_update} to model the dynamics in the system, where the total momentum, $\mu_T$, includes both asynchrony-induced and algorithmic  momentum, $\mu$, in~\eqref{eqn:momentum_gd}.
%\begin{equation}
%	\mathbb{E}[ x_{t+1} - x_t ]
%	= \mu_T \mathbb{E}[x_t - x_{t-1}] - \alpha \mathbb{E}\nabla f(x_{t})
%\label{equ:exp_async_update}
%\end{equation}
%We first use this expression to design an robust estimator $\hat{\mu}_T$ for the value of total momentum.
%%This estimator is a basic building block of \asynctuner, that {\em removes the need to manually compensate for the effects of asynchrony}.
%Given $\hat{\mu}_T$,
%we use it to adjust the value of algorithmic momentum so that the total momentum matches the \emph{target momentum} decided by \tuner in Algorithm~\ref{alg:basic-algo}. Specifically, we %\asynctuner  %(in Appendix~\ref{sec:async_yf})
%uses a simple negative feedback loop to achieve the adjustment. We refer to Appendix~\ref{sec:async_app} for details on estimator $\hat{\mu}_T$ and \Asynctuner in Algorithm~\ref{alg:async-algo}.
%%\Asynctuner in Algorithm~\ref{alg:async-algo} (in Appendix~\ref{sec:async_app}) %(in Appendix~\ref{sec:async_yf})
%%uses a simple negative feedback loop to achieve the adjustment.
%Figure~\ref{fig:we-can-measure} visually demonstrates the mechanism of \Asynctuner in handling the momentum dynamics under asynchrony. In asynchronous-parallel setting, the measured total momentum is strictly higher than the algorithmic momentum (middle plot), as expected from theory.
%Closing the feedback loop (right plot) leads to total momentum matching the target momentum and, as we will see, improves performance significantly.
%
%\begin{figure}
%\centering
%\includegraphics[width=0.95\linewidth]{experiment_results/resnet/mom_dynamic_3_annotated.pdf}
%	\caption{
%%	Momentum dynamics on CIFAR100 ResNet.
%	Running \tuner on a ResNet, total momentum equals algorithmic value in a synchronous setting (left). Total momentum is greater than algorithmic value on 16 asynchronous workers, due to asynchrony-induced momentum (middle).
%	\asynctuner automatically lowers algorithmic momentum and brings total momentum to match the target value (right).
%	Red dots are total momentum estimates, $\hat{\mu}_T$, at each iteration.
%The solid red line is a running average of $\hat{\mu}_T$.
%	}
%	\label{fig:we-can-measure}
%\end{figure}
%

%%%%%%%%%%%%%%%%%%%%%% below are old backup versions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%\paragraph{Measuring the momentum dynamics}
%\Asynctuner estimates total momentum $\mu_{T}$ on a running system and uses a negative feedback loop to adjust algorithmic momentum accordingly.
%Equation~\eqref{equ:exp_async_update} gives an estimate of $\hat{\mu_T}$ on a system with staleness $\tau$, based on \eqref{equ:exp_async_update}.
%\begin{align}
%\hat{\mu_T}
%					= \mathop{\mathsf{median}}\left(
%							\frac{x_{t - \tau} - x_{t - \tau-1} + \alpha \nabla_{S_{t-\tau -1}} f(x_{t - \tau - 1} )}
%							{x_{t - \tau-1} - x_{t - \tau-2}}
%					\right)
%\label{eqn:momentum_measurement}
%\end{align}
%We use $\tau$-stale model values to match the staleness of the gradient,  and perform all operations in an elementwise fashion.
%This way we get a total momentum measurement from each variable;
%the median combines them into a more robust estimate.

%\label{subsec:closed_loop_YF}
%\begin{figure}
%\centering
%\includegraphics[width=0.95\linewidth]{experiment_results/resnet/mom_dynamic_3_annotated.pdf}
%	\caption{
%	Momentum dynamics on CIFAR100 ResNet.
%	Running \tuner, total momentum is equal to algorithmic momentum in a synchronous setting (left). Total momentum is greater than algorithmic momentum on 16 asynchronous workers, due to asynchrony-induced momentum (middle).
%	Additionally applying the momentum feedback loop of \asynctuner, lowers algorithmic momentum and matches total momentum to target value (right).
%	Red dots are individual total momentum estimates, $\hat{\mu}_T$, at each iteration, with
%the solid red line as its running average.
%	}
%	\label{fig:we-can-measure}
%\end{figure}

%\paragraph{Closing the asynchrony loop}
%Given a reliable measurement of $\mu_{T}$,
%we can use it to adjust the value of algorithmic momentum so that the total momentum matches the \emph{target momentum} as decided by \tuner in Algorithm~\ref{alg:basic-algo}.
%\Asynctuner in Algorithm~\ref{alg:async-algo} (in Appendix~\ref{sec:async_app}) %(in Appendix~\ref{sec:async_yf})
%uses a simple negative feedback loop to achieve the adjustment.
%Figure~\ref{fig:we-can-measure} demonstrates that under asynchrony the measured total momentum is strictly higher than the algorithmic momentum (middle plot), as expected from theory;
%closing the feedback loop (right plot) leads to total momentum matching the target momentum.
%Closing the loop, as we will see, improves performance significantly.
%

%To handle the momentum dynamics of asynchronous parallelism, we propose a {\em closed momentum loop} variant of \tuner.
%After some preliminaries, we show the mechanism of the extension:
%it measures the dynamics on a running system and controls momentum with a negative feedback loop.
%\paragraph{Preliminaries}
%Asynchrony is a popular parallelization technique \citep{recht2011hogwild} that avoids synchronization barriers.
%When training on $M$ asynchronous workers, staleness (the number of model updates between a worker's read and write operations) is on average $\tau=M-1$,
%i.e., the gradient in the SGD update is delayed by $\tau$ iterations as $\nabla f_{S_{t - \tau}}(x_{t - \tau} )$.
%Asynchrony yields faster steps, but can
%increase the number of iterations to achieve the same solution,
%a tradeoff between hardware and statistical
%efficiency~\citep{DBLP:journals/pvldb/ZhangR14}.
%\citet{mitliagkas2016asynchrony} interpret asynchrony as added momentum dynamics.
%Experiments in \citet{hadjis2016omnivore} support this finding, and demonstrate that reducing algorithmic momentum can compensate for asynchrony-induced momentum
%and significantly reduce the number of iterations for convergence.
%Motivated by that result, we use the model
%in~\eqref{equ:exp_async_update}, where the total momentum, $\mu_T$, includes both asynchrony-induced and algorithmic  momentum, $\mu$, in~\eqref{eqn:momentum_gd}.
%\begin{equation}
%	\mathbb{E}[ x_{t+1} - x_t ]
%	= \mu_T \mathbb{E}[x_t - x_{t-1}] - \alpha \mathbb{E}\nabla f(x_{t})
%\label{equ:exp_async_update}
%\end{equation}
%We will use this expression to design an estimator for the value of total momentum, $\hat{\mu_T}$.
%This estimator is a basic building block of \asynctuner, that {\em removes the need to manually compensate for the effects of asynchrony}.
%
%
%
%\paragraph{Measuring the momentum dynamics}
%\Asynctuner estimates total momentum $\mu_{T}$ on a running system and uses a negative feedback loop to adjust algorithmic momentum accordingly.
%Equation~\eqref{equ:exp_async_update} gives an estimate of $\hat{\mu_T}$ on a system with staleness $\tau$, based on \eqref{equ:exp_async_update}.
%\begin{align}
%\hat{\mu_T}
%					= \mathop{\mathsf{median}}\left(
%							\frac{x_{t - \tau} - x_{t - \tau-1} + \alpha \nabla_{S_{t-\tau -1}} f(x_{t - \tau - 1} )}
%							{x_{t - \tau-1} - x_{t - \tau-2}}
%					\right)
%\label{eqn:momentum_measurement}
%\end{align}
%We use $\tau$-stale model values to match the staleness of the gradient,  and perform all operations in an elementwise fashion.
%This way we get a total momentum measurement from each variable;
%the median combines them into a more robust estimate.
%
%\label{subsec:closed_loop_YF}
%\begin{figure}
%\centering
%\includegraphics[width=0.95\linewidth]{experiment_results/resnet/mom_dynamic_3_annotated.pdf}
%	\caption{
%	Momentum dynamics on CIFAR100 ResNet.
%	Running \tuner, total momentum is equal to algorithmic momentum in a synchronous setting (left). Total momentum is greater than algorithmic momentum on 16 asynchronous workers, due to asynchrony-induced momentum (middle).
%	Using the momentum feedback mechanism of \asynctuner, lowers algorithmic momentum and brings total momentum to match the target value on 16 asynchronous workers (right).
%	Red dots are individual total momentum estimates, $\hat{\mu}_T$, at each iteration.
%The solid red line is a running average of those estimates.
%	}
%	\label{fig:we-can-measure}
%\end{figure}
%
%\paragraph{Closing the asynchrony loop}
%Given a reliable measurement of $\mu_{T}$,
%we can use it to adjust the value of algorithmic momentum so that the total momentum matches the \emph{target momentum} as decided by \tuner in Algorithm~\ref{alg:basic-algo}.
%\Asynctuner in Algorithm~\ref{alg:async-algo} (in Appendix~\ref{sec:async_app}) %(in Appendix~\ref{sec:async_yf})
%uses a simple negative feedback loop to achieve the adjustment.
%Figure~\ref{fig:we-can-measure} demonstrates that under asynchrony the measured total momentum is strictly higher than the algorithmic momentum (middle plot), as expected from theory;
%closing the feedback loop (right plot) leads to total momentum matching the target momentum.
%Closing the loop, as we will see, improves performance significantly.
%%Note for asynchronous-parallel training, as the estimates and parameter tuning is unstable in the beginning when there are only a small number of iterations, we use initial learning $\frac{1}{\tau + 1}$ instead of $1.0$ to prevent overflow in the beginning.
%
%%\begin{algorithm}[H]
%%	\caption{\Asynctuner}
%%	\begin{algorithmic}[1]
%%%	\State Input: $\mu\gets0$, $\alpha \gets \frac{1}{\tau + 1}$, $\gamma\gets0.01, \tau$ (staleness)
%%	\State Input: $\mu\gets0$, $\alpha \gets 0.0001$, $\gamma\gets0.01, \tau$ (staleness)
%%	\For { $t\gets1$ to $T$}
%%	\State $x_t\!\gets\!x_{t - 1} + \mu (x_{t - 1} - x_{t - 2} ) - \alpha \nabla_{S_t} f(x_{t - \tau - 1} )$
%%	\State $\mu^*,\alpha \gets \Call{\tuner}{\nabla_{S_t} f(x_{t - \tau - 1} ), \beta}$ %(get momentum from the dynamic range)
%%	\State $\hat{\mu_T}
%%					\gets \mathop{\mathsf{median}}\left(
%%							\frac{x_{t - \tau} - x_{t - \tau-1} + \alpha \nabla_{S_{t-\tau-1}} f(x_{t - \tau - 1} )}
%%							{x_{t - \tau-1} - x_{t - \tau-2}}
%%					\right)$ \Comment{Measuring total momentum}
%%	\State $\mu \leftarrow \mu + \gamma \cdot (\mu^* - \hat{\mu_T})$ \Comment{Closing the loop}
%%	\EndFor
%%\end{algorithmic}
%%\label{alg:async-algo}
%%\end{algorithm}
%


%auto-ignore
\providecommand{\MainFolder}{..}
\documentclass[\MainFolder/Text.tex]{subfiles}
\begin{document}

\section{Homotopy transfer and effective action}\label{Sec:HPL}

In \cite{Doubek2018}, they consider multilinear operations $l_{lg}: V^{\otimes l} \rightarrow \R$ for $l\ge 1$, $g\ge 0$ on an odd symplectic vector space~$V$ and write down an action $\Action\in \Fun(V)$ in the form of~\eqref{Eq:MyAction} with~$\MC_{lg}$ replaced by~$l_{lg}^+$. We remark that their ``vertices'' are $l^+_{lg}(v,\dotsc,v)$ for a ``field'' $v\in V$, whereas ours are $\MC_{lg}(v_1,\dotsc,v_l)$ for a ``string of fields'' $v_1\dotsb v_l\in \CycB(V)$. They consider the Schwarz's canonical $\BV$-operator on $\Fun(V)$ from~\cite{Schwarz1992} and show that $\Action$ satisfies the quantum master equation if and only if $(l_{lg})$ satisfy the relations of a quantum $\LInfty$-algebra.\footnote{This is equivalent to the notion of a loop homotopy algebra from \cite{Markl1997} and to string brackets in closed string field theory from \cite{Zwiebach1992}.} On the other hand, we have the string $\BV$-operator on $\Fun(\CycB(V))$ and our action \eqref{Eq:MyAction} satisfies the quantum master equation if and only if $\MC=(\MC_{lg})$ is a Maurer-Cartan element for $\dIBL(\CycC(V))$. Next, they consider a deformation retract ($\eqqcolon\mathrm{DR}$) as in \eqref{Eq:DefRetr} and obtain explicit formulas for the homotopy transfered quantum $\LInfty$-algebra on $V'$ together with all maps and homotopies via the Homological Perturbation Lemma ($\eqqcolon\mathrm{HPL}$). In what follows, we will sketch how to apply their construction to $\IBLInfty$-algebras. We stress that the details have NOT been done yet!

A DR \eqref{Eq:DefRetr} induces a DR
\begin{equation*}
\begin{tikzcd}
\bigl(\CycC(V),\OPQ_{110}\bigr) \arrow[loop left]{l}{K_\CycC}\arrow[shift left]{r}{P_\CycC} & \arrow[shift left]{l}{I_\CycC} \bigl(\CycC(V'),\OPQ_{110}'\bigr),
\end{tikzcd}
\end{equation*}
which further induces a DR
\begin{equation}\label{Eq:SDRSDR}
\begin{tikzcd}
\bigl(\Fun(B(V)),\hat{\OPQ}_{110}\bigr) \arrow[loop left]{l}{K}\arrow[shift left]{r}{P} & \arrow[shift left]{l}{I} \bigl(\Fun(B(V')),\hat{\OPQ}_{110}'\bigr).
\end{tikzcd}
\end{equation}
In~\cite[Remark~3]{Doubek2018}, they write down a formula for $K$ on $\Fun(V)$ given $\Htp$ using a ``tensor trick'' due to Eilenberg Mac-Lane; the same method may apply to get~$K_C$ from~$\Htp$ on $\CycC(V)$ and $K$ on $\Fun(\CycB(V))$ from $K_C$ in our case. With such $K$'s, one can take $P_{\CycC}$ and $P$, resp.~$I_{\CycC}$ and~$I$ to be the natural extensions of $\iota^*$, resp.~$\pi^*$. Note that any surjective quasi-isomorphism over $\R$ is a deformation retraction, but their formula is explicit and preserves special DR's, i.e., DR's satisfying $\Htp^2 = \Htp \iota = \pi \Htp = 0$ ($\eqqcolon\mathrm{SDR}$). \ToDo[caption={injectiv qi},noline]{Are injective quasi-isomorphisms sections of deformatino retractions?} The crucial idea of~\cite{Doubek2018} translated to our situation is to view $\BVOp$ and $\BVOp^\MC$ as perturbations of $\{\FreeAction,\cdot\} = \hat{\OPQ}_{110}$, which we denote by~$\delta^{(1)}$ and~$\delta^{(2)}$, respectively, and apply the HPL from~\cite{Crainic2004}:

For $i=1$, $2$, suppose that $\delta^{(i)}$ is ``small'', i.e., that $(\Id - \delta^{(i)} K)$ is invertible, and consider the maps
\begin{align*}
 \BVOp^{(i)} &\coloneqq \hat{\OPQ}_{110}' + P(\Id - \delta^{(i)}K)^{-1}\delta^{(i)} I = \hat{\OPQ}_{110}' + P \delta^{(i)} I + P \delta^{(i)} K \delta^{(i)} I + \dotsb ,\\
 I^{(i)} &\coloneqq I + K(\Id - \delta^{(i)}K)^{-1}\delta^{(i)} I = I + K \delta^{(i)} I + K \delta^{(i)} K \delta^{(i)} I + \dotsb,\\
 P^{(i)} &\coloneqq P + P(\Id-\delta^{(i)}K)^{-1}\delta^{(i)} K = P + P \delta^{(i)} K + P \delta^{(i)} K \delta^{(i)} K + \dotsb,\\
 K^{(i)} &\coloneqq K + K(\Id-\delta^{(i)}K)^{-1}\delta^{(i)}K = K + K\delta^{(i)} K + K \delta^{(i)} K \delta^{(i)} K + \dotsb.
\end{align*}
The HPL asserts that if \eqref{Eq:SDRSDR} is an SDR, then the following are SDR's as well:
\begin{equation*}\begin{tikzcd}[execute at end picture={
\draw[->,dashed] (3.5,1.5) to[out=0,in=90] node[midway,right,xshift=.5cm]{$\delta^{(1)} = \BVOp_0$} (5,.75) to[out=-90,in=0] (3.5,0);
\draw[->,dashed] (3.5,1.5) to[out=0,in=0] node[pos=0.8,right,xshift=.3cm]{$\delta^{(2)} = \BVOp_0 + \{\IntAction,\cdot\}$} (3.5,-1.5);
}]
\bigl(\Fun(B(V)),\hat{\OPQ}_{110}\bigr) \arrow[loop left]{l}{K}\arrow[shift left]{r}{P} & \arrow[shift left]{l}{I} \bigl(\Fun(B(V')),\hat{\OPQ}_{110}'\bigr) \\
\bigl(\Fun(B(V)),\BVOp \bigr) \arrow[loop left]{l}{K^{(1)}}\arrow[shift left]{r}{P^{(1)}} & \arrow[shift left]{l}{I^{(1)}} \bigl(\Fun(B(V')),\BVOp^{(1)}\bigr)\\
\bigl(\Fun(B(V)),\BVOp^\MC \bigr) \arrow[loop left]{l}{K^{(2)}}\arrow[shift left]{r}{P^{(2)}} & \arrow[shift left]{l}{I^{(2)}} \bigl(\Fun(B(V')),\BVOp^{(2)}\bigr)
\end{tikzcd}\end{equation*}
One defines the \emph{effective action}
$$W \coloneqq \log\bigl(P^{(1)}(e^{\IntAction})\bigr) \in \Fun(\CycB(V'))$$
and the \emph{path integral}
$$ Z \coloneqq L_{e^{-W}} \circ P^{(1)} \circ L_{e^{\IntAction}}: \Fun(\CycB(V)) \rightarrow \Fun(\CycB(V')). $$
Under certain circumstances, the following formulas hold:
\begin{equation}\label{Eq:NiceEqns}
\BVOp^{(1)} = P\BVOp_0 I,\quad \BVOp^{(2)}=\BVOp^{(1)} + \{W,\cdot\}^{(1)}\quad\text{and}\quad P^{(2)} = Z.
\end{equation}
This is proven in \cite{Doubek2018} on $\Fun(V)$ when $V'=\Harm$ is a ``harmonic'' subspace in a Hodge decomposition $V = \Harm \oplus C$ into odd symplectic subspaces, $\pi$ and $\iota$ are the canonical projection and inclusion, respectively, the homotopy $\Htp$ is such that $(\pi,\iota,\Htp)$ is an SDR, and the homotopy $K$ was constructed from $\Htp$ via the tensor trick. Since we deal with the string $\BV$-operator on $\Fun(\CycB(V))$, we can not talk about this ``symplectic compatibility'' and the proof of \eqref{Eq:NiceEqns} might be based on another arguments.

\begin{Question}[HPL and $\IBLInfty$]\label{Q:EqForm}
If one picks an SDR of $V$ onto a harmonic subspace~$\Harm$ as above (basically equivalent to the setting of \cite[Section~11]{Cieliebak2015}) and constructs $K_\CycC$ and $K$ in a particular way, can one achieve that \eqref{Eq:NiceEqns} and the following identities hold?
$$ \BVOp^{(1)} = \BVOp_0',\quad \BVOp^{(2)} = \BVOp^\MC, \quad W = \Action_{\HTP_* \MC},\quad P^{(1)} = e^\HTP,\quad P^{(2)}= e^{\HTP^\MC}. $$
Here, $\Action_{\HTP_*\MC}$ denotes the action \eqref{Eq:MyAction} for the Maurer-Cartan element $\HTP_*\MC$.
\end{Question}

\begin{Remark}[On $\BV$-formalism for $\IBLInfty$]
\begin{RemarkList}
\item As summarized in \cite[Section~5]{Doubek2018}, given a $\BV$-action $\Action$, there are various approaches to obtain $W$ and $Z$ as summations over Feynman graphs (see \cite{Mnev2017} for the stationary phase formula approach).
\item The appearance of Feynman graphs can be explained from the proof of \cite[Theorem~2]{Doubek2018}, where they show that in the special setting above, it holds
$$ P^{(1)} = P e^{D_{\Prpg}} $$
for an order $\le 2$ differential operator $D_{\Prpg}$ which ``connects'' two legs with the propagator~$\Prpg$ (obtained by ``rising one index'' of $\Htp$ using the odd symplectic form). This is reminiscent of the Wick's Theorem for the (formal) perturbative expansion of the path integral (the classical approach to quantum field theories).
\item Having a $\BV$-formulation of the $\IBLInfty$-theory, it is intriguing to compare it to~\cite{Muenster2011}, where certain $\IBLInfty$-structures are considered in the context of open-closed string field theory.\qedhere
\end{RemarkList}
\end{Remark}
\end{document}


\documentclass[]{article}

\usepackage{graphicx}
\usepackage{longtable}
\usepackage{hyperref}
\usepackage{color}
\usepackage{soul}
\usepackage{amsmath}
\usepackage{amssymb}

\DeclareRobustCommand{\hlcyan}[1]{{\sethlcolor{cyan}\hl{#1}}}
\DeclareRobustCommand{\hlgreen}[1]{{\sethlcolor{green}\hl{#1}}}
\DeclareRobustCommand{\hlred}[1]{{\sethlcolor{red}\hl{#1}}}
\DeclareRobustCommand{\hlyellow}[1]{{\sethlcolor{yellow}\hl{#1}}}
\DeclareRobustCommand{\hlorange}[1]{{\sethlcolor{orange}\hl{#1}}}

%opening
\title{Session 9}
\author{Fakhir}

\begin{document}

\maketitle

\section*{Solutions}

\begin{enumerate}
	\item \begin{enumerate}
		\item For continuity $\lim_{x\to0^{-}} f(x) = \lim_{x\to0^{+}} f(x) = 6$.
		\item First derivative of the first piece must be equal to the first derivative of the second piece at 0. This means:
		$$ f'(0^{-}) = f'(0^{+})$$
		$$ 2ax + b = 10x^{4}+12x^{3}+8x+5 $$
		Putting $x = 0$ we get:
		$$b = 5$$
		\item \textit{(According to MiT solutions, no need to check this)} Second derivative of the first piece must be equal to the second derivative of the second piece at 0. This means:
		$$ f''(0^{-}) = f''(0^{+})$$
		$$ 2a = 40x^{3}+36x^{2}+8 $$
		Putting $x = 0$ we get:
		$$a = 4$$

		\hlred{MiT Solution says: The first derivative has to be equal on both sides. We do not need to check the second derivative. So $b=5$ and a can be any real number.}
	\end{enumerate}

	\item \begin{enumerate}
		\item For continuity $\lim_{x\to1^{-}} f(x) = \lim_{x\to1^{+}} f(x)$. So plugging in $x=1$ in both pieces we get:

		$$ a+b+6 = 20 $$
		$$ a+b = 14 $$
		\item First derivative of the first piece must be equal to the first derivative of the second piece at 1. This means:
		$$ f'(1^{-}) = f'(1^{+})$$
		$$ 2a+b = 10+12+8+5 $$
		$$ 2a+b = 35 $$

		Solving these two simultaneous equations we get:
		$$ 2a + (14-a) = 35 $$
		$$ a + 14 = 35 $$
		$$ a = 21 $$
		$$ b = -7 $$

		Hence $a=21$ and $b=-7$.

	\end{enumerate}

\end{enumerate}

\end{document}

mit-ocw/[in progress] Single Variable Calculus, 2010/my solutions/1. Differentiation/Session9.tex


\input{../../style/preamble}
\input{../../latex-math/basic-math}
\input{../../latex-math/basic-ml}
\input{../../latex-math/ml-bagging.tex}
\input{../../latex-math/ml-boosting.tex}
\input{../../latex-math/ml-trees.tex}

\newcommand{\titlefigure}{figure_man/adaboost_example_adjusted.PNG}
\newcommand{\learninggoals}{
  \item Briefly cover the older AdaBoost and its general idea
  \item Understand difference between bagging and boosting
}

\title{Introduction to Machine Learning}
\date{}

\begin{document}

\lecturechapter{Introduction to Boosting / AdaBoost}
\lecture{Introduction to Machine Learning}

% ------------------------------------------------------------------------------

\begin{vbframe}{Introduction to boosting}
  \begin{itemize}
    \item
      Boosting is one of the most powerful learning ideas since 1990.
    \item
      Originally designed for classification, (especially gradient) boosting handles regression (and many other supervised tasks) naturally nowadays.
    \item
      Homogeneous ensemble method (like bagging), but fundamentally different approach.
    \item
      {\bf Idea:} Take a weak classifier and sequentially apply it to modified versions of the training data.
    \item
      We will begin by describing an older, simpler boosting algorithm designed for binary classification, the popular \enquote{AdaBoost}.
  \end{itemize}
\end{vbframe}

% ------------------------------------------------------------------------------

\begin{vbframe}{Boosting vs. Bagging}

  \begin{itemize}
      \item Homogeneous ensemble method (like bagging).
  \item {\bf Idea:} Take a weak classifier and sequentially apply it to modified versions of the training data.
    \item Sequential not parallel model building, to minimize loss instead of variance reduction.
  \end{itemize}

% The general concept of boosting is a sequential fitting of weak learner on the error:
\begin{center}
\includegraphics[width=0.40\textwidth]{figure_man/bagging_vs_boosting.png}
\end{center}

% In bagging, the models are fitted parallel and not sequential.

\end{vbframe}

\begin{vbframe}{Boosting as a theoretical problem}

Boosting was developed as the answer to a theoretical problem:

\lz

\enquote{Does the existence of a weak learner for a certain problem imply
the existence of a strong learner?} (Kearns, 1988)

\lz

\begin{itemize}
\item \textbf{Weak learners} are defined as a prediction rule with a correct classification rate that is at least slightly better than random guessing (> 50\% accuracy on a balanced binary problem).
\item We call a learner a \textbf{strong learner} \enquote{if there exists a polynomial-time algorithm that achieves low error with high confidence for all concepts in the class} (Schapire, 1990).

\end{itemize}

% In practice it is typically easy to construct weak learners, but difficult to build a strong one.

% The proof of this ground-breaking idea generated the first boosting algorithm.

\begin{itemize}
    \item Any weak (base) learner can be iteratively boosted to become
a strong learner (Schapire and Freund, 1990).
  \item Idea was refined into \textbf{AdaBoost} (Adaptive Boosting).
\end{itemize}

\end{vbframe}

% ------------------------------------------------------------------------------

% \section{AdaBoost}

\begin{vbframe}{AdaBoost}

% Any weak (base) learner can be iteratively boosted to become
% a strong learner (Schapire and Freund, 1990).
% The proof of this ground-breaking idea generated the first boosting algorithm.

% Leo Breiman (referring to the success of AdaBoost):
% \enquote{Boosting is the best off-the-shelf classifier in the world.}

% \framebreak

\begin{itemize}
  \item Assume binary classification with $y$ encoded as $\setmp$.
  \item We use binary classifiers as weak base learners (e.g., tree stumps) from a hypothesis space $\mathcal{B}$,
      denoted $\bmm$ (or $\bmm(\xv)$ or $\bmmxth$), which are hard labelers and output from $\setmp$.
  \item Decision score of the ensemble of size $M$ is weighted average:
    $$
    \fx = \sum_{m=1}^{M} \betam \bmmxth \in \R
    $$
  \item The base learner is sequentially applied to weighted training observations.
      After each base learner fit, currently misclassified observations receive a higher weight for
    the next iteration, so we focus more on instances that are harder to classify.
\item BLs with higher predictive accuracy receive higher weights $\betam$.
  % \item The number of iterations $M$ is the main tuning parameter.
  % \item The discrete prediction function is $h(\xv) = \text{sign}(\fx) \in \setmp$.
\end{itemize}

% \framebreak
%
% \begin{algorithm}[H]
%   \begin{algorithmic}[1]
%     \State Initialize observation weights: $w^{[1](i)} = \frac{1}{n} \quad \forall i \in \nset$
%     \For {$m = 1 \to M$}
%       \State Fit classifier to training data with weights $\wm$ and get $\bmmh$
%       \State Calculate weighted in-sample misclassification rate
%       $$
%         \errm = \frac{\sumin \wmi \cdot \mathds{1}_{\{\yi \,\neq\, \bmmh(\xi)\}}}{\sumin \wmi}
%       $$
%       \State Compute: $ \betamh = \frac{1}{2} \log \left( \frac{1 - \errm}{\errm}\right)$
%       \State Set: $w^{[m+1](i)} = \wmi \cdot \exp\left(\betamh \cdot
%         \mathds{1}_{\{\yi \,\neq\, \bmmh(\xi)\}} \right)$
%     \EndFor
%     \State Output: $\fxh = \sum_{m=1}^{M} \betamh \bmmh(\xv)$
%   \end{algorithmic}
%   \caption{AdaBoost}
% \end{algorithm}

%\end{vbframe}

\framebreak

\begin{algorithm}[H]
  \begin{algorithmic}[1]
    \State Initialize observation weights: $w^{[1](i)} = \frac{1}{n} \quad \forall i \in \nset$
    \For {$m = 1 \to M$}
      \State Fit classifier to training data with weights $\wm$ and get $\bmmh$
      \State Calculate weighted in-sample misclassification rate
      $$
        \errm = \sumin \wmi \cdot \mathds{1}_{\{\yi \,\neq\, \bmmh(\xi)\}}
      $$
      \State Compute: $ \betamh = \frac{1}{2} \log \left( \frac{1 - \errm}{\errm}\right)$
      \State Set: $w^{[m+1](i)} = \wmi \cdot \exp\left(- \betamh \cdot
        \yi \cdot \hat{h}(\xi)\right) $
      \State Normalize $w^{[m+1](i)}$ such that $\sumin w^{[m+1](i)} = 1$
    \EndFor
    \State Output: $\fxh = \sum_{m=1}^{M} \betamh \bmmh(\xv)$
  \end{algorithmic}
  \caption{AdaBoost}
\end{algorithm}

\end{vbframe}

% ------------------------------------------------------------------------------

\begin{vbframe}{Adaboost illustration}
\begin{footnotesize}

\textbf{Example}

\begin{itemize}
  \item $n = 10$ observations and two features $x_1$ and $x_2$
  \item Tree stumps as base learners $\bmm(\xv)$
  \item Balanced classification task with $y$ encoded as $\setmp$
  \item $M = 3$ iterations $\Rightarrow$ initial weights
  $w^{[1](i)} = \frac{1}{10} \quad \forall i \in 1,\dots ,10$.
  % \item The label of every observation is represented by triangle or circle.
  % \item Dark grey area: Base model in iteration $m$ predicts triangle.
  % \item Light grey area: Base model in iteration $m$ predicts circle.
\end{itemize}

\vfill

\begin{minipage}[b]{0.45\textwidth}
  \includegraphics[width=0.9\textwidth]{figure/adaboost_viz_mlr3_1.png}\\
  \textbf{Iteration} $m$ = 1:
  \begin{itemize}
    \item $\text{err}^{[1]} = 0.3$
    % \item The ratio $(1 - \errm) / \errm$ used to calculate the weights
    \item $\hat{\beta}^{[1]} = \frac{1}{2} \log \left( \frac{1 - 0.3}{0.3}
    \right) \approx 0.42$
  \end{itemize}
\end{minipage}%
\begin{minipage}[b]{0.55\textwidth}
  New observation weights:
  \begin{itemize}
    %\item $\wmi \cdot \exp \left(\betamh \cdot \mathds{1}_{\{\yi \neq \bmmh(\xi)\}} \right)$
    \item Prediction correct: \\
      $w^{[2](i)} = w^{[1](i)} \cdot \exp \left(-\hat \beta^{[1]} \cdot 1
      \right)$\\ $\approx 0.065.$
    \item For 3 misclassified observations: \\
      $w^{[2](i)} = w^{[1](i)} \cdot \exp \left(-\hat \beta^{[1]} \cdot (-1)
      \right)$\\ $\approx 0.15.$
    \item After normalization:
    \begin{itemize}
      \begin{footnotesize}
      \item correctly classified: $w^{[2](i)} \approx 0.07$
      \item misclassified: $w^{[2](i)} \approx 0.17$
      \end{footnotesize}
    \end{itemize}
\end{itemize}
\end{minipage}

\end{footnotesize}

% ------------------------------------------------------------------------------

\framebreak

\begin{minipage}[c]{0.4\textwidth}
  \includegraphics[width = \textwidth]{figure/adaboost_viz_mlr3_2.png}
\end{minipage}%
\begin{minipage}[c]{0.05\textwidth}
  \phantom{foo}
\end{minipage}%
\begin{minipage}[c]{0.6\textwidth}
  \begin{footnotesize}
  \textbf{Iteration} $m = 2$:
  \begin{itemize}
    \item $\text{err}^{[2]} = 3 \cdot 0.07 = 0.21$
    \item $\hat{\beta}^{[2]} \approx 0.65$
    % \item The ratio $(1 - \errm) / \errm$ used to calculate the weights
  \end{itemize}
  New observation weights:
  \begin{itemize}
    \item E.g., for 3 misclassified observations:
      $w^{[3](i)} = w^{[2](i)} \cdot \exp \left(-\hat \beta^{[1]} \cdot (-1)
      \right) \approx 0.14.$
    \item After normalization: $w^{[3](i)} \approx 0.17$ (misclassified)
  \end{itemize}
  \textbf{Iteration} $m = 3$:
  \begin{itemize}
    \item $\text{err}^{[3]} = 3 \cdot 0.045 \approx 0.14$
    \item $\hat{\beta}^{[3]} \approx 0.92$
  % \item The ratio $(1 - \errm) / \errm$ used to calculate the weights
  \end{itemize}
  \end{footnotesize}
\end{minipage}

\vfill

\begin{footnotesize}
\textbf{Note:} the smaller the error rate of a base learner, the larger the
weight, e.g., $\text{err}^{[3]} \approx 0.14 < \text{err}^{[1]} \approx 0.3$
and $\hat \beta^{[3]} \approx 0.92 > \hat \beta^{[1]} \approx 0.42.$
\end{footnotesize}

\framebreak

% ------------------------------------------------------------------------------

With $\fxh = \sum_{m=1}^{M} \betamh \bmmh(\xv)$ and $h(\xv) = \text{sign}(\fx)
\in \setmp$, \\we get:

\begin{center}
  \includegraphics[trim = 0 20 0 10, clip, width = 0.8\textwidth]{figure_man/adaboost_example_adjusted.png}
\end{center}

Hence, when all three base classifiers are combined, all samples are classified
correctly.

\end{vbframe}

%\begin{vbframe}{Adaboost illustration}

%\begin{columns}
%\column{7cm}

%%\includegraphics[width=7cm]{figure_man/adaboost_example2.png}
%\includegraphics[width=7cm]{figure_man/adaboost_example_adjusted.PNG}
%%

%{\footnotesize Schapire, Boosting, 2012.}

%\column{3cm}

%The three base models are combined into one classifier.

%All observations are correctly classified.

%\end{columns}

%\end{vbframe}

% ------------------------------------------------------------------------------

\begin{vbframe}{Bagging vs Boosting}

\begin{minipage}[t]{0.47\textwidth}
  \textbf{Random forest}
  \begin{itemize}
    \item Base learners are typically deeper decision trees (not only stumps!)
    \item Equal weights for BL
    \item BLs fitted independently
    \item Aim: variance reduction
    \item Tends \textbf{not} to overfit if ensemble size grows
  \end{itemize}
\end{minipage}%
\begin{minipage}[t]{0.06\textwidth}
  \phantom{foo}
\end{minipage}%
\begin{minipage}[t]{0.47\textwidth}
  \textbf{AdaBoost}
  \begin{itemize}
    \item BLs are weak learners, e.g., only stumps
    \item BL are weighted
    \item Sequential fitting of BLs
    \item Aim: loss reduction
    \item Tends to overfit with more iterations
  \end{itemize}
\end{minipage}

\end{vbframe}

% ------------------------------------------------------------------------------

\begin{vbframe}{Bagging vs Boosting Stumps}

Random forest versus AdaBoost (both with stumps) on \texttt{spirals} data from
\texttt{mlbench} ($n=200$, $sd=0$), with $5 \times 5$ repeated CV.

% \begin{minipage}[c]{0.6\textwidth}
%   \includegraphics[width = \textwidth]{figure_man/stump_plot_ntree.png}
% \end{minipage}%
% \begin{minipage}[c]{0.4\textwidth}
%   \begin{minipage}[c]{\textwidth}
%     \includegraphics[width = \textwidth]{figure_man/stump_plot_rf.png}
%   \end{minipage}
%   \begin{minipage}[c]{\textwidth}
%     \includegraphics[width = \textwidth]{figure_man/stump_plot_boost.png}
%   \end{minipage}
% \end{minipage}

\vfill

\includegraphics[width=\textwidth]{figure/stump_plots.png}

\vfill

Weak learners do not work well with bagging as only variance, but no bias reduction happens.

\end{vbframe}

% ------------------------------------------------------------------------------

\begin{vbframe}{Overfitting behavior}

Historically, the overfitting behavior of AdaBoost was often discussed.
% Random Forest versus AdaBoost on \texttt{Spirals} data from \texttt{mlbench} ($n=200$, $sd=0.3$).
% With $5 \times 5$ repeated CV
Increasing standard deviation in \texttt{spirals} to $sd = 0.3$ and allowing for more flexibility in
the base learners, AdaBoost overfits with increasing number of trees while the
RF only saturates.
The overfitting of AdaBoost here is quite typical as data is very noisy.

\vfill

\includegraphics[width=\textwidth]{figure/stump_plots_noisy.png}

\end{vbframe}

% ------------------------------------------------------------------------------

% \begin{vbframe}{Overfitting behavior}

% A long-lasting discussion in the context of AdaBoost is its overfitting behavior.

% % \lz

% % \emph{When a prediction rule concentrates too much on peculiarities of the specific sample of training observations, it will often perform poorly on a new data set.}

% % \lz

% The main instrument to avoid overfitting is the stopping iteration $M$:
% \begin{itemize}
% \item High values of $M$ lead to complex solutions. Overfitting?
% \item Small values of $M$ lead to simple solutions. Underfitting?
% \end{itemize}
% Although it will overfit eventually, AdaBoost in general shows a rather slow overfitting behavior.
% \framebreak

% As an example we have a look on random forest vs. AdaBoost on the \textit{Spiral} data
% from \texttt{mlbench} (n = 200, sd = 0.3). Performance (mmce) is measured
% with 5-fold CV.

% <<comparing-rf-ada1, echo=FALSE, fig.width=8, fig.height=4>>=
% load("rsrc/overfitting_ada_rf.Rdata")
% result$M = as.numeric(as.character(result$M))
% p = ggplot(result, aes(x = M, y = mmce, group = learner)) +
%   geom_line()  +
%   xlab("number trees")
% p
% @

% AdaBoost overfits with increasing number of trees while
% the mmce of the random forest fluctuates on a constant level for the test data.
% \framebreak

% This becomes even more apparent when we take a look on the prediction surface.

% <<comparing-rf-ada2, echo=FALSE, fig.width=13, fig.height=4.5 >>=
% load("rsrc/overfitting_plots.RData")
% learnerPredPlot1 = learnerPredPlot1  + ggplot2::ggtitle("rf")
% learnerPredPlot2 = learnerPredPlot2 + ggplot2::ggtitle("adaboost")
% gridExtra::grid.arrange(learnerPredPlot1, learnerPredPlot2, ncol = 2)
% @

% \end{vbframe}

% % \begin{vbframe}{Software in R}
% %   \begin{itemize}
% %     \item Bagging: Package \pkg{mlr} is able to bag any learner with \code{makeBaggingWrapper()}.
% %   \item Random Forests: Package \pkg{randomForest} with function
% %     \code{randomForest()} based on CART.
% %   \item AdaBoost: included in the packages \code{ada} and \code{boosting}.
% % \end{itemize}
% % \end{vbframe}

\endlecture
\end{document}


\newappendix{Gridworld example where multi-step outperforms one-step}\label{sec:app_grid}

As explained in the main text, this section presents an example that is only a slight modification of the one in Figure \ref{fig:gridworld}, but where a multi-step approach is clearly preferred over just one step. The data-generating and learning processes are exactly the same (100 trajectories of length 100, discount 0.9, $ \alpha = 0.1$ for reverse KL regularization). The only difference is that rather than using a behavior that is a mixture of optimal and uniform, we use a behavior that is a mixture of maximally suboptimal and uniform. If we call the suboptimal policy $ \pi^-$ (which always goes down and left in our gridworld), then the behavior for the modified example is $ \beta = 0.2 \cdot \pi^- + 0.8 \cdot u$, where $ u $ is uniform. Results are shown in Figure \ref{fig:multi_gridworld}.

\begin{figure}[h]
    \centering
    \includegraphics[width=\textwidth]{figures/offline-rl/gridworld/multi_gridworld_flat.png}
    \includegraphics[width=\textwidth]{figures/offline-rl/gridworld/multi_gridworld_error.png}
    \caption{A gridworld example with modified behavior where multi-step is much better than one-step.}
    \label{fig:multi_gridworld}
\end{figure}

By being more likely to go to the noisy states, this behavior policy allows us to get lower variance estimates of the rewards. Essentially, the coverage of the behavior policy in this example reduces the magnitude of the evaluation errors. This allows for more aggressive planning using multi-step methods. Moreover, since the behavior is less likely to go to the good state, the behavior Q function does not propagate the signal from the rewarding state as far, harming the one-step method.

\newappendix{Connection to policy improvement guarantees}\label{sec:app_improvement}

The regularized or constrained one-step algorithm performs an update that directly inherits guarantees from the literature on conservative policy improvement \citep{kakade2002approximately, schulman2015trust, achiam2017constrained}. These original papers consider an online setting where more data is collected at each step, but the guarantee at each step applies to our one-step offline algorithm.

The key idea of this line of work begins with the performance difference lemma of \cite{kakade2002approximately}, and then lower bounds the amount of improvement over the behavior policy. Define the discounted state visitation distribution for a policy $ \pi$ by $ d^\pi(s) := (1-\gamma) \sum_{t=0}^\infty \gamma^t \Prob_{\rho, P, \pi}(s_t = s)$. We will also use the shorthand $ Q(s, \pi) $ to denote $ \E_{a\sim\pi|s}[Q(s,a)]$. Then we have the performance difference lemma as follows.

\begin{lemma}[Performance difference, \cite{kakade2002approximately}]
For any two policies $ \pi$ and $ \beta$,
\begin{align}
    J(\pi) - J(\beta) = \frac{1}{1-\gamma} \E_{\substack{s \sim d^\pi}}[ Q^\beta(s,\pi) - Q^\beta(s, \beta)]. %:= \frac{1}{1-\gamma} \E_{\substack{s \sim d_\pi }}[ A^\beta_\pi(s)].
\end{align}
\end{lemma}

Then, Corollary 1 from \cite{achiam2017constrained} (reproduced below) gives a guarantee for the one-step algorithm. The key idea is that when $ \pi $ is sufficiently close to $ \beta$, we can use $ Q^\beta$ as an approximation to $ Q^\pi$.
\begin{lemma}[Conservative Policy Improvement, \cite{achiam2017constrained}]
For any two policies $ \pi$ and $ \beta$, let $ \|A^\beta_\pi\|_{\infty} = \sup_s  |Q^\beta(s,\pi) - Q^\beta(s, \beta)|$. Then,
    \begin{align}
        J(\pi) - J(\beta) \geq \frac{1}{1-\gamma} \E_{\substack{s \sim d^\beta}}\left[ \left(Q^\beta(s,\pi) - Q^\beta(s, \beta)\right) - \frac{2\gamma \|A^\beta_\pi\|_\infty}{1-\gamma} D_{TV}(\pi(\cdot|s)\|\beta(\cdot|s)) \right]
    \end{align}
where $ D_{TV}$ denotes the total variation distance.
\end{lemma}

Replacing $ Q^\beta$ with $ \widehat Q^\beta$ and the TV distance by the KL, we get precisely the objective that we optimize in the one-step algorithm. This shows that the one-step algorithm indeed optimizes a lower bound on the performance difference. Of course, in practice we replace the potentially large multiplier on the divergence term by a hyperparameter, but this theory at least motivates the soundness of the approach.

We are not familiar with similar guarantees for the iterative or multi-step approaches that rely on off-policy evaluation.

\newappendix{Experimental setup}\label{sec:app_exp_setup}

\subsection{Benchmark experiments (Tables \ref{tab:d4rl} and \ref{tab:multi}, Figure \ref{fig:learning_curves})}

\paragraph{Data.} We use the datasets from the D4RL benchmark \citep{fu2020d4rl}. We use the latest versions, which are v2 for the mujoco datasets and v1 for the adroit datasets.

\paragraph{Hyperparameter tuning.}
\begin{table}[ht]
\vspace{-0.2cm}
    \centering
    \caption{Hyperparameter sweeps for each algorithm.}
    \begin{small}
    \begin{tabular}{lc}
        \toprule
        Algorithm & Hyperparameter set \\
        \midrule
        Reverse KL ($ \alpha$) &
                    \{0.03, 0.1, 0.3, 1.0, 3.0, 10.0\}\\
        Easy BCQ ($ M$) &
                    \{2, 5, 10, 20, 50, 100\}\\
        Exponentially weighted ($ \tau$) &
                    \{0.1, 0.3, 1.0, 3.0, 10.0, 30.0\}\\
        \bottomrule
    \end{tabular}
    \end{small}
    \label{tab:hyperparams}
\end{table}
We follow the practice of \cite{fu2020d4rl} and tune a small set of hyperparameters by interacting with the simulator to estimate the value of the policies learned under each hyperparameter setting. The hyperparameter sets for each algorithm can be seen in Table \ref{tab:hyperparams}.

This may initially seem like ``cheating'', but can be a reasonable setup if we are considering applications like robotics where we can feasibly test a small number of trained policies on the real system. Also, since prior work has used this setup, it makes it easiest to compare our results if we use it too. While beyond the scope of this work, we do think that better offline model selection procedures will be crucial to make offline RL more broadly applicable. A good primer on this topic can be found in \cite{paine2020hyperparameter}.

\paragraph{Models.} All of our Q functions and policies are simple MLPs with ReLU activations and 2 hidden layers of width 1024. Our policies output a truncated normal distribution with diagonal covariance where we can get reparameterized samples by sampling from a uniform distribution and computing the differentiable inverse CDF \citep{Burkhardt2014truncated}. We found this to be more stable than the tanh of normal used by e.g. \cite{fu2020d4rl}, but to achieve similar performance when both are stable. We use these same models across all experiments.

\paragraph{One-step training procedure.} For all of our one-step algorithms, we train our $ \hat \beta $ behavior estimate by imitation learning for 500k gradient steps using Adam \citep{kingma2014adam} with learning rate 1e-4 and batch size 512. We train our $ \widehat Q^\beta$ estimator by fitted Q evaluation with a target network for 2 million gradient steps using Adam with learning rate 1e-4 and batch size 512. The target is updated softly at every step with parameter $ \tau = 0.005$. All policies are trained for 100k steps again with Adam using learning rate 1e-4 and batch size 512.

Easy BCQ does not require training a policy network and just uses $ \hat \beta$ and $ \widehat Q^\beta$ to define it's policy. For the exponentially weighted algorithm, we clip the weights at 100 to prevent numerical instability. To estimate reverse KL at some state we use 10 samples from the current policy and the density defined by our estimated $ \hat \beta$.

Each random seed retrains all three models (behavior, Q, policy) from different initializations. We use three random seeds.

\paragraph{Multi-step training procedure.} For multi-step algorithms we use all the same hyperparameters as one-step. We initialize our policy and Q function from the same pre-trained $ \hat \beta$ and $ \widehat Q^\beta$ as we use for the one-step algorithm trained for 500k and 2 million steps respectively. Then we consider 5 policy steps. To ensure that we use the same number of gradient updates on the policy, each step consists of 20k gradient steps on the policy followed by 200k gradient steps on the Q function. Thus, we take the same 100k gradient steps on the policy network. Now the Q updates are off-policy so the next action $a'$ is sampled from the current policy $ \pi_i$ rather than from the dataset.

\paragraph{Iterative training procedure.} For iterative algorithms we again use all the same hyperparameters and initialize from the same $ \hat \beta$ and $ \widehat Q^\beta$. We again take the same 100k gradient steps on the policy network. For each step on the policy network we take 2 off-policy gradient steps on the Q network.

\paragraph{Evaluation procedure.} To evaluate each policy we run 100 trajectories in the environment and compute the mean. We then report the mean and standard deviation over three training seeds.

\subsection{MSE experiment (Figure 3)}

\paragraph{Data.} To get an independently sampled dataset of the same size as the training set, we use the behavior cloned policy $ \hat \beta$ to sample 1000 trajectories. The checkpointed policies are taken at intervals of 5000 gradient steps from each of the three training seeds.

\paragraph{Training procedure.} The $\widehat Q^{\pi_i}$ training procedure is the same as before so we use Adam with step size 1e-4 and batch size 512 and a target network with soft updates with parameter 0.005. We train for 1 million steps.

\paragraph{Evaluation procedure.} To evaluate MSE, we sample 1000 state, action pairs from the original training set and from each state, action pair we run 3 rollouts. We take the mean over the rollouts and then compute squared error at each state, action pair and finally get MSE by taking the mean over state, action pairs. The reported reverse KL is evaluated by samples during training. At each state in a batch we take 10 samples to estimate the KL at that state and then take the mean over the batch.

\subsection{Gridworld experiment (Figure 4)}

\paragraph{Environment.} The environment is a 15 x 15 gridworld with deterministic transitions. The rewards are deterministically 1 for all actions taken from the state in the top right corner and stochastic with distribution $ \mathcal{N}(-0.5, 1)$ for all actions taken from states on the left or bottom walls. The initial state is uniformly random. The discount is 0.9.

\paragraph{Data.} We collect data from a behavior policy that is a mixture of the uniform policy (with probability 0.8) and an optimal policy (with probability 0.2). We collect 100 trajectories of length 100.

\paragraph{Training procedure.} We give the agent access to the deterministic transitions. The only thing for the agent to do is estimate the rewards from the data and then learn in the empirical MDP. We perform tabular Q evaluation by dynamic programming. We initialize with the empirical rewards and do 100 steps of dynamic programming with discount 0.9. Regularized policy updates are solved for exactly by setting $ \pi_i(a|s) \propto \beta(a|s) \exp(\frac{1}{\alpha} \widehat Q^{\pi_{i-1}}(s,a))$.

\subsection{Overestimation experiment (Figure 5)}

This experiment uses the same setup as the MSE experiment. The main difference is we also consider the Q functions learned during training and demonstrate the overestimation relative to the Q functions trained on the evaluation dataset as in the MSE experiment.

\subsection{Mixed data experiment (Figure 6)}

We construct datasets with $ p_m = \{0.0, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0\}$ by mixing the random and medium datasets from D4RL and then run the same training procedure as we did for the benchmark experiments. Each dataset has the same size, but a different proportion of trajectories from the medium policy.

\newappendix{Learning curves}\label{sec:app_extra_exp}

In this section we reproduce the learning curves and hyperparameter plots across the one-step, multi-step, and iterative algorithms with reverse KL regularization, as in Figure \ref{fig:learning_curves}.

\vspace{-0.2cm}
\begin{figure}[h]
    \centering
    \includegraphics[width=0.85\textwidth]{figures/offline-rl/learning curves/lc-halfcheetah-medium-v2.png}
    \includegraphics[width=0.85\textwidth]{figures/offline-rl/learning curves/lc-walker2d-medium-v2.png}
    \includegraphics[width=0.85\textwidth]{figures/offline-rl/learning curves/lc-hopper-medium-v2.png}
    \vspace{-0.2cm}
    \caption{Learning curves on the medium datasets.}
    \label{fig:app_lc_medium}
\end{figure}

\begin{figure}[h]
    \centering
    \includegraphics[width=0.85\textwidth]{figures/offline-rl/learning curves/lc-halfcheetah-medium-expert-v2.png}
    \includegraphics[width=0.85\textwidth]{figures/offline-rl/learning curves/lc-walker2d-medium-expert-v2.png}
    \includegraphics[width=0.85\textwidth]{figures/offline-rl/learning curves/lc-hopper-medium-expert-v2.png}
    \vspace{-0.2cm}
    \caption{Learning curves on the medium-expert datasets.}
    \label{fig:app_lc_medium-expert}
\end{figure}

\begin{figure}[h]
    \centering
    \includegraphics[width=0.85\textwidth]{figures/offline-rl/learning curves/lc-halfcheetah-random-v2.png}
    \includegraphics[width=0.85\textwidth]{figures/offline-rl/learning curves/lc-walker2d-random-v2.png}
    \includegraphics[width=0.85\textwidth]{figures/offline-rl/learning curves/lc-hopper-random-v2.png}
    \vspace{-0.2cm}
    \caption{Learning curves on the random datasets.}
    \label{fig:app_lc_random}
\end{figure}

\printendnotes