\part{Lecture 12: Policy Gradient Methods}
\title[RL Lecture 12]{Lecture 12: Policy Gradient Methods}
\date{}
\frame{\titlepage}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Preface (1) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Preface (1)}
Shift from (indirect) value-based approaches
\begin{equation}
			\hat{q}(\bm{x}, u, \bm{w}) \approx q(\bm{x}, u)
\end{equation}
to (direct) policy-based solutions:
\begin{equation}
	\bm{\pi}(\bm{u}|\bm{x})=\Pb{\bm{U} = \bm{u} | \bm{X} = \bm{x}}\approx\bm{\pi}(\bm{u}|\bm{x}, \bm{\theta})\, .
\end{equation}\pause
\begin{itemize}
	\item Above, $\bm{\theta}\in \mathbb{R}^d$ is the policy parameter vector.
	\item Note, that $\bm{u}$ might contain multiple continuous quantities.
\end{itemize}
\pause
\begin{block}{Goal of today's lecture}
\begin{itemize}
	\item Introduce an algorithm class based on a parameterizable policy $\pi(\bm{\theta})$.
	\item Extend the action space to continuous actions.
	\item Combine the policy-based and value-based approach.
\end{itemize}
\end{block}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Preface (2) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Preface (2)}
\begin{figure}
	\includegraphics[width=7cm]{fig/lec01/RL_Agent_Taxonomy.pdf}
	\caption{Main categories of reinforcement learning algorithms\\  \SilverLectureSource}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Policy Approximation and its Advantages}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Table of Contents}
\tableofcontents
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Motivating Example (1): Short Corridor Problem %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Motivating Example (1): Short-Corridor Problem}
\begin{itemize}
	\onslide<1->\item Gridworld style problem with two actions: left (l), right (r)
	\onslide<2->\item Second-left state's action execution is reversed
	\onslide<3->\item Feature representation: $\tilde{x}(x, u=\mbox{r})=\begin{bmatrix}1 & 0\end{bmatrix}\T$, $\tilde{x}(x, u=\mbox{l})=\begin{bmatrix}0 & 1\end{bmatrix}\T$
	\onslide<4->\item $\varepsilon$-greedy value-based policy performs actions with $1-\varepsilon/2$ probability
	\onslide<5->\item A policy-based approach can search for the optimal probability split
\end{itemize}
\onslide<1->\begin{figure}
\includegraphics[width=8.5cm]{fig/lec12/Short_Corridor_Problem.pdf}
\caption{Short-corridor problem with $\varepsilon=0.1$ (source: R. Sutton and G. Barto, Reinforcement learning: an introduction, 2018, \href{https://creativecommons.org/licenses/by-nc-nd/2.0/}{CC BY-NC-ND 2.0})}
\label{fig:Short_Corridor_Problem}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Motivating Example (2): Strategic Gaming %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Motivating Example (2): Strategic Gaming}
\onslide<1->Task: Two-player game of extended rock-paper-scissors
\begin{itemize}
	\onslide<2->\item A deterministic policy (i.e., value-based with given feature representation) can be easily exploited by the opponent.
	\onslide<3->\item	Conversely, a uniform random policy would be unpredictable.
\end{itemize}

\onslide<1->
\begin{figure}
	\includegraphics[width=4.3cm]{fig/lec01/Rock_paper_scissors_lizard_spock.pdf}
	\caption{Rock paper scissors lizard Spock game mechanics\\(source: \href{https://commons.wikimedia.org/wiki/File:Rock_paper_scissors_lizard_spock.svg}{www.wikipedia.org},  by \href{https://en.wikipedia.org/wiki/User:Diriector_Doc}{Diriector Doc} \href{https://creativecommons.org/licenses/by-sa/4.0/deed.en}{CC BY-SA 4.0})}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Example Policy Function: Discrete Action Space %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Example Policy Function: Discrete Action Space}
Assumption:
\begin{itemize}
	\item Action space is discrete and compact.
\end{itemize}\pause
A typical policy function is:
\begin{itemize}
	\item \hl{Soft-max in action preferences}
	\begin{equation}
		\pi(u|\bm{x},\bm{\theta}) = \frac{e^{h(\bm{x},u,\bm{\theta})}}{\sum_i e^{h(\bm{x},i,\bm{\theta})}}
		\label{eq:soft_max_preference}
	\end{equation}
	with $h(\bm{x},u,\bm{\theta}):\mathcal{X}\times\mathcal{U}\times\mathbb{R}^d \rightarrow \mathbb{R}$ being the numerical preference per state-action pair.\pause
	\item Denominator of \eqref{eq:soft_max_preference} sums up action probabilities to one per state. \pause
	\item Is designed as a stochastic policy but can approach deterministic behavior in the limit. \pause
	\item The preference is parametrized via a function approximator, e.g., linear in features
	\begin{equation}
		h(\bm{x},u,\bm{\theta}) = \bm{\theta}\T\tilde{\bm{x}}(\bm{x}, u).
	\end{equation}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Example Policy Function: Continuous Action Space (1) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Example Policy Function: Continuous Action Space (1)}
Assumption:
\begin{itemize}
	\item Action space is continuous and there is only one scalar action $u\in\mathbb{R}$.
\end{itemize}\pause
A typical policy function is:
\begin{itemize}
	\item \hl{Gaussian probability density}
	\begin{equation}
		\pi(u|\bm{x},\bm{\theta}) = \frac{1}{\sigma(\bm{x},\bm{\theta})\sqrt{2\pi}}\exp\left(-\frac{(u-\mu(\bm{x},\bm{\theta}))^2}{2\sigma(\bm{x},\bm{\theta})^2}\right)
		\label{eq:gaussian_action}
	\end{equation}
	with mean $\mu(\bm{x},\bm{\theta}):\mathcal{X}\times\mathbb{R}^d \rightarrow \mathbb{R}$ and standard deviation $\sigma(\bm{x},\bm{\theta}):\mathcal{X}\times\mathbb{R}^d \rightarrow \mathbb{R}$ given by parametric function approximation.\pause
		\item Variants regarding function $\mu$ and $\sigma$:
	\begin{enumerate}
		\item Both  share a mutual parameter set $\bm{\theta}$ (e.g., artificial neural network with multiple outputs).\pause
		\item Both are parametrized independently $\bm{\theta}=\begin{bmatrix}\bm{\theta}_\mu & \bm{\theta}_\sigma\end{bmatrix}\T$ (e.g., by two linear regression functions).\pause
		\item Only $\mu(\bm{x},\bm{\theta})$ is parametrized while $\sigma$ is scheduled externally.
	\end{enumerate}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Example Policy Function: Continuous Action Space (2) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Example Policy Function: Continuous Action Space (2)}
\begin{itemize}
	\item Output of the functions $\mu_k=(\bm{x}_k,\bm{\theta}_k)$ and $\sigma_k=(\bm{x}_k,\bm{\theta}_k)$ can change in every time step.
	\item Depending on $\sigma$ exploration is an inherent part of the (stochastic) policy.
\end{itemize}
\begin{figure}
\includegraphics[width=5.5cm]{fig/lec12/Gaussian_Distri.pdf}
\caption{Exemplary univariate Gaussian probability density functions (source: R. Sutton and G. Barto, Reinforcement learning: an introduction, 2018, \href{https://creativecommons.org/licenses/by-nc-nd/2.0/}{CC BY-NC-ND 2.0})}
\label{fig:Gaussian_Distri}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Example Policy Function: Continuous Action Space (3) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Example Policy Function: Continuous Action Space (3)}
Assumption:
\begin{itemize}
	\item Action space is continuous and there are multiple actions $\bm{u}\in\mathbb{R}^m$.
\end{itemize} \pause
A typical policy function is:
\begin{itemize}
	\item \hl{Multivariate Gaussian probability density}
	\begin{equation}
		\pi(\bm{u}|\bm{x},\bm{\theta}) = \frac{1}{\sqrt{(2\pi)^m\det(\bm{\Sigma})}}\exp\left(-\frac{1}{2}(\bm{u}-\bm{\mu})\T\bm{\Sigma}^{-1}(\bm{u}-\bm{\mu})\right)
		\label{eq:gaussian_action_multivariate}
	\end{equation}
	with mean $\bm{\mu}(\bm{x},\bm{\theta}):\mathcal{X}\times\mathbb{R}^d \rightarrow \mathbb{R}^m$ and covariance matrix $\bm{\Sigma}(\bm{x},\bm{\theta}):\mathcal{X}\times\mathbb{R}^d \rightarrow \mathbb{R}^{m \times m }$ given by parametric function approximation. \pause
		\item Same parametrization variants apply to $\bm{\mu}$ and $\bm{\Sigma}$ as in the scalar action case. \pause
		\item In addition, $\bm{\Sigma}$ can be considered a diagonal matrix and clipped to reduce complexity as well as ensure nonsingularity.
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Example Policy Function: Continuous Action Space (4) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Example Policy Function: Continuous Action Space (4)}
\begin{itemize}
	\item Below we find an example for
\end{itemize}
\begin{equation*}
	\bm{\mu}=\begin{bmatrix}-0.4 & 0.3\end{bmatrix}\T \quad \mbox{and} \quad \bm{\Sigma}=\begin{bmatrix}0.04 & 0\\ 0 & 0.02 \end{bmatrix} .
\end{equation*}
\begin{figure}
\includegraphics[width=6.5cm]{fig/lec12/Gaussian_Policy_Multivariate.pdf}
\caption{Exemplary bivariate Gaussian probability density function}
\label{fig:Gaussian_Policy_Multivariate}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Objective Function %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Objective Function}
\begin{itemize}
	\item Goal: find optimal $\bm{\theta}^*$ given the policy $\bm{\pi}(\bm{u}|\bm{x}, \bm{\theta})$. \pause
	\item Problem: which measure of optimality should we use? \pause
\end{itemize}
\vspace{0.5cm}
Possible optimality metrics:
\begin{itemize}
	\item \hl{Start state value} (in episodic tasks):
	\begin{equation}
		J(\bm{\theta}) = v_{\pi_{\bm{\theta}}}(\bm{x}_0)=\E{v|\bm{X}=\bm{x}_0, \bm{\theta}}
		\label{eq:performance_metric_episodic}
	\end{equation}\pause
	\item \hl{Average reward} (in continuing tasks):
	\begin{equation}
		J(\bm{\theta}) = \overline{r}_{\pi_{\bm{\theta}}} = \int_\mathcal{X}\mu_\pi(\bm{x})\int_\mathcal{U}\pi(\bm{u}|\bm{x}, \bm{\theta})\int_{\mathcal{X},\mathcal{R}}p(\bm{x}', r|\bm{x},\bm{u})r
		\label{eq:performance_metric_continuing}
	\end{equation}
	\begin{itemize}
		\item Above, $\mu_\pi(\bm{x})$ is again the steady-state distribution $\mu_\pi(\bm{x})=\lim_{k\rightarrow\infty}\Pb{\bm{X}_k=\bm{x}|\bm{U}_{0:k-1}\sim\pi}$.
	\end{itemize}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Optimization %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Optimization}
\begin{itemize}
	\onslide<1->\item In essence, policy-based RL is an \hl{optimization problem}.
	\onslide<2->{\item Depending on the policy function and task, finding $\bm{\theta}^*$ might be a
	\begin{itemize}
		\item non-linear,
		\item multidimensional and
		\item non-stationary problem.
	\end{itemize}}
	\onslide<3->{\item Hence, we might consider global optimization techniques\footnote[1]{\onslide<3->{Recommended reading: J. Stork et al., \textit{A new Taxonomy of Continuous Global Optimization Algorithms}, \href{https://arxiv.org/abs/1808.08818}{https://arxiv.org/abs/1808.08818}, 2020}} like
		\begin{itemize}
		\item Simple heuristics: random search, grid search,...}
		\onslide<4->\item Meta-heuristics: evolutionary algorithms, particle swarm,....
		\onslide<5->\item Surrogate-model-based optimization: Bayes opt.,...
		\onslide<6->\item Gradient-based techniques with multi-start initialization.
	\end{itemize}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% How to Retrieve the Gradient? %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Gradient}
\begin{columns}[t,onlytextwidth]
\begin{column}{0.475\textwidth}
\begin{minipage}[c]{\linewidth}
\begin{figure}
	\includegraphics[width=4.5cm]{fig/lec12/Gradient_ascent.pdf}
	\caption{Exemplary optimization paths for (stochastic) gradient ascent \\(derivative work of \href{https://commons.wikimedia.org/wiki/File:Gradient_descent.svg}{www.wikipedia.org}, \href{https://creativecommons.org/publicdomain/zero/1.0/deed.en}{CC0 1.0})}
	\label{fig:Gradient_ascent}
\end{figure}
\end{minipage}
\end{column}
\hfill
\begin{column}{0.54\textwidth}
\begin{minipage}[c]{\linewidth}
\begin{itemize}
	\item We will focus on gradient-based methods (\hl{policy gradient}).\pause
	\item Hence, we will assume that the gradient
	\begin{equation*}
		\nabla_{\bm{\theta}} J(\bm{\theta})=\begin{bmatrix}\frac{\partial J}{\partial \theta_1} & \cdots & \frac{\partial J}{\partial \theta_d} \end{bmatrix}\T
	\end{equation*}
	required for \hl{gradient ascent optimization} always exists:
	\begin{equation*}
		\bm{\theta} \leftarrow\bm{\theta} + \alpha \nabla_{\bm{\theta}} J(\bm{\theta}).
	\end{equation*}\pause\vspace{-0.4cm}
	\item True gradient $\nabla_{\bm{\theta}} J(\bm{\theta})$ is usually approximated, e.g., by stochastic gradient descent (SGD) or derived variants.
\end{itemize}
\end{minipage}
\end{column}
\end{columns}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Gradient Theorem %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Gradient Theorem}
\begin{theo}{Policy Gradient}{policy_gradient}
Given a metric $J(\bm{\theta})$ for the undiscounted episodic \eqref{eq:performance_metric_episodic} or continuing tasks \eqref{eq:performance_metric_continuing} and a parameterizable policy $\pi(\bm{u}|\bm{x},\bm{\theta})$ the policy gradient is
\begin{equation}
	\nabla_{\bm{\theta}} J(\bm{\theta}) = \El{q_{\pi}(\bm{x},\bm{u})\frac{\nabla_{\bm{\theta}}\pi(\bm{u}|\bm{x},\bm{\theta})}{\pi(\bm{u}|\bm{x},\bm{\theta})}}{\pi}.
	\label{eq:policy_gradient_theo}
\end{equation}
\end{theo}\pause
\begin{itemize}
 \item Having samples $\left\langle\bm{x}_i, \bm{u}_i \right\rangle$, an estimate of $q_{\pi}$ and the policy function $\pi(\bm{\theta})$ available we receive an \hl{analytical solution for the policy gradient!} \pause
\item Using identity $\nabla \ln a = \frac{\nabla a}{a}$ we can re-write to
\begin{equation}
	\nabla_{\bm{\theta}} J(\bm{\theta}) = \El{q_{\pi}(\bm{x},\bm{u})\nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})}{\pi}
\end{equation}
 with $\nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})$ also called the \hl{score function}.\pause
	\item Derivation available in chapter 13.2 / 13.6 in the lecture book of Barto and Sutton.
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Intuitive Interpretation of Policy Parameter Update %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Intuitive Interpretation of Policy Parameter Update}
\begin{itemize}
	\item Inserting the policy gradient theorem into gradient ascent approach:
\end{itemize}
\begin{equation*}
	\bm{\theta} \leftarrow \bm{\theta} + \alpha \El{\textcolor{blue}{q_{\pi}(\bm{x},\bm{u})} \frac{\textcolor{blue}{\nabla_{\bm{\theta}}\pi(\bm{u}|\bm{x},\bm{\theta})}}{\textcolor{red}{\pi(\bm{u}|\bm{x},\bm{\theta})}}}{\pi}.
\end{equation*}
\begin{itemize}
	\item \textcolor{blue}{Move in the direction that favor actions that yield an increased value}.\pause
	\item \textcolor{red}{Scale the update step size inversely to the action probability to compensate that some actions are selected more frequently}.\pause
\end{itemize}
\vspace{0.5cm}
Also note:
\begin{itemize}
	\item The policy gradient is not depending on the state distribution!\pause
	\item Hence, we do not need any knowledge of the environment and receive a \hl{model-free RL approach!}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Simple Score Function Examples %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Simple Score Function Examples}
\onslide<1->{Soft-max policy with linear function approximation:
\begin{equation*}
\begin{split}
	\pi(u|\bm{x},\bm{\theta}) &= \frac{e^{\bm{\theta}\T\tilde{\bm{x}}(\bm{x}, u)}}{\sum_i e^{\bm{\theta}\T\tilde{\bm{x}}(\bm{x}, i)}}\\}
	\onslide<2->{\Leftrightarrow\quad \nabla_{\bm{\theta}}\ln \pi(u|\bm{x},\bm{\theta}) &= \nabla_{\bm{\theta}}\left(\bm{\theta}\T\tilde{\bm{x}}(\bm{x}, u) - \ln\left(\sum_i e^{\bm{\theta}\T\tilde{\bm{x}}(\bm{x}, i)}\right)\right)\\}
			\onslide<3->{ & = \tilde{\bm{x}}(\bm{x},\bm{u}) - \El{\tilde{\bm{x}}(\bm{x},\cdot)}{\pi}}
\end{split}
\end{equation*}
\onslide<4->{Univariate Gaussian policy with linear function approximation and given $\sigma$:
\begin{equation*}
\begin{split}
	\pi(u|\bm{x},\bm{\theta}) &= \frac{1}{\sigma \sqrt{2\pi}}\exp\left(-\frac{(u-\bm{\theta}\T\tilde{\bm{x}}(\bm{x}, u))^2}{2\sigma^2}\right)\\}
	\onslide<5->{\Leftrightarrow\quad \nabla_{\bm{\theta}}\ln \pi(u|\bm{x},\bm{\theta}) &= \nabla_{\bm{\theta}}\left(\ln\left(\frac{1}{\sigma \sqrt{2\pi}}\right) -\frac{(u-\bm{\theta}\T\tilde{\bm{x}}(\bm{x}, u))^2}{2\sigma^2}\right)\\}
			\onslide<6->{& = \frac{\left(u - \bm{\theta}\T\tilde{\bm{x}}(\bm{x}, u)\right)\tilde{\bm{x}}(\bm{x}, u)}{\sigma^2}}
\end{split}
\end{equation*}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Pro and Cons %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Pro and Cons: Policy vs. Value-Based Approaches}
\hl{Pro value-based solutions} (e.g., $Q$-learning):\pause
\begin{itemize}
	\item Estimated value is an intuitive performance metric.\pause
	\item Considered sample-efficient (cf. replay buffer or bootstrapping).\pause
\end{itemize}
\hl{Pro policy-based solutions} (e.g., using policy gradient):\pause
\begin{itemize}
	\item Policy gradient theorem provides strong convergence properties.\pause
	\item Seamless integration of stochastic and dynamic policies.\pause
	\item Straightforward applicable to large/continuous action spaces. \pause In contrast, value-based approaches would require explicit optimization
	\begin{equation*}
		\bm{u}^*=\argmax_{\bm{u}} q(\bm{x}, \bm{u}, \bm{w}).
	\end{equation*}
\end{itemize}\pause
Mutual hassle:
\begin{itemize}
	\item Gradient-based optimization with (non-linear) function approximation is likely to \hl{deliver only suboptimal and local policy optima}.
\end{itemize}

}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Monte Carlo Policy Gradient}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Table of Contents}
\tableofcontents[currentsection]
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Gradient Theorem %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Basic Concept}
Initial situation:
\begin{itemize}
	\item Score function $\nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})$ can be calculated analytically using suitable policy and chain rule (e.g., by algorithmic differentiation).
	\item Open question: how to retrieve $q_{\pi}(\bm{x},\bm{u})$ in
	\begin{equation*}
	\nabla_{\bm{\theta}} J(\bm{\theta}) = \El{q_{\pi}(\bm{x},\bm{u})\nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})}{\pi} \,\,\mbox{?}
\end{equation*}
\end{itemize}
\vspace{0.5cm}\pause
Monte Carlo policy gradient:
\begin{itemize}
	\item Use sampled episodic return $g_k$ to approximate $q_{\pi}(\bm{x},\bm{u})$:
		\begin{equation*}
		\begin{split}
			 q_{\pi}(\bm{x},\bm{u}) &\approx g_k\\
			\bm{\theta}_{k+1} &= \bm{\theta}_k + \alpha\gamma^k g_k \nabla_{\bm{\theta}}\ln \pi(\bm{u}_k|\bm{x}_k,\bm{\theta}_k).
		\end{split}
		\end{equation*}\pause\vspace{-0.3cm}
		\item The discounting of the policy gradient is introduced as an extension to \theoref{theo:policy_gradient} (which assumed an undiscounted episodic task).\pause
		\item Also known as \hl{\textit{REINFORCE}} approach.
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Algorithmic Implementation: Monte Carlo Policy Gradient %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Algorithmic Implementation: Monte Carlo Policy Gradient}
\begin{itemize}
	\item Usual technical convergence requirements regarding $\alpha$ apply.
	\item Use sampled return as unbiased estimate of $q$.
	\item Recall previous MC-based methods: high variance, slow learning.
\end{itemize}
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\small
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{a differentiable policy function $\pi(\bm{u}|\bm{x},\bm{\theta})$}
\Param{step size $\alpha\in\left\{\mathbb{R}|0<\alpha<1\right\}$}
\Init{parameter vector $\bm{\theta}\in\mathbb{R}^d$ arbitrarily}
 \For{$j=1,2,\ldots,$ episodes}{
		generate an episode following $\pi(\cdot|\cdot,\bm{\theta})$: $\bm{x}_{0}, \bm{u}_{0}, r_{1},\ldots,\bm{x}_{T}$ \;
		\For{$k=0,1,\ldots, T-1$ time steps}{
			$g\leftarrow \sum_{i=k+1}^T\gamma^{i-k-1}r_i$\;
			$\bm{\theta} \leftarrow \bm{\theta} + \alpha \gamma^k g \nabla_{\bm{\theta}}\ln \pi(\bm{u}_k|\bm{x}_k,\bm{\theta})$\;
		}
	}
\caption{Monte Carlo policy gradient (output: parameter vector $\bm{\theta}^*$ for $\pi^*(\bm{u}|\bm{x},\bm{\theta}^*)$)}
\label{algo:MC_policy_gradient}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% REINFORCE Comparison %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{REINFORCE Example: Short-Corridor Problem}
\begin{figure}
\includegraphics[width=11cm]{fig/lec12/REINFORCE_short_corridor.pdf}
\caption{Comparison of Monte Carlo policy gradient approach on short-corridor problem from \figref{fig:Short_Corridor_Problem} for different learning rates (source: R. Sutton and G. Barto, Reinforcement learning: an introduction, 2018, \href{https://creativecommons.org/licenses/by-nc-nd/2.0/}{CC BY-NC-ND 2.0})}
\label{fig:REINFORCE_short_corridor}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Baseline %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Baseline}
\begin{itemize}
	\item Motivation: add a comparison term to the policy gradient to reduce variance while not affecting its expectation.\pause
	\item Introduce the \hl{baseline $b(\bm{x})$}:
\end{itemize}
\begin{equation}
	\nabla_{\bm{\theta}} J(\bm{\theta}) = \El{\left(q_{\pi}(\bm{x},\bm{u})\hl{-b(\bm{x})}\right)\nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})}{\pi}.
\end{equation}\pause
\vspace{-0.3cm}
\begin{itemize}
	\item Since $b(\bm{x})$ is only depending on the state but not on the actions/policy we did not change the policy gradient in expectation:
\end{itemize}
\vspace{0.3cm}
\begin{equation*}
	\nabla_{\bm{\theta}} J(\bm{\theta}) = \El{q_{\pi}(\bm{x},\bm{u})\nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})}{\pi}- \underbrace{\El{b(\bm{x})\nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})}{\pi}}_{=0}.
\end{equation*}\pause
\begin{itemize}
	\item Consequently, the Monte Carlo policy parameter update yields:
\end{itemize}
		\begin{equation*}
			\bm{\theta}_{k+1} = \bm{\theta}_k + \alpha\gamma^k \left(g_k-b(\bm{x}_k)\right) \nabla_{\bm{\theta}}\ln \pi(\bm{u}_k|\bm{x}_k,\bm{\theta}_k).
		\end{equation*}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Advantage %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Advantage Function}
\begin{itemize}
	\item Intuitive choice of the baseline is the state value $b(\bm{x})=v_{\pi}(\bm{x})$.
	\item The resulting policy gradient becomes
	\begin{equation}
	\nabla_{\bm{\theta}} J(\bm{\theta}) = \El{\left(q_{\pi}(\bm{x},\bm{u})-v_{\pi}(\bm{x})\right)\nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})}{\pi}.
\end{equation}\pause
\item Here, the difference between action and state value is the \hl{advantage function}
	\begin{equation}
	a_\pi(\bm{x},\bm{u})= q_{\pi}(\bm{x},\bm{u})-v_{\pi}(\bm{x}) .
\end{equation}\pause
\item Interpretation: value difference taking (arbitrary) action $\bm{u}$ and thereafter following policy $\pi$ compared to the state value following same policy (i.e., choosing $\bm{u}\sim\pi$) given the state.\pause
\item Hence, we might rewrite to:
	\begin{equation}
	\nabla_{\bm{\theta}} J(\bm{\theta}) = \El{a_\pi(\bm{x},\bm{u})\nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})}{\pi}.
\end{equation}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Algorithmic Implementation: Monte Carlo Policy Gradient with Baseline%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Algo. Implementation: MC Policy Gradient with Baseline}
\begin{itemize}
	\item Implementation requires approximation $b(\bm{x})\approx \hat{v}(\bm{x},\bm{w})$.
	\item Hence, we are learning two parameter sets $\bm{\theta}$ and $\bm{w}$.
	\item Keep using sampled return as action-value estimate: $q_{\pi}(\bm{x},\bm{u}) \approx g_k$.
\end{itemize}
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{a differentiable policy function $\pi(\bm{u}|\bm{x},\bm{\theta})$}
\Input{a differentiable state-value function $\hat{v}(\bm{x},\bm{w})$}
\Param{step sizes $\{\alpha_{w}, \alpha_{\theta}\}\in\left\{\mathbb{R}|0<\alpha<1\right\}$}
\Init{parameter vectors $\bm{w}\in\mathbb{R}^{\zeta}$ and $\bm{\theta}\in\mathbb{R}^d$ arbitrarily}
 \For{$j=1,2,\ldots,$ episodes}{
		generate an episode following $\pi(\cdot|\cdot,\bm{\theta})$: $\bm{x}_{0}, \bm{u}_{0}, r_{1},\ldots,\bm{x}_{T}$ \;
		\For{$k=0,1,\ldots, T-1$ time steps}{
			$g\leftarrow \sum_{i=k+1}^T\gamma^{i-k-1}r_i$\;
			$\delta \leftarrow g - \hat{v}(\bm{x}_k, \bm{w})$\;
			$\bm{w} \leftarrow \bm{w}+\alpha_w\delta\nabla_{\bm{w}}\hat{v}(\bm{x}_k,\bm{w})$\;
			$\bm{\theta} \leftarrow \bm{\theta} + \alpha_{\theta} \gamma^k \delta \nabla_{\bm{\theta}}\ln \pi(\bm{u}_k|\bm{x}_k,\bm{\theta})$\;
		}
	}
\caption{Monte Carlo policy gradient with baseline (output: parameter vector $\bm{\theta}^*$ for $\pi^*(\bm{u}|\bm{x},\bm{\theta}^*)$) and $\bm{w}^*$ for $\hat{v}^*(\bm{x}, \bm{w}^*))$}
\label{algo:MC_policy_gradient_baseline}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% REINFORCE Comparison %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{REINFORCE Comparison w/o Baseline}
\begin{figure}
\includegraphics[width=11cm]{fig/lec12/REINFORCE_short_corridor_baseline.pdf}
\caption{Comparison of Monte Carlo policy gradient on short-corridor problem from \figref{fig:Short_Corridor_Problem} where both algorithms' learning rates have been tuned (source: R. Sutton and G. Barto, Reinforcement learning: an introduction, 2018, \href{https://creativecommons.org/licenses/by-nc-nd/2.0/}{CC BY-NC-ND 2.0})}
\label{fig:REINFORCE_short_corridor_baseline}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Actor-Critic Methods (Episodic Tasks)}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Table of Contents}
\tableofcontents[currentsection]
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% General Actor-Critic Idea %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{General Actor-Critic Idea}
Conclusion of Monte Carlo policy gradient with baseline:
\begin{itemize}
	\item Will learn an unbiased policy gradient.
	\item As the other MC-based methods: learns slowly due to high variance.
	\item Updates only available after full episodes.
\end{itemize}\pause
\vspace{0.5cm}
Alternative: use an additional function approximator, the so-called \hl{critic}, to estimate $q_{\pi}$ (i.e., approximate policy gradient):
\begin{equation*}
\begin{split}
	 v(\bm{x})&\approx \hat{v}(\bm{x}, \bm{w}_v),\\
	q(\bm{x},\bm{u})&\approx \hat{q}(\bm{x}, \bm{u}, \bm{w}_q),\\
	a(\bm{x},\bm{u})&\approx \hat{q}(\bm{x}, \bm{u}, \bm{w}_q) - \hat{v}(\bm{x}, \bm{w}_v).
\end{split}
\end{equation*}\pause
\begin{itemize}
  \item Realization: any prediction tool discussed so far (TD(0), LSTD,...).\pause
	\item Potential: we can use online step-by-step updates to estimate $\hat{q}$.\pause
	\item Disadvantage: we would train two value estimates by $\bm{w}_v$ and $\bm{w}_q$.
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Integrating the Advantage Function %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Integrating the Advantage Function}
\begin{itemize}
	\item The TD error is
\begin{equation}
	\delta_\pi = r + \gamma v_\pi(\bm{x}') - v_\pi(\bm{x}). \pause\vspace{-0.3cm}
\end{equation}
\item In expectation the TD error is equivalent to the advantage function
\begin{equation}
\begin{split}
	\El{\delta_\pi|\bm{x}, \bm{u}}{\pi} &= \El{r + \gamma v_\pi(\bm{x}')|\bm{x}, \bm{u}}{\pi} - v_\pi(\bm{x})\\
																			&= q_\pi(\bm{x},\bm{u})- v_\pi(\bm{x})\\
																			&= a_\pi(\bm{x},\bm{u}) .
\end{split}
\end{equation}\pause
\item Hence, the TD error can be used to calculate the policy gradient:
	\begin{equation}
	\nabla_{\bm{\theta}} J(\bm{\theta}) = \El{\delta_\pi \nabla_{\bm{\theta}}\ln \pi(\bm{u}|\bm{x},\bm{\theta})}{\pi}.
\end{equation} \pause\vspace{-0.3cm}
\item This results in requiring only one function parameter set:
	\begin{equation}
	\delta_\pi \approx r + \gamma \hat{v}_\pi(\bm{x}', \bm{w}) - \hat{v}_\pi(\bm{x}, \bm{w}).
\end{equation}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Actor-Critic Structure %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Actor-Critic Structure}
\begin{itemize}
	\item Critic (policy evaluation) and actor (policy improvement) can be considered another form of generalized policy iteration (GPI).
	\item Online and on-policy algorithm for discrete and continuous action spaces with built-in exploration by stochastic policy functions.
\end{itemize}
\begin{figure}
\includegraphics[width=9.5cm]{fig/lec12/Actor-Critic.pdf}
\caption{Simplified flow diagram of actor-critic-based RL}
\label{fig:Actor-Critic}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Algorithmic Implementation:Actor-Critic with TD(0) Targets %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Algo. Implementation: Actor-Critic with TD(0) Targets}
\begin{itemize}
	\item Analog to MC-based policy gradient optional discounting on the gradient updates is introduced.
\end{itemize}
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{a differentiable policy function $\pi(\bm{u}|\bm{x},\bm{\theta})$}
\Input{a differentiable state-value function $\hat{v}(\bm{x},\bm{w})$}
\Param{step sizes $\{\alpha_{w}, \alpha_{\theta}\}\in\left\{\mathbb{R}|0<\alpha<1\right\}$}
\Init{parameter vectors $\bm{w}\in\mathbb{R}^{\zeta}$ and $\bm{\theta}\in\mathbb{R}^d$ arbitrarily}
 \For{$j=1,2,\ldots,$ episodes}{
		initialize $\bm{x}_0$\;
		\For{$k=0,1,\ldots, T-1$ time steps}{
			apply $\bm{u}_k \sim \pi(\cdot|\bm{x}_k, \bm{\theta})$ and observe $\bm{x}_{k+1}$ and $r_{k+1}$\;
			$\delta \leftarrow r_{k+1} + \gamma \hat{v}(\bm{x}_{k+1}, \bm{w})- \hat{v}(\bm{x}_k, \bm{w})$\;
			$\bm{w} \leftarrow \bm{w}+\alpha_w\delta\nabla_{\bm{w}}\hat{v}(\bm{x}_k,\bm{w})$\;
			$\bm{\theta} \leftarrow \bm{\theta} + \alpha_{\theta} \gamma^k \delta \nabla_{\bm{\theta}}\ln \pi(\bm{u}_k|\bm{x}_k,\bm{\theta})$\;
		}
	}
\caption{Actor-critic for episodic tasks using TD(0) targets (output: parameter vector $\bm{\theta}^*$ for $\pi^*(\bm{u}|\bm{x},\bm{\theta}^*)$) and $\bm{w}^*$ for $\hat{v}^*(\bm{x}, \bm{w}^*))$}
\label{algo:actor_critic_TD0}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Actor-Critic Generalization %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Actor-Critic Generalization}
\begin{itemize}
	\item Using the TD(0) error as the target to train the critic is convenient. \pause
	\item However, the usual alternatives can be applied to train $\hat{v}(\bm{x}, \bm{w})$.\pause
	\item $n$-step bootstrapping:
	\begin{equation*}
		 v(\bm{x}_k) \approx r_{k+1}+\gamma r_{k+2}+\cdots+\gamma^{n-1} r_{k+n}+\gamma^{n} \hat{v}_{k+n-1}(\bm{x}_{k+n}, \bm{w}).
	\end{equation*}\pause\vspace{-0.3cm}
	\item $\lambda$-return (forward view):
	\begin{equation*}
		 v(\bm{x}_k) \approx(1-\lambda)\sum_{n=1}^{T-k-1}\lambda^{(n-1)}g_{k:k+n} + \lambda^{T-k-1}g_k.
	\end{equation*}\pause\vspace{-0.3cm}
	\item TD($\lambda$) using eligibility traces (backward view):
		\begin{equation*}
			\begin{split}
				\bm{z}_k &=\gamma\lambda\bm{z}_{k-1}+\nabla_{\bm{w}}\hat{v}(\bm{x}_k,\bm{w}_k),\\
				\delta_k &=r_{k+1} + \gamma \hat{v}(\bm{x}_{k+1}, \bm{w}_k) -\hat{v}(\bm{x}_{k}, \bm{w}_k).
			\end{split}
		\end{equation*}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Algorithmic Implementation:Actor-Critic with TD($\lambda$) Targets %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Algo. Implementation: Actor-Critic with TD($\lambda$) Targets}
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{a differentiable policy function $\pi(\bm{u}|\bm{x},\bm{\theta})$}
\Input{a differentiable state-value function $\hat{v}(\bm{x},\bm{w})$}
\Param{$\{\alpha_{w}, \alpha_{\theta}\}\in\left\{\mathbb{R}|0<\alpha<1\right\}$, $\{\lambda_{w}, \lambda_{\theta}\}\in\left\{\mathbb{R}|0 \leq \lambda \leq 1\right\}$}
\Init{parameter vectors $\bm{w}\in\mathbb{R}^{\zeta}$ and $\bm{\theta}\in\mathbb{R}^d$ arbitrarily}
 \For{$j=1,2,\ldots,$ episodes}{
		initialize $\bm{x}_0$, $\bm{z}_w=0$, $\bm{z}_{\theta}=0$\;
		\For{$k=0,1,\ldots, T-1$ time steps}{
			apply $\bm{u}_k \sim \pi(\cdot|\bm{x}_k, \bm{\theta})$ and observe $\bm{x}_{k+1}$ and $r_{k+1}$\;
			$\delta \leftarrow r_{k+1} + \gamma \hat{v}(\bm{x}_{k+1}, \bm{w})- \hat{v}(\bm{x}_k, \bm{w})$\;
			$\bm{z}_w \leftarrow \gamma\lambda_w\bm{z}_w + \nabla_{\bm{w}}\hat{v}(\bm{x}_k,\bm{w})$\;
			$\bm{z}_{\theta} \leftarrow \gamma\lambda_d\bm{z}_{\theta} + \gamma^k \nabla_{\bm{\theta}}\ln \pi(\bm{u}_k|\bm{x}_k,\bm{\theta})$\;
			$\bm{w} \leftarrow \bm{w}+\alpha_w\delta\bm{z}_w$\;
			$\bm{\theta} \leftarrow \bm{\theta} + \alpha_{\theta} \delta \bm{z}_{\theta}$\;
		}
	}
\caption{Actor-critic for episodic tasks using TD($\lambda$) targets (output: parameter vector $\bm{\theta}^*$ for $\pi^*(\bm{u}|\bm{x},\bm{\theta}^*)$) and $\bm{w}^*$ for $\hat{v}^*(\bm{x}, \bm{w}^*))$}
\label{algo:actor_critic_TD_lambda}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Compatible Function Approximation (1)%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Compatible Function Approximation (1)}
\begin{itemize}
	\item Most default prediction techniques will add a bias to $\hat{q}(\bm{w}) \approx q$.\pause
	\item A biased policy gradient may not find the right solution. \pause
	\item However, following the below theorem we can prevent any bias. \pause
\end{itemize}
\stepcounter{mytheo}
\begin{theo}{Compatible Function Approximation}{Comp_Fct_Approx}
If the value function approximator is compatible to the policy
\begin{equation}
	\hat{q}(\bm{x}, \bm{u}, \bm{w})= \left(\nabla_{\bm{\theta}} \ln \pi_{\bm{\theta}}(\bm{x}, \bm{u})\right)\T\bm{w}
\end{equation}
and the value function parameters $\bm{w}$ minimize the mean-squared error
\begin{equation}
	\bm{w}^* = \argmax_{\bm{w}}\,\, \E{q_{\pi}(\bm{x}, \bm{u}) - \hat{q}(\bm{x}, \bm{u}, \bm{w})}^2
\end{equation}
then the policy gradient using $\hat{q}(\bm{w})$ is exact.
\end{theo}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Compatible Function Approximation (2)%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Compatible Function Approximation (2)}
Interpretation:
\begin{itemize}
	\item The policy gradient $\nabla_{\bm{\theta}} \ln \pi_{\bm{\theta}}(\bm{x}, \bm{u})$ delivers input features for a linear mapping of the value function approximation:
	\begin{equation*}
	\hat{q}(\bm{x}, \bm{u}, \bm{w})= \left(\nabla_{\bm{\theta}} \ln \pi_{\bm{\theta}}(\bm{x}, \bm{u})\right)\T\bm{w}.
\end{equation*}\pause
 \item The unknown parameters $\bm{w}$ are the solution to a linear regression problem estimating $q_{\pi}(\bm{x}, \bm{u})$:
\begin{equation*}
	\bm{w}^* = \argmax_{\bm{w}}\,\, \E{q_{\pi}(\bm{x}, \bm{u}) - \hat{q}(\bm{x}, \bm{u}, \bm{w})}^2.
\end{equation*}\pause
	\item The latter condition may be solved using a batch LSTD approach or relaxed in favor of policy evaluation algorithms using TD learning.
\end{itemize}

}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Actor-Critic Methods (Continuing Tasks)}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Table of Contents}
\tableofcontents[currentsection]
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Introducing Average Rewards for Continuing Tasks %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Introducing Average Rewards for Continuing Tasks}
\begin{itemize}
	\item For continuing tasks the differential reward is used as the performance
\begin{equation}
	\begin{split}
		J(\bm{\theta})=\overline{r}_\pi &= \lim_{h\rightarrow\infty}\frac{1}{h}\sum_{k=1}^{h}\E{R_k|\bm{X}_0, \bm{U}_{0:k-1}\sim\pi},\\
		&=\lim_{k\rightarrow\infty}\E{R_k|\bm{X}_0, \bm{U}_{0:k-1}\sim\pi}.
	\end{split}
\end{equation}\pause\vspace{-0.3cm}
	\item Consequently, the value definitions are adapted using the differential return:
\begin{equation}
		\begin{split}
				G_k &= R_{k+1}-\overline{r}_\pi + R_{k+2}-\overline{r}_\pi+R_{k+3}-\overline{r}_\pi+\ldots,\\
				v_\pi(\bm{x})&=\El{G_k|\bm{X}_k=\bm{x}}{\pi},\\
				q_\pi(\bm{x}, \bm{u}) &= \El{G_k|\bm{X}_k=\bm{x},\bm{U}_k=\bm{u}}{\pi}.
		\end{split}
		\label{eq:diff_values}
	\end{equation}\pause\vspace{-0.3cm}
\item With these modifications the policy gradient theorem \eqref{eq:policy_gradient_theo} still holds. Although we do not have to consider discounting anymore.
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Actor-Critic with Differential Targets %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Actor-Critic with Differential Targets}
\begin{itemize}
	\item The episodic actor-critic approach can be directly transferred to continuing tasks. \pause
	\item The critic is estimating the differential state value from \eqref{eq:diff_values}:
	\begin{equation}
		 v_\pi(\bm{x}) \approx \hat{v}(\bm{x},\bm{w}).
	\end{equation}
	\vspace{-0.5cm}
	\begin{itemize}
		\item The target as the basis for estimating the state value is again flexible: TD(0), TD($\lambda$),....
	\end{itemize}\pause
	\item The differential TD error is also approximated straightforward
	\begin{equation}
		 \delta_\pi \approx r -\hat{\overline{r}} + \hat{v}(\bm{x}',\bm{w}) - \hat{v}(\bm{x},\bm{w}).
	\end{equation}\pause\vspace{-0.3cm}
	\item The policy parameter update is then:
	\begin{equation}
		 \bm{\theta}_{k+1} = \bm{\theta}_k + \alpha \delta_k \nabla_{\bm{\theta}} \ln \pi(\bm{u}_k|\bm{x}_k,\bm{\theta}_k).
	\end{equation}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Algorithmic Implementation:Actor-Critic with Diff. TD(0) Targets %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Implementation: Actor-Critic with Diff. TD(0) Targets}
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{a differentiable policy function $\pi(\bm{u}|\bm{x},\bm{\theta})$}
\Input{a differentiable state-value function $\hat{v}(\bm{x},\bm{w})$}
\Param{step sizes $\{\alpha_{w}, \alpha_{\theta}, \beta \}\in\left\{\mathbb{R}|0<\alpha, \beta <1\right\}$}
\Init{parameter vectors $\bm{w}\in\mathbb{R}^{\zeta}$ and $\bm{\theta}\in\mathbb{R}^d$ arbitrarily}
\Init{avg. return estimate $\hat{\overline{r}}\in\mathbb{R}$, starting state $\bm{x}_{0}$}
\For{$k=0,1,\ldots$ time steps}{
		apply $\bm{u}_k \sim \pi(\cdot|\bm{x}_k, \bm{\theta})$ and observe $\bm{x}_{k+1}$ and $r_{k+1}$\;
		$\delta \leftarrow r_{k+1} -\hat{\overline{r}} + \hat{v}(\bm{x}_{k+1}, \bm{w}) - \hat{v}(\bm{x}_k, \bm{w})$\;
		$\hat{\overline{r}}\leftarrow\hat{\overline{r}}+\beta\delta$\;
		$\bm{w} \leftarrow \bm{w}+\alpha_w\delta\nabla_{\bm{w}}\hat{v}(\bm{x}_k,\bm{w})$\;
		$\bm{\theta} \leftarrow \bm{\theta} + \alpha_{\theta} \delta \nabla_{\bm{\theta}}\ln \pi(\bm{u}_k|\bm{x}_k,\bm{\theta})$\;
}
\caption{Actor-critic for continuing tasks using diff. TD(0) targets (output: parameter vector $\bm{\theta}^*$ for $\pi^*(\bm{u}|\bm{x},\bm{\theta}^*)$) and $\bm{w}^*$ for $\hat{v}^*(\bm{x}, \bm{w}^*))$}
\label{algo:actor_critic_TD0_cont}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Algorithmic Implementation:Actor-Critic with Diff. TD($\lambda$) Targets %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Implementation: Actor-Critic with Diff. TD($\lambda$) Targets}
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\small
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{a differentiable policy function $\pi(\bm{u}|\bm{x},\bm{\theta})$}
\Input{a differentiable state-value function $\hat{v}(\bm{x},\bm{w})$}
\Param{step sizes $\{\alpha_{w}, \alpha_{\theta}, \beta \}\in\left\{\mathbb{R}|0<\alpha, \beta <1\right\}$}
\Param{trace decay rates $\{\lambda_{w}, \lambda_{\theta}\}\in\left\{\mathbb{R}|0 \leq \lambda \leq 1\right\}$}
\Init{parameter vectors $\bm{w}\in\mathbb{R}^{\zeta}$ and $\bm{\theta}\in\mathbb{R}^d$ arbitrarily}
\Init{avg. return estimate $\hat{\overline{r}}\in\mathbb{R}$, starting state $\bm{x}_{0}$, $\bm{z}_w=0$, $\bm{z}_{\theta}=0$}
\For{$k=0,1,\ldots$ time steps}{
		apply $\bm{u}_k \sim \pi(\cdot|\bm{x}_k, \bm{\theta})$ and observe $\bm{x}_{k+1}$ and $r_{k+1}$\;
		$\delta \leftarrow r_{k+1} -\hat{\overline{r}} + \hat{v}(\bm{x}_{k+1}, \bm{w}) - \hat{v}(\bm{x}_k, \bm{w})$\;
		$\hat{\overline{r}}\leftarrow\hat{\overline{r}}+\beta\delta$\;
		$\bm{z}_w \leftarrow \lambda_w\bm{z}_w + \nabla_{\bm{w}}\hat{v}(\bm{x}_k,\bm{w})$\;
		$\bm{z}_{\theta} \leftarrow \lambda_d\bm{z}_{\theta} + \nabla_{\bm{\theta}}\ln \pi(\bm{u}_k|\bm{x}_k,\bm{\theta})$\;
		$\bm{w} \leftarrow \bm{w}+\alpha_w\delta\bm{z}_w$\;
		$\bm{\theta} \leftarrow \bm{\theta} + \alpha_{\theta} \delta \bm{z}_{\theta}$\;
}
\caption{Actor-critic for continuing tasks using differential TD($\lambda$) targets with eligibility traces (output: parameter vector $\bm{\theta}^*$ for $\pi^*(\bm{u}|\bm{x},\bm{\theta}^*)$) and $\bm{w}^*$ for $\hat{v}^*(\bm{x}, \bm{w}^*))$}
\label{algo:actor_critic_TD_lambda_cont}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Deterministic Gradient Policy}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Table of Contents}
\tableofcontents[currentsection]
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Background and Motivation %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Background and Motivation}
Recap on policy gradient so far:
\begin{itemize}
	\item The previously discussed policy functions and the policy gradient theorem were assuming stochastic polices. \pause
	\item The resulting on-policy algorithms may not provide top-class learning performance: \pause
	\begin{itemize}
		\item Non-guided exploration with step-by-step updates and
		\item Greedy actions only in the limit (i.e., infeasible long learning).
	\end{itemize}\pause
\end{itemize}
\vspace{0.5cm}
The alternative:
\begin{itemize}
	\item Apply a deterministic policy with separate exploration. \pause
	\item Enable off-policy learning (with experience replay as a possible extension). \pause
	\item Hence, we will focus on a \hl{deterministic policy function}
	\begin{equation}
		\pi(\bm{x}, \bm{\theta}) = \mu(\bm{x}, \bm{\theta}) .
	\end{equation}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Deterministic Policy Gradient Theorem %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Deterministic Policy Gradient (DPG) Theorem}
\begin{theo}{Deterministic Policy Gradient}{determ_policy_gradient}
Given a metric $J(\bm{\theta})$ for the undiscounted episodic \eqref{eq:performance_metric_episodic} or continuing tasks \eqref{eq:performance_metric_continuing} and  a parameterizable policy $\mu(\bm{x},\bm{\theta})$ the deterministic policy gradient is
\begin{equation}
	\nabla_{\bm{\theta}} J(\bm{\theta}) = \El{\nabla_{\bm{\theta}} \mu(\bm{x},\bm{\theta}) \nabla_{\bm{u}} q(\bm{x},\bm{u})\vert_{\bm{u}=\mu(\bm{x})}}{\mu}.
	\label{eq:policy_gradient_theo_det}
\end{equation}
\end{theo}\pause
\begin{itemize}
\item Again, $q$ needs to be approximated using samples, e.g., implementing a critic via TD learning.\pause
\item Bias problem applies as in the stochastic case and can be compensated using compatible function approximation w.r.t. $\hat{q}$.\pause
\item It turns out that \eqref{eq:policy_gradient_theo_det} is also (approximately) valid in the off-policy case, i.e., if the sample distribution is obtained from a behavior policy.\pause
\item Proof can be found in \href{http://proceedings.mlr.press/v32/silver14.pdf}{D. Silver et al., \textit{Deterministic Policy Gradient Algorithms}, International Conference on Machine Learning, 2014}

\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Exploration %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Exploration with a Deterministic Policy}
\begin{itemize}
	\item If the DPG approach is applied on-policy there is no inherent exploration.
	\item How to learn something?\pause
	\begin{itemize}
		\item The environment itself is sufficiently noisy (random impacts, measurement noise).\pause
		\item Or we have to add noise to the actions, i.e., making the approach off-policy.\pause
		\item Hence, utilizing a behavior policy is also possible.\pause
	\end{itemize}
	\item That additional action noise could be:
	\begin{itemize}
		\item Simple Gaussian noise or\pause
		\item a shaped noise process like a discrete-time Ornstein-Uhlenbeck (OU) process
		\begin{equation*}
			\nu_{k+1} = \lambda \nu_k+ \sigma \epsilon_k
		\end{equation*}
		where $\nu_k$ is the OU noise output, $0 < \lambda < 1$ is a  smoothing factor and $\sigma$ is the variance scaling a standard Gaussian sequence (no mean) $\epsilon_k$.
	\end{itemize}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Algorithmic Implementation: Deterministic Actor-Critic %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Algo. Implementation: Deterministic Actor-Critic (1)}
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\small
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{a differentiable deterministic policy function $\mu(\bm{x},\bm{\theta})$}
\Input{a differentiable action-value function $\hat{q}(\bm{x},\bm{u},\bm{w})$}
\Param{step sizes $\{\alpha_{w}, \alpha_{\theta}\}\in\left\{\mathbb{R}|0<\alpha<1\right\}$}
\Init{parameter vectors $\bm{w}\in\mathbb{R}^{\zeta}$ and $\bm{\theta}\in\mathbb{R}^d$ arbitrarily}
 \For{$j=1,2,\ldots,$ episodes}{
		initialize $\bm{x}_0$\;
		\For{$k=0,1,\ldots, T-1$ time steps}{
			$\bm{u}_k \leftarrow$ apply from $\mu(\bm{x}_k, \bm{\theta})$ w/wo noise or from behavior policy\;
			observe $\bm{x}_{k+1}$ and $r_{k+1}$\;
			choose $\bm{u}'$ from $\mu(\bm{x}_{k+1}, \bm{\theta})$\;
			$\delta \leftarrow r_{k+1} + \gamma \hat{q}(\bm{x}_{k+1}, \bm{u}', \bm{w})- \hat{q}(\bm{x}_k, \bm{u}_k, \bm{w})$\;
			$\bm{w} \leftarrow \bm{w}+\alpha_w\delta\nabla_{\bm{w}}\hat{q}(\bm{x}_k, \bm{u}_k, \bm{w})$\;
			$\bm{\theta} \leftarrow \bm{\theta} + \alpha_{\theta} \gamma^k \nabla_{\bm{\theta}}\mu(\bm{x}_k,\bm{\theta})\nabla_{\bm{u}}\hat{q}(\bm{x}_k, \bm{u}_k, \bm{w})\vert_{\bm{u}=\mu(\bm{x})}$\;
		}
	}
\caption{Deterministic actor-critic for episodic tasks using Sarsa(0) targets applicable on- and off-policy (output: parameter vector $\bm{\theta}^*$ for $\mu^*(\bm{x},\bm{\theta}^*)$) and $\bm{w}^*$ for $\hat{q}^*(\bm{x}, \bm{u}, \bm{w}^*))$}
\label{algo:det_actor_critic_Sarsa0_episode}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Algorithmic Implementation: Deterministic Actor-Critic %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Algo. Implementation: Deterministic Actor-Critic (2)}
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{a differentiable deterministic policy function $\mu(\bm{x},\bm{\theta})$}
\Input{a differentiable action-value function $\hat{q}(\bm{x},\bm{u},\bm{w})$}
\Param{step sizes $\{\alpha_{w}, \alpha_{\theta}, \beta \}\in\left\{\mathbb{R}|0<\alpha, \beta <1\right\}$}
\Init{parameter vectors $\bm{w}\in\mathbb{R}^{\zeta}$ and $\bm{\theta}\in\mathbb{R}^d$ arbitrarily}
\Init{avg. return estimate $\hat{\overline{r}}\in\mathbb{R}$, starting state $\bm{x}_{0}$}
\For{$k=0,1,\ldots$ time steps}{
		$\bm{u}_k \leftarrow$ apply from $\mu(\bm{x}_k, \bm{\theta})$ w/wo noise or from behavior policy\;
		observe $\bm{x}_{k+1}$ and $r_{k+1}$\;
		choose $\bm{u}'$ from $\mu(\bm{x}_{k+1}, \bm{\theta})$\;
		$\delta \leftarrow r_{k+1} -\hat{\overline{r}} + \hat{q}(\bm{x}_{k+1}, \bm{u}', \bm{w})- \hat{q}(\bm{x}_k, \bm{u}_k, \bm{w})$\;
		$\hat{\overline{r}}\leftarrow\hat{\overline{r}}+\beta\delta$\;
		$\bm{w} \leftarrow \bm{w}+\alpha_w\delta\nabla_{\bm{w}}\hat{q}(\bm{x}_k,\bm{u}_k, \bm{w})$\;
		$\bm{\theta} \leftarrow \bm{\theta} + \alpha_{\theta} \nabla_{\bm{\theta}}\mu(\bm{x}_k,\bm{\theta})\nabla_{\bm{u}}\hat{q}(\bm{x}_k, \bm{u}_k, \bm{w})\vert_{\bm{u}=\mu(\bm{x})}$\;
		}
\caption{Deterministic actor-critic for continuing tasks using Sarsa(0) targets applicable on- and off-policy (output: parameter vector $\bm{\theta}^*$ for $\mu^*(\bm{x},\bm{\theta}^*)$) and $\bm{w}^*$ for $\hat{q}^*(\bm{x}, \bm{u}, \bm{w}^*))$}
\label{algo:det_actor_critic_Sarsa0_cont}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Exemplary Comparison to Stochastic Policy Gradient %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Exemplary Comparison to Stochastic Policy Gradient}
\begin{itemize}
	\item DPG-based approach uses compatible function approximation, i.e., suitable linear $\hat{q}$ estimation. A fixed Gaussian behavior policy is applied for exploration.
	\item SAC uses a Gaussian policy with linear function approximation.
\end{itemize}
\begin{figure}
\includegraphics[width=11cm]{fig/lec12/SLH_Benchmarks.pdf}
\caption{Comparison of stochastic on-policy actor-critic (SAC), stochastic off-policy actor-critic (OffPAC), and deterministic off-policy
actor-critic (COPDAC) on continuous-action reinforcement learning (source: D. Silver et al., \textit{Deterministic Policy Gradient Algorithms}, International Conference on Machine Learning, 2014)}
\label{fig:SLH_Benchmarks}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Summary %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Summary: What You've Learned Today}
\begin{itemize}
	\item Policy-based methods are a new class within the RL toolbox.
	\begin{itemize}
		\item Instead of learning a policy indirectly from a value the policy is directly parametrized.\pause
		\item The policy function allows discrete and continuous actions with inherent stochastic exploration.\pause
	\end{itemize}
	\item Solving the underlying optimization task is complex. However, the policy gradient theorem provides a suitable theoretical baseline for gradient-based optimization.\pause
	\item Anyhow, to calculate policy gradients we require a value estimate.
	\begin{itemize}
		\item Monte Carlo prediction is straightforward, but comes with high variance and slow learning.\pause
		\item Adding a state-dependent baseline comparison does not change the policy gradient in expectation but enables decreasing the variance.\pause
	\end{itemize}
	\item Extending this idea naturally leads to integrating a critic network, i.e., an additional function approximation to estimate the value.\pause
	\item The critic can be fed by the usual targets (TD(0), TD($\lambda$),...).
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Final Slide %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{The End for Today}
\vspace{-0.25cm}
\begin{figure}
\hspace*{-0.5cm}
\includegraphics[width=11cm]{fig/lec12/dilbert.jpg}
\end{figure}
\vspace{1cm}
\centering
Thanks for your attention and have a nice week!
}


\chapter{clean} \label{clean}

\section{Introduction}

An LPE may contain redundant information, for example because some of its summands contain all behavior of other summands or because some of its summands will never be (or become) enabled.
Obviously, such summands can be removed without hesitation, which may result in a speedup in other computations in which the LPE is input.

The \texttt{clean} command attempts to detect redundant summands and removes the ones that it finds.
The state space of the LPE is not affected by the changes.

\section{Formal background}

\subsection{Summand containment}

Consider two summands, $s_\alpha$ and $s_\beta$, and reference their elements conform \ref{summandelements}.

Summand $s_\alpha$ is said to \emph{contain} $s_\beta$ if these two conditions hold:

\begin{itemize}
\item $s_\alpha$ and $s_\beta$ must communicate over the exact same channel with exactly as many channel variables; that is, $C_\alpha = C_\beta \land m_\alpha = m_\beta$.

\item Define the mapping
\begin{align*}
X_\alpha = [x_\beta(j) \rightarrow x_\alpha(j) \;|\; 1 \leq j \leq \text{min}(m_1, m_2)]
\end{align*}

The following conditions must also hold:
\begin{align*}
g_\beta[X_\alpha] &\rightarrow g_\alpha \\
v_\alpha(p) = v_\beta(p)[X_\alpha] &\leftrightarrow \text{\textit{True}} \text{ for all } p \in [p_1, \cdots{}, p_k]
\end{align*}

The first condition demands that whenever $s_\beta$ is enabled, $s_\alpha$ must also be enabled; consequently, $s_\alpha$ is `always ready' to perform the same behavior as $s_\beta$.
The second condition demands that the state after the application of $s_\alpha$ is the same as the state after the application of $s_\beta$, regardless of the values of communication variables and LPE parameters.
\end{itemize}

Note that $s_\alpha$ could contain all behavior of $s_\beta$ without these two conditions being true (we under-approximate by tolerating false negatives)!

\subsection{Summand reachability}

Consider summands $s_\alpha$ and $s_\beta$, referencing their elements conform \ref{summandelements}.
Summand $s_\alpha$ is said to be a \emph{possible predecessor} of $s_\beta$ if the following expression \emph{could be} satisfiable:
\begin{align*}
g_\alpha \land {g_\beta}[v \rightarrow q(v) \;|\; v \in \varsof{g_\beta} \setminus P][p \rightarrow v_\alpha(p) \;|\; p \in P]
\end{align*}

where $q(v)$ is a bijective function that relates variable $v$ to a fresh variable.

A summand $s_\beta$ is said to be \emph{reachable} if at least one of these conditions holds:

\begin{itemize}
\item It is possible that summand $s_\beta$ is enabled in the initial state of the LPE.
Formally, $s_\beta$ is reachable if the LPE is initialized as
\begin{align*}
P(v_I(p_1), \cdots{}, v_I(p_m))
\end{align*}

and if
\begin{align*}
g_\beta[p \rightarrow v_I(p) \;|\; p \in P]
\end{align*}

\emph{could be} satisfiable.

\item There exists at least one other summand $s_\alpha$ in the same LPE that is a possible predecessor of $s_\beta$.
It is \emph{not} sufficient if the only possible predecessor of $s_\beta$ is $s_\beta$ itself!
\end{itemize}

Summand reachability is \emph{over-approximated}.
For example, if it is uncertain whether a summand is enabled in the initial state, the summand is considered to be reachable.

\section{Algorithm}

The algorithm consists of 2 phases.

In the first phase, the algorithm does a pairwise comparison of all summands of the LPE.
If it discovers that some summand $s_1$ contains another summand $s_2$, it removes $s_2$ from the LPE.

In the second phase, the algorithm starts by determining which summands are reachable in the initial state of the LPE (the first condition for reachability in the previous section).
This becomes the initial value of the set $X$.
Then, until a fixpoint is reached, to $X$ all summands are added of which there exists a possible predecessor in $X$.
The output LPE of the algorithm contains all summands that are in the fixpoint of $X$.

\section{Example}

Consider the following LPE:

\begin{lstlisting}
//Process definition:
PROCDEF example[A :: Int](x :: Int)
  = A ? i [[x==0]] >-> example[A](1)
  + A ? j [[x==1]] >-> example[A](2)
  + A ? k [[x==1]] >-> example[A](2)
  + A ? l [[x==2]] >-> example[A](0)
  + A ? m [[x>=2]] >-> example[A](0)
  ;

//Initialization:
example[A](0);
\end{lstlisting}

The \texttt{clean} command will detect that the second and third summands of the LPE are equivalent and remove one of them.
It will also detect that the fifth summand contains the fourth summand, and therefore remove the fourth summand.

Now consider

\begin{lstlisting}
//Process definition:
PROCDEF example[A :: Int, B](x :: Int)
  = A ? i [[i != 13]] >-> example[A, B](i)
  + B [[x == 13]] >-> example[A, B](x)
  ;

//Initialization:
example[A, B](0);
\end{lstlisting}

The second summand is unreachable: it is not enabled in the initial state (since \texttt{x = 0}) and it is never enabled after the application of the first summand (since \texttt{i != 13}).

\section{Benchmark results}

The following durations were measured with a benchmark for several models:
\begin{itemize}
\item The average duration of \txs{} to make 500 steps in a model without converting it to LPE form or applying LPE operations;
\item The average duration of \txs{} to make 500 steps in a model after it has been converted to LPE form;
\item The average duration of \txs{} to make 500 steps in a model after it has been converted to LPE form and after the \texttt{clean} operation has been applied.
\end{itemize}

When plotting the second series of measurements against the first (see Figure~\ref{lpe-only-vs-original:fig}), the following conclusions can be drawn:
\begin{itemize}
\item The effect of the LPE transformation on performance is negative in most cases.
\item The effect of the LPE transformation on performance is positive in some cases, but not dramatically so.
\item For one model (\texttt{Adder3}), the effect of the LPE transformation is extremely positive (5.2 seconds instead of 12.5).
Unfortunately, this model is not very realistic since it simply consists of three parallel processes without any synchronization.
\end{itemize}

\begin{figure}[!ht]
\begin{center}
\includegraphics[width=0.8\linewidth]{charts/lpe-only-vs-original}
\caption{Benchmark results: LPE transformation vs original}
\label{lpe-only-vs-original:fig}
\end{center}
\end{figure}

It is possible that the \texttt{clean} operation has a significant positive influence on the performance of \txs{}.
Based on a plot of the third series of measurements against the second (see Figure~\ref{clean-vs-lpe-only:fig}), however, this cannot be concluded: only part of the models show slight (but significant) improvements, whereas other models suffer an (also slight) additional performance loss.

\begin{figure}[!ht]
\begin{center}
\includegraphics[width=0.5\linewidth]{charts/clean-vs-lpe-only}
\caption{Benchmark results: clean vs LPE transformation}
\label{clean-vs-lpe-only:fig}
\end{center}
\end{figure}


\documentclass[conference]{IEEEtran}

\input{../../utils/header_final.tex}

\begin{document}

\title{Machine Learning Project Proposal}
\author{
\IEEEauthorblockN{William Clark}
\IEEEauthorblockA{will.clark@chicagobooth.edu\\
University of Chicago Booth School of Business}
\and
\IEEEauthorblockN{Matthew DeLio}
\IEEEauthorblockA{mdelio@chicagobooth.edu\\
University of Chicago Booth School of Business}}

\maketitle

\section{Summary}
For our final project we will enter a \href{https://www.kaggle.com/c/facial-keypoints-detection}{Kaggle contest} called Facial Keypoints Detection. The goal of the contest is to predict the $(x,y)$ coordinates of the following facial keypoints for a given set of portrait pictures:
\begin{itemize}
    \item center of each eye (2),
    \item inner and outer corner of each eye (4),
    \item end of each eyebrow (4),
    \item tip of nose (1),
    \item corners of mouth (2), and
    \item center of each lip (2),
\end{itemize}
for a total of 15 features per face. Our submission will be scored by the average root mean square error (RMSE) across all keypoints and images.

\section{Data Set}
Our data are a set of \num{8832} 96-by-96 grayscale images of tightly cropped portraits (sample images are shown in \cref{fig:sample_faces}). The images have been vectorized and each pixel is represented by its grayscale intensity on a $(0,255)$ scale. The $(x,y)$ coordinates of all facial keypoints are given, so there are 30 variables to predict for each image. Our models/algorithms will be tested on a testing set of \num{1783} similarly formatted images.
\begin{figure}[!ht]
    \centering
    \includegraphics[width=3.25in]{sample_faces.pdf}
    \caption{Sample Faces in Training Data Set}
    \label{fig:sample_faces}
\end{figure}

\section{Project Description}
This project can be thought of in two discrete parts: (1) image pre-processing to reduce the dimensionality of our data set and make our modeling efforts more robust, and (2) a predictive modeling exercise to identify facial keypoints.

\subsection{Image Processing}
A common practice in other computer vision studies is to use principal components analysis to reduce the dimensionality of the images. In most cases, nearly all the variation across images can be explained with a fraction of the original data set. Rather than defining each image as a vector of length $p=96\cdot96$, we can express it as a linear combination of the first $x$ principal components where $x \ll p$. This process should reduce the noise in each vectorized image and increase the accuracy of the predictive algorithms we need to train and tune.

In addition to PCA, we will also employ edge-detection to help us better identify the key components needed to perform the facial keypoint detection.  Some of the filtering kernels we have already employed on the data-set can be seen in \cref{fig:sample_faces} (see the second and third lines).  We may end up combining this with neural network to create a multi-stage convolutional neural network.  Unfortunately, since h2o does not support this, we may need to research some other languages (like python) with appropriate libraries/packages.  We may explore the techniques used in the LeNet-5 by Yann LeCun as he has a \href{http://deeplearning.net/tutorial/lenet.html}{tutorial} on how to use Python to do something similar to what we're trying to do.

\subsection{Predictive Modeling}
This project requires training and tuning 30 different predictive algorithms/models, one for each $x$ and $y$ coordinate of all 15 facial features. The following list of models/algorithms is neither exclusive nor exhaustive, but represents our best current estimate of what the final paper might include:
\begin{itemize}
    \item linear regression with L1/L2 penalization,
    \item nearest neighbor searches,
    \item regression trees and bagged regression trees,
    \item random forests,
    \item boosting trees, and
    \item neural networks.
\end{itemize}
With this variety of forecasting models, we could make our prediction based on the best performing model or best performing set of models (such that our final algorithm is an ensemble of some combination of listed models).

\end{document}


% Created 2019-11-12 mar 23:36
\documentclass[presentation,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{fixltx2e}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{float}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{marvosym}
\usepackage{wasysym}
\usepackage{amssymb}
\usepackage{hyperref}
\tolerance=1000
\usepackage{khpreamble}
\usepackage{xcolor}
\newcommand{\sign}{\mathrm{sign}}
\renewcommand{\transp}{^{\mathrm{T}}}
\usetheme{default}
\author{Kjartan Halvorsen}
\date{2019-11-13}
\title{Robust Kalman filter}
\hypersetup{
  pdfkeywords={},
  pdfsubject={},
  pdfcreator={Emacs 25.3.50.2 (Org mode 8.2.10)}}
\begin{document}

\maketitle
\section{Introduction}
\label{sec-1}
\begin{frame}[label=sec-1-1]{Why a robust version of the Kalman filter?}
\end{frame}
\begin{frame}[label=sec-1-2]{Why a robust version of the Kalman filter?}
The Kalman filter assumes Gaussian measurement noise and so it is very sensitive to outliers.
\end{frame}
\begin{frame}[label=sec-1-3]{Example 1}
\begin{columns}
\begin{column}{0.3\textwidth}
The target moves in a circle. Observations are noisy with one outlier
\end{column}
\begin{column}{0.7\textwidth}
\begin{center}
\includegraphics[width=0.9\linewidth]{circular-movement-crop}
\end{center}
\end{column}
\end{columns}
\end{frame}
\begin{frame}[label=sec-1-4]{Example 1 contd.}
The model of the dynamics: \emph{Nearly constant velocity model}
\begin{equation*}
x(k+1) = \bbm I & hI\\ 0 & I \ebm x(k) + \bbm \frac{h^2}{2} I\\hI\ebm v(k),
\end{equation*}
where the state vector contains the position and velocity of the target
\[ x = \bbm p\\\dot{p} \ebm.\]
\end{frame}
\begin{frame}[label=sec-1-5]{Example 1 contd.}
\begin{columns}
\begin{column}{0.3\textwidth}
Result of tracking using standard Kalman filter
\end{column}
\begin{column}{0.7\textwidth}
\begin{center}
\includegraphics[width=0.9\linewidth]{circular-movement-kf-crop}
\end{center}
\end{column}
\end{columns}
\end{frame}
\section{Convex optimization}
\label{sec-2}
\begin{frame}[label=sec-2-1]{Convex optimization}
\begin{center}
\includegraphics[width=0.7\linewidth]{cvx-book.png}
\end{center}
\end{frame}

\begin{frame}[label=sec-2-2]{Preparation example}
\begin{columns}
\begin{column}{0.4\textwidth}
Linear regression model
\begin{equation*}
y(k) = ax(k) + b + e(k) + w(k),
\end{equation*}
where $e(k)$ is Gaussian noise and $w(k)$ is a sparse vector of outliers.
\end{column}
\begin{column}{0.6\textwidth}
\begin{center}
\includegraphics[width=\linewidth]{least_squares_example}
\end{center}
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label=sec-2-3]{Preparation example, contd}
Least squares estimation:
\begin{equation*}
 \text{minimize} \; ||y - ax - b||_2\\
\end{equation*}
Or, equivalently
\begin{align*}
 \text{minimize} \; & ||\epsilon||_2\\
 \text{subject to} \; & \epsilon = y - ax-b
\end{align*}
\end{frame}

\begin{frame}[label=sec-2-4]{Preparation example, contd}
Least squares estimation:
\begin{equation*}
 \text{minimize} \; ||y - ax - b||_2\\
\end{equation*}
Solved by forming
\[ A = \bbm x(1) & 1\\x(2) & 1\\ \vdots & \vdots\\ x(N) & 1\ebm \]
and
\[ z = \bbm a\\b\ebm, \]
and solving for $z$ in the (over-determined) system of equations
\[ Az = y. \]
\end{frame}

\begin{frame}[label=sec-2-5]{Preparation example, contd}
\begin{columns}
\begin{column}{0.4\textwidth}
\begin{equation*}
 \text{minimize} \; ||y - ax - b||_2\\
\end{equation*}
\end{column}
\begin{column}{0.6\textwidth}
\begin{center}
\includegraphics[width=\linewidth]{least_squares_regression}
\end{center}
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label=sec-2-6]{The problem with least squares}
\begin{columns}
\begin{column}{0.4\textwidth}
\begin{align*}
 \text{minimize} \; &\sum_k \phi_{S}(\epsilon_k)\\
 \text{where} \; \phi_S(u) &= u^2
\end{align*}
\end{column}

\begin{column}{0.6\textwidth}

\begin{center}
  \begin{tikzpicture}
    \begin{axis}[
      width=8cm,
      height=6cm,
      ylabel=penalty,
      xlabel=residual,
      ]
      \addplot[red, thick, no marks, domain=-4:4, samples=201] {x^2};
    \end{axis}
  \end{tikzpicture}
\end{center}
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label=sec-2-7]{More robust: The Huber penalty function}
\begin{columns}
\begin{column}{0.4\textwidth}
Also known as \alert{robust least squares}
\begin{align*}
 \text{minimize} \; &\sum_k \phi_{hub}(\epsilon_k)\\
 \text{where}\; \phi_{hub}(u) &= \begin{cases} u^2 & |u| \le M\\ M(2|u|-M) & |u| > M \end{cases}
\end{align*}
\end{column}

\begin{column}{0.6\textwidth}
\begin{center}
  \begin{tikzpicture}
    \begin{axis}[
      width=8cm,
      height=6cm,
      ylabel=penalty,
      xlabel=residual,
      ]
      \addplot[red, thick, no marks, domain=-4:4, samples=201] {x^2};
      \addplot[orange!90!black, thick, no marks, domain=-4:-1, samples=201] {2*abs(x)-1};
      \addplot[orange!90!black, thin, no marks, domain=-1:1, samples=201] {x^2};
      \addplot[orange!90!black, thick, no marks, domain=1:4, samples=201] {2*abs(x)-1};
    \end{axis}
  \end{tikzpicture}
\end{center}
\end{column}
\end{columns}
\end{frame}
\begin{frame}[label=sec-2-8]{Preparation example: Robust least squares}
\begin{columns}
\begin{column}{0.4\textwidth}
\begin{equation*}
 \text{minimize} \; &\sum_k \phi_{hub}(\epsilon_k)\\
\end{equation*}
\end{column}
\begin{column}{0.6\textwidth}
\begin{center}
\includegraphics[width=\linewidth]{robust_least_squares_regression}
\end{center}
\end{column}
\end{columns}
\end{frame}

\section{Robust Kalman Filter}
\label{sec-3}
\begin{frame}[label=sec-3-1]{Robustifying the Kalman filter}
\end{frame}
\begin{frame}[label=sec-3-2]{The measurement update of the Kalman filter}
We have the state space model
\begin{align*}
x(k+1) &= Fx(k) + v(k)\\
y(k) &= Hx(k) + e(k) + z(k)\\
e &\sim \mathcal{N}(0,R)\\
v &\sim \mathcal{N}(0,Q)
\end{align*}
The measurement update of the Kalman filter can be shown to be equivalent to solving the problem
\[ \text{minimize} \; (y-Hx)^{\mathrm{T}} R^{-1}(y-Hx) + (x-\hat{x}_{k|k-1})\transp P_{k|k-1}^{-1}(x-\hat{x}_{k|k-1}) \]
The optimal solution is \( x^*=\hat{x}_{k|k} = \hat{x}_{k|k} + K(y-H\hat{x}_{k|k-a}),\) where $K$ is the Kalman gain.
\end{frame}
\begin{frame}[label=sec-3-3]{The measurement update of the Kalman filter}
Introduce $\tilde{x} = x - \hat{x}_{k|k-1}$ and $\tilde{y} = y - H\hat{x}_{k|k-1}$.
The minimization problem can then be written
\begin{align*}
\text{minimize} \quad  &(y-Hx)^{\mathrm{T}} R^{-1}(y-Hx) + (x-\hat{x}_{k|k-1})\transp P_{k|k-1}^{-1}(x-\hat{x}_{k|k-1})\\
  & \quad = (\tilde{y} -H\tilde{x})\transp R^{-1}(\tilde{y}-H\tilde{x}) + \tilde{x}\transp P_{k|k-1}^{-1}\tilde{x}
 \end{align*}
\end{frame}
\begin{frame}[label=sec-3-4]{The measurement update of the Kalman filter}
We now define the residuals $\epsilon$ for the system of equations
\[ \begin{bmatrix} Z_R & 0\\0 & Z_P \end{bmatrix} \begin{bmatrix}(\tilde{y}-H\tilde{x})\\ \tilde{x} \end{bmatrix} = \epsilon,\]
where $Z_R\transp Z_R = R^{-1}$ and $Z_P\transp Z_P = P_{k|k-1}^{-1}$.

The minimization problem can now be written
\begin{align*}
\text{minimize} \quad  & \epsilon\transp\epsilon \\
\text{subject to} \quad & \begin{bmatrix} Z_R & 0\\0 & Z_P \end{bmatrix} \begin{bmatrix}(\tilde{y}-H\tilde{x})\\ \tilde{x} \end{bmatrix} = \epsilon,
 \end{align*}
which is a least-squares problem.
\end{frame}
\begin{frame}[label=sec-3-5]{Robustifying the measurement update}
The idea is to use the Huber penalty function $\phi_{hub}$ instead of the quadratic criterion $\epsilon\transp \epsilon$.
\begin{align*}
 \text{minimize} \quad & \sum_{i=1}^{n+m} \phi_{hub}\big( \epsilon(i)\big) \\
\text{subject to} \quad & \begin{bmatrix} Z_R & 0\\0 & Z_P \end{bmatrix} \begin{bmatrix}(\tilde{y}-H\tilde{x})\\ \tilde{x} \end{bmatrix} = \epsilon,
 \end{align*}
\end{frame}

\begin{frame}[label=sec-3-6]{Tracking example again}
\begin{columns}
\begin{column}{0.3\textwidth}
10\% chance of outlier with 10 times normal standard deviation
\end{column}

\begin{column}{0.7\textwidth}
\begin{center}
\includegraphics[width=0.9\linewidth]{circular_movement_rkf}
\end{center}
\end{column}
\end{columns}
\end{frame}
% Emacs 25.3.50.2 (Org mode 8.2.10)
\end{document}