\part{Lecture 03: Dynamic Programming}
\title[RL Lecture 03]{Lecture 03: Dynamic Programming}
\date{}
\frame{\titlepage}
\frame{\frametitle{Table of Contents}\tableofcontents}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% What is Dynamic Programming? %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{What is Dynamic Programming (DP)?}

\begin{block}{Basic DP definition}
\begin{itemize}
	\item \hl{Dynamic}: sequential or temporal problem structure
	\item \hl{Programming}: mathematical optimization, i.e., numerical solutions
\end{itemize}
\end{block}
\pause
\vspace{1cm}
Further characteristics:
\begin{itemize}
	\item DP is a collection of algorithms to solve MDPs and neighboring problems.
	\begin{itemize}
		\item \hl{We will focus only on finite MDPs.}
		\item In case of continuous action/state space: apply quantization.
	\end{itemize}\pause
	\item Use of value functions to organize and structure the search for an optimal policy.
	\item Breaks problems into subproblems and solves them.
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Requirements %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Requirements for DP}
DP can be applied to problems with the following characteristics.
\begin{itemize}
	\item Optimal substructure:
	\begin{itemize}
		\item Principle of optimality applies.
		\item Optimal solution can be derived from subproblems.
	\end{itemize}\pause
\end{itemize}
\begin{itemize}
	\item Overlapping subproblems:
	\begin{itemize}
		\item Subproblems recur many times.
		\item Hence, solutions can be cached and reused.
	\end{itemize}
\end{itemize}\pause
\vspace{1cm}
How is that connected to MDPs?
\begin{itemize}
	\item MDPs satisfy above's properties:
	\begin{itemize}
		\item Bellman equation provides recursive decomposition.
		\item Value function stores and reuses solutions.
	\end{itemize}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Example: DP vs. Exhaustive Search (1)%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Example: DP vs. Exhaustive Search (1)}
\begin{figure}
	\animategraphics[loop,controls,width=10cm]{0.75}{fig/lec03/gif/PB_Bi_ES-}{0}{6}
	\caption{Shortest path problem to travel from Paderborn to Bielefeld: Eshaustive search requires 14 travel segment evaluations since every possible travel route is evaluated independently.}
	\label{fig:PB_Bi_ES}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Example: DP vs. Exhaustive Search (2)%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Example: DP vs. Exhaustive Search (2)}
\begin{figure}
	\animategraphics[loop,controls,width=10cm]{0.75}{fig/lec03/gif/PB_Bi_DP-}{0}{5}
	\caption{Shortest path problem to travel from Paderborn to Bielefeld: DP requires only 10 travel segment evaluations in order to calculate the optimal travel policy due to the reuse of subproblem results.}
	\label{fig:PB_Bi_DP}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Utility DP %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Utility of DP in the RL Context}
DP is used for iterative \hl{planning} (i.e., \hl{model-based} prediction and control) in an MDP.
\begin{itemize}
	\item Prediction:
	\begin{itemize}
		\item Input: MDP $\left\langle\mathcal{X}, \mathcal{U}, \bm{\mathcal{P}}, \mathcal{R}, \gamma \right\rangle$ and policy $\pi$
		\item Output: (estimated) value function $\hat{v}_{\pi} \approx v_{\pi}$
	\end{itemize}\pause
	\item Control:
	\begin{itemize}
		\item Input: MDP $\left\langle\mathcal{X}, \mathcal{U}, \bm{\mathcal{P}}, \mathcal{R}, \gamma \right\rangle$
		\item Output: (estimated) optimal value function $\hat{v}_{\pi}^* \approx v_{\pi}^*$ or policy $\hat{\pi}^*\approx\pi^*$
	\end{itemize}\pause
\end{itemize}
\vspace{1cm}
In both applications \hl{DP requires full knowledge of the MDP} structure.
\begin{itemize}
	\item Feasibility in real-world engineering applications (model vs. system) is therefore limited.\pause
	\item But: \hl{following DP concepts are largely used in modern data-driven RL algorithms.}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Policy Evaluation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Table of Contents}
\tableofcontents[currentsection]
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Evaluation Background (1) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Evaluation Background (1)}
\begin{itemize}
	\item Problem: evaluate a given policy $\pi$ to predict $v_\pi$. \pause
	\item Recap: Bellman equation for $x_k\in\mathcal{X}$ is given as
	\begin{align*}
	v_\pi(x_k) &= \El{G_k|X_k=x_k}{\pi},\\
									&= \El{R_{k+1}+ \gamma G_{k+1}|X_k=x_k}{\pi},\\
									&= \El{R_{k+1}+ \gamma v_\pi(X_{k+1})|X_k=x_k}{\pi}.
	\end{align*}\vspace{-0.5cm}\pause
	\item Or in matrix form:
		\begin{equation*}
			\begin{split}
					\bm{v}_{\mathcal{X}}^{\pi}&=\bm{r}_{\mathcal{X}}^{\pi}+\gamma\bm{\mathcal{P}}_{xx'}^{\pi}\bm{v}_{\mathcal{X}}^{\pi},\\
					\begin{bmatrix} v_{1}^{\pi} \\ \vdots \\ v_{n}^{\pi} \end{bmatrix} &= \begin{bmatrix} \mathcal{R}_{1}^{\pi} \\ \vdots \\ \mathcal{R}_{n}^{\pi} \end{bmatrix} + \gamma\begin{bmatrix} p_{11}^{\pi} & \cdots & p_{1n}^{\pi}\\ \vdots &  & \vdots\\ p_{n1}^{\pi} & \cdots & p_{nn}^{\pi}\end{bmatrix}\begin{bmatrix} v_{1}^{\pi} \\ 			\vdots \\ v_{n}^{\pi} \end{bmatrix}.
			\end{split}
			\end{equation*}\pause
			\item Solving the Bellman equation for $v_\pi$ requires handling a linear equation system with $n$ unknowns (i.e., number of states).
			\item Remember that the reward function $\mathcal{R}_{x}^{\pi}$ might also contains stochastic influences depending on the MDP structure (see \defref{defi:Markov_decision_process}).
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Evaluation Background (2) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Evaluation Background (2)}
\begin{itemize}
	\item Problem: directly calculating $v_\pi$ is numerically costly for high-dimensional state spaces (e.g., by matrix inversion). \pause
	\item General idea: \hl{apply iterative approximations} $\hat{v}_{i}(x_{k})=v_{i}(x_{k})$ of $v_\pi(x_{k})$ with decreasing errors:
	\begin{equation}
	\left\|v_{i}(x_{k}) - v_\pi\right\|_{\infty}\rightarrow 0 \quad \mbox{for} \quad i=1,2,3,\ldots
\end{equation}\pause
	\item The Bellman equation in matrix form can be rewritten as:
	\begin{equation}
	\label{eq:Bellman_matrix_Ab}
	\underbrace{\left(\bm{I}-\gamma\bm{\mathcal{P}}_{xx'}^{\pi}\right)}_{\bm{A}}\underbrace{\bm{v}_{\mathcal{X}}^{\pi}}_{\bm{\zeta}} =\underbrace{\bm{r}_{\mathcal{X}}^{\pi}}_{\bm{b}}.
\end{equation}\pause
\item To iteratively solve this linear equation $\bm{A}\bm{\zeta}=\bm{b}$, one can apply numerous methods such as
\begin{itemize}
	\item General gradient descent,
	\item Richardson iteration,
	\item Kyrlov subspace methods.
\end{itemize}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Richardson Iteration (1)%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Richardson Iteration (1)}
In the MDP context, the Richardson iteration became the default solution approach to iteratively solve:
\begin{equation*}
	\bm{A}\bm{\zeta} =\bm{b}.
\end{equation*}
The \hl{Richardson iteration} is
\begin{equation}
\label{eq:richardson_general}
	\bm{\zeta}_{i+1}= \bm{\zeta}_{i} + \omega(\bm{b}-\bm{A}\bm{\zeta}_i)
\end{equation}
with $\omega$ being a scalar parameter that has to be chosen such that the sequence $\bm{\zeta}_{i}$ converges. \pause To choose $\omega$ we inspect the series of approximation errors $\bm{e}_i=\bm{\zeta}_{i}-\bm{\zeta}$ and apply it to \eqref{eq:richardson_general}:
\begin{equation}
	\bm{e}_{i+1}= \bm{e}_{i} - \omega\bm{A}\bm{e}_i=\left(\bm{I}-\omega\bm{A}\right)\bm{e}_{i}.
\end{equation}\pause
To evaluate convergence we inspect the following norm:
\begin{equation}
\label{eq:richardson_error_sequence}
	\left\|\bm{e}_{i+1}\right\|_{\infty}= \left\|\left(\bm{I}-\omega\bm{A}\right)\bm{e}_{i}\right\|_{\infty}.
\end{equation}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Richardson Iteration (2)%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Richardson Iteration (2)}
Since any induced matrix norm is sub-multiplicative, we can approximate \eqref{eq:richardson_error_sequence} by the inequality:
\begin{equation}
	\left\|\bm{e}_{i+1}\right\|_{\infty} \leq \left\|\left(\bm{I}-\omega\bm{A}\right)\right\|_{\infty} \left\|\bm{e}_{i}\right\|_{\infty}.
\end{equation}\pause
Hence, the series converges if
\begin{equation}
	\left\|\left(\bm{I}-\omega\bm{A}\right)\right\|_{\infty}<1 .
\end{equation}\pause
Inserting from \eqref{eq:Bellman_matrix_Ab} leads to:
\begin{equation}
	\left\|\left(\bm{I}(1-\omega)+\omega\gamma\bm{\mathcal{P}}_{xx'}^{\pi}\right)\right\|_{\infty}<1 .
\end{equation}\pause
For $\omega =1$ we receive:
\begin{equation}
	\gamma\left\|\left(\bm{\mathcal{P}}_{xx'}^{\pi}\right)\right\|_{\infty}<1 .
\end{equation}\pause
Since the row elements of $\bm{\mathcal{P}}_{xx'}^{\pi}$ always sum up to 1,
\begin{equation}
	\gamma < 1
\end{equation}
follows. Hence, \hl{when discounting the Richardson iteration always converges for MDPs} even if we assume $\omega=1$.
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Iterative Policy Evaluation by Richardson Iteration (1)%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Iterative Policy Evaluation by Richardson Iteration (1)}
General form for any $x_k\in\mathcal{X}$ at iteration $i$ is given as:
\begin{equation}
	v_{i+1}(x_k)	= \sum_{u_k\in\mathcal{U}}\bm{\pi}(u_k|x_k)\left(\mathcal{R}^u_x + \gamma\sum_{x_{k+1}\in\mathcal{X}}p_{xx'}^u v_{i}(x_{k+1})\right)\, .
\end{equation}\pause
Matrix form then is:
\begin{equation}
\label{eq:iterative_policy_eval_matrix}
	\bm{v}_{\mathcal{X},i+1}^{\pi} =\bm{r}_{\mathcal{X}}^{\pi}+\gamma\bm{\mathcal{P}}_{xx'}^{\pi}\bm{v}_{\mathcal{X},i}^{\pi}\, .
\end{equation}\pause
\vspace{0.5cm}
\begin{figure}
	%\hspace*{-1.5cm}
	\includegraphics[width=7cm]{fig/lec03/Back_Up_Value_Policy_Evaluation.pdf}
	\caption{Backup diagram for iterative policy evaluation}
	\label{fig:Back_Up_Value_Policy_Evaluation}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Iterative Policy Evaluation by Richardson Iteration (2)%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Iterative Policy Evaluation by Richardson Iteration (2)}
\begin{itemize}
	\item During one Richardson iteration the 'old' value of $x_k$ is replaced with a 'new' value from the 'old' values of the successor state $x_{k+1}$.
	\begin{itemize}
		\item Update $v_{i+1}(x_k)$ from $v_{i}(x_{k+1})$, see \figref{fig:Back_Up_Value_Policy_Evaluation}.
		\item Updating estimates $(v_{i+1})$ on the basis of other estimates $(v_{i})$ is often called \hl{bootstrapping}.
	\end{itemize}\pause
	\item The Richardson iteration can be interpreted as a gradient descent algorithm for solving \eqref{eq:Bellman_matrix_Ab}.\pause
	\item This leads to \hl{synchronous, full backups} of the entire state space $\mathcal{X}$.\pause
	\item Also called \hl{expected update} because it is based on the expectation over all possible next states (utilizing full knowledge).\pause
	\item In subsequent lectures, the expected update will be supplemented by data-driven samples from the environment.
\end{itemize}

}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Iterative Policy Evaluation Example: Forest Tree MDP%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Iterative Policy Evaluation Example: Forest Tree MDP}
Let's reuse the forest tree MDP example from \figref{fig:Forest_Markov_Decision_Process_State_Value} with \textit{fifty-fifty policy} and discount factor $\gamma=0.8$
plus disaster probability $\alpha=0.2$:
\begin{equation*}
	\bm{\mathcal{P}}_{xx'}^{\pi} = \begin{bmatrix}0 & \frac{1-\alpha}{2} & 0 & \frac{1+\alpha}{2}  \\ 0 & 0 &\frac{1-\alpha}{2} & \frac{1+\alpha}{2} \\ 0 & 0 &\frac{1-\alpha}{2} & \frac{1+\alpha}{2} \\ 0 & 0 & 0 & 1\end{bmatrix}, \quad\bm{r}_{\mathcal{X}}^{\pi} = \begin{bmatrix}0.5 \\ 1 \\ 2 \\ 0 \end{bmatrix} \, .
\end{equation*}
\pause
\begin{table}
	\centering
		\begin{tabular}{l|l|l|l|l}
			$i$ & $v_i(x=1)$ & $v_i(x=2)$ & $v_i(x=3)$ & $v_i(x=4)$\\
			\hline
			0  & 0 & 0 & 0 & 0\\ \pause
			1  & 0.5 & 1 & 2 &0\\ \pause
			2  & 0.82 & 1.64 & 2.64 & 0\\ \pause
			3  & 1.03 & 1.85 & 2.85 & 0\\ \pause
			\vdots & \vdots & \vdots & \vdots & \vdots \\
			$\infty$ & 1.12 & 1.94 & 2.94 & 0
		\end{tabular}
	\label{tab:Forest_tree_policy_eval}
	\caption{Policy evaluation by Richardson iteration \eqref{eq:iterative_policy_eval_matrix} for forest tree MDP}
\end{table}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Variant: In-Place Updates %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Variant: In-Place Updates}
Instead of applying \eqref{eq:iterative_policy_eval_matrix} to the entire vector $\bm{v}_{\mathcal{X},i+1}^{\pi}$ in 'one shot' (synchronous backup), an elementwise \hl{in-place} version of the policy evaluation can be carried out:
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{full model of the MDP, i.e., $\left\langle\mathcal{X}, \mathcal{U}, \bm{\mathcal{P}}, \mathcal{R}, \gamma \right\rangle$ including policy $\pi$}
\Param{$\delta>0$ as accuracy termination threshold}
\Init{$v_0(x)\, \forall \, x\in\mathcal{X}$ arbitrary except $v_0(x)=0$ if $x$ is terminal}
 \Repeat{$\Delta<\delta$}{
		$\Delta \leftarrow 0 $\;
		\For{$\forall \, x_k\in\mathcal{X}$}{
			$\tilde{v}\leftarrow \hat{v}(x_k)$\;
			$\hat{v}(x_k)\leftarrow  \sum_{u_k\in\mathcal{U}}\pi(u_k|x_k)\left(\mathcal{R}^u_x + \gamma\sum_{x_{k+1}\in\mathcal{X}}p_{xx'}^u \hat{v}(x_{k+1})\right)$\;
			$\Delta \leftarrow \max\left(\Delta, |\tilde{v}-\hat{v}(x_k)|\right)$\;
		}
	}
\caption{
 Iterative policy evaluation using in-place updates (output: estimate of $\bm{v}_{\mathcal{X}}^{\pi}$)}
\label{algo:in_place_policy_update}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% In-Place Policy Evaluation Updates for Forest Tree MDP%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{In-Place Policy Evaluation Updates for Forest Tree MDP}
\begin{itemize}
	\item In-place algorithms allow to update states in a beneficial order. \pause
	\item May converge faster than regular Richardson iteration if state update order is chosen wisely (sweep through state space).\pause
	\item For forest tree MDP: reverse order, i.e., start with $x=4$.\pause
	\item As can be seen in \tabref{tab:Forest_tree_policy_eval_in_place} the in-place updates especially converge faster for the 'early states'.
\end{itemize}

\begin{table}
	\centering
		\begin{tabular}{l|l|l|l|l}
			$i$ & $v_i(x=1)$ & $v_i(x=2)$ & $v_i(x=3)$ & $v_i(x=4)$\\
			\hline
			0  & 0 & 0 & 0 & 0\\
			1  & 1.03 & 1.64 & 2 &0\\
			2  & 1.09 & 1.85 & 2.64 & 0\\
			3  & 1.11 & 1.91 & 2.85 & 0\\
			\vdots & \vdots & \vdots & \vdots & \vdots \\
			$\infty$ & 1.12 & 1.94 & 2.94 & 0
		\end{tabular}
	\caption{In-place updates for forest tree MDP}
	\label{tab:Forest_tree_policy_eval_in_place}
\end{table}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Policy Improvement}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Table of Contents}
\tableofcontents[currentsection]
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% General Idea on Policy Improvement%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{General Idea on Policy Improvement}
\vspace{-0.1cm}
\begin{itemize}
	\item If we know $v_{\pi}$ of a given MDP, how to improve the policy?\pause
	\item The simple idea of policy improvement is:
	\begin{itemize}
		\item Consider a new (non-policy conform) action $u\neq{\pi}(x_k)$.
		\item Follow thereafter the current policy $\pi$.
		\item Check the action-value of this 'new move'. If it is better than the 'old' value, take it.
	\end{itemize}
\end{itemize}
	\begin{equation}
	\label{eq:policy_improv_theo01}
			q_\pi(x_k,u_k)	= \E{R_{k+1} + \gamma 	v_{\pi}(X_{k+1})|X_{k}=x_{k},U_{k}=u_{k}}\,.
	\end{equation}\pause\vspace{-0.3cm}
	\setcounter{mytheo}{0}
\vspace{-0.1cm}
\begin{theo}{Policy improvement}{Policy_improvement}
If for any deterministic policy pair $\pi$ and $\pi'$
\begin{equation}
\label{eq:policy_improv_theo02}
	q_{\pi}(x,\pi'(x))\geq v_{\pi}(x) \quad \forall x\in\mathcal{X}
\end{equation}
applies, then the policy $\pi'$ must be as good as or better than $\pi$. Hence, it obtains greater or equal expected return
\begin{equation}
\label{eq:policy_improv_theo03}
	v_{\pi'}(x) \geq v_{\pi}(x) \quad \forall x\in\mathcal{X} .
\end{equation}
\end{theo}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Proof of Policy Improvement Theorem %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Proof of Policy Improvement Theorem}
Start with \eqref{eq:policy_improv_theo02} and recursively reapply \eqref{eq:policy_improv_theo01}:
\begin{equation}
\hspace{-0.35cm}
	\begin{split}
		v_{\pi}(x_k) &\leq q_{\pi}(x_k,\pi'(x_k)),\\
											&= \E{R_{k+1} + \gamma 	v_{\pi}(X_{k+1})|X_{k}=x_{k},U_{k}=\pi'(x_{k})},\\
											&= \El{R_{k+1} + \gamma 	v_{\pi}(X_{k+1})|X_{k}=x_{k}}{\pi'},\\
											&\leq \El{R_{k+1} + \gamma 	q_{\pi}(x_{k+1},\pi'(x_{k+1}))|X_{k}=x_{k}}{\pi'},\\
											&= \El{R_{k+1} + \gamma 	\El{R_{k+2} + \gamma 	v_{\pi}(X_{k+2})|X_{k+1},\pi'(x_{k+1})}{\pi'}|X_{k}=x_{k}}{\pi'},\\
											&= \El{R_{k+1} + \gamma 	R_{k+2} + \gamma^2 v_{\pi}(X_{k+2})|X_{k}=x_{k}}{\pi'},\\
											&\leq \El{R_{k+1} + \gamma 	R_{k+2} + \gamma^2 R_{k+3}+ \gamma^3 v_{\pi}(X_{k+3})|X_{k}=x_{k}}{\pi'},\\
											&\hspace{0.2cm}\vdots\\
											&\leq \El{R_{k+1} + \gamma 	R_{k+2} + \gamma^2 R_{k+3}+ \gamma^3 R_{k+4}+\cdots|X_{k}=x_{k}}{\pi'},\\
											&=v_{\pi'}(x_k).
	\end{split}
\end{equation}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Greedy Policy Improvement (1) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Greedy Policy Improvement (1)}
\begin{itemize}
	\item So far, policy improvement addressed only changing the policy at a single state.\pause
	\item Now, extend this scheme to all states by selecting the best action according to $q_\pi(x_{k}, u_{k})$ in every state (\hl{greedy policy improvement}):\pause
\end{itemize}
\vspace{0.25cm}
\begin{equation}
	\begin{split}
		\pi'(x_{k})&	=\argmax_{u_k\in\mathcal{U}} q_\pi(x_{k}, u_{k}),\\
										& = \argmax_{u_k\in\mathcal{U}} \E{R_{k+1} + \gamma 	v_{\pi}(X_{k+1})|X_{k}=x_{k},U_{k}=u_{k}},\\
										& = \argmax_{u_k\in\mathcal{U}} \mathcal{R}^u_x + \gamma \sum_{x_{k+1}\in\mathcal{X}}p_{xx'}^u v_\pi(x_{k+1}) \, .
	\end{split}
\end{equation}
\begin{itemize}
	\item Again, consider that $\mathcal{R}^u_x$ could be of deterministic or stochastic nature.
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Greedy Policy Improvement (2) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Greedy Policy Improvement (2)}
\begin{itemize}
	\item Each greedy policy improvement takes the best action in a one-step look-ahead search and, therefore, satisfies \theoref{theo:Policy_improvement}.\pause
	\item If after a policy improvement step $v_{\pi}(x_k) = v_{\pi'}(x_k)$ applies, it follows:
\end{itemize}
\vspace{0.25cm}
\begin{equation}
	\begin{split}
		v_{\pi'}(x_k) &=	\max_{u_k\in\mathcal{U}} \E{R_{k+1} + \gamma 	v_{\pi'}(X_{k+1})|X_{k}=x_{k},U_{k}=u_{k}},\\
										& = \max_{u_k\in\mathcal{U}} \mathcal{R}^u_x + \gamma \sum_{x_{k+1}\in\mathcal{X}}p_{xx'}^u v_{\pi'}(x_{k+1}) \, .
	\end{split}
\end{equation}\pause
\begin{itemize}
	\item This is the Bellman optimality equation, which guarantees that $\pi'=\pi$ must be optimal policies.\pause
	\item Although proof for policy improvement theorem was presented for deterministic policies, transfer to stochastic policies $\pi(u_k|x_k)$ is possible.\pause
	\item Takeaway message: \hl{policy improvement theorem guarantees finding optimal policies in finite MDPs} (e.g., by DP).
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Policy and Value Iteration}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Table of Contents}
\tableofcontents[currentsection]
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Concept of Policy Iteration %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Concept of Policy Iteration}
\begin{itemize}
	\item Policy iteration \hl{combines the previous policy evaluation and policy improvement} in an iterative sequence:
\end{itemize}
\vspace{0.25cm}
\begin{equation}
\label{eq:policy_iter}
	\pi_0 \rightarrow v_{\pi_0} \rightarrow \pi_1 \rightarrow v_{\pi_1} \rightarrow \cdots \pi^* \rightarrow v_{\pi^*}
\end{equation}
\begin{itemize}
	\item Evaluate $\rightarrow$ improve $\rightarrow$ evaluate $\rightarrow$ improve ...\pause
	\item In the 'classic' policy iteration, each policy evaluation step in \eqref{eq:policy_iter} is fully executed, i.e., for each policy $\pi_i$ an exact estimate of $v_{\pi_i}$ is provided either by iterative policy evaluation with a sufficiently high number of steps or by any other method that fully solves \eqref{eq:Bellman_matrix_Ab}.
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Iteration Example: Forest Tree MDP (1) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Iteration Example: Forest Tree MDP (1)}
\begin{figure}
	\includegraphics[width=11cm]{fig/lec03/Forest_Markov_Decision_Process.pdf}
\end{figure}
\begin{itemize}
	\item Two actions possible in each state:
	\begin{itemize}
		\item Wait $u=w$: let the tree grow.
		\item Cut $u=c$: gather the wood.
	\end{itemize}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Iteration Example: Forest Tree MDP (2) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Iteration Example: Forest Tree MDP (2)}
Assume $\alpha=0.2$ and $\gamma=0.8$ and start with \hl{'tree hater' initial policy}:\pause
\begin{enumerate}
	\item $\pi_0=\pi(u_k =\mbox{c}|x_k) \quad \,\forall x_k\in\mathcal{X}$.\pause
	\item Policy evaluation: $v_\mathcal{X}^{\pi_0}=\begin{bmatrix}1 & 2 & 3 & 0\end{bmatrix}^T$\pause
	\item Greedy policy improvement:
	\begin{equation*}
	\begin{split}
		\pi_1(x_{k})& = \argmax_{u_k\in\mathcal{U}} \E{R_{k+1} + \gamma 	v_{\pi_0}(X_{k+1})|X_{k}=x_{k},U_{k}=u_{k}},\\
										& = \left\{\pi(u_k =\mbox{w}|x_k=1), \pi(u_k =\mbox{c}|x_k=2), \pi(u_k =\mbox{c}|x_k=3)\right\}
	\end{split}
\end{equation*}\pause
\item Policy evaluation: $v_\mathcal{X}^{\pi_1}=\begin{bmatrix}1.28 & 2 & 3 & 0\end{bmatrix}^T$\pause
\item Greedy policy improvement:
	\begin{equation*}
	\begin{split}
		\pi_2(x_{k})& = \argmax_{u_k\in\mathcal{U}} \E{R_{k+1} + \gamma 	v_{\pi_1}(X_{k+1})|X_{k}=x_{k},U_{k}=u_{k}},\\
										& = \left\{\pi(u_k =\mbox{w}|x_k=1), \pi(u_k =\mbox{c}|x_k=2), \pi(u_k =\mbox{c}|x_k=3)\right\},\\
										& = \pi_1(x_{k})\\
										& = \pi^*
	\end{split}
\end{equation*}
\end{enumerate}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Iteration Example: Forest Tree MDP (3) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Iteration Example: Forest Tree MDP (3)}
Assume $\alpha=0.2$ and $\gamma=0.8$ and start with \hl{'tree lover' initial policy}:\pause
\begin{enumerate}
	\item $\pi_0=\pi(u_k =\mbox{w}|x_k) \quad \,\forall x_k\in\mathcal{X}$.\pause
	\item Policy evaluation: $v_\mathcal{X}^{\pi_0}=\begin{bmatrix}1.14 & 1.78 & 2.78 & 0\end{bmatrix}^T$\pause
	\item Greedy policy improvement:
	\begin{equation*}
	\begin{split}
		\pi_1(x_{k})& = \argmax_{u_k\in\mathcal{U}} \E{R_{k+1} + \gamma 	v_{\pi_0}(X_{k+1})|X_{k}=x_{k},U_{k}=u_{k}},\\
										& = \left\{\pi(u_k =\mbox{w}|x_k=1), \pi(u_k =\mbox{c}|x_k=2), \pi(u_k =\mbox{c}|x_k=3)\right\}
	\end{split}
\end{equation*}\pause
\item Policy evaluation: $v_\mathcal{X}^{\pi_1}=\begin{bmatrix}1.28 & 2 & 3 & 0\end{bmatrix}^T$\pause
\item Greedy policy improvement:
	\begin{equation*}
	\begin{split}
		\pi_2(x_{k})& = \argmax_{u_k\in\mathcal{U}} \E{R_{k+1} + \gamma 	v_{\pi_1}(X_{k+1})|X_{k}=x_{k},U_{k}=u_{k}},\\
										& = \left\{\pi(u_k =\mbox{w}|x_k=1), \pi(u_k =\mbox{c}|x_k=2), \pi(u_k =\mbox{c}|x_k=3)\right\},\\
										& = \pi_1(x_{k})\\
										& = \pi^*
	\end{split}
\end{equation*}
\end{enumerate}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Iteration Example: Jack's Car Rental (1) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Iteration Example: Jack's Car Rental (1)}
\begin{figure}
	\includegraphics[width=2cm]{fig/lec03/car_rental.pdf}
\end{figure}
\begin{itemize}
	\item States: Two rental locations, maximum of 20 cars each\pause
	\item Actions: Move up to 5 cars between locations overnight\pause
	\item Reward:
	\begin{itemize}
		\item  +10 \$ for each car rented (if available at location)
		\item -2 \$ for each overnight car transfer
		\item Discount: $\gamma=0.9$
	\end{itemize}\pause
	\item Dynamics: Cars returned and requested randomly following Poisson distribution
	\begin{itemize}
		\item $P_\lambda(n)=\frac{\lambda^n}{n!}e^{-\lambda}$
		\item $P_\lambda(n)=$ probability of observing $n$ events with mean event rate $\lambda$
		\item 1st location: $\lambda_{\mbox{req.}}=3$, $\lambda_{\mbox{ret.}}=3$
		\item 2nd location: $\lambda_{\mbox{req.}}=4$, $\lambda_{\mbox{ret.}}=2$
	\end{itemize}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Policy Iteration Example: Jack's Car Rental (2) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Policy Iteration Example: Jack's Car Rental (2)}
\begin{figure}
		\includegraphics[width=9cm]{fig/lec03/Jack_Car_Rental.pdf}
		\caption{Sequence of policies found by policy iteration including optimal state value after termination (source: R. Sutton and G. Barto, Reinforcement learning: an introduction, 2018, \href{https://creativecommons.org/licenses/by-nc-nd/2.0/}{CC BY-NC-ND 2.0})}
		\label{fig:Jack_Car_Rental}
	\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Value Iteration (1) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Value Iteration (1)}
\begin{itemize}
	\item Policy iteration involves full policy evaluation steps between policy improvements.
	\item In large state-space MDPs the full policy evaluation may be numerically very costly.\pause
	\item Using a limited number of iterative policy evaluations steps and then apply policy improvement may speed up the entire DP process.\pause
	\item \hl{Value iteration}: One step iterative policy evaluation followed by policy improvement.\pause
	\item Allows simple update rule which \hl{combines policy improvement with truncated policy evaluation}:
	\begin{equation}
	\begin{split}
		v_{i+1}(x_k) &=	\max_{u_k\in\mathcal{U}} \E{R_{k+1} + \gamma 	v_{i}(X_{k+1})|X_{k}=x_{k},U_{k}=u_{k}},\\
										&= \max_{u_k\in\mathcal{U}} \mathcal{R}^u_x + \gamma \sum_{x_{k+1}\in\mathcal{X}}p_{xx'}^u v_{i}(x_{k+1}) \, .
	\end{split}
\end{equation}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Value Iteration (2) %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Value Iteration (2)}
\setlength{\algomargin}{0.5em}
\begin{algorithm}[H]
\SetKwInput{Input}{input}
\SetKwInput{Output}{output}
\SetKwInput{Init}{init}
\SetKwInput{Param}{parameter}
\Input{full model of the MDP, i.e., $\left\langle\mathcal{X}, \mathcal{U}, \bm{\mathcal{P}}, \mathcal{R}, \gamma \right\rangle$}
\Param{$\delta>0$ as accuracy termination threshold}
\Init{$v_0(x)\, \forall \, x\in\mathcal{X}$ arbitrary except $v_0(x)=0$ if $x$ is terminal}
 \Repeat{$\Delta<\delta$}{
		$\Delta \leftarrow 0 $\;
		\For{$\forall \, x_k\in\mathcal{X}$}{
			$\tilde{v}\leftarrow \hat{v}(x_k)$\;
			$\hat{v}(x_k)\leftarrow  \max_{u_k\in\mathcal{U}}\left(\mathcal{R}^u_x + \gamma\sum_{x_{k+1}\in\mathcal{X}}p_{xx'}^u \hat{v}(x_{k+1})\right)$\;
			$\Delta \leftarrow \max\left(\Delta, |\tilde{v}-\hat{v}(x_k)|\right)$\;
		}
	}
\Output{Deterministic policy $\pi\approx\pi^*$, such that}
	$\pi(x_k)\leftarrow  \argmax_{u_k\in\mathcal{U}}\left(\mathcal{R}^u_x + \gamma\sum_{x_{k+1}\in\mathcal{X}}p_{xx'}^u \hat{v}(x_{k+1})\right)$\;
\caption{Value iteration (note: compared to policy iteration, value iteration doesn't require an initial policy but only a state-value guess)}
\end{algorithm}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Value Iteration for Forest Tree MDP %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Value Iteration for Forest Tree MDP}
\begin{figure}
	\includegraphics[width=6cm]{fig/lec03/Forest_Markov_Decision_Process.pdf}
\end{figure}
\begin{itemize}
	\item Assume again $\alpha=0.2$ and $\gamma=0.8$.\pause
	\item Similar to in-place update policy evaluation, reverse order and start value iteration with $x=4$.\pause
	\item As shown in \tabref{tab:Forest_tree_value_iteration} value iteration converges in one step (for the given problem) to the optimal state-value.
\end{itemize}
\begin{table}
	\centering
		\begin{tabular}{l|l|l|l|l}
			$i$ & $v_i(x=1)$ & $v_i(x=2)$ & $v_i(x=3)$ & $v_i(x=4)$\\
			\hline
			0  & 0 & 0 & 0 & 0\\
			1  & 1.28 & 2 & 3 &0\\
			*  & 1.28 & 2 & 3 &0
		\end{tabular}
	\caption{Value iteration for forest tree MDP}
	\label{tab:Forest_tree_value_iteration}
\end{table}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Further Aspects}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Table of Contents}
\tableofcontents[currentsection]
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Summarizing DP Algorithms %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Summarizing DP Algorithms}
\begin{itemize}
	\item All DP algorithms are based on the state-value $v(x)$.
	\begin{itemize}
		\item Complexity is $\mathcal{O}(m\cdot n^2)$ for $m$ actions and $n$ states.
		\item Evaluate all $n^2$ state transitions while considering up to $m$ actions per state.
	\end{itemize}\pause
	\item Could be also applied to action-values $q(x,u)$.
	\begin{itemize}
		\item Complexity is inferior with $\mathcal{O}(m^2\cdot n^2)$.
		\item There are up to $m^2$ action-values which require $n^2$ state transition evaluations each.
	\end{itemize}
\end{itemize}
\pause
\begin{table}
	\centering
		\begin{tabular}{|M{1.65cm}||M{4.75cm}|M{4cm}|}
			\hline
			Problem & Relevant Equations & Algorithm\\
			\hline\hline
			prediction & Bellman expectation eq. &  policy evaluation\\
			\hline
			control & Bellman expectation eq. \& greedy policy improvement & policy iteration\\
			\hline
			control & Bellman optimality eq. & value iteration\\
			\hline
		\end{tabular}
		\caption{Short overview addressing the treated DP algorithms}
		\label{tab:DP_syn_algos}
\end{table}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%Asynchronous DP%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Asynchronous DP}
\begin{itemize}
	\item DP algorithms considered so far used \hl{synchronous backups}:
	\begin{itemize}
		\item In one iteration the entire state space is updated.
		\item May be computational expensive for large MDPs.
		\item Some state-values or policy parts may converge faster than other but are updated as often as slowly converging states.
	\end{itemize}
	\vspace{0.5cm}\pause
	\item In contrast, \hl{asynchronous backups} update states individually in an (arbitrary) order:
	\begin{itemize}
		\item Choose smart order to achieve faster overall convergence rate.
		\item Some states may be updated more frequently than others.\pause
		\item Overall algorithms converges if all states are still visited to some extent (\hl{important requirement to ensure convergence}).\pause
		\item Simple example: in-place policy evaluation where only a subset of all states are updated each iterations (cf. \algoref{algo:in_place_policy_update}).
	\end{itemize}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Asynchronous DP: Prioritized Sweeping %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Asynchronous DP: Prioritized Sweeping}
\begin{itemize}
	\item Use magnitude of \hl{Bellman error} as an indicator which state should be updated next:
	\begin{equation}
		\argmax_{x_k\in\mathcal{X}}\left|\max_{u_k\in\mathcal{U}} \left(\mathcal{R}^u_x + \gamma \sum_{x_{k+1}\in\mathcal{X}}p_{xx'}^u v_{i}(x_{k+1})\right) - v_i(x_k)\right|\, .
	\end{equation}\pause
	\item Update the state with the largest Bellman error first.
	\item Build up a priority queue of most relevant states by refreshing the Bellman error after each state update.
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Asynchronous DP: Real-Time Updates %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Asynchronous DP: Real-Time Updates}
\begin{itemize}
	\item Update those states which are frequently visited by the agent.
	\item Utilizes agent's experience to guide the asynchronous DP updates.
	\item After each time step $\left\langle x_k, u_k, r_{k+1}\right\rangle$ update $x_k$:
	\begin{equation}
		v_i(x_k) \leftarrow \max_{u_k\in\mathcal{U}} \left(\mathcal{R}^u_{x} + \gamma \sum_{x_{k+1}\in\mathcal{X}}p_{xx'}^u v_{i}(x_{k+1})\right).
	\end{equation}
\end{itemize}
\begin{figure}
		\includegraphics[width=5.5cm]{fig/lec03/RTDP.pdf}
		\caption{Real-time DP updates focus on reachable states (source: R. Sutton and G. Barto, Reinforcement learning: an introduction, 2018, \href{https://creativecommons.org/licenses/by-nc-nd/2.0/}{CC BY-NC-ND 2.0})}
		\label{fig:RTDP}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Generalized Policy Iteration %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Generalized Policy Iteration (GPI)}
\begin{itemize}
	\item Almost all RL methods are well-described as GPI.
	\item \hl{Push-pull}: Improving the policy will deteriorate value estimation.
	\item Well balanced \hl{trade-off between evaluating and improving} is required.
\end{itemize}
\begin{figure}
	\subfloat{
		\includegraphics[height=4cm]{fig/lec03/GPI_01.pdf}
	}
	\hspace{1cm}
	\subfloat{
		\includegraphics[height=4cm]{fig/lec03/GPI_02.pdf}
	}
\caption{Interpreting generalized policy iteration to switch back and forth between (arbitrary) evaluations and improvement steps (source: R. Sutton and G. Barto, Reinforcement learning: an introduction, 2018, \href{https://creativecommons.org/licenses/by-nc-nd/2.0/}{CC BY-NC-ND 2.0})}
\end{figure}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Curse of Dimensionality %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{Curse of Dimensionality}
\begin{itemize}
	\item DP is much more efficient than an exhaustive search over all $n$ states and $m$ actions in finite MDPs in order to find an optimal policy.
	\begin{itemize}
		\item Exhaustive search for deterministic policies: $m^n$ evaluations.
		\item DP results in polynomial complexity regarding $m$ and $n$.
	\end{itemize}
	\pause
	\item Nevertheless, DP uses full-width backups:
	\begin{itemize}
		\item For each state update, every successor state and action is considered.
		\item While utilizing full knowledge of the MDP structure.
	\end{itemize}
	\pause
	\item Hence, DP is can be effective up to medium-sized MDPs (i.e., million states)\pause
	\item For large problems DP suffers from the \hl{curse of dimensionality}:
	\begin{itemize}
		\item Number of finite states $n$ grows exponentially with the number of state variables.
		\item Also: if continuous variables need quantization typically a large number of states results.
		\item Single state update may become computational infeasible.
	\end{itemize}
\end{itemize}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Summary %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Summary: What You've Learned Today}
\begin{itemize}
	\item DP is applicable for prediction and control problems in MDPs.\pause
	\item But requires always full knowledge about the environment (i.e., it is a model-based solution also called planning).\pause
	\item DP is more efficient than exhaustive search.\pause
	\item But suffers from the curse of dimensionality for large MDPs.\pause
	\item (Iterative) policy evaluations and (greedy) improvements solve MDPs.\pause
	\item Both steps can be combined via value iteration.\pause
	\item This idea of (generalized) policy iteration is a basic scheme of RL.\pause
	\item Implementing DP algorithms comes with many degrees of freedom.
	\item For example how to order the state updates (asyn. vs. sync.).
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Final Slide %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\frame{\frametitle{The End for Today}
\begin{figure}
\includegraphics[width=10cm]{fig/lec03/dilbert.jpg}
\end{figure}
\vspace{1cm}
\centering
Thanks for your attention and have a nice week!
}


\documentclass[a4paper,12pt]{article} % declaration
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsfonts}
\usepackage{xcolor}
\usepackage{enumitem}
\usepackage{graphicx}
\usepackage{fancyhdr}
\usepackage{color}
\usepackage{amsthm}
\usepackage{CJKutf8}

%\def\pgfsysdriver{pgfsys-dvipdfmx.def}
\usepackage{tikz}

\newtheorem{definition}{Definition}[section]
\newtheorem{example}{Example}[section]
\newtheorem{theorem}{Theorem}[section]
\newtheorem{proposition}{Proposition}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}

\title{Introduction}
\author{Guoning Wu}

\begin{document}
\graphicspath{{../Figs/}}

%\tableofcontents
\setcounter{tocdepth}{2}

%\listoffigures
%\listoftables

\maketitle

\section{The Creation of the Calculus}
\begin{quote}
    {\color{red}Who, by a vigor of mind almost divine, the motions and figures
    of the planets, the paths of comets, and the tides of seas first
    demonstrated. -----Newton's Epitaph}
\end{quote}
\subsection{The Motivation for the Calculus}
Following hard on the adoption of the function concept came the
calculus, which, next to Euclidean geometry, is the greatest
in all of mathematics. Though it was to some extent the answer
to problems already tackled by the Greeks, the calculus was
created primarily to treat the major scientific problems of the
seventeenth century.

There were four major types of problems. The first was: Given
the formula for the distance a body covers as a function of time,
to find the velocity and acceleration at any instant; and,
conversely, given the formula describing the acceleration of a
body as a function of the time, to find the velocity and the
distance traveled. This problem arose directly in the study of motion and the difficulty it posed was that the velocities and the acceleration of concern to the seventeenth century varied from instant to instant. In calculating an instantaneous velocity, for example, one cannot, as one can in the case of average velocity, divide the distance traveled by the time of travel, because at a given instant both the distance traveled and time are zero, and $0/0$ is meaningless. Nevertheless, it was clear on physical grounds that moving objects do have a velocity at each instant of their travel. The inverse problem of finding the distance covered, knowing the formula for velocity, involves the corresponding difficulty; one cannot multiply the velocity at any one instant by the time of travel to obtain the distance traveled because the velocity varies from instant to instant.

The second type of problem was to find the tangent to a curve.
It was a problem of pure geometry, and it was of great importance
for scientific applications. Optics, as we know, was one of the major scientific pursuits of the seventeenth century; the design of lenses was direct interest for Fermat, Descartes, Huygens, and Newton.

The third problem was that of finding the maximum or minimum
value of a function. When a cannonball is shot from a cannon, the distance it will travel horizontally-the range-depends on the angle at which the cannon is inclined to the ground. One "practical" problem was to find the angle that would maximize the range. Early in the seventeenth century, Galileo determined that(in a vacuum) the maximum range is obtained for an angle of fire of 45 degree. The study of the motion of the planets
involved maxima and minima problems, such as finding the
greatest and least distances of a planet from the sun.

The fourth problem was finding the lengths of curves, for
example, the distance covered by a planet in a given period
of time; the area bounded by curves; volumes bounded by
surfaces; centers of gravity of bodies; and the gravitational
attraction that extended body, a planet for example, exerts on
another body. The Greeks had used the method of exhaustion to find some areas and volumes. Despite the fact that they used it for relatively simple areas and volumes, they had to apply much ingenuity, because the method lack generality. Nor did they often come up with numerical answers. Interest in finding lengths, areas, volumes, and centers of gravity was revived when the work of Archimedes became known in Europe. The method of exhaustion was first modified gradually, and then radically by the invention of the calculus.

\subsection{Early Seventeenth-Century Work on the Calculus}

The problems of the calculus were tackled by at least a dozen
of the greatest mathematicians of the seventeenth century and
by several dozen minor ones. All of their contributions were
crowned by the achievements of Newton and Leibniz. Here we shall
be able to note only the principle contributions of the precursors
of these two masters.

Several methods were advanced to find the tangent to a curve.
Gilles Persone de Roberval(1602-75) generalized a method Archimedes
had used to find the tangent at any point on his spiral, which
the tangent line of a projectile shot from a canon is the resultant
diagonal line of the vertical and horizontal velocities.

While the notion of a tangent as a line having the direction of the
resultant velocity was more complicated than the Greek definition
of a line touching a curve, this newer concept applied to many
curves for which the older one failed.

\section{Sets and Elementary Operations on them}

In this note, we introduce some basic concepts for real
analysis.

\subsection{The Concept of a Set}
Since the late nineteenth and early twentieth centuries
the most universal language of mathematics has been the
language of set theory. This is even manifest in one of
the definitions of mathematics as the science that
studies different structures (relations) on sets.

\emph{"We take a set to be an assemblage of definite,
perfectly distinguishable objects of our intuition or
our thought into a coherent whole."} Thus did Georg
Cantor\footnote{G.Cantor(1845-1918) - German
mathematician, the creator of the theory of infinite
sets and the progenitor of set theoretic language in
mathematics.}, describe the concept of a set.

\begin{itemize}
    \item A set may be consist of any distinguishable
        objects.
    \item A set is unambiguously determined by the
        collection of objects that comprise it.
    \item Any property defines the set of objects
        having that property.
\end{itemize}

If $x$ is an object, $P$ is a property, and $P(x)$
denotes the assertion that $x$ has property $P$,
then the class of objects having the property
$P$ is denoted $\{x\lvert P(x)\}$

And in fact the concept of the set of all sets, for
example, is simply contradictory. This is the
classical paradox of \textbf{Russell}. \footnote{B.Russell (1872-1970) - British logician,
philosopher, sociologist ans social activist.}

\subsection{The Inclusion Relation}
The statement, "$x$ is an element of the set $X$" is written
briefly as
\[
    x\in X
\]
and its negation as
\[
    x \notin X
\]

When statements about sets are written, frequent use is
made of the logical operators $\exists$ ("there exists" or "there
are") and $\forall$ ("every" or "for all") which are called the
\emph{existence} and \emph{generalization} respectively.

Thus two sets are equal if they consist of the same
elements, this statement is usually written briefly as
\[
    A=B,
\]
read as "$A$ equals $B$". The negation of equality is usually
written as
\[
    A \ne B.
\]
If every element of $A$ is an element of $B$, we write
$A\subset B$ and say that $A$ is a subset of
$B$ or that $B$ contains $A$.

Thus
\[
    A\subset B := \forall x\in A \Rightarrow x\in B
\]
If $A\subset B$ and $A\ne B$, we shall say that the inclusion $A\subset B$ is
\emph{strict} or that $A$ is a proper subset of $B$.

Using these definitions, we can now conclude that
\[A=B \Leftrightarrow A\subset B \wedge B\subset A\]
If $M$ is a set, any property of $P$ distinguishes in M the subset
\[\{x\in M \vert P(x)\}\]
consisting of the elements of $M$ that have the property.

For example, it is obvious that
\[ M=\{x\in M \vert x\in M\},\]
and the \emph{empty} subset of $M$ is
\[\emptyset = \{x\in M \vert x\ne x\}\]

\subsection{Elementary Operations on Sets}
\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=0.75]
    \draw  (0,0) rectangle  (4,3);
        \node [below,black] at (2, 0) {(a)};
    \draw (2.5,1.5) circle (1);
    \draw (1.5,1.5) circle (1);
    \node [above left,green] at (1.5, 1.5) {$A$};
    \node [above right,blue] at (2.5, 1.5) {$B$};
    \clip (1.5,1.5) circle (1);
    \clip (2.5,1.5) circle (1);
    \draw [fill, blue!100] (0, 0) rectangle (4, 3);
    \node [black] at (2, 1.5) {$ A\cap B$};
\end{tikzpicture}
    \begin{tikzpicture}[scale=0.75]
    \draw  (0,0) rectangle  (4,3);
        \node [below,black] at (2, 0) {(b)};
    \draw [fill=green](2.5,1.5) circle (1);
    \draw [fill=green] (1.5,1.5) circle (1);
    \node [above left,black] at (1.5, 1.5) {$A$};
    \node [above right,black] at (2.5, 1.5) {$B$};
    \clip (1.5,1.5) circle (1);
    \clip (2.5,1.5) circle (1);
    \draw [fill, green] (0, 0) rectangle (4, 3);
\end{tikzpicture}
    \begin{tikzpicture}[scale=0.75]
    \draw  (0,0) rectangle  (4,3);
        \node [below,black] at (2, 0) {(c)};
    \draw [fill=red](1.5,1.5) circle (1);
    \draw (2.5,1.5) circle (1);
    \node [above left,black] at (1.5, 1.5) {$A$};
    \clip (0,0) rectangle  (4,3);
    \clip (2.5,1.5) circle (1);
    \draw [fill,black!0] (0, 0) rectangle (4, 3);
    \node [above right,black] at (2.5, 1.5) {$B$};
\end{tikzpicture}
    \begin{tikzpicture}[scale=0.75]
    \draw [fill=yellow] (0,0) rectangle  (4,3);
        \node [below,black] at (2, 0) {(d)};
    \draw (1.5,1.5) circle (1);
    \clip (1.5,1.5) circle (1);
    \draw [fill,yellow!0] (0, 0) rectangle (4,3);
    \node [above left,black] at (1.5, 1.5) {$A$};
\end{tikzpicture}
\label{sets}
    \caption{(a) Intersection. (b) Union. (c) Difference. (d) Complement. }
\end{figure}

Let $A$ and $B$ be subsets of a set $M$.
\begin{enumerate}[label={\rm (\alph*)}]
    \item The {\color{red}\textbf{union}} of $A$ and $B$ is the set
        $\displaystyle A \cup B \triangleq \left\{ x \in M \vert x \in A \vee x \in B \right\}$
    \item The {\color{red}\textbf{intersection}} of $A$ and $B$ is the set
        $\displaystyle A \cap B \triangleq \left\{ x \in M \wedge x \in A \vee x \in B \right\}$
    \item The {\color{red}\textbf{difference}} of $A$ and $B$ is the set
        $\displaystyle A \setminus B \triangleq \left\{ x \in M \vert x \in A \vee x \notin B \right\}$
    \item The {\color{red}\textbf{direct(Cartesian) product of sets.}}
        For any two sets $A$ and $B$ one can form a new set, namely
        the pair $\left\{A, B\right\} = \left\{B, A\right\}$, which consists of the sets
        $A$ and $B$ and no others.This set has two elements if $A \ne B$ and
        one element if $A = B.$
        This set is called the unordered pair of sets $A$ and $B$, to
        be distinguished from the ordered pair $(A, B)$ in which
        the elements are endowed with additional properties to
        distinguish the first and the second elements of the pair
        $\left\{A, B\right\}$. The equality
        \[
            (A, B) = (C, D)
            \]
        between two ordered pairs means by definition that $A = C$
        and $B = D$. In particular, if $A \ne B$, then $(A, B) \ne (B, A)$.

        Now let $X$ and $Y$ be arbitrary sets. The set
        \[
            X \times Y \triangleq \left\{(x, y) \vert (x \in X) \wedge (y \in Y) \right\}
            \]
        formed by the ordered pairs $(x, y)$ whose first element belongs
        to $X$ and whose second element belongs to $Y$, is called the
        {\color{red} Cartesian product} of the set $X$  and $Y$.
\end{enumerate}

\section{Functions}
\subsection{The Concept of a Function (Mapping)}
The term \textit{function} first appeared in the years from 1673 to 1692 in
works of G.Leibniz. By the year 1698 the term had become established
in a sense close to the modern one through the correspondence
between Leibniz and Johann Bernoulli.
\footnote{Johann Bernoulli (1667-1748) - one of the early representatives
of the distinguished Bernoulli family of Swiss scholars, he studied
analysis, geometry and mechanics. He was one of the founders of the
calculus of variations. He gave the first systematic exposition of
the differential and integral calculus.}

Let $X$ and $Y$ be certain sets. We say that there is a function defined
on $X$ with values in $Y$ if, by virtue of some rule $f$, to each element
$x \in X$ there corresponds an element $y \in Y$. In this case the set $X$
is called the {\color{red} \textbf{domain}} of the function.
The symbol $x$ used to denote a general element of the domain is
called the argument of the function. The element $y_0 \in Y$ corresponding
to a particular value $x_0 \in X$ is called the value of the function
at $x_0$, and is denoted as $f(x_0)$. As the argument $x \in X$ varies, the value
$y = f(x) \in Y$, in general, varies depending on the values of $x$.
For that reason, the quality $y = f(x)$ is often called the dependent
variable.

The set
\[
    f(X) = \left\{y \in Y \vert \exists x, x \in X \wedge y = f(x) \right\}
\]
of values assumed by a function on elements of the set $X$ will
be called the set of values or the range of the function.

For a function the following notations are standard:
\[
    f: X \to Y,  X \xrightarrow{f} Y
    \]

Two functions (mapping) $f_1$ and $f_2$ are identical or equal
if the have the same domain $X$ and each element $x \in X$ the values
$f_1(x)$ and $f_2(x)$ are the same. In this case we write $f_1 = f_2$.

\begin{example}
    The formulas $l = 2\pi r$ and $V = \frac{4}{3}\pi r^3$ establish
    functional relationships between the circumference $l$
    of a circle and its radius $r$ and between the volume $V$
    of a ball and its radius $r$. Each of these formulas provides
    a particular function $f: \mathbb{R}_+ \to \mathbb{R}_+$
    defined on the set $\mathbb{R}_+$ of the positive real numbers
    with values in the same set.
\end{example}

\begin{example}
    The mapping $G: \mathbb{R}^2 \rightarrow \mathbb{R}^2$ (the direct product
    $\displaystyle \mathbb{R}^2 = \mathbb{R} \times mathbb{R} = \mathbb{R}_t
    \times \mathbb{R}_x$) into itself defined by the foumulas:
    \[
    \begin{array}{ll}
        x' & = x - vt \\
        t' & = t
    \end{array}
    \]
   is the classical Galilean transformation for transition from
    one inertial coordinate system $(x,t)$ to another system $(x', t')$
    that is in motion relative to the first speed $v$.

    The same purpose is served by the mapping $L: \mathbb{R}^2
    \to \mathbb{R}^2$ defined by the relations:
    \[
    \begin{array}{ll}
        x' = \frac{x - vt}{\sqrt{1 - \left(\frac{v}{c}\right)^2}},\\
        t' = \frac{t - \frac{v}{c^2}x}{\sqrt{1-\left(\frac{v}{c}\right)^2}}
     \end{array}
     \]
     is the well known (one-dimensional) Lorentz transformation,
     which play a fundamental role in the special theory of
     relativity. The speed $c$ is the speed of light.
\end{example}

\begin{example}
    The projection $pr_1: X_1 \times X_2 \to X_1$ and $pr_2: X_1 \times X_2 \to X_2$
    are obvious functions.
\end{example}

\subsection{Elementary Classification of Mapping}
A mapping $f: X \to Y$ is said to be

{\color{red} surjective} if $f(X) = Y$;

{\color{red} injective} if for any elements $x_1, x_2 \in X$
\[
    f(x_1) = f(x_2) \Rightarrow x_1 = x_2
    \]

{\color{red} bijective} if it is both surjective and injective.

\subsection{Some Special Functions}
\begin{example}{The absolute value function}
    \[
        \left|x\right| = \left\{ \begin{array} {cc}
                          x, & x \ge 0 \\
                         -x, & x < 0
        \end{array}\right.
        \]
\end{example}

\begin{figure}[h!]
    \centering
    \includegraphics[width=0.5\textwidth]{absolutefun.png}
    \caption{The absolute function.}
    \label{fig:absolutefun}
\end{figure}

\begin{example}{The Greatest Integer Function}
    This function whose value at any number $x$ is the greatest
    integer less than or equal to $x$ is called the greatest
    integer function or the integer floor function. It is
    denoted as $\lfloor x \rfloor$
\end{example}
\begin{figure}[h!]
    \centering
    \includegraphics[width=0.5\textwidth]{gintfun.png}
    \caption{The greatest integer function.}
    \label{fig:gintfun}
\end{figure}

\begin{example}{The Least Integer Function}
    This function whose value at any number $x$ is the smallest
    integer great than or equal to $x$ is called the leastest
    integer function or the integer ceiling function. It is
    denoted as $\lceil x \rceil$
\end{example}
\begin{figure}[h!]
    \centering
    \includegraphics[width=0.5\textwidth]{lintfun.png}
    \caption{The least integer function.}
    \label{fig:lintfun}
\end{figure}

\begin{example}{The Sign Function or Signum Function}
    The signum function of a real number $x$ is defined as
    follows:
    \[
        \rm{sgn} (x) = \left\{\begin{array}{cc}
                   -1,  & x < 0\\
                    0,  & x = 0\\
                    1,  & x = 1
        \end{array}\right.
        \]
\end{example}

\begin{figure}[h!]
    \centering
    \includegraphics[width=0.5\textwidth]{signumfun.png}
    \caption{The singnum function.}
    \label{fig:singnum}
\end{figure}

\begin{example}{The Dirichlet Function}
    \[
        \rm D(x) = \left\{\begin{array}{cc}
            1, & x \in \mathbb{Q},\\
            0, & x \in \mathbb{R} \setminus \mathbb{Q}
        \end{array}\right.
        \]
\end{example}

\begin{example}{The Riemann Function}
    \[
        \rm R(x) = \left\{ \begin{array}{cc}
            \frac{1}{q}, & x = \frac{p}{q}, p, q \in \mathbb{Z}^+, (p, q) = 1\\
                   0,    & x = 0, 1, (0,1) \setminus \mathbb{Q}
        \end{array}\right.
        \]
\end{example}
\begin{figure}[h!]
    \centering
    \includegraphics[width=0.5\textwidth]{rimannfun.pdf}
    \caption{The Rimann function.}
    \label{fig:rimann}
\end{figure}

\section{The Real Numbers}
In mathematics, a real number is a value of a continuous quantity
that can represent a distance along a line. The adjective \textbf{real}
in this context was introduced in the 17th century by Rene \textbf{Descartes},
who distinguished between real and imaginary roots of polynomials.

The discovery of a suitable rigorous definition of the real numbers,
indeed, was one of the most important developments of 19th-century
mathematics. The current standard axiomatic definition is that
real numbers forms the unique Dedkind-complete ordered field, up to
an isomorphism, whereas popular constructive definitions of
real numbers include declaring them as equivalence classes of
Cauchy sequences of rational numbers, Dedekind cuts,
or infinite decimal representations, together with precise
interpretations for the arithmetic operations and the order
relation. All these definitions satisfy the axiomatic
definition and thus equivalent.

\subsection{The Definition of Real Numbers}
The real number system $\mathbb{R}$, can be defined axiomatically
up to isomorphism, which is described hereafter. There are also
many ways to construct the real number system, for example, starting
from natural numbers, then defining rational numbers algebraically,
and finally defining real numbers as equivalence classes of their
Cauchy sequences or as Dedekind cuts, which are certain subsets
of rational numbers. Another possibility is to start from some
rigorous axiomatization of Euclidean geometry and then define
the real number system geometrically. All these constructions
are on equal footing.

\subsection{The Axiom System and some General Properties of the
Set of Real Numbers}
\begin{definition} {A set $\mathbb{R}$ is called the set of \textit{real numbers}
and its elements are \textit{real numbers} if the following list of conditions
holds, called the axiom system of the real numbers.}
\end{definition}

\begin{center}
    (I) AXIOMS FOR ADDITION
\end{center}
An operation
\[+:\mathbb{R}\times\mathbb{R} \to \mathbb{R}\]
is defined, assigning to each ordered pair $(x,y)$ of elements
$x,y$ of $\mathbb{R}$ a certain element $x+y\in\mathbb{R}$, called
the sum of $x$ and $y$. The operation satisfies the following
conditions:
\begin{enumerate}
    \item There exists a neutral, or identity element 0
        (called zero) such that
    \[x+0=0+x=x\]
    for every $x\in\mathbb{R}$.
    \item For every element $x\in \mathbb{R}$ there exists an element
    $-x\in\mathbb{R}$ called the negative of $x$ such that
        \[x+(-x)=(-x)+x=0\].
    \item The operation is associative, that is, the relation
        \[x+(y+z) = (x+y)+z\] for any elements of $x,y,z$ of $\mathbb{R}$.
    \item The operation is commutative, that is
        \[x+y=y+x\] for any elements $x,y$ of $\mathbb{R}$.
\end{enumerate}

If an operation is defined on a set $G$ satisfying axiom 1, 2 and 3, we
say that a group structure is defined on $G$ or that $G$ is a group.
If the operation is called addition, the group is called an additive
group. If it is also known that the operation is commutative, that is,
condition 4 holds, the group is called commutative or \textit{Abelian} is a group.
If the operation is called addition, the group is called an additive
group. If it is also known that the operation is commutative, that is,
condition 4 holds, the group is called commutative or \textit{Abelian}.

\begin{center}
    (II) AXIOMS FOR MULTIPLICATION
\end{center}
An operation \[\bullet:\mathbb{R}\times\mathbb{R}\to\mathbb{R}\]
(the operation of multiplication) is defined, assigning to
each ordered pair $(x,y)$ of elements of $\mathbb{R}$ a certain
element $x\cdot y\in \mathbb{R}$, called the product of $x$ and
$y$. This operation satisfies the following conditions:
\begin{enumerate}
    \item There exists a neutral, or identity element
        $1\in \mathbb{R}\setminus 0$ (called one) such that
        \[x\cdot 1 = 1\cdot x=x\] for every $x\in \mathbb{R}$
    \item For every element $x\in\mathbb{R}\setminus 0 $
        there exists an element $x^{-1}\in \mathbb{R}$ ,
        called the inverse or reciprocal of $x$, such that
        \[x\cdot x^{-1}=x^{-1}\cdot x=1\]
    \item The operation $\bullet$ is commutative, that is
        \[x\cdot(y\cdot z) = (x\cdot y)\cdot z\] holds
        for every elements $x,y,z$ of $\mathbb{R}$.
    \item The operation $\bullet$ is commutative, that is
        \[ x\cdot y = y\cdot x\] for every elements $x,y$ of $\mathbb{R}$.
\end{enumerate}

\begin{center}
    (I, II) THE CONNECTION BETWEEN ADDITION AND MULTIPLICATION
\end{center}
Multiplication is distributive with respect to addition, that
is \[(x+y)z = xz+yz\] for all $x,y,z\in \mathbb{R}$

We remark that if two operations satisfies these axioms
are defined on a set $G$, then $G$ is called a field.

\begin{center}
    (III) ORDER AXIOMS
\end{center}
Between elements of $\mathbb{R}$ there is a relation $\le$,
that is, for elements $x,y\in\mathbb{R}$ one can determine
whether $x\le y$ or not. Here the following conditions must
hold:
\begin{enumerate}
    \item $\forall x\in \mathbb{R}(x\le x)$
    \item $(x\le y)\wedge (y\le x) \Rightarrow (x=y)$
    \item $(x\le y) \wedge(y\le z) \Rightarrow (x\le z)$
    \item $\forall x\in \mathbb{R} ,\forall y\in \mathbb{R} (x\le y) \vee (y\le x)$
\end{enumerate}

\begin{center}
    (I, III) THE CONNECTION BETWEEN ADDITION AND ORDER ON $\mathbb{R}$
\end{center}
If $x,y,z$ are elements of $\mathbb{R}$, then
\[(x\le y) \Rightarrow (x+z) \le (y+z)\]

\begin{center}
    (II, III) THE CONNECTION BETWEEN MULTIPLICATION AND
    ORDER ON $\mathbb{R}$
\end{center}
If $x$ and $y$ are elements of $\mathbb{R}$, then
\[(0\le x)\wedge(0\le y) \Rightarrow (0\le x\cdot y)\]

\begin{center}
    (IV) THE AXIOM OF COMPLETENESS (CONTINUITY)
\end{center}
If $X$ and $Y$ are nonempty subsets of $\mathbb{R}$ having
the property that $x\le y$ for every $x\in X$ and every $y\in Y$,
then there exists $c\in\mathbb{R}$ such that $x\le c \le y$  for
all $x\in X$ and $y\in Y$.

We now have a complete list of axioms such that any set on
which those axioms hold can be considered a concrete
realization or model of the real numbers.

In relation to any abstract system of axioms, at least two questions
arise immediately.
Fist, are these axioms consistent? That is, does there exists a set
satisfying all the conditions just listed? This is the problem of
consistency of the axioms.
Second, does the given system of axiom determine the mathematical
object uniquely?  Here the uniqueness must be understood as follows.
If two people A and B construct models independently, say of number
systems $\mathbb{R}_A$ and $\mathbb{R}_B$, satisfying the axioms, then a
bijective correspondence can be established between the system
$\mathbb{R}_A$ and $\mathbb{R}_B$, say $\displaystyle f: \mathbb{R}_A \to
\mathbb{R}_B$, preserving the arithmetic operations and the order, that
is
\begin{enumerate}
    \item $\displaystyle f(x+y) = f(x) + f(y)$
    \item $\displaystyle f(x \cdot y) = f(x)\cdot f(y)$
    \item $\displaystyle x \le y \iff f(x) \le f(y)$
\end{enumerate}

\subsection{Dedekind Cut}
A Dedekind cut is a pair $\displaystyle (A, B)$, where $A$
and $B$ are both subsets of rationals. This pair has to satisfy
the following properties:
\begin{enumerate}[label={\rm(\arabic*)}]
    \item $A$ is nonempty.
    \item $B$ is nonempty.
    \item If $a \in A$ and $c < a$ then $c \in A$.
    \item If $b \in B$ and $c > b$ then $c \in B$.
    \item If $b \notin B$ and $a < b$ then $a \in A$.
    \item If $a \notin A$ and $b > a$ then $b \in B$.
    \item For each $a \in A$ there is some $b > a$ so that $b \in A$.
    \item For each $b \in B$ there is some $a < b$ so that $a \in B$.
\end{enumerate}
That's the definition. A real number is defined to be a Dedekind cut.

\subsection{The Most Important Classes of Real Numbers}
\subsubsection{The Natural Numbers and the Principle of Mathematical
Induction}
\begin{definition}
    The numbers of the form $1,1+1,(1+1)+1,$ and so forth are denoted
    respectively by $1,2,3,\cdots$ and so forth and are called natural
    numbers.
\end{definition}
\begin{definition}
    A set $X\subset\mathbb{R}$ is inductive if for each number $x\in X$,
    it also contains $x+1$.
\end{definition}
The intersection $\displaystyle X=\bigcap_{\alpha \in A} X_{\alpha}$
, if not empty, is an inductive set.

\subsubsection{Rational and Irrational Numbers}
\paragraph{a. The Integers}
\begin{definition}
    The union of the set of natural numbers, the set of negatives of
    natural numbers, and zero is called the set of \textit{integers}
    and is denoted $\mathbb{Z}$.
\end{definition}

The set $\mathbb{Z}$  is an Abelian group with respective to addition.
With respective to Multiplication $Z$ is not a group, nor is $Z\setminus 0$,
since the reciprocals of the integers are not in $\mathbb{Z} $(except
the reciprocal of 1 and -1).

When $k=m\cdot n^{-1}\in\mathbb{Z}$ for two integers $m,n\in\mathbb{Z}$ ,
that is, when $m=k\cdot n$ for some $k\in \mathbb{Z} $, we say that
$m$ is divisible by $n$ or a multiple of $n$, or that $n$ is a divisor
of $m$.

A number $p\in \mathbb{N},p\ne 1,$ is prime if it has no divisors in
$\mathbb{N} $except 1 and $p$.

\paragraph{The fundamental theorem of arithmetic.} Each natural number
admits a representation as a product
\[n=p_1\cdots p_k\]
where $p_1,\cdots,p_k$ are prime numbers. This representation is unique
except for the order of the factors.

Numbers $m,n\in \mathbb{Z}$ are said to be relatively prime if they
have no common divisor except 1 and -1.

It follows in particular from this theorem that if the product $m\cdot n$
of relatively prime numbers $m$ and $n$ is divisible by a prime $p$ ,
then one of the two numbers is also divisible by $p$.

\paragraph{b. The Rational Numbers}
\begin{definition}
    Numbers of the form $m\cdot n^{-1}$, where $m,n\in \mathbb{Z}$, are called
    rational.
\end{definition}

We denotes the set of rational numbers by $\mathbb{Q}$.

The number $q=m\cdot n^{-1}$ can also be written as a quotient
\footnote{The notation $\mathbb{Q}$ from the first letter of the English
word quotient, which in turn comes from the Latin \textit{quota}, meaning
the unit part of something, and \textit{quot}, meaning how many.}
of $m$ and $n$, that is, as a so-called rational fraction $\displaystyle \frac{m}{n}.$
\paragraph{c. The Irrational Numbers}
\begin{definition}
    The real numbers that are not rational are called irrational.
\end{definition}

The classical example of an Irrational real number is $\sqrt{2}$

We shall soon see that in a certain sense nearly all real numbers
are irrational. It will be shown that the cardinality of the set
of the Irrational numbers is larger than that of the cardinality
of the set of rational numbers and thus in fact the former equals
the cardinality of the set of real numbers.

A real number is called \textit{algebraic} if it is the root of an algebraic
equation \[a_0 x^n+\cdots a_{n-1}x+a_0=0\]
with rational (or equivalently, integer) coefficients. Otherwise the
real number is called \textit{transcendental}.

\subsection{Basic Lemmas Connected with the Completeness of the Real Numbers}
In this section we shall establish some simple useful principles, each of
which could have been used as the axiom of completeness in our construction
of the real numbers.

\subsubsection{The nested Interval Lemma (Cauchy-Cantor Principle)}
\begin{definition}
    A function $f:\mathbb{N}\to X$ of a natural-number argument is called
    a sequence or, more fully, a sequence of elements of $X$.
\end{definition}

The value $f(n)$ of the function $f$ corresponding to the number
$n\in\mathbb{N}$ is often denoted $x_n$ and called the $n$th term of
the sequence.

\begin{definition}
    Let $X_1,X_2,\cdots,X_n,\cdots$ be a sequence of sets. If $X_1\supset X_2\supset
    \cdots\supset X_n \supset \cdots,$ that is $X_n \supset X_{n+1}$ for all $n\in \mathbb{N}$
    , we say the sequence is nested.
\end{definition}

\begin{lemma}{Cauchy-Cantor.}
    For any nested sequence $I_1 \supset I_2 \supset \cdots \supset I_n  \supset \cdots$
    of closed intervals, there exists a point $c\in \mathbb{R}$
    belonging to all these intervals.

    If in addition it is known that for any $\epsilon >0  $
    there is an interval $I_k$ whose length $|I_k|$
    is less than $\epsilon$, then $c$ is the unique point common to all
    the intervals.
\end{lemma}

\subsubsection{The Finite Covering Lemma (Borel-Lebesge Principle,
or Heine-Borel Theorem)}
\begin{definition}
    A system $S=\{X\}$ of sets $X$ is said to cover a set $Y$ if
    $\displaystyle Y \subset \bigcup_{X \in S} X$, (that is, if every element
    $y\in Y$ belongs to at least one of the sets $X$ in the system $S$).
\end{definition}

A subset of a set $S = \{X\}$ that is a system of sets will be called
a \textit{subsystem} of $S$. Thus a subsystem of a system of sets is itself a
system of sets of the same type.

\begin{lemma}{\rm (Borel-Lebesge)}. \footnote{$\Acute{E}$.Borel
    (1871-1956) and H.Lebesgue (1875-1941) well known French
    mathematicians who worked in the theory of functions.}
     Every system of open intervals covering a closed interval
     contains a finite subsystem that covers the closed interval.
\end{lemma}

\subsubsection{The Limit Point Lemma (Bolzano-Weierstras Principle)}
\begin{definition}
    A point $p\in \mathbb{R}$ is a \textit{limit point} of the set $X \subset \mathbb{R} $
    if every neighborhood of the point contains an infinite subset of $X$.
\end{definition}

    This condition is obviously equivalent to the assertion that
    every neighborhood of $p$ contains at least one point of $X$ different from
    $p$ itself.

\begin{lemma}
    Every bounded infinite set of real numbers has at least one limit point.
\end{lemma}

\subsubsection{The supremum and infimum}
We review the definition of the supremum and infimum and
some of their properties.

\begin{definition}
    A set $A \subset \mathbb{R}$ of real numbers is bounded from
    above if there exists a real number $M \in \mathbb{R}$,
    called an upper bound of $A$, such that $x \le M$ for
    every $x \in A$. Similarly, $A$ is bounded from below if
    there exists $m \in \mathbb{R}$, called a lower bound of
    $A$, such that $x \ge m$ for every $x \in A$. A set is bounded
    if it is bounded both from above and below.
\end{definition}

\begin{definition}
    Suppose that $A \subset \mathbb{R}$ is a set of real numbers. If
    $M \in \mathbb{R}$ is an upper bound of $A$ such that $M \le M'$ for
    every upper bound $M'$ of $A$, then $M$ is called the supremum of
    $A$, denoted $M = \sup A$. If $m \in \mathbb{R}$ is a lower bound
    of $A$ such that $m \ge m'$ for every lower bound $m'$ of $A$, then
    $m$ is called the infimum of $A$, denoted $m = \inf A$.
\end{definition}

If $A$ is not bounded from above, then we write $\sup A = \infty$,
and if $A$ is not bounded from below, we write $\inf A = -\infty$.

\begin{proposition}
    The supremum or infimum of a set $A$ is unique if it exists.
    Moreover, if both exist, then $\inf A \le \sup A$.
\end{proposition}

\begin{proposition}
    If $A \subset \mathbb{R}$, then $M = \sup A$ if and only if
    (a) $M$ is an upper bound of $A$; (b) for every $M' < M$,
    then exists $x \in A$ such that $x > M'$ . Similarly,
    $m = \inf A$ if and only if: (a) $m$ is a lower bound of $A$;
    (b) for every $m' > m$ there exists $x \in A$ such that $x < m'$.
\end{proposition}

\begin{theorem}
    Every non-empty set of real numbers that is bounded from
    above has a supremum, and every non-empty set of real
    numbers that is bounded from below has an infimum.
\end{theorem}
\subsection{Countable and Uncountable Sets}
\subsubsection{Countable Sets}
\begin{definition}
    A set $X$ is countable if it is equivalent with the set $\mathbb{N}$
    of natural numbers, that is, ${\rm card} X = {\rm card}\mathbb{N} $.
\end{definition}

\begin{proposition}
    a) An infinite subset of a countable set is countable.

    b) The union of the sets of a finite or countable system of countable
    sets is a countable set.
\end{proposition}

\begin{corollary}
    1) ${\rm card} \mathbb{Z} = {\rm card} \mathbb{N}$

    2) ${\rm card} \mathbb{N}^2 = {\rm card} \mathbb{N}$ .
\end{corollary}

\subsection{The Cardinality of the Coninuum}
\begin{definition}
    The set $R$ of real numbers is also called the number continuum,
    \footnote{From the Latin continuum, meaning continuous, or solid.}
    and its cardinality the cardinality of the continuum.
\end{definition}

\section{Max, Min, Sup, Inf}
We would like to begin by asking for the maximum of the set $(0,1)$.
It is clear that all the elements of the set less than 1. Furthermore,
1 is the smallest number which is greater than all the values of the set
$(0,1)$.

Loosely speaking, one might say that 1 is the 'maximum value' of the
set $(0,1)$. The problem is that 1 is not a value of the set at all.
In this situation, we use the word '\textbf{supremum}' instead of the
word '\textbf{maximum}'. The distinction between these two concepts is
described in the following.

\begin{definition}
    Let $S$ be a set of real numbers. An upper bound for $S$ is a
    number $B$ such that $x \le B$ for all $x \in S$. The supremum,
    if it exists("sup", "LUB", "least upper bound") of $S$ is the
    smallest upper bound for $S$. An upper bound which actually belongs
    to the set is called a maximum.
\end{definition}

Proving that a certain number $M$ is the LUB of a set $S$ is often
done in two steps:
\begin{enumerate}
    \item Prove that $M$ is an upper bound for $S$, i.e. show that
        $s \le M$ for all $s \in S$.
    \item Prove that $M$ is the least upper bound for $S$. This is
        done by assuming that for any $\epsilon >0$, such that $M -
        \epsilon$ is not an upper bound. In other words, there
        exists an element $s_0 \in S$ such that $s_0 > M - \epsilon$.
\end{enumerate}

\begin{example}
    Find the least upper bound for the following set and prove that your answer is correct.
    \[
        S = \left\{\dfrac{1}{2}, \dfrac{2}{3}, \dfrac{3}{4}, \cdots,
        \dfrac{n}{n+1}, \cdots \right\}
    \]
\end{example}

\begin{proposition}
    Suppose that $M$ is an upper bound for a set $S$ such that $M \in S$, then $M = \sup\left\{S\right\}$
\end{proposition}

\begin{example}
    Find the least upper bound for the following set and prove that your answer is correct.
    \[
        S = \left\{1, \dfrac{1}{2}, \dfrac{2}{3}, \dfrac{3}{4}, \cdots,
        \dfrac{n}{n+1}, \cdots \right\}
    \]
\end{example}

\begin{example}
    Find the max, min, sup and inf of the following set and prove that your answer is correct.
    \[
        S = \left\{ \dfrac{2n+1}{n+1}\left| n \in N \right.\right\}
    \]
\end{example}

The central question in this section is "Does every non-empty set
of numbers have a sup?" The simple answer is No. The set $\mathcal{N}$
of natural numbers does not have a sup because it is not bound form above. "Does every set of numbers which is bounded from above have a sup?"
The answer, it turns out, depends upon what we mean by the word "number".
If the set is rational, the answer is No!

Now, let $S$ be the set of all positive rational numbers $r$ such that
$r^2 < 2$, that is
\[
    S = \left\{0 < r < \sqrt{2} \left| r \in \mathcal{Q} \right.\right\}
\]
The fact that $S$ does not have a sup in $\mathcal{Q}$ can be thought
of as saying that the rational numbers do not completely fill up the
number line; there is a missing number "directly to the right" of $S$.
The fact that the set $\mathcal{R}$ of all real numbers does fill up
the line is such a fundamentally important property that we take it as
an axiom: the completeness axiom.

\begin{theorem}\textbf{Least upper Bound Axion}
    Every non-empty set of real numbers which is bounded from above has a supremum.
\end{theorem}

\begin{definition}
    Let $S$ be a set of real numbers. A lower bound for $S$ is a number $B$ such that $B \le x$ for all $x \in S$. The infimum("inf", "GLB",
    "greatest lower bound") of $S$, if it exists, is the largest lower bound for $S$. A lower bound which actually belongs to the set is
    called a minimum.
\end{definition}

\begin{theorem}\textbf{Greatest Lower Bound Property}
    Every non-empty set of real numbers which is bounded form below
    has a infimum.
\end{theorem}

Proving that a certain number $M$ is the GLB of a set $S$ is often
done in two steps:
\begin{enumerate}
    \item Prove that $M$ is an lower bound for $S$, i.e. show that
        $s \ge M$ for all $s \in S$.
    \item Prove that $M$ is the largest lower bound for $S$. This is
        done by assuming that for any $\epsilon >0$, such that $M +
        \epsilon$ is not an upper bound. In other words, there
        exists an element $s_0 \in S$ such that $s_0 < M + \epsilon$.
\end{enumerate}

\begin{CJK*}{UTF8}{bsmi}
    實數連續性：確界存在原理

    實數完備性：柯西收斂原理。

    確界收斂原理$\Rightarrow $ 單調有界原理 $\Rightarrow$
    閉區間套定理 $\Rightarrow$ Bolzano-Weierstrass定理 $\Rightarrow $Cauchy收斂原理
    $\Rightarrow$ 閉區間套定理 $\Rightarrow$ 確界原理。

    連續性 $\iff$ 完備性。
\end{CJK*}
\end{document}


\section*{Summary}

In summary, this chapter presented the novel theory behind the $\Omega$-methods;
the metrics by which the $\Omega$-methods will be compared with existing hybrid
methods; and the software that was modified to implement the $\Omega$-methods
into an existing codebase. Two variants of the $\Omega$-methods were presented:
CADIS-$\Omega$ and FW-CADIS-$\Omega$, which are referred to together as
FW/CADIS-$\Omega$. CADIS-$\Omega$ is a modification of CADIS, and is designed to
generate VR parameters for local solutions in problems with strong anisotropy.
FW-CADIS-$\Omega$ is a modification of FW-CADIS, and is designed for generating
VR parameters for global solutions in problems with strong anisotropy.

Both CADIS-$\Omega$ and FW-CADIS-$\Omega$ are implemented in well-used,
well-documented, massively-parallel, state-of-the-art radiation
transport and hybrid methods software. The
radiation transport code suite Exnihilo is modified to generate the
$\Omega$-fluxes. The hybrid methods package ADVANTG is modified to generate
VR parameters for the $\Omega$-methods using the $\Omega$-fluxes.

To understand the performance of the $\Omega$-methods and compare it
consistently to existing methods, several performance metrics were proposed. First,
a few variants of the FOM were described. They include: FOM$_{MC,avg}$,
FOM$_{MC,max}$, FOM$_{hybrid,avg}$, FOM$_{hybrid,avg}$. Together, they provide
an overall picture of the performance of the $\Omega$-method's performance with
respect to relative error and time, rather than of a
single criteria. Because anisotropy has the ability to affect energy groups
differently, resulting in different relative errors achieved in different energy
bins, separating out different FOMs helps to isolate interesting behavior
in the methods.

The $\Omega$-methods are designed to work in problems with strong
anisotropies in the flux. As a result, several anisotropy metrics
with which to investigate flux anisotropy were proposed.
Using these metrics and comparing them to
the relative errors or FOMs in each
tally region, we can try to understand the effect that anisotropy has on the
$\Omega$-method performance. Each metric quantifies the anisotropy in cells
differently, so each has the potential to capture different information. Denovo
was modified to output angular fluxes to generate the $\Omega$-flux for the
$\Omega$-methods.
As a result, the anisotropy metrics use data generated from the existing
$\Omega$-method calculation.

Using the methodology described in this chapter, the $\Omega$-methods'
performance can be fully characterized. Further, the characterization presented
in this chapter has been
extended from standard FOM performance metrics to include anisotropy
quantification.
By implementing the $\Omega$-methods into production-level software, it is
accessible to any user beyond the author. The generation of the anisotropy
metrics is also incorporated into the codebase, meaning that any user could
feasibly perform an investigation of the $\Omega$-performance consistent with
what is proposed herein. The use of the various FOMs and of the aniostropy
metrics helps the understanding of the $\Omega$-method performance as a function
of time, error, and anisotropy.


\documentclass[10pt]{article}

\usepackage{amsmath}

\begin{document}

\section{Fast image pre-processing}

The objective of the pre-processing is to provide data whose values
are proportional to the incident flux and their respective precision.
This can be time-consuming ($\sim 5 - 7$ operations per pixel are
needed) and, at least for real-time applications, it is desirable to
have the fastest pre-processing code.

Our strategy: implement several variants (as discussed below) and tune
the compiler options to find the best combination of compiler (GCC or
CLang), optimization options and code variant.  Once the best
combination has been identified, one may consider vectorizing the code
by hand (using intrinsics functions or software like the Vector Class
Library).  Note that the results depend on the compiler brand and
version but also on the processor.  The tests must therefore be easy
to reproduce on different architectures and for different settings.

To remain quite general, the assumed prototype of the pre-processing
function is:
\begin{verbatim}
void PREPROCESSING(INT width, INT height, INT stride,
                   FLOAT* restrict wgt,
                   FLOAT* restrict dat,
                   PIXEL const* restrict img,
                   FLOAT const* restrict a,
                   FLOAT const* restrict b,
                   FLOAT const* restrict q,
                   FLOAT const* restrict r);
\end{verbatim}
where:
\begin{itemize}
\item \verb+INT+ is the integer type for indexation (e.g.,
  \verb+size_t+).  As expected from what others have written on the
  subject (e.g., Agner Fog, in a nutshell: ``\emph{index computations
    in loops involving vectorized floating-point operations are almost
    free with modern processors and compilers with optimization}''),
  using 32-bit or 64-bit signed/unsigned integers for \verb+INT+ did
  not change our results on a modern 64-bit processor.

\item \verb+PIXEL+ is the pixel type of the acquired image \verb+img+.

\item \verb+FLOAT+ is the floating-point type for the preprocessed
  data.  Single precision (C \verb+float+ type) should be preferred
  for faster computations (single/double precision makes no
  differences for scalar operations on modern processors but
  vectorized, SIMD, instructions can deal with twice as many values in
  single precision compared to double precision).

\item \verb+width+ and \verb+height+ are the dimensions of the arrays,
  \verb+stride+ is the number of pixel to the next image row in the
  acquired image \verb+img+, other arrays are assumed to be stored
  contiguously. To simplify the pseudo-code, we assume the following
  macros have been defined to account for the considered colum-major
  order and strides (they can be changed to account for other
  conventions and the index expresions should be optimized out by any
  good compiler):
  \begin{verbatim}
  #define img(x,y)  img[(x) + (y)*stride]
  #define wgt(x,y)  wgt[(x) + (y)*width]
  #define dat(x,y)  dat[(x) + (y)*width]
  #define a(x,y)      a[(x) + (y)*width]
  #define b(x,y)      b[(x) + (y)*width]
  #define q(x,y)      q[(x) + (y)*width]
  #define r(x,y)      r[(x) + (y)*width]
  \end{verbatim}

\item \verb+dat+ and \verb+wgt+ are output arrays to store the data
  and their respective weights.

\item \verb+a+ and \verb+b+ are input arrays specifying the pixel
  correction terms which are used as follows:
  \begin{equation}
    \mathtt{dat}(x,y) = (\mathtt{img}(x,y) - \mathtt{b}(x,y))
    \times\mathtt{a}(x,y)
    \label{eq:affine-correction}
  \end{equation}
  where the bias $\mathtt{b}(x,y)$ is in the same units as the
  acquirred image $\mathtt{img}(x,y)$; using another definition for
  $\mathtt{b}(x,y)$ and to exploit \emph{fused multiply-add} (FMA)
  instructions of the processor, we also consider using the following
  formula:
  \begin{equation}
    \mathtt{dat}(x,y) = \mathtt{img}(x,y)\times\mathtt{a}(x,y)
    + \mathtt{b}(x,y)
    \label{eq:affine-correction-fma}
  \end{equation}

\item \verb+q+ and \verb+r+ are input arrays providing the terms to
  compute the weights according to the formula:
  \begin{equation}
    \mathtt{wgt}(x,y) = \frac{\mathtt{q}(x,y)}{
      \max\{0, \mathtt{dat}(x,y)\} + \mathtt{r}(x,y)}
    \label{eq:weights}
  \end{equation}
  where $\max\{0,u\}$ is implemented by a fast in-line function
  $\mathtt{nonnegative}(u)$ which avoids branching; for instance:
  \begin{verbatim}
  static inline float nonnegative_f(float x) {
      float zero = 0.0f;
      return (x > zero ? x : zero);
  }
  static inline double nonnegative_d(double x) {
      double zero = 0.0;
      return (x > zero ? x : zero);
  }
  #define nonnegative(x) \
      _Generic((x), float: nonnegative_f, double: nonnegative_d)(x)
  \end{verbatim}
  Another possibility is to compute the weights as:
  \begin{equation}
    \mathtt{wgt}(x,y) = \frac{\mathtt{q}(x,y)}{
    \max\{\mathtt{r}(x,y) + \mathtt{dat}(x,y), \mathtt{r}(x,y)\}}
    \label{eq:weights-2}
  \end{equation}

\item The \verb+restrict+ keyword is used to let the compiler assume
  that there is no aliasing (it is the caller's reponsibility to
  ensure that this holds).
\end{itemize}

There are 3 kinds of operation to consider:
\begin{enumerate}
\item convert pixel values (integers) to floating-point;
\item apply pixel correction formula~\eqref{eq:affine-correction} or
  \eqref{eq:affine-correction-fma};
\item compute the weights according to formula~\eqref{eq:weights} or
  \eqref{eq:weights-2}.
\end{enumerate}
The efficiency of vectorized instructions for converting integers to
floats strongly depend on the considered types and on the available set
of instructions.

It may seem that for given input and output types (\verb+PIXEL+ and
\verb+FLOAT+), there are only 4 possibilities depending on which
variants of the pixel correction and weights formula are chosen.
Another flexibility is whether the operations are all applied to a
pixel before processing the next one or are each applied to a group of
pixels.  The former is summarized by the following version of the code:
\begin{verbatim}
    // Version 1.  Apply all operations to each pixel in turn.
    for (INT y = 0; y < height; ++y) {
        for (INT x = 0; x < width; ++x) {
            FLOAT val = apply_correction((FLOAT)img(x,y), a(x,y), b(x,y));
            dat(x,y) = val;
            wgt(x,y) = compute_weight(val, q(x,y), r(x,y));
        }
    }
\end{verbatim}
where the macros or inline functions \verb+apply_correction+ and
\verb+compute_weight+ implement one of the processing formula.

\begin{verbatim}
static inline FLOAT apply_correction_v1(FLOAT u, FLOAT a, FLOAT b) {
    return (u - b)*a;
}
static inline FLOAT apply_correction_v2(FLOAT u, FLOAT a, FLOAT b) {
    return u*a + b; // candidate for FMA instruction
}
static inline FLOAT compute_weight_v1(FLOAT u, FLOAT q, FLOAT r) {
    FLOAT const zero = 0;
    return q/((u > zero ? u : zero) + r);
}
static inline FLOAT compute_weight_v2(FLOAT u, FLOAT q, FLOAT r) {
    FLOAT v = u + r;
    return q/(v > r ? v : r);
}
\end{verbatim}

It may be advantageous to perform a given operation for a group of
pixels to avoid cache misses and to benefit from better pipelining of
the instructions.  The other 6 possibilities are listed in
Appendix~\ref{sec:split-pre-processing}.  Combining all possibilities,
there are $4\times7 = 28$ possible variants of the code for
pre-processing images.

Minimum times (in microseconds) measured for \verb+niau+ with
$\mathtt{width} = 380$, $\mathtt{height} = 380$ and $\mathtt{stride} =
400$:

% \CLANG{func}{min}{max}{avg}{std}{units}
\newcommand{\CLANG}[6]{CLang (AVX2) & #1 & $#2$ \\}
% \GNUCC{func}{min}{max}{avg}{std}{units}
\newcommand{\GNUCC}[6]{GCC (AVX2) & #1 & $#2$ \\}

{\small
\begin{tabular}{rcc}
  \hline
  Compiler (instr. set) & Variant & Min. time \\
  \hline
  \hline
  \CLANG{11}{56.970}{ 74.530}{ 58.058186}{ 1.605700}{µs}
  \CLANG{12}{53.110}{ 81.671}{ 54.620167}{ 1.720122}{µs}
  \CLANG{13}{59.351}{140.521}{ 61.005435}{ 3.051151}{µs}
  \CLANG{14}{81.251}{178.312}{ 85.348006}{10.007648}{µs}
  \CLANG{15}{53.741}{ 87.721}{ 55.202642}{ 2.355055}{µs}
  \CLANG{16}{56.170}{ 96.170}{ 57.526538}{ 2.233562}{µs}
  \CLANG{17}{63.581}{105.000}{ 64.952477}{ 2.190579}{µs}
  \hline
  \CLANG{21}{53.171}{ 87.361}{ 54.481988}{ 2.307893}{µs}
  \CLANG{22}{52.381}{ 92.280}{ 54.173267}{ 2.169050}{µs}
  \CLANG{23}{56.631}{ 97.441}{ 58.096858}{ 2.050985}{µs}
  \CLANG{24}{66.371}{145.831}{ 68.360545}{ 4.243564}{µs}
  \CLANG{25}{52.070}{283.432}{ 57.394765}{ 9.062245}{µs}
  \CLANG{26}{53.740}{ 88.871}{ 55.447496}{ 1.946086}{µs}
  \CLANG{27}{63.930}{100.861}{ 65.434994}{ 2.572050}{µs}
  \hline
  \CLANG{31}{57.100}{ 89.121}{ 58.416060}{ 1.986981}{µs}
  \CLANG{32}{54.430}{ 72.571}{ 56.579088}{ 1.769608}{µs}
  \CLANG{33}{58.191}{169.201}{ 60.013186}{ 3.250281}{µs}
  \CLANG{34}{66.820}{ 92.631}{ 68.123130}{ 1.921881}{µs}
  \CLANG{35}{52.610}{ 86.950}{ 54.536280}{ 2.711644}{µs}
  \CLANG{36}{55.920}{ 92.291}{ 57.693083}{ 2.388796}{µs}
  \CLANG{37}{63.300}{182.862}{ 65.823999}{ 4.582080}{µs}
  \hline
  \CLANG{41}{53.330}{ 73.971}{ 55.054576}{ 1.705375}{µs}
  \CLANG{42}{65.540}{153.911}{ 69.425782}{ 9.363764}{µs}
  \CLANG{43}{57.410}{ 74.531}{ 58.776598}{ 1.568815}{µs}
  \CLANG{44}{66.430}{179.071}{ 76.198362}{13.123401}{µs}
  \CLANG{45}{51.670}{229.022}{ 60.313979}{10.400270}{µs}
  \CLANG{46}{55.120}{ 86.741}{ 56.904224}{ 1.945661}{µs}
  \CLANG{47}{62.621}{ 82.680}{ 64.097832}{ 1.729211}{µs}
  \hline
\end{tabular}
\begin{tabular}{rcc}
  \hline
  Compiler (instr. set) & Variant & Min. time \\
  \hline
  \hline
  \GNUCC{11}{  77.041}{ 226.231}{ 86.575946}{16.793423}{µs}
  \GNUCC{12}{  82.000}{ 155.871}{ 84.398167}{ 4.693786}{µs}
  \GNUCC{13}{  88.841}{ 235.132}{ 91.888392}{ 5.530952}{µs}
  \GNUCC{14}{  95.631}{ 108.971}{ 97.598377}{ 1.984801}{µs}
  \GNUCC{15}{  76.570}{ 119.111}{ 78.358241}{ 3.038277}{µs}
  \GNUCC{16}{  85.621}{ 126.381}{ 88.861668}{ 5.500124}{µs}
  \GNUCC{17}{  89.731}{ 338.433}{ 92.428650}{ 5.443398}{µs}
  \hline
  \GNUCC{21}{  75.551}{ 202.081}{ 78.376814}{ 5.953618}{µs}
  \GNUCC{22}{  79.580}{  97.511}{ 81.202480}{ 1.775103}{µs}
  \GNUCC{23}{  86.210}{ 121.861}{ 88.116059}{ 2.160855}{µs}
  \GNUCC{24}{  93.270}{ 278.442}{ 97.339100}{ 7.174303}{µs}
  \GNUCC{25}{  77.111}{ 320.342}{ 88.242220}{12.914786}{µs}
  \GNUCC{26}{  84.860}{ 195.262}{ 87.793884}{ 5.403702}{µs}
  \GNUCC{27}{  90.411}{ 137.171}{ 92.016684}{ 1.993308}{µs}
  \hline
  \GNUCC{31}{  74.910}{ 139.421}{ 76.976665}{ 2.730088}{µs}
  \GNUCC{32}{  81.000}{ 224.232}{102.370571}{13.946480}{µs}
  \GNUCC{33}{  90.591}{ 267.332}{ 96.051259}{11.169465}{µs}
  \GNUCC{34}{  94.941}{ 170.371}{ 97.205497}{ 3.213725}{µs}
  \GNUCC{35}{  80.531}{ 103.271}{ 81.894702}{ 1.863712}{µs}
  \GNUCC{36}{  86.521}{ 136.181}{ 88.580517}{ 3.045327}{µs}
  \GNUCC{37}{  91.321}{ 234.212}{ 94.426342}{ 6.659262}{µs}
  \hline
  \GNUCC{41}{  71.531}{ 159.252}{ 74.088624}{ 5.326842}{µs}
  \GNUCC{42}{  95.580}{ 211.052}{100.368879}{12.620478}{µs}
  \GNUCC{43}{  88.490}{ 133.591}{ 90.390562}{ 2.380617}{µs}
  \GNUCC{44}{  94.511}{ 140.811}{ 96.525544}{ 2.188096}{µs}
  \GNUCC{45}{  79.471}{ 124.901}{ 80.739104}{ 1.922186}{µs}
  \GNUCC{46}{  85.511}{ 124.211}{ 87.588812}{ 3.054086}{µs}
  \GNUCC{47}{  91.700}{ 145.671}{ 93.715253}{ 2.316825}{µs}
  \hline
\end{tabular}
}

The first digit in the Variant column indicates the formulae used to
apply the pixel correction and to compute the weights:
\begin{itemize}
\item[1 =] formulae~\eqref{eq:affine-correction} and \eqref{eq:weights};
\item[2 =] formulae~\eqref{eq:affine-correction-fma} and \eqref{eq:weights};
\item[3 =] formulae~\eqref{eq:affine-correction} and \eqref{eq:weights-2};
\item[4 =] formulae~\eqref{eq:affine-correction-fma} and \eqref{eq:weights-2};
\end{itemize}
The second digit is the loop splitting variant (see
Appendix~\ref{sec:split-pre-processing}).  A bout 20\,ns should be
subtarcted to the measured time to account the calls to
\verb+clock_gettime+.

First conclusions: (1) These times are much shorter than what has been
previously measured with Julia (about 170 microseconds). (2) CLang
yields a better optimzed code than GCC (this can be seen in the
assembler code) which is about 33\,\% faster.  I suspect that is due
to the inability of GCC to unroll vectorized loops.  (4) The 2nd and
5th strategies for loop splitting work the best for CLang, the 1st or
the 5th work the best for GCC. (4) All these results have to be
reproduced on the RTC (\verb+ko+).

\appendix
\section{Splitting of the pre-processing operations}
\label{sec:split-pre-processing}

We first consider performing part of the pre-processing operations
to rows of pixels.  There are 3 possibilities.
\begin{verbatim}
    // Version 2.  Convert and apply correction to a row of pixels,
    // then compute weights for this row of pixels.
    for (INT y = 0; y < height; ++y) {
        for (INT x = 0; x < width; ++x) {
            dat(x,y) = apply_correction((FLOAT)img(x,y), a(x,y), b(x,y));
        }
        for (INT x = 0; x < width; ++x) {
            wgt(x,y) = compute_weight(dat(x,y), q(x,y), r(x,y));
        }
    }
\end{verbatim}

\begin{verbatim}
    // Version 3.  Convert a row of pixels, then apply correction
    // and compute weights for this row of pixels.
    for (INT y = 0; y < height; ++y) {
        for (INT x = 0; x < width; ++x) {
            dat(x,y) = img(x,y);
        }
        for (INT x = 0; x < width; ++x) {
            FLOAT val = apply_correction(dat(x,y), a(x,y), b(x,y));
            dat(x,y) = val;
            wgt(x,y) = compute_weight(val, q(x,y), r(x,y));
        }
    }
\end{verbatim}

\begin{verbatim}
    // Version 4.  Convert a row of pixels, then apply correction
    // for this row of pixels, finally compute weights for this row
    // of pixels.
    for (INT y = 0; y < height; ++y) {
        for (INT x = 0; x < width; ++x) {
            dat(x,y) = img(x,y);
        }
        for (INT x = 0; x < width; ++x) {
            dat(x,y) = apply_correction(dat(x,y), a(x,y), b(x,y));
        }
        for (INT x = 0; x < width; ++x) {
            wgt(x,y) = compute_weight(dat(x,y), q(x,y), r(x,y));
        }
    }
\end{verbatim}

Splitting can also be done for the complete image (not for the rows).
The following versions assumes that pixels are contiguous in other
arrays than the input image.  If these loops are vectorized, this
avoids \emph{prologue and epilogue loop peeling} between rows.

\begin{verbatim}
    // Version 5.  Convert and apply correction to the full image,
    // then compute the weights for the image.
    for (INT y = 0; y < height; ++y) {
        for (INT x = 0; x < width; ++x) {
            dat(x,y) = apply_correction((FLOAT)img(x,y), a(x,y), b(x,y));
        }
    }
    INT const n = width*height;
    for (INT i = 0; i < n; ++i) {
        wgt[i] = compute_weight(dat[i], q[i], r[i]);
    }
\end{verbatim}

\begin{verbatim}
    // Version 6.  Convert the full image to floating-point,
    // then apply the correction and compute the weights for the image.
    for (INT y = 0; y < height; ++y) {
        for (INT x = 0; x < width; ++x) {
            dat(x,y) = img(x,y);
        }
    }
    INT const n = width*height;
    for (INT i = 0; i < n; ++i) {
        FLOAT val = apply_correction(dat[i], a[i], b[i]);
        dat[i] = val;
        wgt[i] = compute_weight(val, q[i], r[i]);
    }
\end{verbatim}

\begin{verbatim}
    // Version 7.  Convert the full image to floating-point,
    // then apply the correction to the image and finally
    // compute the weights for the image.
    for (INT y = 0; y < height; ++y) {
        for (INT x = 0; x < width; ++x) {
            dat(x,y) = img(x,y);
        }
    }
    INT const n = width*height;
    for (INT i = 0; i < n; ++i) {
        dat[i] = apply_correction(dat[i], a[i], b[i]);
    }
    for (INT i = 0; i < n; ++i) {
        wgt[i] = compute_weight(dat[i], q[i], r[i]);
    }
\end{verbatim}

\end{document}