\documentclass{beamer}

\usepackage{paratype}
\setbeamerfont{frametitle}{family=\bf}
\usepackage{listings}

% Beamer theme settings
\usecolortheme{seagull}
\usenavigationsymbolstemplate{} % no navigation buttons

\usepackage[utf8]{inputenc}

\usepackage{graphicx}
\usepackage{epic}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}

\newcommand{\basetop}[1]{\vtop{\vskip-1ex\hbox{#1}}}
\newcommand{\source}[1]{\let\thefootnote\relax\footnotetext{\scriptsize\textcolor{kugray1}{Source: #1}}}

% for coloured code citation in text:
\usepackage{fancyvrb}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%    code sections   %%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% code highlighting commands in own block
\DefineVerbatimEnvironment{code}{Verbatim}{fontsize=\scriptsize}
\DefineVerbatimEnvironment{icode}{Verbatim}{fontsize=\scriptsize}

% Fancy code with color commands:
\DefineVerbatimEnvironment{colorcode}%
        {Verbatim}{fontsize=\scriptsize,commandchars=\\\{\}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%    some coloring    %%%%%%%%

\definecolor{Red}{RGB}{220,50,10}
\definecolor{Blue}{RGB}{0,51,102}
\definecolor{Yellow}{RGB}{102,51,0}
\definecolor{Orange}{RGB}{178,36,36}
\definecolor{Grey}{RGB}{180,180,180}
\definecolor{Green}{RGB}{20,120,20}
\definecolor{Purple}{RGB}{160,50,100}
\newcommand{\red}[1]{\textcolor{Red}{{#1}}}
\newcommand{\blue}[1]{\textcolor{Blue}{{#1}}}
\newcommand{\yellow}[1]{\textcolor{Yellow}{{#1}}}
\newcommand{\orange}[1]{\textcolor{Orange}{{#1}}}
\newcommand{\grey}[1]{\textcolor{Grey}{{#1}}}
\newcommand{\green}[1]{\textcolor{Green}{{#1}}}
\newcommand{\purple}[1]{\textcolor{Purple}{{#1}}}

% use "DIKU green" from our color theme for \emph
%\renewcommand{\emph}[1]{\textcolor{structure}{#1}}
\renewcommand{\emph}[1]{\textcolor{CosGreen}{ #1}}
% use some not-too-bright red for an \emp command
\definecolor{DikuRed}{RGB}{130,50,32}
\newcommand{\emp}[1]{\textcolor{DikuRed}{ #1}}
\definecolor{CosGreen}{RGB}{10,100,70}
\newcommand{\emphh}[1]{\textcolor{CosGreen}{ #1}}
\definecolor{CosBlue}{RGB}{55,111,122}
\newcommand{\emphb}[1]{\textcolor{CosBlue}{ #1}}
\definecolor{CosRed}{RGB}{253,1,1}
\newcommand{\empr}[1]{\textcolor{CosRed}{ #1}}

\newcommand{\mymath}[1]{$ #1 $}
\newcommand{\myindx}[1]{_{#1}}
\newcommand{\myindu}[1]{^{#1}}

\newtheorem{mydef}{Definition}
\newtheorem{mytheo}{Theorem}
\newtheorem{mylemma}{Lemma}

\title{OpenCL Day 3: Basic Block of Data-Parallel Programming and Applications}

\author{Cosmin Oancea and Troels Henriksen}

\date{February 4, 2019}

\begin{document}

\frame{\titlepage}

\section{Course Contents}

\begin{frame}
  \tableofcontents
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% Reduction
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Map and Reduce}

\subsection{Types, Semantics and Properties}

\begin{frame}[fragile,t]
   \frametitle{Basic Blocks of Parallel Programming: Map}

\bigskip

\emp{map} : $(\alpha \to \beta, [\alpha]) ~~\to~~ [\beta] $ has \emph{\em inherently parallel semantics}.
\smallskip

Applies a function to every element of the input array producing an array of equal length.

\bigskip

\begin{tabular}{crcccccl}
X = & \emph{map}(~f, ~ [& $a_1$, & $a_2$, & .., & $a_n$ & ])\\
    &      & $\downarrow$ & $\downarrow$ &  & $\downarrow$ & &\\
X $\equiv$ &  [  & \emph{f($a_1$)}, & \emph{f($a_2$)}, & .., & \emph{f($a_n$)} & ] &
\end{tabular}\bigskip

\begin{scriptsize}
Similar: {\tt~~~~map2(f,[$a_1$,$\ldots$,$a_n$],[$b_1$,$\ldots$,$b_n$]) $\equiv$ [f($a_1$,$b_1$),$\ldots$,f($a_n$,$b_n$)]}
\end{scriptsize}\\\bigskip\pause

\emp{Map Fusion:} \emph{{\tt~~~map(f, map(g, A))} $~\equiv~$ {\tt map(f o g, A)}} \bigskip

\begin{tabular}{llllllll}
A & $\equiv$ & [ & $a_1$, & $a_2$, & $\ldots$, & $a_n$ & ] \\
X = map(g, A) & $\equiv$ & [ & g($a_1$), &g($a_2$), &$\ldots$, &g($a_n$) &]\\
\emph{Y = map(f, X)} & $\equiv$ & [& \emph{f(g($a_1$))}, &\emph{f(g($a_2$))}, &$\ldots$, &\emph{f(g($a_n$))} &]\\
\emph{map(f o g, A)} & $\equiv$ & [& \emph{f(g($a_1$))}, &\emph{f(g($a_2$))}, &$\ldots$, &\emph{f(g($a_n$))} &]\\
\end{tabular}
\bigskip

Fusion is a very important optimizations; saves bandwidth!

\end{frame}

\begin{frame}[fragile,t]
   \frametitle{Basic Blocks of Parallel Programming: Reduce}

\bigskip

\emp{reduce} : $((\alpha, \alpha) \to \alpha),~ \alpha, [\alpha]) ~~\to~~ \alpha$

\smallskip

\emp{reduce}(~$\odot$, ~$0_\odot$, ~[$a_1$, $a_2$, ..., $a_n$]) $\equiv$ \emph{$0_\odot ~\odot ~a_1 ~\odot a_2 ~\odot ... \odot~ a_n$}

\smallskip

~~~~~where $\odot$ is an associative binary operator (\red{otherwise bug!})\\
~~~~~~~~~~~~~~~~$0_\odot$ is the neutral element of the monoid induced by $\odot$

\bigskip\pause

\begin{center}
        \includegraphics[height=20ex]{img/day3/ReduceEg.pdf}
\end{center}

Build programs by combining \emp{map}, \emp{reduce} and other such operators.

\end{frame}

\begin{frame}[fragile,t]
   \frametitle{Trivial Examples of Map-Reduce Programming}

\emp{Small Exercise:} write a function that receive as parameters a predicate
$\texttt{p}~: ~ \alpha ~ \to ~ \texttt{bool}$ and an array {\tt A}, and that
results in {\tt true} if all elements satisfy {\tt p} and in {\tt false} otherwise.\\\smallskip
\emp{Try to write (i) a divide-and-conquer and (ii) a map-reduce implementation.}
\medskip\pause

\begin{columns}
\column{0.4\textwidth}
\begin{colorcode}[fontsize=\small]
all(p, [ ])  = True
all(p, [x])  = p(x)
all(p, x++y) = all(p, x) &&
               all(p, y)
\end{colorcode}
\column{0.58\textwidth}
\begin{colorcode}[fontsize=\small]
    all(p, x) =
        y = map(p, x)
        z = reduce(&&, true, y)
        return z
\end{colorcode}
\end{columns}

\bigskip\pause
{\tt ++} denotes array concatenation.\smallskip

A well define divide and conquer implementation requires that any
split of the input array into {\tt x++y} gives the same result.\smallskip

(This is equivalent to the requirement that the binary operator of
reduction is associative.)

\bigskip
Under this conditions, the two are equivalent: if you can write one,
then you can derive the other (list homomorphic $\equiv$ map-reduce).

\end{frame}

\subsection{Efficient Sequentialization: Work, Depth, and Brent Lemma}

\begin{frame}[fragile,t]
   \frametitle{Asymptotic Work and Depth; Brent Lemma}

Assuming an infinity of processors:
\begin{itemize}
    \item \emp{Work Complexity $W(n)$}:       is the total \# of ops performed,\smallskip
    \item \emp{Depth/Step Complexity $D(n)$}: is the \# of sequential steps.\bigskip
    \item A parallel implem is {\bf work efficient} {\it iff} its work complexity
            is equal to the one of the golden sequential implem.\bigskip

    \item Work and Depth are good high-level approximations;
    \item If we know the work and depth asymptotic for a program,
            Brent Theorem offers good complexity bounds for a PRAM.
\end{itemize}

\begin{mytheo}[Brent Theorem]\label{BrentTh}
An algorithm of depth $D(n)$ and work $W(n)$ can be
simulated on a $P$-processor PRAM in time complexity T such that:\\\bigskip
\emp{$\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \frac{W(n)}{P} \leq T \leq \frac{W(n)}{P} + D(n)$}
\end{mytheo}

\end{frame}

\begin{frame}[fragile,t]
   \frametitle{Applying Brent Lemma to Map-Reduce Computation}

\begin{center}
\includegraphics[height=20ex]{img/day3/ReduceEg.pdf}
\end{center}

Reducing an array of length {\tt n} with {\tt n/2} processors requires:
\begin{itemize}
    \item work $W(n) = n$ and
    \item depth $D(n) = lg \ n$, i.e., number of sequential steps.
    \item Brent Theorem states the bounds for optimal runtime $T^{opt}$:\\\smallskip
            $ \frac{W(n)}{P} \leq T^{opt} \leq \frac{W(n)}{P} + D(n)$
\end  {itemize}

\end{frame}

\begin{frame}[fragile,t]
   \frametitle{Applying Brent Lemma to Map-Reduce Computation}

\begin{columns}
\column{0.37\textwidth}
\includegraphics[height=13ex]{img/day3/ReduceEg.pdf}
\column{0.61\textwidth}
Reducing an array of length {\tt n} with {\tt n/2} processors requires:
\begin{itemize}
    \item $W(n) = n$, $D(n) = lg \ n$.
    \item Optimal runtime $T^{opt}$ bounds:\\
            $ \frac{W(n)}{P} \leq T^{opt} \leq \frac{W(n)}{P} + D(n)$
\end  {itemize}
\end{columns}\bigskip

{\bf An optimized map-reduce computation can be implemented as:}
\begin{itemize}
    \item splits the input array into $P$ subarrays, each containing
            about the same number of elements;\smallskip
    \item perform the computation sequentially on each subarray,
            but in parallel across subarrays;\smallskip
    \item reduce the $P$ per-processor results.
\end{itemize}\medskip
{\bf This leads to optimal runtime on $P$ processors:}\pause \emph{$O(\frac{n}{P} + lg \ P)$}
\bigskip

\emp{This kind of chunking is often referred to as:}\\
\emp{``efficient sequentialization of excess parallelism''!}

\end{frame}

%\begin{mytheo}[Optimized Map Reduce]\label{MapRed}
%Assume {\tt split$_p$} splits an array into $p$ subarrays, each containing about
%the same number of elements. Denoting
%$\mbox{~~~~~}{\tt redomap}~\odot ~ f ~ e_{\odot}~ X \equiv \ {\tt reduce}(~\odot,~ e_{\odot}, ~ {\tt map(f, X)~)}$\\
%the following equality holds:\\\smallskip
%
%\emp{${\tt redomap} \ \odot \ f \ e_{\odot} \ \ \equiv$}\\
%\emp{$\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ({\tt reduce} \ \odot \ e_{\odot}) \ \circ \ ({\tt map} \ ({\tt redomap} \ \odot \ f \ e_{\odot})) \ \circ \ {\tt split}_p$}
%\end{mytheo}

\subsection{Maximum Segment Sum Problem (MSSP)}

\begin{frame}[fragile,t]
  \frametitle{Almost Homomorphisms (Gorlatch)}

\emp{``Systematic Extraction and Implementation of Divide-and-Conquer Parallelism'', Sergei Gorlatch, 1996.}
\bigskip

\emph{Intuition}: a non-homomorphic function $g$ can be sometimes lifted
to a homomorphic one $f$, by computing a baggage of \emp{\em extra info}.

\bigskip

The initial fun obtained by projecting the homomorphic result:\\
$g\mbox{ }=\pi~\circ~f$

\bigskip

\emp{\bf Maximum-Segment Sum Problem ({\sc mssp})}: \\
Given a list of integers, find the contiguous segment of the list
whose members have the largest sum among all such segments.\\
The result is only the maximal sum (not the segment's members).
For simplicity lets assume we are interested only in {\bf positive sums}.

\bigskip

E.g., {\tt mss} [1, -2, 3, 4, -1, 5, -6, 1] = 11 \\
(the corresponding segment is [3, 4, -1, 5]).

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Maximum Segment Sum (MSSP): Preliminary Reasoning}

\emp{\bf Maximum-Segment Sum Problem ({\sc mss})}: \\
Given a list of integers, find the contiguous segment of the list
whose members have the largest sum among all such segments.\\
The result is only the maximal sum (not the segment's members).
For simplicity lets assume we are interested only in {\bf positive sums}.
\medskip

\red{A first straightforward/naive attempt:}
\begin{colorcode}
mss [ ]      = 0
mss [a]      = a \mymath{\uparrow} 0  \emp{//\mymath{\uparrow} denotes Max}
mss (x{\tt ++}\mbox{ }y) = mss(x) ??? mss(y)
\end{colorcode}

\bigskip
\bigskip

\begin{columns}
\column{0.58\textwidth}\pause
\includegraphics[height=20ex]{img/day3/mssp1}
\column{0.44\textwidth}
\red{Which case is problematic?}\pause\bigskip

\emph{Answer: when the segment of interest lies partly in {\tt x} and partly in {\tt y}!}
\end{columns}
\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Maximum Segment Sum (MSSP): A Better Reasoning}

\red{The problematic case is when the segment of interest lies
     partly in {\tt x} and partly in {\tt y}!}
\bigskip

We need to compute extra information:\pause
\begin{itemize}
    \item maximum concluding segment
    \item maximum initial segment
    \item total segment sum
\end{itemize}\smallskip\pause

\includegraphics[height=22ex]{img/day3/mssp2}

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{MSSP: Deriving the Implementation}

\includegraphics[height=22ex]{img/day3/mssp2}
\medskip
Lets compute the {\tt mis}, {\tt mcs}, {\tt mss}, and {\tt ts} for
the result of the two concatenating segments. $\uparrow$ denotes {\tt max}.

\bigskip
{\tt mis = \pause mis1 $\uparrow$ (ts1 + mis2)}
\medskip

{\tt mcs = \pause mcs2 $\uparrow$ (mcs1 + ts2)}
\medskip

{\tt mss = \pause mss1 $\uparrow$ mss2 $\uparrow$ (mcs1 + mis2)}
\medskip

{\tt ts~~= ts1 + ts2}
\end{frame}

\begin{frame}[fragile,t]
  \frametitle{MSSP: Map-Reduce C-ish Pseudocode}

\begin{columns}
\column{0.53\textwidth}
\begin{colorcode}
// x \mymath{\uparrow} y \mymath{\equiv} (x >= y) ? x : y
Qtup \blue{\mymath{\odot}} (Qtup x, Qtup y) \{
    Qtup r;
    r.mss = x.mss\mymath{\mbox{ }\uparrow\mbox{ }}y.mss\mymath{\mbox{ }\uparrow\mbox{ }}
            (x.mcs + y.mis);
    r.mis = x.mis\mymath{\mbox{ }\uparrow\mbox{ }}(x.ts + y.mis);
    r.mcs = (x.mcs + y.ts)\mymath{\mbox{ }\uparrow\mbox{ }}y.mcs;
    r.ts  = x.ts + y.ts;
    return r;
\}

Qtup \emp{f}(int x) \{
    Qtuple r;
    int x0 = x\mymath{\mbox{ }\uparrow\mbox{ }}0;
    r.mss = x0; r.mis = x0;
    r.mcs = x0; r.ts = x;
    return r;
\}
\end{colorcode}
\column{0.45\textwidth}
\begin{colorcode}
typedef struct msict \{
  int mss;
  int mis;
  int mcs;
  int ts;
\} Qtup;

int mss(int* xs) \{
    Qtup ne, res;
    ne.mss = 0; ne.mis = 0;
    ne.mcs = 0; ne.ts  = 0;

    Qtup* ys = \emph{map}(\emp{f}, xs);
    res = \emph{reduce}(\blue{\mymath{\odot}}, ne, ys);
    return res.mss;
\}
\end{colorcode}
\end{columns}
\smallskip

The baggage: $3$ extra integers ({\tt misx, mcsx, tsx})
and a constant number of integer operations per communication stage.
\medskip

\emp{For performance:} array {\tt ys} should not be manifested in memory;\\
fuse the {\tt map} with the {\tt reduce} operations.

\end{frame}

\subsection{Exercise 1: OpenCL Implementation of MSSP}

\begin{frame}[fragile,t]
  \frametitle{Exercise 1: OpenCL Implementation of MSSP}

  \begin{itemize}
  \item For this exercise, you will implement MSSP as an OpenCL program.
  \item As inspiration, we will first study four OpenCL
    implementations of \textit{summing an integer array}.
  \end{itemize}

\huge
  \[
    \sum_{i < n} x[i]
  \]

\end{frame}

\begin{frame}
  \frametitle{Binary Tree Reduction}

  \textbf{The idea:} each thread reads two neighbouring elements, adds
  them together, and writes one element.  This halves the array in
  size.  Continue until only a single element is left.

\begin{center}
  \includegraphics[height=20ex]{img/day3/ReduceEg.pdf}
\end{center}

\begin{itemize}
\item Each level becomes a kernel invocation, with number of threads
  equal to half the number of array elements.
\item $O(n)$ work and $O(\log(n))$ span (optimal).
\item \textbf{Why is this not efficient?}
\end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Improving the Tree Reduction}

  \textbf{The idea:} instead of shrinking the array by a factor of two
  for each level, shrink it by the workgroup size.

  \begin{itemize}
  \item Same asymptotic performance.
  \item Avoids kernels with very few threads.  E.g with workgroup size
    $256$: $10000000 \rightarrow 39063 \rightarrow 153 \rightarrow 1$.
  \end{itemize}

\pause\bigskip

  \begin{tabular}{lrr}
    \textbf{Implementation} & $n=1000$ & $n=1000000$ \\\hline
    Tree reduction & $77\mu{}s$ & $363\mu{}s$ \\
    Group reduction & $17\mu{}s$ & $179\mu{}s$ \\
  \end{tabular}

\end{frame}

\begin{frame}
  \frametitle{Applying Brent's Lemma}

  \textbf{The idea:} instead of letting the thread count depend on the
  input size, always launch the same number of threads, and have each
  thread perform an efficient sequential summation of a \textit{chunk}
  of the input.

  \begin{itemize}
  \item GPUs have a maximum (hardware/problem-dependent) capacity for
    exploiting parallelism.  Beyond that limit, parallelism is at best
    worthless, and usually comes with overhead (e.g. excessive
    synchronisation).
  \item \textit{A straightforward implementation of this idea only
      works if the operator is commutative!}
  \end{itemize}

\pause\bigskip

  \begin{tabular}{lrr}
    \textbf{Implementation} & $n=1000$ & $n=1000000$ \\\hline
    Tree reduction & $77\mu{}s$ & $363\mu{}s$ \\
    Group reduction & $17\mu{}s$ & $179\mu{}s$ \\
    Chunked reduction & $70\mu{}s$ & $103\mu{}s$ \\
  \end{tabular}

\end{frame}

\begin{frame}[fragile]
  \frametitle{Using Atomics}

  \textbf{The idea:} GPUs have special hardware support for performing
  certain memory updates atomically.  In OpenCL, this is exposed
  through \textit{atomic operations}.

\begin{lstlisting}[backgroundcolor=\color{lightgray}]
int atomic_add(volatile __global int *p,
               int val)
\end{lstlisting}

  \begin{itemize}
  \item Concise parallel reduction: each thread reads an element and
    uses \texttt{atomic\_add()} to update the same location in memory.
  \item \textbf{Why is this slow for large inputs?}
  \end{itemize}

\pause\bigskip

  \begin{tabular}{lrr}
    \textbf{Implementation} & $n=1000$ & $n=1000000$ \\\hline
    Tree reduction & $77\mu{}s$ & $363\mu{}s$ \\
    Group reduction & $17\mu{}s$ & $179\mu{}s$ \\
    Chunked reduction & $70\mu{}s$ & $103\mu{}s$ \\
    Atomics & $8\mu{}s$ & $1278\mu{}s$ \\
  \end{tabular}

\end{frame}

\begin{frame}
  \frametitle{For MSSP}

  Implement three versions:

  \begin{enumerate}
  \item Tree reduction
  \item Group reduction
  \item Chunked reduction
  \end{enumerate}

  \textbf{Hints:}
  \begin{itemize}
  \item Instead of a \texttt{struct Qtup}, you may want to use
    \texttt{int4} (inside the kernels) and \texttt{cl\_int4} (in host
    code).
  \item The MSSP operator is \textit{not} commutative, so you will
    need to use a sliding window approach for the chunked reduction.
  \item For the chunked reduction, you can fuse the \texttt{map}
    function into the reduction itself.  For the others, a separate
    pass is done.
  \end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% SCAN
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Scan and Applications}

\begin{frame}
  \tableofcontents[currentsection]
\end{frame}

\subsection{Scan: Type, Semantics, Asymptotic, Parallel Implementation}

\begin{frame}[fragile,t]
  \frametitle{Scan: A Basic Block of Data-Parallel Programming}

Scan is also known as parallel prefix sum:
\begin{itemize}
\item computes all partial prefixes of an array;
\item similar type with reduce, except that it returns an array;\medskip
\item \emp{exclusive scan}: result array starts with the neutral element;
\item \emph{inclusive scan}: starts with the first element of the input array;\medskip
\item Inclusive scan is slightly more useful than exclusive scan.
\end{itemize}\bigskip\pause

\emp{{\tt scan$^{exc}$: ( ($\alpha$,$\alpha$)$\rightarrow\alpha$),$~\alpha$, [n]$\alpha$ )$~\rightarrow~$[n]$\alpha$}}\\
        \emph{\tt scan$^{exc}(\odot$,e,[x$_1$,$\ldots$,x$_n$])~=~[e,e$\odot$x$_1$,$\ldots$,e$\odot$x$_1\odot\ldots$x$_{n-1}$]}\\
        i.e., \emp{{\tt{}e:$\alpha$, x$_i$:$\alpha,~\forall i$}}, and
        \emp{\tt~~~$\odot$:$(\alpha,\alpha)\rightarrow\alpha$}.\bigskip

\emp{{\tt scan$^{inc}$: ( ($\alpha$,$\alpha$)$\rightarrow\alpha$),$~\alpha$, [n]$\alpha$ )$~\rightarrow~$[n]$\alpha$}}\\
        \emph{\tt scan$^{inc}(\odot$,e,[x$_1$,$\ldots$,x$_n$])~=~[x$_1$,~x$_1$$\odot$x$_2$,$\ldots$,x$_1\odot\ldots\odot$x$_{n}$]}\\
        i.e., \emp{{\tt{}e:$\alpha$, x$_i$:$\alpha,~\forall i$}}, and
        \emp{\tt~~~$\odot$:$(\alpha,\alpha)\rightarrow\alpha$}.

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Parallel Exclusive Scan with Associative Operator $\oplus$}
\bigskip

\begin{columns}
\column{0.4\textwidth}
        \includegraphics[height=30ex]{img/day3/ScanEg}
\column{0.6\textwidth}
Two Steps:
\begin{itemize}
    \item \blue{Up-Sweep:} similar with reduction
    \item Root is replaced with neutral element.
    \item \emp{Down-Sweep:}
    \begin{itemize}
        \item the left child sends its value to parent and
                updates its value to that of parent.
        \item the right-child value is given by $\oplus$
                applied to the left-child value and
                the (old) value of parent.
        \item note that the right child is in fact the parent,
                i.e., in-place algorithm.
    \end  {itemize}
\end  {itemize}
\end{columns}\bigskip

{\bf Scan's Work and Depth:} \emph{$D(n) = \Theta(lg \ n), W(n) = \Theta(n)$}

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Wavefront-Level Inclusive Scan for GPUs}

\begin{columns}
\column{0.46\textwidth}
\includegraphics[height=15ex]{img/day3/incScanGPU}
\column{0.53\textwidth}
\begin{colorcode}
Input:  array A of n=\mymath{2\myindu{k}} elements
                       of type \mymath{\alpha}
        \mymath{\oplus} : \mymath{(\alpha,\alpha)\to\alpha} associative
Output: B = [a\mymath{\myindx{1}}, a\mymath{\myindx{1}}\mymath{\oplus}a\mymath{\myindx{2}}, \mymath{\ldots} ,\mymath{\oplus\myindx{j=0}\myindu{n-1}} a\mymath{\myindx{j}}]
1.  forall i = 0 : n-1 do
2.    B[i] \mymath{\leftarrow} A[i]
3.  endfor
4.  for d = 0 to k-1 do
5.    h = 2\mymath{\myindu{d}}
6.    forall i = h to n-1 do
7.      B[i] \mymath{\leftarrow} B[i-h] \mymath{\oplus} B[i]
8.    endfor
9.  endfor
\end{colorcode}
\end{columns}\bigskip

\emp{Offers better performance because it operates in one sweep rather than two!}

\end{frame}

\subsection{Exercise 2: Intra-Wave Inclusive Scan Implementation}

\begin{frame}[fragile,t]
  \frametitle{Exercise 2: Intra-Wave Inclusive Scan Implementation}

\begin{columns}
\column{0.46\textwidth}
\includegraphics[height=15ex]{img/day3/incScanGPU}
\column{0.53\textwidth}
\begin{colorcode}
Input:  array A of n=\mymath{2\myindu{k}} elements
                       of type \mymath{\alpha}
        \mymath{\oplus} : \mymath{(\alpha,\alpha)\to\alpha} associative
Output: B = [a\mymath{\myindx{1}}, a\mymath{\myindx{1}}\mymath{\oplus}a\mymath{\myindx{2}}, \mymath{\ldots} ,\mymath{\oplus\myindx{j=0}\myindu{n-1}} a\mymath{\myindx{j}}]
1.  forall i = 0 : n-1 do
2.    B[i] \mymath{\leftarrow} A[i]
3.  endfor
4.  for d = 0 to k-1 do
5.    h = 2\mymath{\myindu{d}}
6.    forall i = h to n-1 do
7.      B[i] \mymath{\leftarrow} B[i-h] \mymath{\oplus} B[i]
8.    endfor
9.  endfor
\end{colorcode}
\end{columns}

\begin{itemize}
    \item Open file "scanapps.cl", and implement function named "incScanWave" (follow the instructions)
    \item Your {\tt n = WAVE} and {\tt k = lgWAVE}; Ignore the init. loop;
    \item Unroll the {\tt for d} loop ({\tt \#pragma unroll});
    \item loop {\tt forall i = h to n-1} is implicit,
    \item it should be replace by a condition {\tt if (i>=h) \{ ... \}},
    \item except that {\tt i} is not exactly the thread id.
    \item Remember, you want to {\tt scan} each wave, independently!
\end{itemize}

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{OpenCL Scan Implementation}

    BLACKBOARD!

\begin{itemize}
    \item CPU stub is in {\tt Day3-Exercises/ScanApps/scan.h}
    \item Hierarchical design:
    \item[1] sequentially scan {\tt ELEMS\_PER\_THREAD} by each thread,
    \item[2] publish in local memory
    \item[3] scan at wavefront level
    \item[4] ``update'' the scan at workgroup level
    \item[5] Gather the last elements in the scan groups into
             a separate array, and scan that!
    \item[5] Virtualize steps [1-4] so that the scan at step 5 fits
             into one workgroup;
    \item[6] add element {\tt i-1} of the result of step 5
                to each element of workgroup {\tt i} resulted from step $4$!
    \item \red{This requires $2$ reads and $2$ writes from/to global memory.}\\
          \emphh{Can you do it better?}
\end{itemize}

\end{frame}

\subsection{Other Data-Parallel Operators: Scatter, Partition}

\begin{frame}[fragile,t]
  \frametitle{Scatter: Parallel Write Operator}

{\bf Scatter} \emph{updates in parallel and in place} an input array with a set of values at specified indices:
\smallskip

\emph{scatter : ([m]$\alpha$, [n]int, [n]$\alpha$) $\rightarrow$ [m]$\alpha$}
\bigskip

A (data vector)    {\tt~=[b0, b1, b2, b3]}\\
I (index vector)   {\tt~=[2,~~4,~~1,~~-1]}\\
X (input array)    {\tt~=[a0,~a1,~a2,~a3,~a4,~a5]}\\
\emp{scatter X I A {\tt~~~=[a0,~b2,~b0,~a3,~b1,~a5]}}
\bigskip\pause

\emph{scatter} has $D(n)=\Theta(1)$ and $W(n)=\Theta(n)$,\\
i.e., requires {\tt n} update operations ({\tt n} is the size of I or A, not of X!).\bigskip
\bigskip

{\bf replicate(n, v)} creates an array of length {\tt n} filled with element {\tt v}!

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Exercise 3: Partition2 Operator}

{\bf Type and Semantics of Partition2}:\\
\emph{\tt partition2 : ($\alpha\rightarrow$Bool, [n]$\alpha$) $\rightarrow$ (int,[n]$\alpha$)}\\\bigskip

Partition2 receives as input a predicate and an array and results in:
\begin{itemize}
    \item an integer denoting the number of elements that succeed under predicate, tupled with
    \item a new array, having the same elements as the input array, but reordered
            such as the elements that succeed under the predicate occur before the others.
    \item The partial order of the elements that succeed/fail should be respected.
\end{itemize}\bigskip

\begin{colorcode}
bool even(int v) \{ return (bool)(1 -(v&1)); \}
partition2(even, [5, 4, 2, 10, 3, 7, 8] ) should result in
             (4, [4, 2, 10, 8, 5, 3, 7])
\end{colorcode}

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Exercise 3, Step 1: Partition2 High-Level Implem}

\begin{colorcode}
bool even(int v) \{ return (bool)(1 -(v&1)); \}
partition2(even, [5, 4, 2, 10, 3, 7, 8] ) should result in
             (4, [4, 2, 10, 8, 5, 3, 7])
\end{colorcode}

Step 1: implement partition2 based on map, scan and scatter (?)\bigskip

\begin{columns}
\column{0.59\textwidth}
\begin{colorcode}
partition2(X: [n]i32): (i32, [n]i32) =
 cs = map(even, X);
 tfs= map (\mymath{\lambda}c->if c then 1 else 0
          ,  cs);\pause
 isT= scan( (+), 0, tfs );
 i  = isT[n-1];

 ffs= map (\mymath{\lambda}c->if c then 0 else 1
          ,  cs );
 tmps = scan( (+), 0, ffs );
 isF= map (\mymath{\lambda}t -> t+i, tmps);

 inds=map3(\mymath{\lambda}(c,iT,iF) ->
              if c then iT-1 else iF-1
          , cs, isT, isF);
 res = scatter(scratch n, inds, X);
 return (i, res);
\end{colorcode}
\column{0.4\textwidth}\vspace{-2ex}
\begin{colorcode}[fontsize=\scriptsize]
\emp{X   = [5, 4, 2, 10,3, 7, 8]}
\emp{n   = 7}
\emp{cs  = [F, T, T, T, F, F, T]}
\emp{tfs = [0, 1, 1, 1, 0, 0, 1]}

\emp{isT = [0, 1, 2, 3, 3, 3, 4]}
\emp{i   = 4}

\emp{ffs = [1, 0, 0, 0, 1, 1, 0]}
\emp{tmps= [1, 1, 1, 1, 2, 3, 3]}
\emp{isF = [5, 5, 5, 5, 6, 7, 7]}

\emp{inds= [4, 0, 1, 2, 5, 6, 3]}
\emp{X   = [5, 4, 2, 10,3, 7, 8]}
\emp{Res = [_, _, _, _, _, _, _]}
\emp{---------------------------}
\emp{Res = [4, 2, 10, 8,5, 3, 7]}
\end{colorcode}
\end{columns}

\end{frame}

\subsection{Exercise 3, Step 2: Partition2 OpenCL Implementation}

\begin{frame}[fragile,t]
  \frametitle{Exercise 3: Implement Partition2 in OpenCL}

\begin{itemize}
    \item CPU stub is {\tt Day3-Exercises/ScanApps/partition2.h},
            function {\tt runPartition}, but nothing to do there!\bigskip
    \item[(1)] In {\tt Day3-Exercises/ScanApps/scanapps.cl}, implement
             from scratch kernel {\tt mapPredPartKer} which is supposed
             to compute {\tt tfs} and {\tt ffs}.\bigskip
    \item[(2)] the two scans are already implemented for you!\bigskip
    \item[(3)] Then implement kernel {\tt scatterPartKer} which is supposed
             to compute the last part! Note that {\tt isT} and {\tt isF}
             are the results of {\tt scan}, i.e., {\tt isF} is actually {\tt tmps}
             in slides!\\
             It is Ok to redundantly compute the condition again.
\end{itemize}

\end{frame}

\section{Segmented Scan and Applications}

\subsection{Segmented Scan: Type, Semantics, Asymptotic, Impleme}

\begin{frame}[fragile,t]
  \frametitle{What is a Segmented Scan?}

\pause
Assume an \emp{irregular} matrix (two-dimensional array),
i.e., rows have different number of elements.\medskip

{\bf Segmented Scan Intuition}: the operation that scans each
row of an irregular matrix and returns the result.\medskip

{\bf Flat Representation of a 2D irregular array:} flag + value arrays:
\pause
\begin{columns}
\column{0.33\textwidth}
\begin{colorcode}
[ [1, 2, 3]
, [4, 5, 6, 7, 8]
, [9, 10]
]
\end{colorcode}
\column{0.58\textwidth}
\begin{colorcode}
Flag Array:
[1, 0, 0, 1, 0, 0, 0, 0, 1, 0 ]
Value Array:
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
\end{colorcode}
\end{columns}\medskip

\emp{The flag array} marks with one (or something different than zero)
the start of a row/segment, the other elements are zero.\medskip

\emp{The value array}: flat sequence of elements. \bigskip

\begin{scriptsize}
\emp{{\tt sgmScan$^{inc/exc}$:~(($\alpha$,$\alpha$)$\rightarrow\alpha$),$~\alpha$,~[n]int,~[n]$\alpha$)$~\rightarrow~$[n]$\alpha$}}\\
\begin{colorcode}
\emp{sgmScan\mymath{\myindu{inc}}((+), 0, [1, 0, 0, 1, 0, 0,  0,  0,  1, 0 ]}
               \emp{ , [1, 2, 3, 4, 5, 6,  7,  8,  9, 10])} \mymath{\equiv}\pause
               \emph{   [1, 3, 6, 4, 9, 15, 22, 30, 9, 19]}
\end{colorcode}
\end{scriptsize}
\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Intuition: How Does the Implementation Looks Like?}

Slide taken from CMU 15-418: Parallel Computer Architecture and Programming (Spring 2012).
Segmented Exclusive Scan:\vspace{-2ex}

\begin{center}
\includegraphics[height=42ex]{img/day3/SgmExcScanFig}
\end  {center}
\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Segmented Exclusive Scan Algorithm And Complexity}
\vspace{-1ex}
\begin{columns}
\column{0.31\textwidth}
\begin{itemize}
    \item While there will be more branches, the asymptotic
            is the same as the one for reduce and scan:
    \item \emph{$D(n) = \Theta(lg \ n)$},\\\emp{$W(n) = \Theta(n)$!}
    \item That's the good news!
    \item The bad news:
\end{itemize}\pause
\column{0.75\textwidth}
\begin{colorcode}[fontsize=\scriptsize]
Input:  flag array F of n=2\mymath{\myindu{k}} of ints
        data array A of n=2\mymath{\myindu{k}} elems of type T
        \mymath{\oplus::T\times T\rightarrow T} associative
Output: B = segmented scan of 2-dim array A
1.  \emph{FORALL i = 0 to n-1 do} B[i] \mymath{\leftarrow} A[i] \emph{ENDDO}
2.  \emp{FOR d = 0 to k-1 DO} \emph{// up-sweep}
3.    \emph{FORALL i = 0 to n-1 by 2\mymath{\myindu{d+1}} DO}
4.      IF F[i+2\mymath{\myindu{d+1}}-1] == 0 THEN
5.          B[i+2\mymath{\myindu{d+1}}-1] \mymath{\leftarrow} B[i+2\mymath{\myindu{d}}-1] \mymath{\oplus} B[i+2\mymath{\myindu{d+1}}-1]
6.      ENDIF
7.      F[i+2\mymath{\myindu{d+1}}-1] \mymath{\leftarrow} F[i+2\mymath{\myindu{d}}-1] .|. F[i+2\mymath{\myindu{d+1}}-1]
8.  \emp{ENDDO} \emph{ENDDO}
9.  B[n-1] \mymath{\leftarrow} 0
10. \emp{FOR d = k-1 downto 0 DO} \emph{// down-sweep}
11.   \emph{FORALL i = 0 to n-1 by 2\mymath{\myindu{d+1}} DO}
12.     tmp \mymath{\leftarrow} B[i+2\mymath{\myindu{d}}-1]
13.     IF \alert{F\_original}[i+2\mymath{\myindu{d}}] \mymath{\neq} 0 THEN
14.          B[i+2\mymath{\myindu{d+1}}-1] \mymath{\leftarrow} 0
15.     ELSE IF F[i+2\mymath{\myindu{d}}-1] \mymath{\neq} 0 THEN
16.          B[i+2\mymath{\myindu{d+1}}-1] \mymath{\leftarrow} tmp
17.     ELSE B[i+2\mymath{\myindu{d+1}}-1] \mymath{\leftarrow} tmp \mymath{\oplus} B[i+2\mymath{\myindu{d+1}}-1]
18.     ENDIF
19.     F[i+2\mymath{\myindu{d+1}}-1] \mymath{\leftarrow} 0
20. \emp{ENDDO} \emph{ENDDO}
\end{colorcode}
\end{columns}

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Segmented Inclusive Scan is a Sort of Scan!}
\pause

Can be implemented as a scan with a modified binary-associative
operator that operates on boolean-value (or int-value) tuples.
\bigskip

We need to define {\tt zip} first, which performs SoA to AoS transform:\\\smallskip
\begin{scriptsize}
\emp{{\tt zip: ( [n]$\alpha$, [n]$\beta$ )$~\rightarrow~$[n]$(\alpha,\beta)$}}\\
\emph{\tt zip([x$_1$,$\ldots$,x$_n$], [y$_1$,$\ldots$,y$_n$])~=~[(x$_1$,y$_1$),$\ldots$,(x$_{n}$,y$_{n}$)]}\\\medskip
\end{scriptsize}
\medskip\pause

\begin{colorcode}[fontsize=\scriptsize]
typedef Tuple \{
    \mymath{\alpha}    val;
    bool flg;
\} Tup;

Tup \mymath{\odot\myindu{sgm}}(Tup x, Tup y) \{
    Tup r;
    r.flg = x.flg || y.flg;
    r.val = y.flg ? y.val : x.val \mymath{\odot} y.val;
    return r;
\}

\mymath{\alpha}* sgmScan\mymath{\myindu{inc}} (\mymath{\odot: (\alpha,\alpha)\to\alpha}, \mymath{\alpha} ne, \mymath{\alpha}* flags, \mymath{\alpha}* vals) \{
    Tup nes; nes.flg = false; nes.val = ne;
    Tup* X = zip(flags, vals);
    return scan\mymath{\myindu{inc}}(\mymath{\odot\myindu{sgm}}, nes, X)
\}
\end{colorcode}

\end{frame}

\subsection{Application: Sparse Matrix-Vector Multiplication}

\begin{frame}[fragile,t]
  \frametitle{Sparse Matrix-Vector Multiplication: Pseudocode}

\begin{colorcode}[fontsize=\scriptsize]
smvMul( mat_cols: [N]int, mat_vals: [N]real
      , vct: [vct_len], shape: [num_rows]int ) : [num_rows]real =
  real res_vct[num_rows];
  int  offset = 0;
  for(int i=0; i < num_rows; i++) \{
    real res = 0;
    \emp{for(int j=offset; j < offset+shape[i]; j++) \{}
        int vct_ind = mat_cols[j];
        int mat_elm = mat_vals[j];
        res += mat_elm * vct[vct_ind];
    \}
    res_vct[i] = res;
    offset += shape[i];
  \}
\end{colorcode}

\bigskip

\begin{colorcode}[fontsize=\scriptsize]
shape:    [2, 3, 1]
mat_cols: [0,    1,    1,   2,    3,   3  ]
mat_vals: [2.0, -1.0, -1.0, 2.0, -1.0, 3.0]
vct:      [1.0,  2.0,  3.0, 4.0]
-------------------------------------------\pause
Result:
[ 2.0*1.0 + (-1.0)*2.0               = 0.0
, (-1.0)*2.0 + 2.0*3.0 + (-1.0)*4.0  = 0.0
, 3.0*4.0                            = 12.0
]
\end{colorcode}

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Sparse Matrix-Vector Multiplication}

Imperative Pseudocode:
\begin{colorcode}[fontsize=\scriptsize]
smvMul( mat_cols: [N]int, mat_vals: [N]real
      , vct: [vct_len], shape: [num_rows]int ) : [num_rows]real =
  real res_vct[num_rows];
  int  offset = 0;
  for(int i=0; i < num_rows; i++) \{
    real res = 0;
    \emp{for(int j=offset; j < offset+shape[i]; j++) \{}
        int vct_ind = mat_cols[j];
        int mat_elm = mat_vals[j];
        res += mat_elm * vct[vct_ind];
    \}
    res_vct[i] = res;
    offset += shape[i];
  \}
\end{colorcode}

We would like to:
\begin{itemize}
    \item exploit all parallelism, \emp{but the inner parallelism is irregular!}
    \item the inner loop:\pause looks like a map-reduce composition,\\
            buts its size is variant to the outer {\tt map};
    \item Hint: compute flags + use segmented scan + extract last elements!
\end{itemize}

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Computing Flag Array from Shape Array}

This works even in the presence of empty rows!\bigskip

\begin{columns}
\column{0.59\textwidth}
\begin{colorcode}
mkFlagArray (aoa_shp: [m]i32) : []i32 =
  shp_rot = map (\mymath{\lambda}i->if i==0 then 0
                      else aoa_shp[i-1]
                , [0,...,m-1]);
  shp_scn = scan( (+), 0, shp_rot );
  aoa_len = shp_scn[m-1] + aoa_shp[m-1];
  shp_ind = map2 (\mymath{\lambda}shp ind ->
                   if shp==0 then -1
                             else ind
                 , aoa_shp, shp_scn);
  return
    scatter( replicate(aoa_len, 0)
           , shp_ind
           , replicate(m, 1) );
\end{colorcode}
\column{0.4\textwidth}
\begin{colorcode}[fontsize=\scriptsize]
\emp{aoa_shp=[0,3,1,0,4,2,0]}\pause
\emp{shp_rot=[0,0,3,1,0,4,2]}\pause

\emp{shp_scn=[0,0,3,4,4,8,10]}\pause
\emp{aoa_len= 10}

\emp{shp_ind=} \pause
      \emp{[-1,0,3,-1,4,8,-1]}
\emp{scatter}\pause
  \emp{[0,0,0,0,0,0,0,0,0,0]}
  \emp{[-1,0,3,-1,4,8,-1]}
  \emp{[1,1,1,1,1,1,1]}
  \emp{---------------------}
  \emp{[1,0,0,1,1,0,0,0,1,0]}
\end{colorcode}
\end{columns}

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Flat-Parallel Sparse Matrix-Vector Mult}

\emphh{Idea: use segmented scan, then extract last element in every row!}\\
We assume no rows of length 0, and also inline flag computation.

\begin{colorcode}
smvMul( mat_cols: [N]int       \emp{// [0,    1,    1,   2,    3,   3  ]}
      , mat_vals: [N]real      \emp{// [2.0, -1.0, -1.0, 2.0, -1.0, 3.0]}
      , vct: [vct_len]         \emp{// [1.0,  2.0,  3.0, 4.0]}
      , shape: [num_rows]int ) \emp{// [2,    3,    1]}
      : [num_rows]real =       \emp{// Expected: [0.0, 0.0, 12.0]}
  \blue{shpscn= scan( (+), 0, shape );}       \emp{// [2, 5, 6]}
  \emphh{len   = shpscn[num_rows-1];}
  \emphh{ones  = replicate(num_rows, 1);}      \emp{// [1, 1, 1]}
  \emphh{indm1 = map(\mymath{\lambda} i -> if i==0 then 0}
                     \emphh{else shpscn[i-1]}
                    \emphh{, [0...num_rows]);} \emp{// [0, 2, 5]}
  \emphh{flags = scatter( replicate(len, 0)}   \emp{// [0, 0, 0, 0, 0, 0]}
                 \emphh{, indm1}
                 \emphh{, ones );}     \emp{// [1,    0,    1,   0,    0,   1  ]}
  \red{prods = map2(\mymath{\lambda}col v ->}       \emp{// [2.0, -2.0, -2.0, 6.0, -4.0, 12.0]}
                    \red{v * vct[col]}
              \red{, mat_cols, mat_vals );}
  \orange{scnarr= sgmScan\mymath{\myindu{inc}}( (+), 0}
              \orange{, flags, prods);} \emp{// [2.0, 0.0, -2.0, 4.0, 0.0, 12.0]}\pause
  return                       \emp{// [scnarr[1], scn_arr[4], scn_arr[5]]}
    \purple{map(\mymath{\lambda} ind -> scnarr[ind-1];}
       \purple{, shpscn );}             \emp{// [0.0, 0.0, 12.0]}
\end{colorcode}

\end{frame}

\subsection{Exercise 4: Sparse Matrix-Vector Mult in OpenCL}

\begin{frame}[fragile,t]
  \frametitle{Exercise 4: Sparse Matrix-Vector Mult in OpenCL}

\begin{itemize}
    \item the CPU stub is in {\tt Day3-Exercises/ScanApps/spMatVecMult.h}, function {\tt runSpMatVectMul};
            nothing to do there! You only need to implement several kernels in {\tt scanapps.cl}.\smallskip
    \item[(1)] \blue{\tt shpscn= scan( (+), 0, shape );}\\ is already implemented!
    \item[(2)] \emphh{Implement kernel {\tt mkFlagsSpMVM}.} Array {\tt flags} has been already zeroed;
                {\tt shape\_scn} is {\tt shpscn} in the previous slide.\smallskip
    \item[(3)] \red{Implement kernel {\tt mulPhaseSpMVM}.}\\
                {\tt mat\_ind} is {\tt mat\_cols} from the previous slide.\smallskip
    \item[(4)] \orange{\tt scnarr= sgmScan\mymath{\myindu{inc}}( (+), 0, flags, prods);}\\
            is already implemented!\smallskip
    \item[(5)] \purple{Implement kernel{\tt getLastSpMVM}.}\\ {\tt shape\_scn} and
                {\tt sgm\_mat} are {\tt shpscn} and {\tt scnarr} in previous slide,
                respectively, and {\tt out} is the result.
\end{itemize}

\end{frame}

\section{Optimized Partition2}

\begin{frame}
  \tableofcontents[currentsection]
\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Back To Partition2}

\begin{columns}
\column{0.59\textwidth}
\begin{colorcode}
partition2(X: [n]i32): (i32, [n]i32) =
 cs = map(even, X);
 tfs= map (\mymath{\lambda}c->if c then 1 else 0
          ,  cs);
 isT= scan( (+), 0, tfs );
 i  = isT[n-1];

 ffs= map (\mymath{\lambda}c->if c then 0 else 1
          ,  cs );
 tmps = scan( (+), 0, ffs );
 isF= map (\mymath{\lambda}t -> t+i, tmps);

 inds=map3(\mymath{\lambda}(c,iT,iF) ->
              if c then iT-1 else iF-1
          , cs, isT, isF);
 res = scatter(scratch n, inds, X);
 return (i, res);
\end{colorcode}
\column{0.4\textwidth}\vspace{-2ex}
\begin{colorcode}[fontsize=\scriptsize]
\emp{X   = [5, 4, 2, 10,3, 7, 8]}
\emp{n   = 7}
\emp{cs  = [F, T, T, T, F, F, T]}
\emp{tfs = [0, 1, 1, 1, 0, 0, 1]}

\emp{isT = [0, 1, 2, 3, 3, 3, 4]}
\emp{i   = 4}

\emp{ffs = [1, 0, 0, 0, 1, 1, 0]}
\emp{tmps= [1, 1, 1, 1, 2, 3, 3]}
\emp{isF = [5, 5, 5, 5, 6, 7, 7]}

\emp{inds= [4, 0, 1, 2, 5, 6, 3]}
\emp{X   = [5, 4, 2, 10,3, 7, 8]}
\emp{Res = [_, _, _, _, _, _, _]}
\emp{---------------------------}
\emp{Res = [4, 2, 10, 8,5, 3, 7]}
\end{colorcode}
\end{columns}
\bigskip

This shows you how to think about it, but a direct
implementation is not very efficient in practice.

\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Partition2: Taking Inspiration From Scan}

\begin{colorcode}
bool even(int v) \{ return (bool)(1 -(v&1)); \}
partition2(even, [5, 4, 2, 10, 3, 7, 8] ) should result in
             (4, [4, 2, 10, 8, 5, 3, 7])
\end{colorcode}

A scan implementation is typically split into three kernels:\medskip
\begin{itemize}
    \item[(1)] reduce at workgroup level (with sequential chunking);\medskip
    \item[(2)] scan the results of (1) in one workgroup; this is typically
                negligible, as it is applied on few elements;\medskip
    \item[(3)] perform the workgroup level scan while starting from the
                result of the previous workgroup.
\end{itemize}
\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Partition2: Deriving an Optimized Implementation}

\begin{colorcode}
bool even(int v) \{ return (bool)(1 -(v&1)); \}
partition2(even, [5, 4, 2, 10, 3, 7, 8] ) should result in
             (4, [4, 2, 10, 8, 5, 3, 7])
\end{colorcode}
\smallskip

\begin{scriptsize}
\begin{itemize}
    \item[(I)] map-reduce at workgroup level; operator is commutative!
        \begin{itemize}
           \item[(a)] \begin{scriptsize}{\tt X = map($\lambda$(e)-> if pred(e) then (1,0) else (0,1), A);}
                      \end{scriptsize}
           \item[(b)] \begin{scriptsize}{\tt reduce$^{wkgp}$($\lambda$((a1,b1),(a2,b2))->(a1+b2, b1+b2)),(0,0),X);}
                      \end{scriptsize}
        \end{itemize}\smallskip
    \item[(II)] scan the results of (I) in one workgroup; after this step we know
                the global offsets of the elements that succeed/fail at the start
                of each workgroup.\smallskip
    \item[(III)] Do all the rest in one kernel:
        \begin{itemize}
        \begin{scriptsize}
            \item[(1)] Copy {\tt ELEMS\_PER\_THREAD} from global to local memory, coalesced (ToDo).\smallskip
            \item[(2)] Save them in per-thread register memory ({\tt chunk}) +\\
                        Reduce sequentially each thread's {\tt ELEMS\_PER\_THREAD}
                        elems in {\tt tf}; (ToDo).\smallskip
            \item[(3)] Perform workgroup-level scan on the {\tt tf} of each thread (DONE).\smallskip
            \item[(4)] The number of elements that succeed/fail per workgroup
                        ({\tt num\_ts/num\_fs}) is the last scanned element (DONE).\smallskip
            \item[(5)] Save in {\tt tf} the result of the previous thread (DONE).\smallskip
            \item[(6)] Redo computation (2) but instead place the elements from
                        register to local memory in the order in which they would occur
                        if the partition was done at workgroup level; remember to add
                        {\tt num\_ts} to the failed-elems index (ToDo).\smallskip
            \item[(7)] Finally copy-out from local to global memory in coalesced fashion (DONE).
        \end{scriptsize}
        \end{itemize}
\end{itemize}
\end{scriptsize}
\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Last Exercise: Optimized Partition2}

\begin{itemize}
    \item CPU stub: in file {\tt Day3-Exercises/Partition2Opt/partition2.h}, function {\tt runPartition};
            that is where the OpenCL kernels are called, but nothing to do there.\bigskip

    \item Open in file {\tt Day3-Exercises/Partition2Opt/kernels.cl}, kernel {\tt scanPhaseKer}.
            Follow the instructions, which correspond to implementing the ToDo items from
            the previous slide.\bigskip

    \item The performance of the optimized partition2 should come close to that of inclusive scan!
\end{itemize}

\end{frame}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\documentclass[10pt,twocolumn,letterpaper]{article}

\usepackage{wacv}
\usepackage{times}
\usepackage{epsfig}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{bm}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{multirow}

% Include other packages here, before hyperref.
\input{notation.tex}

\graphicspath{{images/}}

% If you comment hyperref and then uncomment it, you should delete
% egpaper.aux before re-running latex.  (Or just hit 'q' on the first latex
% run, let it finish, and you should be clear).
\usepackage[pagebackref=true,breaklinks=true,letterpaper=true,colorlinks,bookmarks=false]{hyperref}

\wacvfinalcopy % *** Uncomment this line for the final submission

\def\wacvPaperID{287} % *** Enter the ICCV Paper ID here
\def\httilde{\mbox{\tt\raisebox{-.5ex}{\symbol{126}}}}

% Pages are numbered in submission mode, and unnumbered in camera-ready
\ifwacvfinal\pagestyle{empty}\fi
\begin{document}

%%%%%%%%% TITLE
\title{Forget the checkerboard: practical self-calibration using a planar scene\\Supplementary material}

\author{Daniel Herrera C. \qquad Juho Kannala \qquad Janne Heikkil\"a\\
University of Oulu\\
Finland\\
%{\tt\small firstauthor@i1.org}
% For a paper whose authors are all at the same institution,
% omit the following lines up until the closing ``}''.
% Additional authors and addresses can be added with ``\and'',
% just like the second author.
% To save space, use either the email address or home page, not both
%\and
%Second Author\\
%Institution2\\
%First line of institution2 address\\
%{\tt\small secondauthor@i2.org}
}

\maketitle
%\thispagestyle{empty}

%%%%%%%%% ABSTRACT
%%%%%%%%% BODY TEXT
\section{Additional results}
We present additional calibration results with real cameras in Table \ref{fig:results2}. Camera B is a low-quality Logitech webcam V-UAR33. Camera C is a very old Creative Labs webcam PD-1000. Camera C has significant distortion (see Fig.~\ref{fig:sample}) and severe rolling shutter effects. The rolling shutter problems did not allow video calibration, but the algorithm successfully calibrated the distortion. The two validation image sets consisted of 15 checkerboard images and in both cases the checkerboard corners covered the whole image area in various configurations. Both cameras are calibrated successfully and very accurately, as seen by the low validation reprojection errors. Thus confirming the results presented in the paper.

\begin{table}
\caption{Additional calibration results with real cameras. Our method consistently matches the performance of Bouguet's calibration toolbox.}
\label{fig:results2}
\centering
\begin{tabular}{|c|c|c|c|c|}
\hline
 & & Bouguet & \multicolumn{2}{|c|}{Our method (\%)} \\
& & & user match & automatic match  \\
 \hline
\multirow{6}{*}{B} & $f_x$ & 552.68 & -0.15 & -2.76 \\
& $f_x$ & 548.39 & -0.19 &  -3.06\\
& $u_0$ & 321.97 & 0.02 &  2.65 \\
& $v_0$ & 235.25 & 0.11 &  2.84 \\
& $d_0$ & -0.184 & -6.23  &  6.28 \\
& $d_1$ & 0.254 & -8.54 &  -24.13 \\
  \hline
& $e_{\text{train}}$ & 0.17px & 0.16px & 0.66px\\
& $e_{\text{val}}$ & 0.26px & 0.26px & 0.38px \\
  \hline
  \hline
\multirow{6}{*}{C} & $f_x$ & 763.90 & -0.08 &  \\
& $f_x$ & 759.56 & 0.13 &   \\
& $u_0$ & 306.74 & 2.41 &  no video \\
& $v_0$ & 247.25 & 7.17 &  available \\
& $d_0$ & -0.409 & -5.26  &  \\
& $d_1$ & 0.287 & -25.99 &   \\
  \hline
& $e_{\text{train}}$ & 0.47px & 0.46px & n/a\\
& $e_{\text{val}}$ & 0.50px & 0.50px & n/a \\
\hline
\end{tabular}
\vspace*{-0.5cm}
\end{table}

\section{Limitations of the feature extractor}

Although the feature matching stage is outside of the scope of our paper, we briefly address the limitation of the off-the-shelf feature extractor used and how this limits calibration accuracy. We used the ORB feature extractor implemented in OpenCV. The feature matching stage limits the calibration accuracy for two reasons. First, the localization accuracy of the feature extractor is naturally lower than that of carefully selected checkerboard corners. But more importantly, the current implementation leaves the borders of the image unconstrained.

Figure \ref{fig:sample} shows that this implementation doesn't extract features around the borders of the image. This leaves these areas unconstrained during calibration. The distortion function is a fourth degree polynomial and when unconstrained can introduce severe errors in this area. The validation image set has features (i.e. checkerboard corners) in these border areas and therefore the aforementioned issue is a likely explanation for the observed higher validation errors with automatically detected features. However, it is important to note that this detail is not related to the contributions of our paper but is just a particular feature of our current ORB-based camera tracking system used with videos. As shown in the tables of the paper and supplementary material, the accuracy of our self-calibration method is similar to that of Bouguet’s toolbox when similar image features are used in both approaches.

\begin{figure}
\centering
\includegraphics[height=2.5cm]{features.png}
\includegraphics[height=2.5cm]{sampleOldCam.jpg}
\caption{\textbf{Left:} Features extracted using an off-the-shelf implementation. Point color denotes different scales. There is a big area around the border where no features are extracted. \textbf{Right:} Images from camera C have significant distortion.}
\label{fig:sample}
\vspace*{-0.25cm}
\end{figure}

\end{document}


% !TEX engine = xelatex
%
% Notes on reversible jump Markov chain Monte Carlo
%

\documentclass[11pt,a4paper]{article}

\usepackage{mathtools}
\usepackage{amsthm}
\usepackage{iftex}
\ifPDFTeX
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
\else
  \usepackage{unicode-math}
\fi
\usepackage{hyperref}

% Custom maths commands %=======================================================

\newcommand\ds{\displaystyle}                    % large maths.
\newcommand\ts{\textstyle}                       % small maths.
\newcommand\mb[1]{\mathbb{#1}}                   % mathbb shorthand.
\newcommand\mc[1]{\mathcal{#1}}                  % mathcal shorthand.
\newcommand\ff[1]{^{\underline{#1}}}             % falling factorial.
\newcommand\rf[1]{^{\overline{#1}}}              % rising factorial.
\newcommand\ul[1]{\underline{#1}}                % underline.
\newcommand\ol[1]{\overline{#1}}                 % overline.
\DeclareMathOperator\Pb{P}                       % probability.
\DeclareMathOperator\Ex{E}                       % expected value.
\DeclareMathOperator\Va{V}                       % variance.
\DeclarePairedDelimiter\lr{\lparen}{\rparen}     % sized parentheses.
\DeclarePairedDelimiter\lrb{\lbrack}{\rbrack}    % sized brackets.
\DeclarePairedDelimiter\abs{\lvert}{\rvert}      % absolute value symbol.
\DeclarePairedDelimiter\cl{\lceil}{\rceil}       % ceiling symbol.
\DeclarePairedDelimiter\fl{\lfloor}{\rfloor}     % floor symbol.

\newtheorem{lem}{Lemma}
\newtheorem{thm}{Theorem}
\theoremstyle{definition}
\newtheorem{defn}{Definition}

% Title %=======================================================================

\title{Notes on reversible jump Markov chain Monte Carlo}
\author{}
\date{February 2018}

% Document %====================================================================

\begin{document}

\maketitle

\section[]{Reversible jump Markov chain Monte Carlo\footnote{See Peter Green,
\textit{Reversible jump Markov chain Monte Carlo computation and Bayesian model
determination}, 1995. Much of the current exposition is also based on Rasmus
Waagepetersen, \textit{A tutorial on reversible jump MCMC with a view towards
applications in QTL mapping}, 2001.}} %=========================================
\label{sec:rjmc}

\subsection{Introduction} %=====================================================
\label{sec:intro}

Consider the scenario in which we have a data set that we assume has been
generated by a particular model. Let~$\mc{X}$ be the state space of this model,
so that each state~$x \in \mc{X}$ is a realised parameter vector---often, but
not always, of real numbers.

Let~$\pi(dx)$ denote the target distribution that we are interested in---usually
the posterior distribution for the model parameters given the data. We
assume~$\pi$ is a probability measure\footnote{See Section~\ref{sec:measure} for
some introductory measure theory.} on~$\mc{X}$, known up to a normalising
constant. In standard Markov chain Monte Carlo, we define a conditional
probability measure~$P(x,dx')$ (a probability kernel) that gives transition
probabilities between states in~$\mc{X}$; if~$P$ is irreducible (so all states
can be reached from any initial state), aperiodic (there are no states that can
only be returned to in multiples of a certain number of steps), and reversible,
then samples drawn from a run of the Markov chain defined by~$P$ approximate the
target distribution~$\pi$. More specifically, $\pi$ is the stationary
distribution of the Markov chain.

Practically, the most important of the above properties is reversibility. The
Markov chain defined by~$P$ is said to be reversible with respect to~$\pi$ if it
satisfies a property called detailed balance, which requires that for all
measurable $A,B \subset \mc{X}$,
\begin{equation}\label{eqn:balance mh}
  \int_A \int_B \pi(dx)P(x,dx') = \int_B \int_A \pi(dx')P(x',dx)
\end{equation}
In Metropolis-Hastings MCMC, the transition measure~$P(x,dx')$ is made up of a
proposal distribution~$q(x,A)$ and an acceptance density~$\alpha(x,x')$, such
that $P(x,dx') = q(x,dx')\alpha(x,x')$, and in this case, there is a choice
of~$\alpha$ that always leads to detailed balance:
\begin{equation}\label{eqn:alpha mh}
  \alpha(x,x') = \min\lr*{1,\frac{\pi(dx')q(x',dx)}{\pi(dx)q(x,dx')}}.
\end{equation}

To be clear, we have assumed that~$\pi(dx)$ is only known as a density function
up to a constant factor, say~$\pi(dx) = p(x)d\pi/Z$. Fortunately, this is no
real restriction, since $\pi(dx')/\pi(dx)$ can still be computed as
\[ \frac{\pi(dx')}{\pi(dx)} = \frac{p(x')/Z}{p(x)/Z} = \frac{p(x')}{p(x)}. \]
Usually, $\pi(dx)$ is the posterior distribution of a model~$M$'s parameters~$x
\equiv \theta$, given data~$y$:
\[ \pi(A) = \int_A p(\theta \mid y,M)\,d\theta
  = \int_A p(y \mid \theta,M)p(\theta \mid M)/p(y \mid M)\,d\theta, \]
in which case~$Z = p(y \mid M)$.

\subsection{Multiple parameter spaces} %========================================
\label{sec:multiple}

In the more general case addressed by reversible jump MCMC, the state
space~$\mc{X}$ may consist of disjoint, disparate subspaces corresponding to a
countable set of different models, and the Markov chain may move between them in
much the same way as it does within them.

In single-model Metropolis-Hastings simulations, it is quite common to break the
transition process into simpler parts by applying a number of different moves,
either in sequence or according to a predefined probability vector. One of the
notable contributions of reversible jump MCMC is the possibility for
state-dependent mixing, in which the set of proposed moves may depend on the
current state of the Markov chain.

For example, we might have a countable set of moves indexed by a set~$M$, and a
conditional probability~$q_m(x,dx')$ of proposing a move of type~$m$ to
state~$x'$, given that the current state of the chain is~$x$. These
probabilities need not sum to 1, so that with probability~$1 - \sum_m
q_m(x,\mc{X})$ no move is proposed. Also, it is reasonable to restrict the moves
that are available from any given state, so it is quite possible
that~$q_m(x,\mc{X}) = 0$ for arbitrarily many~$m$. As in the previous section,
each move is paired with an acceptance probability~$\alpha_m(x,x')$, which will
be derived abstractly below, and in full for the real case later on. The
probability of remaining at the current state~$x$ is then:
\[ s(x) = 1 - \sum_{m\in M} q_m(x,\mc{X})
  + \sum_m \int_{\mc{X}} q_m(x,dx')\lr*{1 - \alpha_m(x,x')}, \]
and the transition measure~$P$ takes the form:
\[ P(x,B) = \sum_m \int_B q_m(x,dx')\alpha_m(x,x') + s(x)[x \in B]. \]
Here, $B$ is a measurable subset of~$\mc{X}$, and~$[x \in B]$ is the Iverson
bracket, or indicator function.

In terms of reversibility of the chain, state-dependent mixing imposes no real
restrictions, because for detailed balance to hold overall, it need only hold
for each move individually. Substituting the above form for~$P(x,B)$
into~\eqref{eqn:balance mh} gives:
\begin{multline*}
  \sum_m \int_A \pi(dx) \int_B q_m(x,dx')\alpha_m(x,x')
    + \int_{A\cap B} \pi(dx)s(x) \\
  = \sum_m \int_B \pi(dx') \int_A q_m(x',dx)\alpha_m(x',x)
    + \int_{B\cap A} \pi(dx')s(x'),
\end{multline*}
which is satisfied if for each~$m$, $A$, and~$B$,
\begin{equation}\label{eqn:balance rj}
\begin{multlined}
  \int_A \pi(dx) \int_B q_m(x,dx')\alpha_m(x,x') \\
  = \int_B \pi(dx') \int_A q_m(x',dx)\alpha_m(x',x).
\end{multlined}
\end{equation}

There is an important difference between this equation and~\eqref{eqn:balance
mh}: in contrast with the previous section, the measures being integrated with
respect to here are not necessarily interchangeable. If~$q_m$ is a
model-switching move, and~$A$ and~$B$ belong to parameter spaces~$S_1$ and~$S_2$
respectively, then in the first integral we are concerned with a measure
on~$S_1\times S_2$ (it may be null elsewhere on~$\mc{X}\times\mc{X}$), whereas
the measure in the second integral has to do with~$S_2\times S_1$.
For~\eqref{eqn:balance rj} to hold, we must first know that both integrals can
at least be expressed with respect to the same measure; we thus make the
following assumption: let~$\mu_m$ be a measure on~$\mc{X}\times\mc{X}$ relative
to which~$\pi q_m$ has a finite, computable density on~$(S_1\times S_2) \cup
(S_2\times S_1)$. Also assume that~$\mu_m$ is symmetric, in the sense that it
matters not whether it is applied on~$S_1\times S_2$ or~$S_2\times S_1$, i.e.,
$\mu_m(dx,dx') = \mu_m(dx',dx)$. (This is all not quite as abstract and
circuitous as it may sound---as Section~\ref{sec:reals} will hopefully show.)

With respect to~$\mu_m$, assume that~$\pi(dx)q_m(x,dx')$ has density~$f_1(x,x')$
and~$\pi(dx')q_m(x',dx)$ density~$f_2(x',x)$. Then detailed balance holds (for
move~$m$) if we let
\begin{equation}\label{eqn:alpha rj}
\begin{gathered}
  \alpha_m(x,x') = \min\lr*{1,\frac{f_2(x',x)}{f_1(x,x')}} \\
  \text{and} \\
  \alpha_m(x',x) = \min\lr*{1,\frac{f_1(x',x)}{f_2(x,x')}},
\end{gathered}
\end{equation}
(we have simply made explicit the fact that~$\alpha$ is defined separately
on~$S_1\times S_2$ and~$S_2\times S_1$), because this implies that
\[ f_1(x,x')\alpha_m(x,x') = f_2(x',x)\alpha_m(x',x), \]
which in turn is all that was required to prove~\eqref{eqn:balance rj}:
\begin{align*}
  \int_A \int_B \pi(dx)q_m(x,dx')\alpha_m(x,x')
  &= \int_A \int_B \mu_m(dx,dx')f_1(x,x')\alpha_m(x,x') \\
  &= \int_B \int_A \mu_m(dx',dx)f_2(x',x)\alpha_m(x',x) \\
  &= \int_B \int_A \pi(dx')q_m(x',dx)\alpha_m(x',x).
\end{align*}

\subsection{Real parameter spaces} %============================================
\label{sec:reals}

Let~$\mc{X}$ be a state space consisting of real-valued vectors, so $\mc{X} =
\cup_{i\in I} S_i$, with each~$S_i \subseteq \mb{R}^{n_i}$, for some countable
index set~$I$. We will assume the Borel $\sigma$-algebra on~$\mc{X}$, but won't
refer to it directly. Let~$\pi$ again be the target distribution we wish to
sample from (a probability measure on~$\mc{X}$), and assume that for each
subspace~$S_i$ the measure~$\pi$ has a non-negative, finite density~$p_i/Z_i$
with respect to Lebesgue measure~$\lambda_{n_i}$ on~$S_i$.

Following on from the previous section, we can restrict ourselves to moves
between a pair of subspaces, say~$S_1$ and~$S_2$. Let~$q_1(x,dx')$
and~$q_2(x',dx)$ be the proposal measures (technically, these are kernels)
on~$S_1 \times S_2$ and~$S_2 \times S_1$ (i.e., for moves~$S_1 \to S_2$ and~$S_2
\to S_1$) respectively. Also assume that these moves are only attempted with
probabilities~$j_1(x)$ and~$j_2(x')$, respectively.

The most obvious practical scenario (and the only one we address) is one in
which moves can be made with the aid of auxiliary variables, and we have an idea
of how these random variables should be generated. In the simplest case, we
might have~$n_1 < n_2$, and need $n_2-n_1$ random numbers~$u$ to map a state~$x
\in S_1$ to a proposed state~$x' \in S_2$. More generally, $u$ might
contain~$m_1$ auxiliary numbers, so that~$(x,u) \in \mb{R}^{n_1} \times
\mb{R}^{m_1} = \mb{R}^n$ (say).

We assume that the proposed state~$x'$ is obtained with the aid of some
bijection~$h\colon \mb{R}^n \to \mb{R}^n$ that we have access to, so that
$h(x,u) = (x',u')$, where $u'$ is a vector of~$m_2 = n-n_2$ auxiliary variables.
The so-called `dimension-matching' assumption of reversible jump MCMC is that we
use~$h^{-1}$ when we perform moves in the reverse direction; it implies that a
vector of auxiliary variables~$u'$ will need to be generated when proposing a
state in~$S_1$ given~$x' \in S_2$ as well. We therefore need to assume that we
have access to densities~$g_1(u)$ and~$g_2(u')$, with respect to~$\lambda_{m_1}$
and~$\lambda_{m_2}$, that can be used to augment states in~$S_1$ and~$S_2$
respectively.

It is important (for reasons we'll point out below) that the bijection~$h$ is a
diffeomorphism---that is, both~$h$ and~$h^{-1}$ are differentiable---and also
that the intermediate dimension~$n$ is minimal, so that there are no superfluous
auxiliary variables. To be specific, $(x,x')$ must determine
$(u,u')$---otherwise the move~$x \to x'$ could not be expressed using~$g_1(u)$
for a particular~$u$. We already know that $(x,u)$ determines~$x'$ via~$h$ and a
projection onto~$S_2$; if we set~$r_x(u) = x'$, the fact that $(x,x')$
determines $(u,u')$ will imply that~$r_x^{-1}$ exists.

Let~$A \subseteq S_1$ and~$B \subseteq S_2$. We have two measures, on~$S_1
\times S_2$ and~$S_2 \times S_1$ respectively:
\begin{equation}\label{eqn:measures}
\begin{gathered}
  \int_A \lambda_{n_1}(dx)\,p_1(x)j_1(x) q_1(x,B) \\
    \text{and} \\ \int_B \lambda_{n_2}(dx')\,p_2(x')j_2(x') q_2(x',A).
\end{gathered}
\end{equation}
Consider the first measure, which gives the probability of proposing a move from
a state in~$A$ to one in~$B$. We wish to define the proposal measure~$q_1$ in
such a way that its density (with respect to some new measure on~$S_2$) can be
expressed in terms of~$g_1$, which is a density on~$\mb{R}^{m_1}$. In
particular, we require a measure~$\mu_1$ on~$S_2$ for which
\begin{equation}\label{eqn:mu_1}
  \int_B (g_1 \circ r_x^{-1})\,d\mu_1 = \int_{r_x^{-1}(B)} g_1\,d\lambda_{m_1},
\end{equation}
since this implies that the probability of proposing some~$x' \in S_2$ is
simply~$g_1(u')$, where~$u = r_x^{-1}(x')$. The change of variables theorem for
integrals has just this form: set
\[ \mu_1(B) = \lambda_{m_1}(r_x^{-1}(B))
  = \lambda_{m_1}(\{\, u : r_x(u) \in B \,\}); \]
then $\mu_1$ is a positive measure on~$S_2$, and for any measurable~$f\colon S_2
\to \mb{R}$:
\[ \int_{S_2} f\,d\mu_1 = \int_{\mb{R}^{m_1}} (f \circ r_x)\,d\lambda_{m_1}. \]
Equation~\ref{eqn:mu_1} follows when~$f = g_1 \circ r_x^{-1}$ on~$B$:
\begin{align*}
  \int_B (g_1 \circ r_x^{-1})\,d\mu_1
    &= \int_{S_2} \symbf{1}_B \cdot (g_1 \circ r_x^{-1})\,d\mu_1 \\
    &= \int_{\mb{R}^{m_1}} (\symbf{1}_B \circ r_x) \cdot g_1\,d\lambda_{m_1}
    = \int_{r_x^{-1}(B)} g_1\,d\lambda_{m_1}.
\end{align*}

The first measure in~\eqref{eqn:measures} can now be rewritten as
\begin{equation}\label{eqn:measure 1}
  \int_A \lambda_{n_1}(dx)\,p_1(x)j_1(x)
      \int_B \mu_1(dx')\,g_1(r_x^{-1}(x')),
\end{equation}
and since the product of two measures is a measure on the relevant product
space, we have
\[ \int_A \int_B \mu(dx,dx')\,p_1(x)j_1(x)g_1(r_x^{-1}(x')), \]
in which the product measure is
\[ \mu(A \times B) = \lambda_{n_1}(A)\mu_1(B)
  = \lambda_n(\{\, (x,u) : x \in A, r_x(u) \in B \,\}). \]
This measure is in fact sufficient for our purposes, for two reasons: one,
because it can also be applied to~$S_2 \times S_1$, making it symmetric, and
two, because both measures in~\eqref{eqn:measures} have computable densities
with respect to~$\mu$.

The density for moves from~$S_1$ to~$S_2$ is clear:
\[ f_1(x,x') = p_1(x)j_1(x)g_1(r_x^{-1}(x')). \]
The second density---on~$S_2 \times S_1$---must exist by the Lebesgue
decomposition and Radon-Nikodym theorems, and can also be given explicitly,
because in this case the Radon-Nikodym derivative is the determinant of the
Jacobian of~$h^{-1}\colon \mb{R}^n \to \mb{R}^n$.

To see this, note firstly that if we apply all of the above steps to the second
expression in~\eqref{eqn:measures}, then the resulting measure on~$S_2 \times
S_1$ is
\begin{equation}\label{eqn:measure 2}
  \int_B \lambda_{n_2}(dx')\,p_2(x')j_2(x')
    \int_A \mu_2(dx)\,g_2(r_{x'}^{-1}(x)),
\end{equation}
mirroring~\eqref{eqn:measure 1}. The new functions are straightforward: we have
$r_{x'}(u') = x'$, and~$\mu_2(A) = \lambda_{m_2}(r_{x'}^{-1}(A))$. Together,
$\lambda_{n_2}$ and~$\mu_2$ provide a measure on~$S_2 \times S_1$ (as above):
\[ \mu^*(B \times A) = \lambda_{n_2}(B)\mu_2(A)
  = \lambda_n(\{\, (x',u') : x' \in B, r_{x'}(u') \in A \,\}). \]
If~$\mu(B \times A) = \mu(A \times B)$ is the Lebesgue measure of the region~$R
\subseteq \mb{R}^n$, then~$\mu^*(B \times A)$ is the Lebesgue measure of~$h(R)$.
In addition, $\mu^*$ is absolutely continuous relative to~$\mu$, because if~$R$
is a null set of~$\mu$, one of its elements is determined by the other~$n-1$,
and since~$h$ is one-to-one, the same must be true of~$h(R)$. Hence~$\mu^*$ has
a Radon-Nikodym derivative with respect to~$\mu$, and~\eqref{eqn:measure 2}
implies
\begin{multline*}
  \int_B \int_A \mu^*(dx',dx)\,p_2(x')j_2(x')g_2(r_{x'}^{-1}(x)) \\
    = \int_B \int_A \mu(dx,dx')\,\frac{d\mu^*}{d\mu}
        p_2(x')j_2(x')g_2(r_{x'}^{-1}(x)).
\end{multline*}
Since $d\mu^*/d\mu$ is the relative rate of change of~$\mu^*$ with respect
to~$\mu$, it is the factor by which~$h$ scales volume, and is thus given by the
absolute value of the determinant of the Jacobian matrix of~$h$. The Jacobian
exists for any differentiable mapping, but its determinant is only non-zero if
the function's inverse is also differentiable---hence the diffeomorphism
requirement above. In particular then, the density of the joint target-proposal
measure on~$S_2 \times S_1$ relative to~$\mu$ is
\[ f_2(x',x) = \abs*{\frac{\partial(x',u')}{\partial(x,u)}}\,
    p_2(x')j_2(x')g_2(r_{x'}^{-1}(x)), \]
and we are now in a position to prove that detailed balance holds for moves
between~$S_1$ and~$S_2$, as long as we make use of the acceptance probability as
formulated in~\eqref{eqn:alpha rj}. We only work out the case in
which~$\alpha_m(x,x') = f_2(x',x)/f_1(x,x') \le 1$, the other one being entirely
analogous.

Recall that we are trying to prove that~\eqref{eqn:balance rj} holds, which in
the current context means showing that~\eqref{eqn:measure 1}
and~\eqref{eqn:measure 2} are equal when paired, respectively,
with~$\alpha_m(x,x')$ and~$\alpha(x',x)$:
\begin{multline*}
  \int_A \lambda_{n_1}(dx)\,p_1(x)j_1(x)
      \int_B \mu_1(dx')\,g_1(r_x^{-1}(x'))\alpha_m(x,x') \\
  \begin{aligned}
  &= \int_A \int_B \mu(dx,dx')\,p_1(x)j_1(x)g_1(r_x^{-1}(x'))\alpha_m(x,x') \\
  &= \int_A \int_B \mu(dx,dx')\,\frac{d\mu^*}{d\mu}
      p_2(x')j_2(x')g_2(r_{x'}^{-1}(x)). \\
  &= \int_B \int_A \mu^*(dx',dx)\,p_2(x')j_2(x')g_2(r_{x'}^{-1}(x)) \\
  &= \int_B \lambda_{n_2}(dx')\,p_2(x')j_2(x')
      \int_A \mu_2(dx)\,g_2(r_{x'}^{-1}(x))\alpha_m(x',x).
  \end{aligned}
\end{multline*}

\section{Some measure theory} %=================================================
\label{sec:measure}

These definitions and results all come from the
\emph{Random}\footnote{\url{http://www.randomservices.org/random}} website.

\subsection %===================================================================
  {\href{http://www.randomservices.org/random/foundations/Measure.html}
  {Algebras and measurable functions}}

Let~$S$ be a set, and~$\mc{S}$ a non-empty collection of subsets of~$S$.
\begin{defn}
$\mc{S}$ is an \emph{algebra} if it is closed under complements and unions.
\end{defn}
In particular, an algebra of sets is closed under a \emph{finite} number of
unions, intersections, or complements (or a combination thereof).

\begin{defn}
$\mc{S}$ is a \emph{$\sigma$-algebra} if it is closed under complements and
countable unions.
\end{defn}
It follows that a $\sigma$-algebra of sets is closed under countable
intersections too. If~$S$ is a set and~$\mc{S}$ a $\sigma$-algebra of subsets
of~$S$, then the pair~$(S,\mc{S})$ is called a \emph{measurable space}.

\begin{lem}
The intersection of a (possibly uncountable) collection of $\sigma$-algebras of
subsets of~$\mc{S}$ is itself a $\sigma$-algebra of subsets of~$\mc{S}$.
\end{lem}

Let~$\mc{B}$ be a collection of subsets of~$S$.
\begin{defn}
The $\sigma$-algebra \emph{generated by~$\mc{B}$} is the intersection of all
$\sigma$-algebras of subsets of~$S$ that contain~$\mc{B}$.
\end{defn}
The $\sigma$-algebra generated by the collection of open subsets of~$S$ is
called the \emph{Borel} $\sigma$-algebra of~$S$.

Suppose that~$(S,\mc{S})$ and~$(T,\mc{T})$ are measurable spaces.
\begin{defn}
A function~$f\colon S \to T$ is \emph{measurable} if $f^{-1}(B) \in \mc{S}$ for
every~$B \in \mc{T}$.
\end{defn}

\subsection %===================================================================
  {\href{http://www.randomservices.org/random/prob/Measure.html}
  {Measures}}

As above, let~$(S,\mc{S})$ and~$(T,\mc{T})$ be measurable spaces, and~$f\colon S
\to T$ be a measurable function.
\begin{defn}
A function~$\mu\colon \mc{S} \to [0,\infty]$ is a \emph{positive measure}
on~$(S,\mc{S})$ if $\mu(\emptyset) = 0$ and countable additivity holds (i.e.,
the measure of a countable union of disjoint subsets is the sum of the measures
of the subsets).
\end{defn}
A positive measure on~$(S,\mc{S})$ that also satisfies $\Pb(S) = 1$ is called a
\emph{probability measure}. The triple~$(S,\mc{S},\mu)$ is called a
\emph{measure space}.

\begin{lem}
If~$\mu$ is a positive measure on~$(S,\mc{S})$ and~$c > 0$, then~$c\mu$ is also
a positive measure on~$(S,\mc{S})$.
\end{lem}

\begin{lem}
If~$(R,\mc{R})$ is a measurable subspace of~$(S,\mc{S})$---i.e., $(R,\mc{R})$ is
a measurable space and~$\mc{R} \subseteq \mc{S}$---then~$\mu$ restricted
to~$\mc{R}$ is a positive measure on~$(R,\mc{R})$.
\end{lem}

\begin{thm}\label{thm:measures change}
The function~$\nu$ defined by $\nu(B) = \mu\lr*{f^{-1}(B)}$ for~$B \in \mc{T}$
is a positive measure on~$(T,\mc{T})$.
\end{thm}
The construction of a measure according to Theorem~\ref{thm:measures change} is
sometimes referred to as a \emph{change of variables}.

\subsection %===================================================================
  {\href{http://www.randomservices.org/random/dist/Integral.html}
  {Integrals}}

Let~$(S,\mc{S},\mu)$ be a measure space. Then the integral of a measurable
function~$f\colon S \to \mb{R}$ is denoted by~$\int_S f\,d\mu$, or~$\int_S
f(x)\,d\mu(x)$, or even~$\int_S \mu(dx)f(x)$, and we say that $f$ is
\emph{integrable} if the integral exists as a number in the set~$\mb{R} \cup
\{-\infty,\infty\}$.
\begin{defn}
If~$A \in \mc{S}$ then~$\int_S \symbf{1}_A\,d\mu = \mu(A)$.
\end{defn}

The integral with respect to a measure is defined in three stages: for simple
functions, non-negative functions, and then general functions.
\begin{defn}
A \emph{simple function} is a measurable, real-valued function with finite
range.
\end{defn}

\begin{lem}
A simple function~$f$ can be represented as a sum~$\sum_{i\in I}
a_i\symbf{1}_{A_i}$, where~$I$ is a finite index set, $a_i \in \mb{R}$ for
each~$i \in I$, and $\{\, A_i : i \in I \,\}$ is a collection of sets
in~$\,\mc{S}$ that partition~$S$.
\end{lem}
If the~$a_i$ are distinct and the~$A_i$ non-empty, the representation is called
\emph{canonical}.

Let~$f$ be a non-negative simple function with representation~$\sum_i
a_i\symbf{1}_{A_i}$, where~$a_i \ge 0$ for each~$i \in I$.
\begin{defn}
The integral of~$f$ with respect to~$\mu$ is
\[ \int_S f\,d\mu = \sum_{i\in I} a_i\mu(A_i). \]
\end{defn}
The above definition is consistent, in that the value of the integral is the
same for all representations of~$f$.

Next, suppose that $f\colon S \to [0,\infty)$ is measurable.
\begin{lem}
There exists an increasing sequence~$(f_1,f_2,\dots)$ of non-negative simple
functions such that~$f_n \to f$ on~$S$ as~$n \to \infty$.
\end{lem}
That is, every non-negative measurable function is the limit of a sequence of
non-negative simple functions.

\begin{defn}
The integral of a non-negative function~$f$ is
\[ \int_S f\,d\mu = \sup\left\{\, \int_S g\,d\mu
  : g \text{ is simple and } 0 \le g \le f \,\right\}. \]
\end{defn}

The above lemma and definition are linked by the \emph{monotone convergence
theorem}.
\begin{thm}
If~$(f_1,f_2,\dots)$ is an increasing sequence of non-negative simple functions
such that~$f_n \to f$ as~$n \to \infty$, then
\[ \int_S \lim_{n\to\infty} f_n\,d\mu = \lim_{n\to\infty} \int_S f_n\,d\mu. \]
\end{thm}

Finally, let~$f\colon S \to \mb{R}$ be any measurable function. For~$x \in
\mb{R}$, write the positive and negative parts of~$x$ as~$x^+ = \max(x,0)$
and~$x^- = \max(-x,0)$, so that $x = x^+ - x^-$ and $\abs{x} = x^+ + x^-$. This
definition extends to functions: $f^+$ is~$0$ wherever~$f \le 0$, and~$f^-$
is~$0$ wherever~$f \ge 0$.
\begin{defn}
The integral of a measurable function~$f$ is
\[ \int_S f\,\mu = \int_S f^+\,d\mu - \int_S f^-\,d\mu. \]
\end{defn}
The function~$f$ is integrable if and only if~$\int_S \abs{f}\,d\mu < \infty$.

\begin{defn}
The integral of~$f$ over a measurable subset~$A \in \mc{S}$ is
\[ \int_A f\,d\mu = \int_S \symbf{1}_A f\,d\mu. \]
\end{defn}
The integral over the union of finitely many disjoint, measurable subsets is the
sum of the integrals over the subsets.

(From \href{http://www.randomservices.org/random/dist/Properties.html}
{Section~2.11}.) Given a measure space~$(S,\mc{S},\mu)$, a measurable
space~$(T,\mc{T})$, and a measurable function~$f\colon S \to T$,
Theorem~\ref{thm:measures change} states that the function~$\nu(B) =
\mu(f^{-1}(B))$ is a positive measure on~$(T,\mc{T})$. This change of variables
theorem extends to integrals as follows.
\begin{thm}
If~$g\colon T \to \mb{R}$ is a measurable function, then, assuming the
integrals exist,
\[ \int_T g\,d\nu = \int_S (g \circ f)\,d\mu. \]
\end{thm}

\subsection %===================================================================
  {\href{http://www.randomservices.org/random/dist/Density.html}
  {Density functions}}

Let~$(S,\mc{S},\mu)$ be a measure space.
\begin{defn}
A subset~$A \in \mc{S}$ is a \emph{null} set of~$\mu$ if~$\mu(B) = 0$ for
every~$B \in \mc{S}$ such that $B \subseteq A$.
\end{defn}
\begin{defn}
Let~$\nu$ be another measure on~$(S,\mc{S})$. Then~$\nu$ is \emph{absolutely
continuous} with respect to~$\mu$---or \emph{dominated} by~$\mu$---if every null
set of~$\mu$ is also a null set of~$\nu$. On the other hand, $\mu$ and~$\nu$ are
\emph{mutually singular} if there exists an~$A \in \mc{S}$ such that~$A$ is null
for~$\mu$ and~$A^c$ is null for~$\nu$.
\end{defn}
Two measures~$\mu$ and~$\nu$ on~$(S,\mc{S})$ are said to be \emph{equivalent} if
they are absolutely continuous with respect to one another.

Let~$f\colon S \to \mb{R}$ be a measurable function whose integral with respect
to~$\mu$ exists.
\begin{defn}
The function~$\nu$ defined by
\[ \nu(A) = \int_A f\,d\mu \]
is a measure on~$(S,\mc{S})$ that is absolutely continuous with respect
to~$\nu$. The measurable function~$f$ is called a \emph{density function}
of~$\nu$ with respect to~$\mu$.
\end{defn}
If~$f$ is integrable, then $\nu$ is a finite measure; if~$f$ is non-negative,
$\nu$ is a positive measure; and if~$f$ is both non-negative and~$\int_S
f\,d\mu = 1$, then~$\nu$ is a probability measure. In the final case, $f$ is a
\emph{probability density function} of~$\nu$ relative to~$\mu$.

\begin{lem}
If~$f$ is a density function of~$\nu$ with respect to~$\mu$, then~$g\colon S \to
\mb{R}$ is a density function of~$\nu$ relative to~$\mu$ if and only if~$f = g$
almost everywhere on~$S$ with respect to~$\mu$.
\end{lem}

\begin{thm}
Let~$\nu$ be a measure on~$(S,\mc{S})$. Then~$\nu$ can be decomposed as~$\nu =
\nu_c + \nu_s$ such that~$\nu_c$ is absolutely continuous with respect to~$\mu$,
and~$\nu_s$ and~$\mu$ are mutually singular. Furthermore, $\nu_c$ has a density
function with respect to~$\mu$.
\end{thm}
The first and second parts of this theorem are called the \emph{Lebesgue
decomposition} and \emph{Radon-Nikodym} theorems respectively. In particular,
$\nu$ has a density function with respect to~$\mu$ if and only if~$\mu$
dominates~$\nu$. The derivative in this case is also referred to as the
\emph{Radon-Nikodym derivative} of~$\nu$ with respect to~$\mu$, and is sometimes
written as~$d\nu/d\mu$.

Suppose that~$\nu$ is absolutely continuous and has density function~$f$
relative to~$\mu$.
\begin{thm}
If~$g$ is a measurable function whose integral with respect to~$\nu$ exists,
then
\[ \int_S g\,d\nu = \int_S gf\,d\mu. \]
\end{thm}
This is the \emph{density theorem} for integrals, and is the reason for the
notation~$f = d\nu/d\mu$.

\subsection{The reals} %========================================================

(From \href{http://www.randomservices.org/random/foundations/Other.html}
{Section~0.12}.) Consider the set of open intervals on the real line:
\[ \mc{B} = \{\, (-\infty,a] : a \in \mb{R} \,\}
  \cup \{\, (a,b] : a,b \in \mb{R}, a < b \,\}
  \cup \{\, (b,\infty) : b \in \mb{R} \,\}. \]
\begin{lem}
The $\sigma$-algebra generated by~$\mc{B}$ is the $\sigma$-algebra of Borel sets
of~$\mb{R}$.
\end{lem}
In~$\mb{R}^n$, the collection that gives rise to the Borel $\sigma$-algebra
on~$\mb{R}^n$ is
\[ \mc{B}_n = \left\{\, \ts\prod_{i=1}^n A_i
  : A_i \in \mc{B} \text{ for each } i \in \{1,\dots,n\} \,\right\}. \]

(From \href{http://www.randomservices.org/random/prob/Probability.html}
{Section~1.3}.)
\begin{defn}
The standard $n$-dimensional measure on~$\mb{R}^n$ is called \emph{Lebesgue
measure}, defined by
\[ \lambda_n(A) = \int_A 1\,dx. \]
\end{defn}
In particular, $\lambda_1, \lambda_2$, and $\lambda_3$ correspond to length,
area, and volume, respectively.

\end{document}


\documentclass[USEnglish,ignorenonframetext,notheorems,aspectratio=1610]{beamer}
\usetheme[compress]{Madrid}
\usecolortheme{iwr}
\usepackage{tikz,tikzscale}
\usetikzlibrary{snakes}
\usepackage{../mathsim}
\usepackage{times}
\usepackage{xr}
\externaldocument{awa}
\externaldocument{explicit}
\externaldocument{implicit}
\externaldocument{lmm}
\externaldocument{rwa}
\externaldocument{newton}
\externaldocument{fd}
\externaldocument{appendix}
\usepackage{mfirstuc}
\usepackage{mathtools}
\mathtoolsset{showonlyrefs}

\def\footnote#1{}

% Definitions for Runge-Kutta-Koefficients
\def\rks{s}  % Number of steps
\def\rka{a}  % Coefficient matrix
\def\rkb{b}  % Quadrature weights
\def\rkc{c}  % Quadrature points
\def\rkg{g}  % Intermediate points

\def\lmms{s}

% Definitions for boundary value problems
\def\rwaa{B_a}
\def\rwab{B_b}

\title{\textbf{Numerical Analysis \\of\\
    Ordinary Differential Equations}}
\author{Guido Kanschat}
\date{\today}
\begin{document}
\frame{\maketitle}
\frame{\frametitle{Overview}\tableofcontents[hideallsubsections]}
\section{Initial Value Problems and their Properties}
\frame{\tableofcontents[currentsection,hideothersubsections]}
\subsection{Modeling with ordinary differential equations}
\subsection{Introduction to initial value problems}

\frame {\input {blocks/Definition-ode.tex}}
\frame {\input {blocks/Definition-explicit-ode.tex}}
\frame {\input {blocks/Lemma-first-order.tex}}
\frame {\input {blocks/Definition-autonomization.tex}}
\frame {\input {blocks/Definition-IVP.tex}}
\frame {\input {blocks/Definition-local-solution.tex}}
\frame {\input {blocks/Lemma-volterra.tex}}
\frame {\input {blocks/Theorem-peano.tex}}
\frame {\input {blocks/Theorem-peano-continuation.tex}}

\subsection{Linear differential equations
  and Grönwall's inequality}

\frame {\input {blocks/Definition-linear-ode.tex}}
\frame {\input {blocks/Definition-integrating factor.tex}
  \input {blocks/Lemma-linear-representation.tex}}
\frame {\input {blocks/Lemma-gronwall.tex}}
\frame {\input {blocks/Corollary-awa:unique-linear.tex}}
\frame {\input {blocks/Lemma-solution-space.tex}}
\frame {\input {blocks/Definition-fundamental-system.tex}}
\frame {\input {blocks/Corollary-fundamental-regular.tex}}

\subsection{Well-posedness of the IVP}

\frame {\input {blocks/Definition-hadamard.tex}}
\frame {\input {blocks/Definition-Lipschitz-condition.tex}}
\frame {\input {blocks/Theorem-awa-stability.tex}}
\frame {\input {blocks/Theorem-picardlindelof.tex}}

\section{Explicit One-Step Methods and Convergence}
\frame{\tableofcontents[currentsection,hideothersubsections]}
\subsection{Introduction}

\frame {\input {blocks/Definition-partitioning.tex}}
\frame {\input {blocks/Definition-one-step.tex}}
\frame {\input {blocks/Example-euler-linear-1.tex}}

\subsection{Error analysis}
\begin{frame}
  \frametitle{Error decomposition}
  \centering
  \includegraphics[width=.49\textwidth]{fig/forward-error.tikz}
  \includegraphics[width=.49\textwidth]{fig/backward-error.tikz}
\end{frame}

\frame {\input {blocks/Definition-truncation-error.tex}}
\frame {\input {blocks/Lemma-gronwall-discrete.tex}}
\frame {\input {blocks/Theorem-stability-discrete.tex}}
\frame {\input {blocks/Corollary-finite-precision}}
\frame {\input {blocks/Theorem-convergence-one-step.tex}}

\subsection{Runge-Kutta methods}

\frame {\input {blocks/Definition-erk.tex}}
\frame {\input {blocks/Definition-butcher-tableau-erk.tex}}
\frame {\input {blocks/Example-rk2.tex}
  \input {blocks/Lemma-Heun-consistency.tex}}
\frame {\input {blocks/Example-rk3.tex}}
\frame {\input {blocks/Lemma-rk-order-3-4.tex}}
\frame {\input {blocks/Lemma-taylor-u.tex}}
\frame {\input {blocks/Lemma-taylor-y.tex}}
\frame {\input {blocks/Example-rk4.tex}}
\begin{frame}
  \begin{block}{Butcher barriers}
    \input{tables/butcher-barriers}
  \end{block}
\end{frame}
\frame {\input {blocks/Lemma-erk-Lipschitz.tex}}

\subsection{Estimate of the truncation error and time step control}

\frame {\input {blocks/Theorem-richardson.tex}}
\frame {\input {blocks/Definition-embedded-butcher.tex}}
\frame {\input {blocks/Definition-dormand-prince-45.tex}}
\subsection{Continuous Runge-Kutta methods}

\frame {\input {blocks/Example-rk4-continuous.tex}}
\frame {\input {blocks/Example-dormand-prince-45-continuous.tex}}

\section{Implicit One-Step Methods and Long-Time Stability}
\frame{\tableofcontents[currentsection,hideothersubsections]}

\subsection{Monotonic initial value problem}

\frame {\input {blocks/Definition-lipschitz-one-sided.tex}}
\frame {\input {blocks/Theorem-stability-monotonic.tex}}
\frame {\input {blocks/Lemma-linear-monotonic.tex}}
\frame {\input {blocks/Definition-stiff.tex}}

\subsection{A- and B-stability}

\frame {\input {blocks/Definition-stability-function.tex}}

\begin{frame}
  \frametitle{Stability regions of the implicit and explicit Euler
    methods}
  \begin{center}
    \includegraphics[width=.47\textwidth]{fig/stability-euler}
    \includegraphics[width=.47\textwidth]{fig/stability-euler2}
  \end{center}
\end{frame}

\frame {\input {blocks/Definition-a-stability.tex}
  \input {blocks/Theorem-a-stability.tex}}
\frame {\input {blocks/Theorem-implicit:a-stability-erk.tex}}
\frame {\input {blocks/Definition-b-stability.tex}}
\frame {\input {blocks/Theorem-b-stability.tex}
  \input {blocks/Corollary-b-stable-a-stable.tex}}
\frame {\input {blocks/Definition-l-stability.tex}}

\subsection{General Runge-Kutta methods}

\frame {\input {blocks/Definition-rk.tex}}
\begin{frame}
  \begin{block}{Two-stage SDIRK methods of order 3}
    \begin{gather*}
      \def\arraystretch{1.5}
      \begin{array}{c|cc}
        \frac12 - \frac{\sqrt{3}}{6} & \frac12 - \frac{\sqrt{3}}{6} & 0 \\
        \frac12 + \frac{\sqrt{3}}{6} & \frac{\sqrt{3}}{3} & \frac12 - \frac{\sqrt{3}}{6} \\
        \hline
                                     & \frac12 & \frac12
      \end{array}
      \qquad
      \begin{array}{c|cc}
        \frac12 + \frac{\sqrt{3}}{6} & \frac12 + \frac{\sqrt{3}}{6} & 0 \\
        \frac12 - \frac{\sqrt{3}}{6} & -\frac{\sqrt{3}}{3} & \frac12 + \frac{\sqrt{3}}{6} \\
        \hline
                                     & \frac12 & \frac12
      \end{array}
    \end{gather*}
  \end{block}
\end{frame}
\frame {\input {blocks/Lemma-stability-rk.tex}}
\frame {\input {blocks/Example-stability-examples-rk.tex}}

\begin{frame}
  \frametitle{Stability domains of ERK}
  \begin{tabular}{ccc}
    \includegraphics[width=.3\textwidth]{fig/stability-RK2}
    &
    \includegraphics[width=.3\textwidth]{fig/stability-RK4}
    &
    \includegraphics[width=.3\textwidth]{fig/stability-DOPRI5}
    \\
    mod. Euler & RK4 & DOPRI5
  \end{tabular}
\end{frame}

\frame {\input {blocks/Definition-theta-scheme.tex}}
\frame {\input {blocks/Theorem-theta-a-stability.tex}}

\begin{frame}
  \frametitle{Stability domains of the $\theta$-scheme}
  \begin{center}
    \begin{tabular}{cccc}
      \includegraphics[width=.22\textwidth]{fig/stability-CR}
      &
      \includegraphics[width=.22\textwidth]{fig/stability-theta6}
      &
      \includegraphics[width=.22\textwidth]{fig/stability-theta7}
      &
      \includegraphics[width=.22\textwidth]{fig/stability-euler2}
      \\
      $\theta= 0.5$
      & $\theta = 0.6$
      & $\theta = 0.7$
      & $\theta = 1.0$
      \\
      Crank-Nicolson &&& impl. Euler
    \end{tabular}
  \end{center}
\end{frame}

\frame {\input {blocks/Lemma-existence-implicit-1.tex}}
\frame {\input {blocks/Lemma-existence-implicit-2.tex}}
\frame {\input {blocks/Theorem-existence-implicit-3.tex}}
\frame {\input {blocks/Definition-simplifying-conditions.tex}
  \input {blocks/Theorem-order-conditions.tex}}

\subsection{Methods based on quadrature}

\frame {\input {blocks/Definition-gauss-quadrature.tex}}
\frame {\input {blocks/Definition-radau-lobatto-quadrature.tex}}
\frame {\input {blocks/Lemma-gauss-quadrature.tex}}
\frame {\input {blocks/Definition-collocation.tex}}
\frame {\input {blocks/Lemma-collocation.tex}}
\frame {\input {blocks/Lemma-collocation-equivalence.tex}}
\frame {\input {blocks/Theorem-collocation-order.tex}}
\frame {\input {blocks/Theorem-collocation-continuous.tex}}
\frame {\input {blocks/Definition-gauss-collocation.tex}}
\frame {\input {blocks/Example-Gauss-collocation-2-3.tex}}
\frame {\input {blocks/Theorem-gauss-consistency.tex}
  \input {blocks/Theorem-gauss-stability.tex}}
\frame {\input {blocks/Example-Radau-right-2-3.tex}}
\begin{frame}
  \frametitle{Stability regions of Radau-collocation methods}
    \centering
  \includegraphics[width=.3\textwidth]{fig/stability-radau1}
  \hfill
  \includegraphics[width=.3\textwidth]{fig/stability-radau2}
  \hfill
  \includegraphics[width=.3\textwidth]{fig/stability-radau3}
\end{frame}

\frame {\input {blocks/Theorem-DIRK-order-barrier.tex}}

\section{Newton and quasi-Newton methods}

\frame {\input {blocks/Definition-nonlinear.tex}}
\frame {\input {blocks/Definition-iteration-order.tex}}
\frame {\input {blocks/Definition-newton.tex}}
\frame {\input {blocks/Theorem-newton-kantorovich.tex}}
\frame {\input {blocks/Definition-gradient-method.tex}}
\frame {\input {blocks/Theorem-gradient-method.tex}}
\frame {\input {blocks/Definition-newton-line-search.tex}}
\frame {\input {blocks/Definition-newton-step-size.tex}}
\frame {\input {blocks/Definition-descent-methods.tex}}
\frame {\input {blocks/Lemma-downhill.tex}}

\section{Linear Multi-Step Methods}
\frame{\tableofcontents[currentsection,hideothersubsections]}

\begin{frame}
  \begin{block}{Adams-Moulton methods (implicit)}
  \begin{gather*}
    y_k = y_{k-1} + \sum_{r=0}^\lmms f_{k-r} \int_{t_{k-1}}^{t_k}
    L_r(t) \dt,
  \end{gather*}
    \begin{center}
      \includegraphics[width=.9\textwidth]{fig/adams-moulton.tikz}
    \end{center}
    \input{definitions/adams-moulton}
  \end{block}
\end{frame}
\begin{frame}
  \begin{block}{Adams-Bashforth methods (explicit)}
  \begin{gather*}
    y_k = y_{k-1} + \sum_{r=1}^\lmms f_{k-r} \int_{t_{k-1}}^{t_k}
    L_r(t) \dt.
  \end{gather*}
    \begin{center}
      \includegraphics[width=.9\textwidth]{fig/adams-bashforth.tikz}
    \end{center}
    \input{definitions/adams-bashforth}
  \end{block}
\end{frame}
\begin{frame}
  \begin{block}{Backward Differencing Formulas}
  \begin{gather*}
    y'(t_k) = f(t_k, y_k) = \sum_{r=0}^\lmms y_{k-r} L'_{k-r}(t_k).
  \end{gather*}
    \input{definitions/bdf}
  \end{block}
\end{frame}

\frame {\input {blocks/Definition-lmm.tex}}
\frame {\input {blocks/Definition-lmm-errors.tex}}
\frame {\input {blocks/Definition-lmm-consistency.tex}}
\frame {\input {blocks/Lemma-lmm-bramble-hilbert.tex}}
\frame {\input {blocks/Theorem-lmm-consistency.tex}}
\frame {\input {blocks/Definition-difference-equation.tex}}
\frame {\input {blocks/Lemma-lmm:1.tex}}
\frame {\input {blocks/Lemma-difference-equation-solutions.tex}}
\frame {\input {blocks/Theorem-difference-equation-basis.tex}}
\frame {\input {blocks/Corollary-root-test.tex}}
\frame {\input {blocks/Definition-lmm-stability.tex}
  \input {blocks/Theorem-lmm-stability.tex}}
\frame {\input {blocks/Corollary-adams-stability}
  \input {blocks/Theorem-BDF-stability.tex}}
\frame {\input {blocks/Definition-lmm-convergence.tex}}
\frame {\input {blocks/Lemma-lmm-one-step.tex}}
\frame {\input {blocks/Lemma-lmm-one-consistency.tex}}
\frame {\input {blocks/Lemma-lmm-one-stability.tex}}
\frame {\input {blocks/Theorem-lmm-convergence.tex}}
\frame {\input {blocks/Definition-lmm-stability-region.tex}}
\frame {\input {blocks/Definition-lmm-stability-polynomial.tex}
  \input {blocks/Lemma-lmm-stability.tex}}
\frame {\input {blocks/Theorem-dahlquist2.tex}}
\frame {\input {blocks/Definition-aa-stability.tex}}

\frame{
\begin{figure}[tp]
  \centering
  \includegraphics[width=.45\textwidth]{fig/stability-bdf.png}
  \hfill
  \includegraphics[width=.45\textwidth]{fig/stability-bdf-zoom.png}
  \caption{Boundaries of stability regions of BDF1 to BDF6. Unstable
    region right of the origin. Zoom on the right}
  \label{fig:bdf-stability}
\end{figure}
}

\frame {\input {blocks/Definition-predictor-corrector.tex}}

\section{Boundary Value Problems}
\frame{\tableofcontents[currentsection,hideothersubsections]}

\frame {\input {blocks/Definition-bvp.tex}
  \input {blocks/Definition-bvp-linear.tex}}
\frame {\input {blocks/Definition-gateaux-derivative.tex}}
\frame {\input {blocks/Definition-variational-equation.tex}}
\frame {\input {blocks/Lemma-fundamental.tex}}
\frame {\input {blocks/Theorem-derivative.tex}}
\frame {\input {blocks/Theorem-derivative-f.tex}}
\frame {\input {blocks/Definition-local-uniqueness.tex}}
\frame {\input {blocks/Lemma-bvp-derivative.tex}}
\frame {\input {blocks/Theorem-local-uniqueness.tex}}
\frame {\input {blocks/Theorem-bvp-stability-1.tex}}
\frame {\input {blocks/Theorem-bvp-stability-2.tex}}
\frame {\input {blocks/Corollary-bvp-linear.tex}}
\frame {\input {blocks/Definition-single-shooting.tex}}
\frame {\input {blocks/Algorithm-single-shooting.tex}}
\frame {\input {blocks/Definition-multiple-shooting.tex}}
\frame {\input {blocks/Definition-multiple-shooting-newton.tex}}
\frame {\input {blocks/Definition-multiple-shooting-multiple.tex}}
\frame {\input {blocks/Definition-bvp-parameter.tex}}
\frame {\input {blocks/Definition-multiple-shooting-parameter.tex}}

\section{Finite Differences}
\frame{\tableofcontents[currentsection,hideothersubsections]}

\frame {\input {blocks/Definition-bvp-second.tex}}
\frame {\input {blocks/Definition-difference-operators.tex}}
\frame {\input {blocks/Definition-fd-consistency.tex}}
\frame {\input {blocks/Lemma-fd-consistency.tex}}
\frame {\input {blocks/Definition-fd.tex}}
\frame {\input {blocks/Example-fd-example.tex}}
\frame {\input {blocks/Example-fd-matrix.tex}}

\begin{frame}
  \begin{block}{The reduced system}
    \begin{gather*}
      L_h y =
    \begin{pmatrix}
      \lambda_1 & \nu_1\\
      \mu_2 & \lambda_2 & \nu_2 \\
      & \ddots & \ddots & \ddots \\
      &&\mu_{n-1}& \lambda_{n-1}
    \end{pmatrix}
    \begin{pmatrix}
      y_1\\\vdots\\y_{n-1}
    \end{pmatrix}
    =
    \begin{pmatrix}
      f_1 \\\vdots\\f_{n-1}
    \end{pmatrix} = f_h.
  \end{gather*}
  \end{block}
\end{frame}

\frame {\input {blocks/Definition-m-matrix.tex}}
\frame {\input {blocks/Lemma-m-matrix-fd1.tex}}
\frame {\input {blocks/Lemma-m-matrix-inverse.tex}}
\frame {\input {blocks/Theorem-fd-stability.tex}}
\frame {\input {blocks/Example-upwind.tex}}
\frame {\input {blocks/Theorem-fd-convergence.tex}}
\frame {\input {blocks/Example-square-domain.tex}}
\frame {\input {blocks/Definition-laplace-equation.tex}}
\frame {\input {blocks/Example-dirichlet.tex}}
\frame {\input {blocks/Theorem-mean-value.tex}}
\frame {\input {blocks/Theorem-maximum.tex}}
\frame {\input {blocks/Definition-cartesian-grid.tex}}
\frame {\input {blocks/Definition-lexicographic.tex}}
\frame {\input {blocks/Definition-5-point-stencil.tex}}
\frame {\input {blocks/Lemma-fd-matrix.tex}}
\frame {\input {blocks/Theorem-5-point-m-matrix.tex}}
\frame {\input {blocks/Lemma-fd2-consistency.tex}}
\frame {\input {blocks/Theorem-fd2-convergence.tex}}
\frame {\input {blocks/Theorem-fd2-maximum.tex}}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End: