\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[linesnumbered, lined, boxed, commentsnumbered, ruled, vlined]{algorithm2e}
\usepackage{amsmath} % http://ctan.org/pkg/amsmath
\newcommand{\thickhat}[1]{\mathbf{\hat{\text{$#1$}}}}
% \usepackage{accents}
% \newcommand{\ubar}[3]{\underaccent{\bar}{#1}}

\title{Variant I Pseudocode: Adam Can play any strategy }
\author{Karan Muvvala}
\date{April 2020}

\begin{document}

\maketitle

\section{Introduction}

Algorithm 1 computes the $Reg$ of a Game $G$ for a given value of $b$ and a payoff function $Val$. The construction of $\hat{G}$ depends on computing $W'$ which is primarily computing $cVal$ for each edge $\{e = (u,v) \in G$ \vert $u \in V_\exists \}$ such that Eve plays an alternate strategy $\sigma'$ and $\tau$ for Adam that maximizes the payoff for any of the alternative strategy $\sigma'$ except the original strategy $\sigma$. \\

We then use values($b \in W'$) to construct $G^b$ with the new weight function $\hat{w} = w(e) - b$ to construct $\hat{G}$. On this game it is sufficient to play a memoryless/positional strategies for either players to ensure an antagonistic value of at least(resp. at most) $aVal(\hat{G})$. From \textbf{Section 3 Claim 1.} we conclude that $Reg(G) = -aVal(\hat{G})$. Algorithm 4 initially computes the memoryless strategy for Eve and Adam where Eve looks for the edge with the highest weight and Adam plays adversarially looking for the edge with the minimum weight, to compute the $aVal$ of the game $\hat{G}$. We then can compute $\sigma$ and $\tau$ for the game $G$ as there is an one-to-one correspondence with strategies $\hat{\sigma}$ and $\hat{\tau}$ of the game $\hat{G}$.

\section{Algorithms}

    \begin{algorithm}[H]
        \SetAlgoLined
        \SetKwData{Left}{Left}\SetKwData{This}{This}\SetKwData{Up}{Up}
        \SetKwFunction{ComputeGmin}{ComputeGmin}\SetKwFunction{ComputeGmax}{ComputeGmax}
        \SetKwFunction{ConstructWprime}{ConstructWprime}
        \SetKwFunction{ConstructGhat}{ConstructGhat}
        \SetKwFunction{ComputeAVal}{ComputeAVal}
        \SetKwInOut{Input}{Input}\SetKwInOut{Output}{Output}
        % \KwData{$G=(X,U)$ such that $G^{tc}$ is an order.}
        % \KwResult{{$G’=(X,V)$ with $V\subseteq U$ such that $G’^{tc}$ is an interval order.}
        \Input{A weighted game $G = (V, V_\exists, v_I, E, w)$, $b$,  and $Val$ the payoff function \in $\{Sup, Inf, LimInf, LimSup, \overline{MP},\underline{MP}\}$}
        \Output{$Reg, \sigma, \tau$}

        % begin the algorithm
        \Begin{
            %\uIf{$Val = Inf$} {
            %  $G$ = \ComputeGmin{$G$}
            %}

            % \uElseIf{$Val = Sup $}{
            %  $G$ = \ComputeGmax{$G$}%\tcc*[r]{else if}
            % }

            $W^'$ \longleftarrow \ConstructWprime{$G$}

            $\^{G}$ \longleftarrow \ConstructGhat{$G, W'$}

            $Reg, \sigma, \tau$ \longleftarrow \ComputeAVal{$\^{G}, b, Val$}

        }
        % title of the algorithm
    \caption{Compute Regret for Eve given Adam can play any strategy}
    \end{algorithm}

    % \begin{algorithm}
    % \SetAlgoLined
    %     \SetKwData{Left}{Left}\SetKwData{This}{This}\SetKwData{Up}{Up}
    %     \SetKwInOut{Input}{Input}\SetKwInOut{Output}{Output}

    %     % print i/p and o/p
    %     \Input{Weighted Game Arena $G = (V, V_\exists, v_I, E, w)$}
    %     \Output{Weighted Game Arena $G_{min}$ / $G_{max}$}

    %     \Begin{
    %         Compute $V^'$ \longleftarrow{} $V$ \times $\{w(e) \vert e \in E\}$

    %         Assign Vertex to Eve: $V_\exists^'$  \longleftarrow{} $\{(v,n) \in  $V^'$ \vert v \in V_\exists \}$

    %         Set $v_I^'$ \longleftarrow{} $(v_I, W)$

    %         Construct edges $E'$ \exists $((u, n), (v, m)) iff (u,v) \in E$ and $m$ = min \{$n,$ $w(u,v)$\} %\\\ use min\{\} if Val is Inf

    %     Assign weights $w^{'}((u,n), (v,m))$ \longleftarrow{} $m$

    %     }

    % \caption{Compute Gmin or Gmax graph}
    % \end{algorithm}

    \begin{algorithm}[H]
        \SetAlgoLined
        \SetKwData{Left}{Left}\SetKwData{This}{This}\SetKwData{Up}{Up}
        \SetKwInOut{Input}{Input}\SetKwInOut{Output}{Output}

        % print i/p and o/p
        \Input{Weighted Game Arena $G = (V, V_\exists, v_I, E, w)$}
        \Output{Weight function $W^'$}

        \Begin{
            \For{$e = (u, v)$ \in $E$}{
                \If{$u$ \in $V \setminus V_\exists$}{
                    $W^{'}(e)$ \longleftarrow  -\infty \\
                }

                \Else{
                    $W^{'}(e)$ \longleftarrow max\{cVal$^{v^'}$ \vert (u,$v^'$) \in\; $E$ \;\setminus \{e\}\} \\
                }
            }
        } \\

    \caption{ ConstructWprime: Construct $W^'$ that represents the best value obtainable for a strategy of Eve that differs at the given edge}
    \end{algorithm}

    \begin{algorithm}[H]
    \SetKwData{Left}{Left}\SetKwData{This}{This}\SetKwData{Up}{Up}
        \SetKwInOut{Input}{Input}\SetKwInOut{Output}{Output}

        % print i/p and o/p
        \Input{Weighted Game Arena $G = (V, V_\exists, v_I, E, w)$ and weight function $W'$}
        \Output{$\^{G}$}

        \Being{

            Initialize $v_0$ \in $\^{V}$ \setminus $\^{V}_\exists$

            Initialize $v_1, v_\bot$ \in $\^{V}_\exists $

            \For{$b \in Range(W^{'}) \setminus -\infty $}{
                construct $G^b = (V^b, V_\exists^b, v_I^b, E^b ,\^{w})$ \vert{}  $G^b$ is a copy of $G$ with edges $e$ restricted by $W^{'}(e) \leq b$
                \\
                Initialize weight function $\^{w(e)}$ \longleftarrow w(e) - b \; $\forall\; e \in E $ \\
            }

            \For{e in $\^{G}$}{
                w($v_0$, $v_0$) = 0

                w($v_\bot$, $v_\bot$) = $-2w_{max} - 1$

                w($v_0$ , $v_1$) = 0

                \For{$v_I^b$ in $G^b$}{
                    w($v_1$, $v_I^b$)= 0 \\
                }

                \For{$v^b$ \in $V^b$ that don't have a successor}{

                    w($v^b$, $v_\bot$) = 0 \\
                }
            }
        }

    \caption{ConstructGhat: construct $\^{G}$}
    \end{algorithm}

    % $\^{\sigma(v_1)}$ = $v_I^b$ and avoids $v_\bot$
    % $\^{\tau}(v_o)$ = $v_I$

    \begin{algorithm}
        \SetAlgoLined
        \SetKwData{Left}{Left}\SetKwData{This}{This}\SetKwData{Up}{Up}
        \SetKwInOut{Input}{Input}\SetKwInOut{Output}{Output}
        % \KwData{$G=(X,U)$ such that $G^{tc}$ is an order.}
    % \KwResult{{$G’=(X,V)$ with $V\subseteq U$ such that $G’^{tc}$ is an interval order.}
        \Input{An antagonistic game $\^{G}$, $b$, and $Val$ the payoff function \in $\{Sup, Inf, LimInf, LimSup, \overline{MP},\underline{MP}\}$}
        \Output{$Reg, \sigma, \tau $}

        \Beign{

            $\hat{\sigma}(v_1)$ = $v_I^b$ and avoids $v_{\bot}$ \\
            $\hat{\tau}(v_o)$ = $v_I$ \\

            \For{b \in $Range(W^{'}) \setminus -\infty $}{

                \For{$u^b$ \in $V^b$}{
                    \If{$u^b$ \in $V^b$ \setminus ${V}_\exists^b$ }{

                        $\hat{\tau}(u^b)$ = \{$v^b$ \vert \min(\hat{w}($u^b, v^b$)) and ($u^b, v^b$) \in $E^b$ \} \\
                    }

                    \ElseIf{$u^b$ \in ${V}_\exists^b$}{

                    $\hat{\sigma}(u^b)$ = \{$v^b$ \vert \max(\hat{w}($u^b, v^b$)) and ($u^b, v^b$) \in $E^b$ \} \\
                    }
                }
            }

            $Reg$ = $Val_{\hat{G}}^v$($\hat{\sigma}, \hat{\tau}$) \\
            $\sigma$ = $\hat{\sigma}$ \setminus v_1 \\
            $\tau$ = $\hat{\tau}$ \setminus v_o \\
        }
    \caption{ComputeAVal: Compute the antagonistic value of $\^{G}$}
    \end{algorithm}

\end{document}


\chapter{Preparation}\label{preparation}

\ifpdf
    \graphicspath{{Chapter2/Figs/Raster/}{Chapter2/Figs/PDF/}{Chapter2/Figs/}}
\else
    \graphicspath{{Chapter2/Figs/Vector/}{Chapter2/Figs/}}
\fi

\section{Balls-into-Bins}

Balls-into-bins models have been studied since the 20th century in probability theory under several different names, such as ``urn processes'' or ``occupancy problems''~\cite{kolchin1978coined}. A few years later its applicability to real world problems, such as load balancing, has been highlighted, and further research led to the analyses of even more realistic and efficient models. Apart from practical applications, the results derived for balls-into-bins models turned out to be useful in the design and analysis of several other algorithms~\cite{edmonds2006cakecutting}. It is still an actively researched field gaining much attention, and in particular, the main model I will study has been first analysed theoretically in 2019~\cite{dwivedi2019firstthinning} (but known since 1986~\cite{derek1986twothinningfirstattempt}). Now I present a definition of the balls-into-bins abstraction.

\begin{definition}[balls-into-bins] \label{definition: balls-into-bins}
In the balls-into-bins problem, there are $n$ bins (servers), and there are $m$ balls (jobs) arriving sequentially. Whenever a ball arrives, it is placed in one of the bins according to a (usually randomised) protocol. The protocol is assessed according to its expected final maximum load.
\end{definition}

\subsection{Assumptions} \label{assumptions}

Load balancing in the real-world is a complex task, whose optimal solution depends the exact characteristics of the system. Building a theoretical model that captures all the characteristics is infeasible. Balls-into-bins are no exception, often making the following assumptions:

\begin{itemize}
    \item
    The balls never leave the bins. While in most real-world scenarios jobs do end, this model can still provide a good local approximation, or represent the overall work assigned over a time period. Also, most of the theoretical work in the literature analyses the \textit{normalised} maximum load, i.e.\ the difference between the maximum load and the average load over all the servers, and this subtraction has a similar type of effect as finishing jobs over time. For other applications, such as distributed hash tables~\cite{wieder2017ballsintobinslandscape}, persistent balls are realistic.
    \item
    The number of balls $m$ is known in advance. As shown in~\cite{feldheim2021longtermthinning}, there are strategies achieving an asymptotically smaller normalised maximum load if $m$ is known. In many applications $m$ can be estimated from previous runs.
    \item
    The balls and the bins are homogeneous. In practice, servers might have different processing speed or might crash, and jobs might be of different types. Assuming unit-sized jobs is reasonable in several cases; for example, even though Google searches are different, the built-in timeout makes them approximately unit-sized.
    \item
    Different objective functions may be of interest. As the balls-into-bins process is stochastic, possible other metrics include: expected $\ell_2$-norm of the final load, variance of the final maximum load or expected maximum normalised load throughout the whole run.
\end{itemize}

There are other variants handling deletion~\cite{azar1999twochoice}, unknown $m$~\cite{feldheim2021longtermthinning}, heterogeneous balls and bins~\cite{berenbrink2008weighted} and different objective functions~\cite{feldheim2021longtermthinning}.

\subsection{Protocols} \label{protocols}

I now define the randomised protocols that I will study in later chapters.

\paragraph{\OneChoice}

This is the simplest randomised protocol. The arriving ball is allocated uniformly at random into one of the bins.

\paragraph{\TwoChoice}
For each ball, two bins are sampled uniformly at random, and it is allocated in the lesser loaded of the two, breaking ties randomly. Implementing \TwoChoice in practice is challenging due to concurrency and communication delays: the servers need to report their loads by interrupting their currently running process; the client has to wait for the response by both servers and then allocate the job to one of them.

\paragraph{\TwoThinning}

For each ball, a primary bin is sampled uniformly at random, and if the decision strategy accepts that bin, the ball is allocated there, otherwise it is allocated into a secondary bin sampled uniformly at random.

\paragraph{\KThinning}

\KThinning generalises \TwoThinning. For each ball, up to $k$ bin samples are taken sequentially uniformly at random until one of them is accepted by the decision strategy. In theory, \KThinning provides an asymptotic improvement over \TwoThinning~\cite{feldheim2020dthinning, los2021quantilethreshold}.

\paragraph{\GraphicalTwoChoice}
This protocol generalises \TwoChoice, where only a subset of the pairs of bins can be sampled. Treating the bins as nodes and the pairs that can be sampled as edges of a graph, \GraphicalTwoChoice samples an edge uniformly at random for each ball (see Figure~\ref{graphical-two-choice-intro}). It is often assumed that the graph is regular (i.e.\ all nodes have the same degree), otherwise the symmetry of the final maximum load objective function may not be reasonable; take the star graph as an example.

\begin{figure}[hbt!]
    \centering
    \includegraphics[scale=1.5]{Chapter2/Figs/graphical_two_choice_intro.pdf}
    \caption{One step of the \GraphicalTwoChoice protocol. The decision strategy had to decide between the endpoints of the orange edge, and chose the green one.}
    \label{graphical-two-choice-intro}
\end{figure}

The real-world motivation for this protocol stems from the geographical locality of the servers~\cite{krishnaram2006graphicaltwochoiceoriginal}. One possible implementation of \GraphicalTwoChoice for load balancing is that the client chooses a server $A$ uniformly at random, server $A$ queries the current load of a neighbouring server $B$ chosen uniformly at random, and then $A$ decides whether itself or $B$ should complete the job. In this case the edges correspond to nearby servers, but different underlying graphs are also possible~\cite{peres2015oneplusbeta}, and there are other applications as well, including the efficient storage of hash tables~\cite{krishnaram2006graphicaltwochoiceoriginal}. For this dissertation, I will make the same simplifying assumptions for \GraphicalTwoChoice as discussed for \TwoThinning in Section~\ref{my-approach}.

Note that remarkably, the \Greedy strategy, which allocates the ball into the lesser loaded endpoint of the edge has been shown not to be optimal in general~\cite{bansal2021twochoicegraphical}, therefore this protocol requires a decision strategy, unlike \TwoChoice. I present the first known concrete counterexample for the suboptimality of Greedy in Lemma~\ref{lemma: greedy-suboptimal}.

Analogously, \textsc{Graphical Two-Thinning} is a reasonable protocol too, but I chose to focus on \GraphicalTwoChoice, since there is more literature available on that, so a more thorough comparative evaluation is possible.\\

\subsection{Definitions and Notation} \label{notation}

For the lemmas and conjectures presented in later chapters, it is useful to define and extend some of the above notions formally as well. Let's denote the integers between $0$ and $n-1$ by $[n]$, indicating the indices of the bins.

\begin{definition} [\TwoThinning decision strategy]
A \TwoThinning decision strategy is a function $f$, such that for any load vector $v$ and bin $i\in[n]$, $f(v, i)\in\{\mathrm{accept},\ \mathrm{reject}\}$.
\end{definition}

\begin{definition} [\KThinning decision strategy]
A \KThinning decision strategy is a function $f$, such that for any load vector $v$, bin $i\in[n]$ and $0\leq c<k-1$ indicating the number of bins already rejected for the current ball at $v$, $f(v, i, c)\in\{\mathrm{accept},\ \mathrm{reject}\}$.
\end{definition}

\begin{definition} [slicing strategy]
We call a \TwoThinning (or in general \KThinning) decision strategy \textit{slicing}, if it accepts bin $i$ at load vector $v$ if and only if its load $v[i]$ is less than or equal to a threshold $a$, that can depend on $v$, and for \KThinning also on the number of bins already rejected for the current ball.
\end{definition}

\begin{definition} [threshold function]
Note that a \KThinning \textit{slicing} strategy $f$ can be alternatively defined by a corresponding \textit{threshold function} denoted by $h^f$, such that $h^f(v,c)$ is the threshold used at load $v$ after $c$ rejected bins.
\end{definition}

\begin{definition} [monotone strategy]
We call a \KThinning slicing strategy $f$ \textit{monotone}, if $h^f(v,c)\leq h^f(v,c+1)$ for each $v$ and $0\leq c<k-2$. Intuitively, a monotone strategy never becomes more selective after rejecting a ball.
\end{definition}

\begin{definition} [expected final maximum load]
Let us denote the expected final maximum load of a (decision) strategy $f$ for a protocol $h$ by $E^f_h$, or $E^f$ when $h$ is implicit from the context.
\end{definition}

\subsection{Notes on Related Work and Background Reading}

As part of the preparation, I have read several papers about the theory of balls-into-bins. I present my main findings in Appendix~\ref{related-work}.

\section{Reinforcement Learning} \label{RLintro}

In the Part IB Artificial Intelligence course we have been introduced to supervised learning. While RL is also about optimising an objective function, it differs in several aspects (e.g.\ no labelled data, interactive environment). A very readable introduction to RL can be found in~\cite{sutton2018RLbook}, expanding on my explanation below.

In RL, there is an agent trying to learn an optimal policy by interacting with an environment. It starts in a start state, carries out an action, receives a reward from the environment, observes a new state, and repeats this sequence until it reaches an end state. This process is executed several times (one such run is called an ``episode''), until a sufficiently good policy is learnt. There are several characterisations of RL, but there is a common underlying mathematical model, the so-called Markov Decision Process (MDP) that I will introduce next.

\subsection{Finite Markov Decision Process}

The main component of a Finite MDP is the environment consisting of
\begin{itemize}[itemsep=0pt]
    \item $S$: state space,
    \item $s_0$: start state,
    \item $S_f$: set of final states,
    \item $A(s)$: action space available at state $s$,
    \item $R(s, a, s')$: possible rewards after executing action $a$ in state $s$ and observing next state $s'$, and
    \item $P(s', r \mid s, a)$: the probability of receiving reward $r$ and transitioning to state $s'$ after executing action $a$ in state $s$.
\end{itemize}

The Markov property has to hold: using timesteps as indices during an execution

\begin{equation} \label{eq:MarkovProperty}
P(s_{t+1},r_{t} \mid s_{t}, a_{t}, s_{t-1}, a_{t-1}, s_{t-2}, a_{t-2}, ..., s_{0}, a_{0}) = P(s_{t+1},r_{t} \mid s_{t}, a_{t})\text{ .}
\end{equation}

The other component of the MDP is an agent that has to learn a \textit{policy}, a function $\pi(a\mid s)$ that assigns probabilities to each of the executable actions in any state. The goal of the agent is to maximise the expected (discounted) cumulative reward collected until the end state:

\begin{equation}\label{eq:cumReward}
\mathbb{E}_{\pi}[r_{0} + \gamma r_{1} + \gamma^2 r_{2} + \gamma^2 r_{3} + \ldots]\text{ ,}
\end{equation}

where $\gamma \in [0, 1]$ is the discount factor, and the sum goes until an end state is reached. Discounting is needed mainly for unbounded MDPs -- where one run can last arbitrarily long -- to avoid divergent rewards. As we will see in Chapter~\ref{implementation}, all our MDPs are bounded -- often even have a fixed number of steps -- so I will set $\gamma=1$ and I will not discuss the more general case from now on.

During training, the agent leans the policy by executing the process several times, i.e.\ starting from the start state, and then interacting with the environment using an carefully chosen policy, until an end state is reached.\\

An example MDP is a simplified version of what I will use for \TwoThinning:

\begin{itemize}[itemsep=0pt]
    \item
    $S$: load vectors $\{v\in \mathbb{N}^n\mid\mathrm{sum}(v)\triangleq \sum_{i=0}^{n-1}v[i]\leq m\}$,
    \item
    $s_0$: the load configuration no balls,
    \item
    $S_f$: load configurations with $m$ balls,
    \item
    $A(s)$: integer thresholds $0\leq a\leq m$ (meaning that bins with load at most $a$ are the ones accepted),
    \item
    $R(s, a, s')$: the reward is $0$ for $\mathrm{sum}(s')<m$ and $-\mathrm{maxload}(s')\triangleq -\max_{i=0}^{n-1} s[i]$ for $\mathrm{sum}(s')=m$,
    \item
    $P(s', r \mid s, a)$: the probabilities are derived according to the rules of \TwoThinning.
\end{itemize}

\subsection{Fundamentals}

Now I present some standard notions that will be useful for the RL and DP algorithms.

The \textit{state-value function} is the expected cumulative reward of a policy starting from state $s$:

\begin{equation}\label{eq:statevalueFunction}
V_{\pi}(s)=\mathbb{E}_\pi[G_t \mid s_t = s] \text{ ,}
\end{equation}

where the random variable $G_t$ is defined as $r_{t} +  r_{t+1} + r_{t+2} + \ldots$.

For $s=s_0$, we get the expected total cumulative reward of a policy $\pi$, exactly what we aim to maximise. Hence, the optimal policy $\pi^*$ is defined by $\pi^* = \argmax_{\pi} V_{\pi}(s_0)$.

Similarly, the \textit{action-value function} is the expected cumulative reward of a policy starting from state $s$, choosing action $a$:

\begin{equation}\label{eq:actionvalueFunction}
Q_{\pi}(s, a)=\mathbb{E}_\pi[G_t \mid s_t = s, a_t = a] \text{ .}
\end{equation}

The basis of most of the learning algorithms are the Bellman equations~\cite{bellman1957bellmanequation} that characterize the optimal policy recursively:

\begin{equation}\label{eq:bellmanState}
V_{\pi^*}(s) = \max_a \mathbb{E} [r_t + V_{\pi^*}(s_{t+1}) \mid s_t=s, a_t=a]
\end{equation}

\begin{equation} \label{eq:bellmanAction}
Q_{\pi^*}(s,a) = \mathbb{E} [r_t + \max_{a'} Q_{\pi^*}(s_{t+1},a') \mid s_t=s, a_t=a ]
\end{equation}

Even with these equations, finding the optimal policy is challenging. The two main difficulties are 1) the possible cycles while moving around the state space and 2) the exponentially large size of the state space. As we will see in Chapter~\ref{implementation}, in our cases the state transition graph is acyclic, so the former is not a problem. However, in general, to deal with the latter, the optimal policy has to be approximated.

\subsection{Algorithms}

In all of our protocols, the state space is large (the number of ways $m$ balls can be allocated into $n$ bins is exponentially large), so the agent can focus only on a subset of the states and actions during training. It has to balance between exploring new actions (possibly leading to new states), and gaining confidence in its top actions. This is more generally known as the exploration-exploitation trade-off~\cite{kaelbling1996explorationexploitation}.

The most common approach for dealing with the exploration-exploitation trade-off is the $\epsilon$-greedy technique. During training, the currently best action is chosen with probability $1-\epsilon$ (exploitation) and otherwise the action is chosen uniformly at random (exploration). After training, we set $\epsilon=0$.

\subsubsection*{Q-Learning}

Q-Learning~\cite{watkins1989qlearning} uses the action-value function based Bellman equation~\eqref{eq:bellmanAction}. The algorithm maintains a so-called $Q$-table, such that $Q(s,a)$ stores the current estimates of the action-value function $Q_{\pi^*}(s,a)$ of the optimal policy. The agent executes the process several times using the $\epsilon$-greedy technique based on the current estimates in the $Q$-table. After every step, it updates the $Q$-table using a linear interpolation with smoothing factor $\alpha \in (0,1]$ according to the following update rule:

\begin{equation} \label{eq:q-learningUpdate}
Q(s_t,a_t) \leftarrow Q(s_t,a_t) + \alpha\left( r_t + \max_{a'} Q(s_{t+1}, a')) - Q(s_t,a_t)\right) \text{ .}
\end{equation}

\subsubsection*{\DQL} \label{deepq-learning}

Q-Learning, and its iterative approximation method is a good choice for moderately sized state spaces, where more direct, mainly DP methods are infeasible. However, for exponentially large state spaces, as in our cases, it is infeasible to store the $Q$-table, and also most of its entries would never be updated during training. In \DQL~\cite{mnih2013DQN}, the $Q$-table is approximated using a neural network (NN) $Q_w$ with a tractable number of weights $w$.

Since with function approximations it is not possible to update $Q_w(s, a)$ in isolation, the update rule also has to be adjusted. A natural choice would be to use the gradient of the squared error between the old and the new estimate:

$$\frac{\partial (( r_t + \max_{a'} Q_w(s_{t+1}, a')) - Q_w(s_t,a_t))^2}{\partial w} \text{ .}$$

In practice, due to stability reasons~\cite{barnard1993semigradient}, a semi-gradient is used instead, which essentially treats $Q_{\mathbf{w_t}}(s_{t+1}, a')$ independent of $w$ for the purpose of the differentiation. This yields the final update rule

\begin{equation} \label{eq:deep-q-learning-update-with-semi-gradient}
\mathbf{w} \leftarrow \mathbf{w} + \alpha\left( r_t+ \max_{a'} Q_{\mathbf{w}}(s_{t+1}, a') - Q_{\mathbf{w}}(s_t,a_t)\right)\nabla Q_{\mathbf{w}}(s_{t}, a_t) \text{ .}
\end{equation}

Following standard terminology, I will call the NN used by \DQL as a Deep Q-Network (DQN). I will call the learnt policy as the \DQN strategy.

\subsubsection*{Alternatives}

There are several other algorithms and variants, some of which I implemented and did not outperform \DQL -- see Appendix~\ref{alternativeRL} for further discussion.

\subsection{Recurrent Neural Networks} \label{RNN}

I experimented with several NN architectures for $Q_w$, and here I briefly introduce Recurrent Neural Networks (RNN)~\cite{hopfield1982RNNoriginal}, as they were not covered in the IB Artificial Intelligence or other courses.

RNNs process sequential information by using a hidden state, and updating it by a common weight matrix on every new input (see Figure~\ref{RNN-image}). I used RNNs to obtain an embedding of load vectors, and not to produce an output sequence, as shown in Figure~\ref{RNN-image} for the more general usecase.

\begin{figure}[ht]
    \centering
    \includegraphics[scale=0.2]{Chapter2/Figs/RNN.png}
    \caption{Two ways of thinking about RNNs~\cite{RNN}.}
     \label{RNN-image}
\end{figure}

\section{Starting Point}

The starting point has not changed since writing the project proposal found in Appendix~\ref{proposal}, no unexpected changes have happened. Before the project, I had no experience in RL and balls-into-bins. I had a solid knowledge of Python and its machine learning libraries, but I had never used them for RL. The project does not rely on any existing codebase, apart from common open-source Python libraries, as discussed in Section~\ref{software-and-hardware}.

\section{Requirements Analysis and Risk Assessment}

In this section I present the requirements of the projects, the corresponding estimated risks before the start, and some reflections after completing the requirements.

\begin{itemize}

    \item \textbf{Background Reading}: \textcolor{YellowOrange}{medium risk}, \textcolor{Red}{high difficulty}

    I had no previous knowledge about RL or balls-into-bins beyond related foundational courses from the Tripos, so I have done extensive background reading in both, gaining a deep understanding of the balls-into-bins literature, and exploring the relevant parts of RL.

    The biggest challenge was understanding the proofs in balls-into-bins papers, which I considered an extension. I eventually succeeded and gained valuable insights.

    \item \textbf{Implementing Deep Reinforcement Learning}: \textcolor{red}{high risk}, \textcolor{red}{high difficulty}

    The main risk involved in this main component was the uncertainty whether RL, being a novel approach to balls-into-bins, will actually perform well in learning decision strategies.

    The main difficulty was optimising \DQL for better results (see Section~\ref{improvementideas}).

    \item \textbf{Implementing Other Strategies}: \textcolor{green}{low risk}, \textcolor{YellowOrange}{medium difficulty}
    Having done much competitive programming, I was confident in my algorithmic knowledge, so I could efficiently develop and implement algorithms, such as dynamic programming.

    \item \textbf{Evaluation}: \textcolor{YellowOrange}{medium risk}, \textcolor{YellowOrange}{medium difficulty}

    The medium risk stemmed from the novelty of my non-asymptotic approach, and the time it took to compare the strategies with acceptable confidence intervals.

    The hardest part about evaluation was finding the most insightful analyses for explaining the behaviour of strategies.

\end{itemize}

\section{Software Engineering Methodology}

\subsection{Software and Hardware} \label{software-and-hardware}

For this project I used Python, due to its flexibility for quick experimentation, its extensive library support with strong community (e.g.\ numpy~\cite{harris2020numpy} for vectorised computation, Pytorch~\cite{paszke2019pytorch} for deep learning), and its object oriented features. As there is no prior work on RL for the discussed balls-into-bins protocols, I implemented the RL algorithms from scratch using PyTorch for the NNs. Even though there are some attempts at general purpose RL libraries, such as Stable Baselines~\cite{hill2018stablebaselines}, they are very hard to customise at the moment. I used the PyCharm IDE for its powerful debugger and its ability to seamlessly parallelise tasks.

I used my laptop for all experiments and its details are: \texttt{Intel(R) Core(TM) i7-8565U CPU @ 1.80GHz, 1992 Mhz, 4 Cores, 8 Logical Processors, NVIDIA GeForce MX150 GPU}. Gaining the expected speedup from GPUs for RL is not an easy problem, due to the difficulty of batching for a MDP~\cite{stooke2018gpudeepRL} (see Section~\ref{evaluationnotes} for my approach).

\subsection{Project Management}

I used Git for version control and Google Drive as a secondary backup, taking monthly copies. I used the work plan in my project proposal as the target schedule, interleaving evaluation and implementation work to gain better insights about what to focus on. Due to the large number of components in the RL algorithms (see Sections~\ref{dqn-implmentation-two-thinning} and~\ref{improvementideas}), I applied an iterative approach to the implementation, making sure each new component added has the desired effect. We had biweekly meetings with my supervisors, where we came up with several ideas and analysed my results from the perspective of the closely related research they are doing. I maintained a logbook that later served as the baseline for my dissertation.


% Document Head
\documentclass[11pt, oneside]{book}
\usepackage{geometry}
\geometry{letterpaper}
\usepackage[parfill]{parskip}
\usepackage{graphicx}

% Essential Packages
\usepackage{ragged2e}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{mathrsfs}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage[hyperref]{ntheorem}

% Theorem Style Customization
\setlength\theorempreskipamount{2ex}
\setlength\theorempostskipamount{3ex}

% hyperref Package Settings
\usepackage{hyperref}
\hypersetup{
	colorlinks = true,
	linkcolor = magenta
}

% ntheorem Declarations
\theoremstyle{break}
\newtheorem{thm}{Theorem}[section]
\newtheorem*{proof}{Proof}
\newtheorem{crly}{Corollary}[thm]
\newtheorem{lemma}[thm]{Lemma}
\newtheorem{propo}{Proposition}[section]
\newtheorem*{remark}{Remark}
\newtheorem*{note}{Note}
\newtheorem{defn}{Definition}[section]
\newtheorem{eg}{Example}[section]

% ntheorem listtheorem style
\makeatletter
\def\thm@@thmline@name#1#2#3#4{%
        \@dottedtocline{-2}{0em}{2.3em}%
                   {\makebox[\widesttheorem][l]{#1 \protect\numberline{#2}}#3}%
                   {#4}}
\@ifpackageloaded{hyperref}{
\def\thm@@thmline@name#1#2#3#4#5{%
    \ifx\\#5\\%
        \@dottedtocline{-2}{0em}{2.3em}%
            {\makebox[\widesttheorem][l]{#1 \protect\numberline{#2}}#3}%
            {#4}
    \else
        \ifHy@linktocpage\relax\relax
            \@dottedtocline{-2}{0em}{2.3em}%
                {\makebox[\widesttheorem][l]{#1 \protect\numberline{#2}}#3}%
                {\hyper@linkstart{link}{#5}{#4}\hyper@linkend}%
        \else
            \@dottedtocline{-2}{0em}{2.3em}%
                {\hyper@linkstart{link}{#5}%
                  {\makebox[\widesttheorem][l]{#1 \protect\numberline{#2}}#3}\hyper@linkend}%
                    {#4}%
        \fi
    \fi}
}
\makeatother
\newlength\widesttheorem
\AtBeginDocument{
  \settowidth{\widesttheorem}{Proposition A.1.1.1\quad}
}

\theoremlisttype{allname}

% Shortcuts
\newcommand{\bb}[1]{\mathbb{#1}}			% using bb instead of mathbb
\newcommand{\floor}[1]{\lfloor #1 \rfloor}	% simplifying the writing of a floor function
\newcommand{\ceiling}[1]{\lceil #1 \rceil}	% simplifying the writing of a ceiling function
\newcommand{\dotp}{\, \cdotp}				% dot product to distinguish from \cdot

% Main Body
\title{Final Exam Content Revision}
\author{Johnson Ng}

\usepackage{hyperref}

\begin{document}
\maketitle

\tableofcontents

\chapter*{List of Theorems}
\theoremlisttype{allname}
\listtheorems{lemma,thm,crly,propo}

\chapter{Important Theorems}
\begin{thm}[Monotone Convergence Theorem]

\end{thm}

\begin{proof}

\end{proof}

\begin{thm}[Nested Interval Theorem]
	For each $k \in \bb{N}$, if each $I_k$ is closed, bounded, and nonempty. If $I_0 \supseteq I_1 \supseteq I_2 \supseteq \hdots$, then $\bigcap I_k \neq \emptyset$. Moreover, if $|I_k| \to 0$ as $k \to \infty$, $\bigcap I_k$ is a single point.
\end{thm}

\begin{proof}

\end{proof}

\begin{thm}[Intermediate Value Theorem]
	Let $a, b \in \bb{R}$ with $a < b$, let $f: [a, b] \to \bb{R}$ be continous.
	\begin{equation*}
		\forall y \in \bb{R} \enspace \min\{f(a), f(b)\} \leq y \leq \max\{f(a), f(b)\} \implies \exists x \in [a, b] \; f(x) = y
	\end{equation*}
\end{thm}

\begin{proof}

\end{proof}

\begin{thm}[Closed Bounded Intervals and Uniform Convergence]
	Let $a, b \in \bb{R}$ with $a < b$. If $f: [a, b] \to \bb{R}$ is continous, then f is uniformly continuous.
\end{thm}

\begin{thm}[Equivalent Definitions of Integrability]
	Let Let $a, b \in \bb{R}$ and $f: [a, b] \to \bb{R}$. TFAE:
	\begin{enumerate}
		\item $f$ is integrable on $[a, b]$.
		\item $\forall \epsilon > 0 \exists \; P \text{ that is a partition} \; U(f, P) - L(f, P) < \epsilon$
		\item $U(f) = L(f)$
	\end{enumerate}
\end{thm}

\begin{proof}

\end{proof}

\begin{thm}[Continuous Functions are Integrable]
	Let $a, b \in \bb{R}$. Every continuous function $f: [a, b] \to \bb{R}$ is integrable.
\end{thm}

\begin{proof}

\end{proof}

\begin{thm}[Cauchy Criterion for Uniform Convergence - Sequence of Functions]
	Let $I \subseteq \bb{R}, \; \forall n \in \bb{Z}^+ \; f_n: I \to \bb{R}$.
	\begin{gather*}
		\exists g: I \to \bb{R} \; f_n \to g \text{ uniformly on } I \\
		\iff \\
		\forall \epsilon > 0 \; \exists N \in \bb{Z}^+ \; \forall x \in I \; \forall k, l \in \bb{Z}^+ \; (k, l \geq N \implies |f_k(x) - f_l(x)| < \epsilon)
	\end{gather*}
\end{thm}

\begin{proof}

\end{proof}

\begin{thm}[Uniform Convergence, Limits and Continuity]
	Supposed $f_n \to f$ uniformly on $I \subseteq \bb{R}$, where $\forall n \in \bb{Z}^+ \; f_n: I \to \bb{R}$. If $\lim\limits_{y \to x} f_n(y)$ exists for each n, then
	\begin{equation*}
	 	\lim_{n \to \infty} \lim_{y \to x} f_n(y) = \lim_{y \to x} \lim_{n \to \infty} f_n(y)
	\end{equation*}
	Furthermore, if $f_n$ is continuous on I for each n, then f is continous on I.
\end{thm}

\begin{proof}

\end{proof}

\begin{thm}[Equivalent Topological Definitions]
	Let $A \subseteq \bb{R}^n$.
	\begin{enumerate}
		\item A is closed iff $A' \subseteq A$
		\item $\bar{A} = A' \cup A$
		\item $A^0$ is equal to the set of all interior points of A
		\item $\partial A = \bar{A} \setminus A^0$
	\end{enumerate}
\end{thm}

\begin{proof}

\end{proof}

\begin{thm}[Heine-Borel Theorem]
	Every subset of $\bb{R}^n$ is compact iff it is closed and bounded.
\end{thm}

\begin{proof}

\end{proof}

\chapter{Important Examples}

\chapter{FAQ for Self}

\chapter{Food for Thought}
\end{document}
% Document End


\documentclass[12pt]{article}
\usepackage[utf8]{inputenc}
\usepackage{braket}
\usepackage{mathtools}
\usepackage{amsmath}
\usepackage{enumitem}
\usepackage{physics}
\usepackage{float}
\usepackage{array}
\usepackage[super]{nth}
\usepackage{amssymb}
\usepackage{mathrsfs}
\usepackage{gensymb}
\usepackage{siunitx}
\usepackage{booktabs}
\usepackage{graphicx}
\usepackage[colorlinks,bookmarks=false,citecolor=blue,linkcolor=blue,urlcolor=blue]{hyperref}
\usepackage[margin=2.5cm]{geometry}

\title{\textbf{Adaptive step-size} \\ \textbf{Runge-Kutta Methods}}

\author{Maitrey Sharma}

\date{\today}

\begin{document}

\maketitle

\section{Introduction}
Before getting into the limitations of the Runge-Kutta methods, we need to understand the concepts of stability and stiffness while dealing with ODEs. \textit{Stability} is concerned with the accumulation of local errors (the error caused by one iteration) while doing numerical integration. If the method is unstable, the global error (that is, the error)

While working with algorithms to solve ODEs, the choice of the step-size $h$ is important as if it is too large, the truncation error may be unacceptable and if it is too small, we are expending unnecessary computer memory. Moreover, a constant step-size may
not be appropriate for the entire range of integration. For example, if the solution curve starts off with rapid changes before becoming smooth we should use a small h at the beginning and increase it as we reach the smooth region. Therefore we need to develop adaptive methods which estimate the truncation error at each integration step and automatically adjust the step size to keep the error within prescribed limits.

\section{Formulation}
The adaptive Runge-Kutta methods use so-called embedded integration formulas. These formulas come in pairs: One formula has the integration order $m$, and the other one is of order $m+1$. The idea is to use both formulas to advance the solution from $x$ to $x+h$. Denoting the results by $y_m(x+h)$ and $y_{m+1}(x+h)$, an estimate of the truncation error in the formula of order $m$ is obtained from
\begin{equation}
\label{eq7.17}
    E(h) = y_{m+1}(x+h) - y_m(x+h)
\end{equation}
Here are Runge-Kutta formulae of order five:
\begin{equation}
\label{eq7.18}
    \begin{split}
        K_0
        &= h F(x, y) \\
        K_i
        &= h F \Bigg( x+A_ih, y + \sum_{j = 0}^{i-1} B_{ij} K_j \Bigg), i = 1, 2, \ldots, 6
    \end{split}
\end{equation}
\begin{equation}
\label{eq7.19a}
    y_5 (x+h) = y(x) + \sum_{i=0}^6 C_i K_i
\end{equation}
The embedded fourth-order formula is
\begin{equation}
    y_4 (x+h) = y(x) + \sum_{i=0}^6 D_i K_i
\end{equation}
The coefficients appearing in these formulas proposed by Dormand and Prince are given in table (\ref{tab:coeff}). Using (\ref{eq7.17}), we have
\begin{equation}
    E(h) = y_5 (x+h) - y_4 (x+h) = \sum_{i=0}^6 (C_i - D_i)K_i
\end{equation}
To control the error measure $e(h)$ in the $E(h)$, out of the many approaches is to equate to the root-mean-square error in $E(h)$, so
\begin{equation}
    e(h) = \Bar{E}(h) = \sqrt{\dfrac{1}{n} \sum_{i=0}^{n-1} E_i^2 (h)}
\end{equation}
Error control is achieved by adjusting the increment $h$ so that the per-step error $e(h)$ is approximately equal to a prescribed tolerance $\varepsilon$. Noting that the truncation error in the fourth-order formula is $\mathcal{O} (h^5)$, we conclude that
\begin{equation}
\label{eq7.23}
    \dfrac{e(h_1)}{e(h_2)} \approx \Big( \dfrac{h_1}{h_2} \Big)
\end{equation}
Let us suppose that we performed an integration step with $h_1$ that resulted in the
error $e(h_1)$. The step size $h_2$ that we should have used can now be obtained from (\ref{eq7.23}) by setting $e(h_2) = \varepsilon$:
\begin{equation}
\label{eq7.234}
    h_2 = h_1 \Bigg[ \dfrac{\varepsilon}{e(h_1)} \Bigg]^{1/5}
\end{equation}
If $h_2 \geq h_1$ we could repeat the integration step with $h_2$, but as the error was
below the tolerance, we can accept current step and try $h_2$ in the next step. However, if $h_2 > h_1$, we must scrap the current step and repeat it with $h_2$.
\par
The equation (\ref{eq7.234}) can be modified slightly just to be safe and not overestimate the step-sizes as
\begin{equation}
\label{eq7.24}
    h_2 = 0.9 h_1 \Bigg[ \dfrac{\varepsilon}{e(h_1)} \Bigg]^{1/5}
\end{equation}
Also to prohibit large fluctuations in values of $h$, following constrains are applied
\begin{equation}
    0.1 \leq \dfrac{h_2}{h_1} \leq 10
\end{equation}
$e(h)$ applies to a single integration step, that is, it is a measure of the local truncation error. The global truncation error is caused by the accumulation of the local errors. Because $e(h)$ is a conservative estimate of the actual error, setting $\varepsilon = \varepsilon_{global}$ is
usually adequate. If the number of integration steps is very large, it is advisable to
decrease $\varepsilon$ accordingly.
\par
Now finally writing the algorithm, consider $h$ as the trial value of the increment for the first integration step and $K_0$ being computed from scratch only there. We use
\begin{equation}
\label{eq7.24c}
    (K_0)_{m+1} = \dfrac{h_{m+1}}{h_m} (K_6)_m
\end{equation}
 If $m$th step was accepted, and
\begin{equation}
\label{eq7.24d}
    (K_0)_{m+1} = \dfrac{h_{m+1}}{h_m} (K_0)_m
\end{equation}
To prove (\ref{eq7.24c}), we let $i = 6$ in (\ref{eq7.18}), obtaining
\begin{equation}
    (K_6)_m = h_m F \Bigg[ x_m + A_6 h_m, y_m + \sum_{i=0}^5 B_{6i} (K_i)_m \Bigg]
\end{equation}
From table (\ref{tab:coeff}) we have last row of $B$-coefficients is identical to the $C$-coefficients
(i.e., $B_{6i} - C_{i}$). Also as $A_6 = 1$. Therefore,
\begin{equation}
\label{eq7.24e}
    (K_6)_m = h_m F \Bigg[ x_m + h_m, y_m + \sum_{i=0}^5 C_i (K_i)_m \Bigg]
\end{equation}
But according to (\ref{eq7.19a}), the fifth-order formula is
\begin{equation}
    y_{m+1} = y_m + m_{i=0}^6 C_i (K_i)_m
\end{equation}
Since $C_6 = 0$ from table (\ref{tab:coeff}), we can reduce the upper limit of the summation from 6
to 5. Therefore, (\ref{eq7.24e}) becomes
\begin{equation}
    (K_6)_m = h_m F(x_{m+1}, y_{m+1}) = \dfrac{h_m}{h_{m+1}} (K_0)_{m+1}
\end{equation}
which completes the proof.
The validity of (\ref{eq7.24d}) is rather obvious by inspection of the first equation of (\ref{eq7.18}). Because step $m+1$ repeats step $m$ with a different value of $h$, we have
\begin{equation}
    (K_0)_m = h_m F(x_m, y_m) \hspace{1cm} (K_0)_{m+1} = h_{m+1} F(x_m, y_m)
\end{equation}
which leads directly to (\ref{eq7.24d}).
\par
This algorithm is coded in accompanying Python file.
% Please add the following required packages to your document preamble:
% \usepackage{booktabs}
\begin{table}[]
\centering
\caption{Dormand-Prince coefficients.}
\vspace{1cm}
\label{tab:coeff}
\setlength\extrarowheight{20pt}

\begin{tabular}{@{}cccccccccc@{}}
\toprule
$i$ & $A_i$ & \multicolumn{6}{c}{$B_{ij}$} & $C_i$ & $D_i$ \\ \midrule
0 & $-$  &  $-$   &  $-$  &  $-$  & $-$   &  $-$ &  $-$ & $\dfrac{35}{384}$  & $\dfrac{5179}{57600}$   \\
1 & $\dfrac{1}{5}$   &  $\dfrac{1}{5}$   & $-$  & $-$   & $-$   & $-$ & $-$ &  0    & 0   \\
2 & $\dfrac{3}{10}$   &  $\dfrac{3}{40}$   &  $\dfrac{9}{40}$  &  $-$ &  $-$  &  $-$  & $-$ &  $\dfrac{500}{1113}$  &  $\dfrac{7571}{16695}$  \\
3 & $\dfrac{4}{5}$   &  $\dfrac{44}{45}$   &  $-\dfrac{56}{15}$  &  $\dfrac{32}{9}$  &  $-$  &  $-$ & $-$ & $\dfrac{125}{192}$  & $\dfrac{393}{640}$   \\
4 & $\dfrac{8}{9}$   &  $\dfrac{19372}{6561}$   & $-\dfrac{25360}{2187}$   & $\dfrac{64448}{6561}$    & $-\dfrac{212}{729}$   & $-$ & $-$ & $-\dfrac{2187}{6784}$   &  $-\dfrac{92097}{339200}$  \\
5 & 1   & $\dfrac{9017}{3168}$    & $-\dfrac{355}{33}$   & $\dfrac{46732}{5247}$    & $\dfrac{49}{176}$   & $-\dfrac{5103}{18656}$    & $-$ & $\dfrac{11}{84}$   & $\dfrac{187}{2100}$    \\
6 & 1   &  $\dfrac{35}{384}$   & 0    &  $\dfrac{500}{1113}$  &  $\dfrac{125}{192}$  &  $-\dfrac{2187}{6784}$  & $\dfrac{11}{84}$   &   0 & $\dfrac{1}{40}$\\ \bottomrule
\end{tabular}
\end{table}

\section{Problem Statement}
Now that we have formulated and coded the algorithm we can consider an application in a real-world physical problem. Consider the aerodynamic drag force acting on a certain object in free fall
\begin{equation}
    F_D = a v^2 e^{-by}
\end{equation}
where $v =$ velocity of object in m/s; $y =$ elevation of the object in metres; $a = \SI{7.45}{\kilogram \per \metre}$; $b = \SI{10.53e-5}{\per \metre}$. The exponential term accounts for the change of air density with elevation. The differential equation describing the fall is
\begin{equation}
    m \Ddot{y} = -mg + F_D
\end{equation}
where $g = \SI{9.80665}{\metre \per \second \squared}$ and $m = \SI{114}{\kilogram}$ is the mass of the object. Taking the initial condition that the object was released from elevation of 9 km, let us use our algorithm to find elevation and speed after a 10 second fall.
\par
The differential equation and the initial conditions are
\begin{equation}
    \begin{split}
        \Ddot{y}
        &= -g + \dfrac{a}{m} \Dot{y}^2 exp(-by) \\
        &= -9.80665 + \dfrac{7.45}{114} \Dot{y}^2 exp(-10.53 \times 10^{-5} y)
    \end{split}
\end{equation}
with $y(0) = \SI{9000}{\metre}$ and $\Dot{y}(0) = 0$
Let us define $\Dot{y}$ as a vector:
\begin{equation}
    \Dot{y} =
    \begin{bmatrix}
        \Dot{y}_0 \\
        \Dot{y}_1 \\
    \end{bmatrix}
    =
    \begin{bmatrix}
        y_1 \\
        -9.80665 + (65.351 \times 10^{-3})y_1^2 exp(-10.53 \times 10^{-5}y_0)
    \end{bmatrix}
\end{equation}
and
\begin{equation}
    y(0) =
    \begin{bmatrix}
        9000 m \\
        0
    \end{bmatrix}
\end{equation}

The driver program for this problem is written in the accompanying Jupyter notebook. We specified a per-step error tolerance of $10^{-2}$ in \texttt{integrate}.  Considering the magnitude of $y$, this should be enough for five decimal point accuracy in the solution.
\par
The first step was carried out with the prescribed trial value $h = \SI{0.5}{\second}$. Apparently
the error was well within the tolerance, so that the step was accepted. Subsequent
step sizes, determined from (\ref{eq7.24}), were considerably larger. Inspecting the output, we see that at $t = \SI{10}{\second}$ the object is moving with the speed $v = -\Dot{y} = \SI{19.52}{\second}$ at an elevation of $y = \SI{8831}{\metre}$.
\end{document}


The Serurity API provides functions for security purposes, such as hashing and CSPRNG\footnote{Cryptographically secure psuedo-random number generator}.

\section{Functions}

\begin{tabularx}{\textwidth}{l l X}
	\textbf{\large Function} & \textbf{\large Return} & \textbf{\large Description}
	\\ \\
	\endhead
	security.toSHA1(string) & string & Returns SHA-256 hash of input string in array of bytes (as a string)
	\\ \\
	security.toSHA256(string) & string & Returns SHA-1 hash of input string in array of bytes
		\\ \\
	security.toMD5(string) & string & Returns MD-5 hash of input string in array of bytes
	\\ \\
	security.randomBytes(\textbf{len}: int) & string & Returns byte array of random values in desired \textbf{len}gth.
	\\ \\
	security.decodeBase64(string) & string & Decodes Base64 string and returns the result as string.
	\\ \\
	security.encodeBase64(string) & string & Encodes input string as Base64 format and returns the result as array of bytes.
\end{tabularx}


\subsection{Practical use}

\begin{frame}{$\mathcal{O}$-Notation}{Practical use}
  \textbf{Practical use:}
  \begin{itemize}
    \item
      It is much easier to determine the runtime of an algorithm by using
      the $\mathcal{O}$-Notation
    \begin{enumerate}
      \item
        Computing rules
      \item
        Practical use
    \end{enumerate}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Characteristics}
  \begin{itemize}
    \item
      Transitivity:
      \begin{eqnarray*}
        f \in \Theta(g) \; \wedge \; g \in \Theta(h)
       \onslide<2- |handout:1>{& \rightarrow & f \in \Theta(h)}\\
       f \in \mathcal{O}(g) \; \wedge \; g \in \mathcal{O}(h)
       \onslide<3- |handout:1>{ & \rightarrow & f \in \mathcal{O}(h)}\\
       f \in \Omega(g) \; \wedge  \; g \in \Omega(h)
       \onslide<4- |handout:1>{& \rightarrow & f \in \Omega(h)}
      \end{eqnarray*}
    \item<5- |handout:1>
      Symmetry:
      \begin{eqnarray*}
        f \in \Theta(g) & \leftrightarrow & g \in \Theta(f)\\
        \onslide<6- |handout:1>{f \in \mathcal{O}(g)}
        \onslide<7- |handout:1>{& \leftrightarrow & g \in \Omega(f)}
      \end{eqnarray*}
    \item<8- |handout:1>
      Reflexivity:
      \begin{eqnarray*}
        f \in \Theta(f) & f \in \Omega(f) & f \in \mathcal{O}(f)
      \end{eqnarray*}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Calculation Rules}
  \begin{itemize}
    \item
      Trivial:
      \begin{eqnarray*}
        f & \in & \mathcal{O} (f)\\
        C \cdot \mathcal{O} (f) & = & \mathcal{O} (f)\\
        \mathcal{O} (f + C) & = & \mathcal{O} (f)
      \end{eqnarray*}
    \item
      Addition:
      \begin{eqnarray*}
        \mathcal{O} (f) + \mathcal{O} (g) & = & \mathcal{O}
        (\mathrm{max}\{f,\;g\})
      \end{eqnarray*}
    \item
      Multiplication:
      \begin{eqnarray*}
        \mathcal{O} (f) \cdot \mathcal{O} (g) & = & \mathcal{O} (f \cdot g)
      \end{eqnarray*}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Runtime Complexity}
  \begin{itemize}
    \item
      The input size for all examples is $n$
    \item
      Basic operations
  \end{itemize}
  \begin{tabularx}{\linewidth}{P{11em}O{4em}}
    i1 = 0 & $\mathcal{O}(1)$
  \end{tabularx}
  \begin{itemize}
    \item
      Sequences of basic operations
  \end{itemize}
  \begin{tabularx}{\linewidth}{P{11em}O{4em}O{1.75em}L}
    i1 = 0 & $\mathcal{O}(1)$ & {} & {}\\
    i2 = 0 & $\mathcal{O}(1)$ & {} & {}\\
    $\cdots$ & $\cdots$ & {} & {}\\
    i327 = 0 & $\mathcal{O}(1)$ &
    \hspace*{-0.5em}%
    \multirow{-4}{*}{$\left.%
        \begin{array}{@{}c@{}}\\[3em]\end{array}%
      \right\rbrace$} &%
    \multirow{-4}{*}{$327 \cdot \mathcal{O}(1) = \mathcal{O}(1)$}%
  \end{tabularx}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Runtime Complexity}
  \begin{itemize}
    \item
      Loops
  \end{itemize}
  \begin{tabularx}{\linewidth}{P{11em}O{4em}O{1.75em}L}
    for i in range(0, n): & $\mathcal{O}(n)$ & {} & {}\\%
    \hhline{~*{1}{-}}%
    $\hspace*{1.5em}$ a[i] = 0 & $\mathcal{O}(1)$ &%
    \multirow{-2}{*}{%
      $\left.\begin{array}{@{}c@{}}\\[1em]\end{array}\right\rbrace$}&%
    \multirow{-2}{*}{$\mathcal{O}(1) \cdot \mathcal{O}(n) = \mathcal{O}(n)$}%
  \end{tabularx}\\[0.5em]
  \begin{tabularx}{\linewidth}{P{11em}O{4em}O{1.75em}O{5em}O{1.75em}L}
    for i in range(0, n): & $\mathcal{O}(n)$ & {} & {} & {} & {}\\
    \hhline{~*{3}{-}}
    $\hspace*{1.5em}$ a1[i] = 0 & $\mathcal{O}(1)$ & {} & {} & {} & {}\\
    $\hspace*{1.5em} \cdots$ & $\cdots$ & {} & {} & {} & {}\\
    $\hspace*{1.5em}$ a137[i] = 0 & $\mathcal{O}(1)$ &
    \multirow{-3}{*}{$\left.
        \begin{array}{@{}c@{}}\\[2em]\end{array}
      \right\rbrace$} &
    \multirow{-3}{*}{
      $\begin{array}{@{}c@{}}
        137 \cdot \mathcal{O}(1)\\
        = \mathcal{O}(1)
      \end{array}$
    } &
    \multirow{-4}{*}{$\left.
        \begin{array}{@{}c@{}}\\[3em]\end{array}
      \right\rbrace$} &
    \multirow{-4}{*}{
      $\begin{array}{@{}c@{}}
      \mathcal{O}(1) \cdot \mathcal{O}(n)\\
      = \mathcal{O}(n)
      \end{array}$
    }
  \end{tabularx}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Runtime Complexity}
  \begin{itemize}
    \item
      Loops
   \end{itemize}
  \begin{tabularx}{\textwidth}{P{11em}O{4.75em}O{1.75em}O{4.5em}O{1.75em}L}
    for i in range(0, n): & $\mathcal{O}(n)$ & {} & {} & {} & {}\\
    \hhline{~*{1}{-}}
    $\hspace*{1.5em}$ for j in range(0, n-1): & $\mathcal{O}(n-1)$ &
    \multirow{-2}{*}{$\left.%
        \begin{array}{@{}c@{}}\\[1em]\end{array}%
      \right\rbrace$} &%
    \multirow{-2}{*}{%
      $\begin{array}{@{}c@{}}%
        \mathcal{O}(n) \cdot \mathcal{O}(n)\\%
        = \mathcal{O}(n^2)%
      \end{array}$%
    } & {} & {}\\
    \hhline{~*{3}{-}}
    $\hspace*{3em}$ a1[i][j] = 0 & $\mathcal{O}(1)$ & {} & {} & {} & {}\\
    $\hspace*{3em} \cdots$ & $\cdots$ & {} & {} & {} & {}\\
    $\hspace*{3em}$ a137[i][j] = 0 & $\mathcal{O}(1)$ &
    \multirow{-3}{*}{$\left.
      \begin{array}{@{}c@{}}\\[2em]\end{array}
      \right\rbrace$} &
    \multirow{-3}{*}{
      $\begin{array}{@{}c@{}}
      137 \cdot \mathcal{O}(1)\\
      = \mathcal{O}(1)
      \end{array}$
    } &%
    \multirow{-5}{*}{$\left.
      \begin{array}{@{}c@{}}\\[4.5em]\end{array}
      \right\rbrace$} &%
    \hspace*{-0.5em}%
    \multirow{-5}{*}{%
      $\begin{array}{@{}c@{}}
      \mathcal{O}(1) \cdot \mathcal{O}(n^2)\\
      = \mathcal{O}(n^2)
      \end{array}$
    }%
  \end{tabularx}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Runtime Complexity}
  \begin{itemize}
    \item
      Conditions
  \end{itemize}
  \begin{tabularx}{\textwidth}{P{11em}O{4em}O{1.75em}O{4.5em}O{1.75em}L}
    if x < 100: & $\mathcal{O}(1)$ & {} & {} & {} & {}\\
    \hhline{~*{1}{-}}
    $\hspace*{1.5em}$ y = x & $\mathcal{O}(1)$ &
    \multirow{-2}{*}{$\left.%
      \begin{array}{@{}c@{}}\\[1em]\end{array}%
      \right\rbrace$} &%
    \multirow{-2}{*}{$\mathcal{O}(1)$} & {} & {}\\
    else: & {} & {} & {} & {} & {}\\
    $\hspace*{1.5em}$ for i in range(0, n): & $\mathcal{O}(n)$ &
    {} & {} & {} & {}\\
    \hhline{~*{1}{-}}
    $\hspace*{3em}$ if a[i] > y: & $\mathcal{O}(1)$ & {} & {} & {} & {}\\
    \hhline{~*{1}{-}}
    $\hspace*{4.5em}$ y = a[i] & $\mathcal{O}(1)$ &
    \multirow{-3}{*}{$\left.
      \begin{array}{@{}c@{}}\\[2em]\end{array}
      \right\rbrace$} &
    \multirow{-3}{*}{
      $\begin{array}{@{}c@{}}
      \mathcal{O}(n) \cdot \mathcal{O}(1)\\
      = \mathcal{O}(n)
      \end{array}$
    } &%
    \multirow{-6}{*}{$\left.
      \begin{array}{@{}c@{}}\\[5.5em]\end{array}
      \right\rbrace$} &%
    \hspace*{-0.5em}%
    \multirow{-6}{*}{%
      $\begin{array}{@{}c@{}}
      \mathcal{O}(\max\{1,n\})\\
      = \mathcal{O}(n)
      \end{array}$
    }%
  \end{tabularx}
\end{frame}

%-------------------------------------------------------------------------------

\codeslide{python}{
\begin{frame}{$\mathcal{O}$-Notation}{Arithmetic mean}
  \begin{itemize}
    \item
      {\color{MainA}Input:} List $x$ with $n$ numbers
    \item
      {\color{MainA}Output:} $a[i]$ is the arithmetic mean of $x[0]$
      to $x[i]$
  \end{itemize}
  \lstinputlisting[
    language=Python,
    style={python-idle-code},
    basicstyle=\small,
    tabsize=4,
    emph={arithMean},
    emphstyle=\color{blue}
  ]{Code/ArithmeticMean/ArithmeticMean.py}
\end{frame}
}

%-------------------------------------------------------------------------------

\codeslide{java}{
\begin{frame}{$\mathcal{O}$-Notation}{Arithmetic mean}
  \begin{itemize}
    \item
      {\color{MainA}Input:} Array $x$ with $n$ numbers
    \item
      {\color{MainA}Output:} $a[i]$ is the arithmetic mean of $x[0]$
      to $x[i]$
  \end{itemize}
  \lstinputlisting[
    language=Java,
    style={java-eclipse-code},
    basicstyle=\small,
    tabsize=4,
    emph={arithMean},
    emphstyle=\color{blue}
  ]{Code/ArithmeticMean/ArithmeticMean.java}
\end{frame}
}

%-------------------------------------------------------------------------------

\codeslide{cpp}{
\begin{frame}{$\mathcal{O}$-Notation}{Arithmetic mean}
  \begin{itemize}
    \item
      {\color{MainA}Input:} Vector $x$ with $n$ numbers
    \item
      {\color{MainA}Output:} $a[i]$ is the arithmetic mean of $x[0]$
      to $x[i]$
  \end{itemize}
  \lstinputlisting[
    language=C++,
    style={cpp-eclipse-code},
    basicstyle=\small,
    tabsize=4,
    emph={arith_mean},
    emphstyle=\color{blue}
  ]{Code/ArithmeticMean/ArithmeticMean.cpp}
\end{frame}
}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation Runtime complexity}
 \begin{tabularx}{\textwidth}{P{11em}O{4.75em}O{1.75em}O{3em}O{1.75em}L}
   for i in range(0, len(x)): & $\mathcal{O}(n)$ & {} & {} & {} & {}\\
   \hhline{~*{1}{-}}
   $\hspace*{1.5em}$ s = 0 & $\mathcal{O}(1)$ &
   \multirow{-2}{*}{$\left.%
     \begin{array}{@{}c@{}}\\[1em]\end{array}%
     \right\rbrace$} &%
    \multirow{-2}{*}{$\mathcal{O}(n)$} & {} & {}\\
    \hhline{~*{3}{-}}
    $\hspace*{1.5em}$ for j in range(0, i+1): & $\mathcal{O}(i+1)$ &
    {} & {} & {} & {}\\
    \hhline{~*{1}{-}}
    $\hspace*{3em}$ s = s + x[j] & $\mathcal{O}(1)$ &
    \multirow{-2}{*}{$\left.%
      \begin{array}{@{}c@{}}\\[1em]\end{array}%
      \right\rbrace$} &%
    \multirow{-2}{*}{$\mathcal{O}(i)$} & {} & {}\\
    \hhline{~*{3}{-}}
    $\hspace*{1.5em}$ a[i] = s / (i+1) & $\mathcal{O}(1)$ & {} & {} &
    \multirow{-5}{*}{$\left.
      \begin{array}{@{}c@{}}\\[4.5em]\end{array}
      \right\rbrace$} &%
    \hspace*{-0.5em}%
    \multirow{-5}{*}{%
      $\begin{array}{@{}c@{}}
      \mathcal{O}(n) \cdot \mathcal{O}(i)\\
      = \mathcal{O}(n^2)
      \end{array}$
    }%
  \end{tabularx}
  \begin{itemize}
    \item
      How often will the instructions in the loop be executed,
      when the problem has size $n$?
  \end{itemize}
  \[1 + 2 + \ldots + n = \dfrac{n \cdot (n + 1)}{2} \in \mathcal{O} (n^2)\]
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Discussion}
  \textbf{Way of speaking:}
  \begin{itemize}
    \item
      With the $\mathcal{O}$-Notation we look at the behavior of a function when
      $n \to \infty$
    \item
      We only analyze the runtime when $n \geq n_0$
    \item
      We talk about {\color{MainA}asymptotic analysis}, when we discuss
      cost, runtime, etc. as $\mathcal{O}(\ldots)$, $\Omega(\ldots)$
      or $\Theta(\ldots)$
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Discussion}
  \begin{alertblock}{\textbf{Attention:}}
    \begin{itemize}
      \item
        If you are using \textbf{asymptotic analysis}, you can not make any
        predictions about the runtime of smaller input sizes ($n < n_0$)
      \item
        For small input sizes (mostly $n < 10$),
        the runtime is predictably small
      \item
        $n_0$ does not necessarily have to be small
    \end{itemize}
  \end{alertblock}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Discussion}
  \textbf{Examples:}
  \begin{itemize}
    \item
    Let A and B be algorithms
    \begin{itemize}
      \item
        A has the runtime $f(n) = 80 \, n$
      \item
        B has the runtime $g(n) = 2 \, n \, \log_2 n$
    \end{itemize}
    \item
      So $f = \mathcal{O}(g)$ but \textbf{not} $\Theta(g)$
      \begin{itemize}
        \item
          $\Rightarrow$ A is asymptotic faster than B
        \item
          $\Rightarrow$
          There is an $n_0$ for that $n \geq n_0\!: \; f(n) \leq g(n)$
      \end{itemize}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{$\mathcal{O}$-Notation}{Discussion}
  \begin{block}{When is A faster then B?}
    We search the minimal $n_0$:
    \begin{align*}%
      f(n_0) & = \; g(n_0)\\
      80 \, n_0 & = \; 2 \, n_0 \, \log_2 n_0\\
      40 & = \; \log_2 n_0\\
      n_0 & = \; 2^{40}\\
      {} & = \; (2^{10})^4 = (1024)^4\\
      {} & \approx \; (10^3)^4 = 10^{12}\\
      {} & \approx \; \SI{1}{trillion}
    \end{align*}
    A ist faster than B if $n_0$ has more than 1 trillion elements
  \end{block}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Runtime Examples}{Continued}
  \begin{itemize}
    \item
      Logarithm of different bases differ only by a constant
      \begin{displaymath}
        \log_a n = \dfrac{\log_b n}{\log_b a}
        = \dfrac{1}{\log_b a}\cdot \log_b n
      \end{displaymath}
    \item
      Hence: $\log_a n \in \Theta(\log_b n)$
    \item
      For exponent this does not hold
      \begin{displaymath}
        3^n \not\in\Theta(2^n)
      \end{displaymath}
    \item
      Proof: Use equation~$(\ref{eq:big-O})$ from Slide~\ref{slide:fraction}
      \begin{displaymath}
        3^n \in \mathcal{O}(2^n)
        \Leftrightarrow \lim_{n \to \infty} \dfrac{3^n}{2^n} < \infty
      \end{displaymath}
    \item
      However:
      \begin{displaymath}
        \lim_{n \to \infty} \dfrac{3^n}{2^n} =  \lim_{n \to \infty}
        \left(\dfrac{3}{2}\right)^n = \infty
      \end{displaymath}
  \end{itemize}
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}<presentation>{Additional Figure}
  \begin{itemize}
  \item
    Figure for slide~\ref{slide:limit}
    \begin{center}
      \includegraphics[width=0.9\textwidth]{Images/limits-epsilon.pdf}
    \end{center}
  \end{itemize}
\end{frame}


\section{Partial differentiation of vector fields}


% !TEX root = ../master/master.tex

%% Dexter Barrows, 2016
%% dbarrows.github.io

\section{Data Setup}

	This section will focus on taking the stochastic SIR model from the previous section, truncating the synthetic data output from realizations of that model, and seeing how well IF2 and HMC can reconstruct out-of-sample forecasts.

	Figure [\ref{sc2dataplot}] shows an example of a simulated system with truncated data.

	\begin{figure}
        \centering
        \captionsetup{width=.8\linewidth}
        \includegraphics[width=0.8\textwidth]{./images/dataplot.pdf}
        \caption{Infection count data truncated at $T = 30$. The solid line shows the true underlying system states, and the dots show those states with added observation noise. Parameters used were $\mathcal{R}_0 = 3.0$, $\gamma = 0.1$, $\eta = .05$, $\sigma_{proc} = 0.5$, and additive observation noise was drawn from $\mathcal{N}(0,10)$. \label{sc2dataplot}}
    \end{figure}

	In essence, we want to be able to give either IF2 of HMC only the data points and have it reconstruct the entirety of the true system states.

\section{IF2}

	For IF2, we will take advantage of the fact that the particle filter will produce state estimates for every datum in the time series given to it, as well as producing ML point estimates for the parameters. Both of these sources of information will be used to produce forecasts by parametric bootstrapping using the final parameter estimates from the particle swarm after the last IF2 pass, then using the newly generated parameter sets along with the system state point estimates from the first fitting to simulate the systems forward into the future.

	We will truncate the data at half the original time series length (to $T = 30$), and fit the model as previously described.

	Figure 2 shows [\ref{if2fit}] the state estimates for each time point produced by the last IF2 pass.

	\begin{figure}
        \centering
        \captionsetup{width=.8\linewidth}
        \includegraphics[width=0.8\textwidth]{./images/if2fit.pdf}
        \caption{Infection count data truncated at $T = 30$ from Figure [\ref{sc2dataplot}]. The dashed line shows IF2's attempt to reconstruct the true underlying state from the observed data points. \label{if2fit}}
    \end{figure}

    Recall that IF2 is not trying to generate posterior probability densities, but rather produce a point estimate. Since we wish to determine the approximate distribution of each of the parameters in addition to the point estimate, we must add another layer atop the IF2 machinery, parametric bootstrapping.

\subsection{Parametric Bootstrapping}

	The goal of the parametric bootstrap is use an initial density sample $\theta^*$ to generate further samples $\theta_1, \theta_2, ..., \theta_M$ from the sampling distribution of $\theta$. It works by using $\theta$ to generate artificial data sets $D_1, D_2, ..., D_M$ to which we can refit our model of interest and generate new parameter sets. The literature suggests the most straightforward way of doing this is to fit the model to obtain $\theta^*$, then use the model's forward simulator to generate new data sets, in essence treating our original estimate $\theta^*$ as the ``truth'' set \cite{Genest2008}.

	An algorithm for parametric bootstrapping using IF2 and our stochastic SIR model is shown in Algorithm [\ref{paraboot}].

	\begin{algorithm}

        \BlankLine

        \SetKwInOut{Input}{Input}
        \SetKwInOut{Output}{Output}
        \DontPrintSemicolon

        \Input{Forward simulator $S(\theta)$, data set D}

        \BlankLine

        \tcc{Initial fit}
        $\theta^* \gets IF2(D)$

        \BlankLine

        \tcc{Generate artificial data sets}
        \For{$i = 1:M$}{
	        $D_i \gets S(\theta^*)$
        }

        \BlankLine

        \tcc{Fit to new data sets}
        \For{$i = 1:M$}{
	        $\theta_i \gets IF2(D_i)$
        }

        \BlankLine

        \Output{Distribution samples $\theta_1, \theta_2, ..., \theta_M$}

        \caption{Parametric Bootstrap \label{paraboot}}

    \end{algorithm}

\subsection{IF2 Forecasts}

	Using the parameter sets $\theta_1, \theta_2, ..., \theta_M$ and the point estimate of the state provided by the initial IF2 fit, we can use use forward simulations from the last estimated state to produce estimates of the future state.

	Figure [\ref{if2combined}] shows a projection of the data from the previous plots in Figures [\ref{sc2dataplot}] and [\ref{if2fit}].

	\begin{figure}
        \centering
        \captionsetup{width=.8\linewidth}
        \includegraphics[width=0.8\textwidth]{./images/if2combined.pdf}
        \caption{Forecast produced by the IF2 / parametric bootstrapping framework. The dotted line shows the mean estimate of the forecasts, the dark grey ribbon shows the 95\% confidence interval based on the 0.025 and 0.975 quantiles on the true state estimates, and the lighter grey ribbon shows the same confidence interval on the true state estimates with added observation noise drawn from $\mathcal{N}(0,\sigma)$. \label{if2combined}}
    \end{figure}

    We can define a metric to gauge overall forecast effectiveness by calculating the sum of squared errors of prediction (SSE). For the data in Figure [\ref{if2combined}] the value was approximately $SSE = 50.1$. Normally we would also want to address questions of forecast coverage, but this would require at least a 100-fold increase in computational cost. This is potentially an avenue of future investigation.

\section{HMC}

	For HMC we can use a simpler approach to approach forecasting. We do not get state estimates directly from the RStan fitting due to the way we implemented the model, but we can construct them using the process noise latent variables as described in Chapter 2. Once we've done this we can forward simulate the system from the state estimate into the future.

	 Figure [\ref{hmcforecast}] shows the result of the HMC forecasting framework as applied to the data from Figure [\ref{sc2dataplot}].

	\begin{figure}
        \centering
        \captionsetup{width=.8\linewidth}
        \includegraphics[width=0.8\textwidth]{./images/hmcforecast.pdf}
        \caption{Forecast produced by the HMC / bootstrapping framework with $M = 200$ trajectories. The dotted line shows the mean estimate of the forecasts, and the grey ribbon shows the 95\% confidence interval on the estimated true states as described in Figure [\ref{if2combined}]. \label{hmcforecast}}
    \end{figure}

    And as before we can evaluate the SSE of the forecast for the data shown, giving approximately $SSE = 608$.

\section{Truncation vs. Error}

	Of course the above mini-comparison only shows one truncation value for one trajectory. Really, we need to know how each method performs on average given different trajectories and truncation amounts. In effect we wish to ``starve'' each method of data and see how poor the estimates become with each successive data point loss.

	Using each method, we can fit the stochastic SIR model to successively smaller time series to see the effect of truncation on forecast averaged SSE. This was performed with 10 new trajectories drawn for each of the desired lengths. The results are shown in Figure [\ref{truncation}].

	\begin{figure}
        \centering
        \captionsetup{width=.8\linewidth}
        \includegraphics[width=0.8\textwidth]{./images/truncation.pdf}
        \caption{Error growth as a function of data truncation amount. Both methods used 200 bootstrap trajectories. Note that the y-axis shows the natural log of the averaged SSE, not the total SSE. \label{truncation}}
    \end{figure}

    IF2 and HMC perform very closely, with IF2 maintaining a small advantage up to a truncation of about 25-30 data points.

    Since the parametric bootstrapping approach used by IF2 requires a significant number of additional fits, its computational cost is significantly higher than the simpler bootstrapping approach used by the HMC framework, about 35.5x as expensive. However the now much longer running time can somewhat alleviated by parallelizing the parametric bootstrapping process; as each of the parametric bootstrap fittings in entirely independent, this can be done without a great deal of additional effort. The code used here has this capability, but it was not utilised in the comparison so as to accurately represent total computational cost, rather than potential running time.


\section{Discussion and Future Work}
\label{sec:Concl}
\label{sec:discussion}

\subsection{Final remarks}

We limit our discussion to dynamic trees. In particular, all data structures we discuss above can solve the dynamic connectivity problem for trees maintaining a forest under a finite sequence of edge insertions and deletions and supporting queries asking whether two vertices belong to the same tree or not.

The experiments we carried out provide a general overview of the strengths and weaknesses of the data structures presented in Sections \ref{sec:TechDes} and \ref{sec:Eval}. Although the results might change slightly if different implementations are tested or if a different architecture is used, some general observations can be made.

Firstly, we have presented \dyntset, a new approach to two existent functional data structures (i.e. finger trees and sets) for maintaining dynamic trees. This structure can manage $k$-degree trees, rooted or unrooted persistently whilst solving the dynamic trees problem.

Although the updates are conceptually very simple, namely \link and \cut, the proof that both indeed take $O(\log n)$ time is rather inherited from the core structures. Such definitions are acyclic and involve $O(1)$ number of steps to perform each.

Our experimental analysis has shown that the three operations we have implemented exceed the theoretical bounds by a $O(\log n)$ factor. Also, a native mechanism in the Haskell programming language, the lazy evaluation, is a crucial factor to achieve such performance. Nevertheless, the \link operation is slower that its imperative counterpart, by a factor of $O(\log n)$, is overcome in practice.

\subsection{Further work}

\tcr{Sets in Haskell are trees themselves, and it is odd to store a tree in every node of a tree. Can you take the finger tree, specialized to this particular monoidal annotation, and optimize it further more?}

Unfortunately, much of the simplicity of \dyntset data structure results from its use of sets as monoidal annotations, which allocate a $O(\log m)$ space per set within the host structure, i.e. finger tree, where $m \leq n$. Potential improvement suggests that benefits may accrue if we were to use higher order programming with effects, such as stateful computation or database-related structures rather than sets.

Uniqueness on edges allow to carry labels, therefore \dyntset could solve the dynamic trees problem from other approaches such as path-decomposition (i.e. link-cut trees) and tree-contraction.

Parallelism can play an important speed up when calling functions such \conn. Recalling the its first lines, we have
\begin{lstlisting} [mathescape]
conn x y forest = case (nodeIn x forest, nodeIn y forest) of
$\ldots$
\end{lstlisting}
Since the result of the leftist \code{nodeIn} is independent from the right one, both are suitable candidates to be evaluated in parallel. The pending research here is the time and space analysis between the sequential (both strict and lazy) against the parallel cases.

Finally, there is an evident need for a library of well-crafted test cases against which implementations of dynamic trees, graphs and applications can be tested. At present, for example, there is little to guide us when generating the update-sequences against which our structures are validated, and this raises a number of obvious issues. For example,

\begin{itemize}
\item Can we find test sets which guarantee coverage of key properties?
\item Which properties are we interested in?
\item If we know the general ratio of updates to queries (say), can we identify a more specific data structure to enhance efficiency still further?
\end{itemize}

These and other questions remain to be addressed, but we believe the quest for efficient dynamic data structures will become ever more important as seek to model, simplify and reason about the increasingly dynamic algorithmic structures in which modern culture is immersed and on which it depends.


% !TeX root = ../main.tex

\chapter{Theory}

%\section{The Standard Model of Particle Physics}
\section{Elementary Particles and Interactions in the Standard Model}
The standard model (SM) reflects our current understanding of elementary particles and several basic interactions.
It is a gauge quantum field theory containing the internal symmetries of the unitary product group $SU(3) \times SU(2) \times U(1)$,
in which the color group $SU(3)$ presents the strong interaction, and $SU(2) \times U(1)$ describes the electroweak interactions.
Over the past decades, the SM has been widely tested through various experiments with extremely high precision.

\input{chapters/Theory/elementaryparticles}
%\subsection{Elementary particles in the Standard Model}
\input{chapters/Theory/ewktheory}
%\subsection{Electroweak theory}
\input{chapters/Theory/symbreaking}
%\subsection{Higgs mechanics and electroweak symmetry breaking}
\input{chapters/Theory/bsmhiggs}
%\subsection{Beyond the SM Higgs sector}

\section{Phenomenology of Large Hadron Collider}
The Large Hadron Collider (LHC)~\cite{Bruning:2004ej, Buning:2004wk, Benedikt:2004wm} was built as a bridge between the experiments and the theories.
Physicists hope that the LHC can help to answer some open questions in fundamental physics,
such as the basic laws of interactions, the forces among the elementary particles,
the deep structure of space and time, and the interrelation between quantum mechanics and general relativity.
This section will talk about firstly the general introduction of Physics inside hadronic collision,
then followed by two important LHC phenomenologies of the Higgs physics and Diboson physics that are related closely to this dissertation.

\input{chapters/Theory/hadroniccollision}
%\subsection{Physics at hadron colliders}
\input{chapters/Theory/higgs}
%\subsection{Higgs Physics at LHC}
\input{chapters/Theory/diboson}
%\subsection{Diboson physics}


\documentclass[12pt]{article}
\usepackage{graphics}
\newcommand{\kg}{\mathrm{kg}}
\newcommand{\m}{\mathrm{m}}
\newcommand{\s}{\mathrm{s}}
\renewcommand{\deg}{\mathrm{deg}}
\newcommand{\cm}{\mathrm{cm}}
\newcommand{\mps}{\m\,\s^{-1}}
\newcommand{\mpss}{\m\,\s^{-2}}
\newcommand{\kgpmmm}{\kg\,\m^{-3}}
\newcommand{\N}{\mathrm{N}}
\newcommand{\J}{\mathrm{J}}
\newcommand{\Npmm}{\N\,\m^{-2}}
\newcommand{\tv}[1]{\mathbf{\vec{#1}}}
\newcommand{\dd}{\mathrm{d}}
\newcommand{\cell}[1]{\texttt{{#1}}}
\newcounter{problem}
\addtolength{\oddsidemargin}{-1in}
\addtolength{\textheight}{\headheight}
\setlength{\headheight}{0in}
\addtolength{\textheight}{\headsep}
\setlength{\headsep}{0in}
\setlength{\marginparwidth}{2in}
\begin{document}

\section*{NYU Physics 1---midterm exam}

Wednesday 2008 October 15 in lecture.

~ \vfill ~

\section*{Name:}

~ \vfill ~

\begin{equation}
\tv{v} = \frac{\dd}{\dd t}\tv{r}
\quad ; \quad
\tv{a} = \frac{\dd}{\dd t}\tv{v}
\end{equation}
\begin{equation}
a_{\mathrm{circ}} = \frac{v^2}{R} = \omega^2\,R
\end{equation}
\begin{equation}
\tv{F} = m\,\tv{a}
       = \frac{\dd}{\dd t}\tv{p}
       = \frac{\dd}{\dd t}(\gamma\,m\,\tv{v})
\end{equation}
\begin{equation}
|\tv{F}_{\mathrm{grav}}| = \frac{G\,M\,m}{r^2} = m\,g
\end{equation}
\begin{equation}
|\tv{F}_{\mathrm{static}}| \leq \mu\,|\tv{N}|
\quad ; \quad
|\tv{F}_{\mathrm{kinetic}}| = \mu\,|\tv{N}|
\end{equation}
\begin{equation}
\Delta U = m\,g\,\Delta h
\quad ; \quad
K = \frac{1}{2} m\,v^2
\end{equation}

\clearpage

\paragraph{Problem~\theproblem}\refstepcounter{problem}%
This is the layout for a planned roller coaster; one that satisfies
all standard safety concerns.  Draw a free-body diagram for the
roller-coaster cart when it gets to the point marked A.\\
\includegraphics{../eps/roller_coaster.eps}

~ \vfill ~

\paragraph{Problem~\theproblem}\refstepcounter{problem}%
In lecture we considered a car sliding around a banked circular turn
at constant speed, in the absence of friction.  Now imagine that the
car has static friction acting for transverse forces (that is,
opposing sliding up or down the bank) with a coefficient of $0.2$.
Draw the free-body diagram for the car when it is going around the
turn at the maximum speed at which the car can go without sliding
uphill or downhill.

~ \vfill ~

\clearpage

\paragraph{Problem~\theproblem}\refstepcounter{problem}%
Compute the velocity of the Space Shuttle in its orbit around the
Earth in units of $\mps$.  Assume it is orbiting very close to the
surface of the Earth (not a bad approximation, as we discussed in
class).  If you need to assume anything else, state your assumptions
explicitly.

~ \vfill ~

\paragraph{Problem~\theproblem}\refstepcounter{problem}%
The Sun orbits the Milky Way on a close-to-circular orbit at a speed
of $2\times 10^5\,\mps$ and a distance of $R= 3\times 10^{20}\,\m$.
What is the orbital period of the Sun around the Milky Way?

~ \vfill ~

\clearpage

\paragraph{Problem~\theproblem}\refstepcounter{problem}%
Imagine you plan a interstellar voyage in which you increase your
momentum at constant force $F$ (in a constant direction) for a time
$T$, where $T$ is long enough that you have to use the relativistic
formula for momentum.  What speed $v$ do you end up going at the end
of the period, relative to the frame at which you were at rest when
you started?

~ \vfill ~

\paragraph{Problem~\theproblem}\refstepcounter{problem}%
A bullet of mass $m$ moving at speed $v$ hits and lodges in a block of
mass $M$ that is initially at rest.  The combined system ends up
moving at some new speed $v'$.  Ignore all other forces and conserve
linear momentum.  What is the change in kinetic energy (total kinetic
energy after minus total kinetic energy in all components before)?

~ \vfill ~

\clearpage

\paragraph{Problem~\theproblem}\refstepcounter{problem}%
Imagine a particle of mass $m$ moving back and forth in the $x$
direction according to the equation
\begin{equation}
x(t)= A\,\cos\left(\omega\,t\right) \quad .
\end{equation}
What is the force as a function of time $F(t)$?

~ \vfill ~

\paragraph{Problem~\theproblem}\refstepcounter{problem}%
What formulae should go into cells \cell{E9} and \cell{F9} in this
spreadsheet, which is integrating a trajectory?  State your formulae
in terms of the cell numbers (such as ``\cell{I8}''), not variables
(such as ``$a_x$''). \\
\resizebox{\textwidth}{!}{\includegraphics{../xls/exam.eps}}

~ \vfill ~

\clearpage

\paragraph{Problem~\theproblem}\refstepcounter{problem}%
For this block-and-pulley system, I started to give---in lecture---an
argument that the $7\,\kg$ block will accelerate downwards.  Give this
argument, in words.
\marginpar{\includegraphics{../eps/blocks_3_4_7.eps}}%

~ \vfill ~

\end{document}


\documentclass{article}

\usepackage[open-square,define-standard-theorems,roman-theorems,dont-number-theorems]{QED}
\usepackage{basic,enumerate}

\title{A Deconstruction of the Fundamental Theorem Galois Theory}
\author{Sean McLaughlin}

%-------------------------------------------------------------------------------
% Defs
%-------------------------------------------------------------------------------

\newcommand{\Extension}[2][F]{#2/#1}
\newcommand{\GalGroup}[1]{\mbox{Gal}(#1)}

%-------------------------------------------------------------------------------
% Top
%-------------------------------------------------------------------------------

\begin{document}
\maketitle
\begin{abstract}
This document breaks down the proof of the Fundamental Theorem of Galois Theory
into a sequence of lemmas amenable to formalization in the Coq proof assistant.
\end{abstract}

This document breaks down the proof of the Fundamental Theorem of Galois Theory
into a sequence of lemmas amenable to formalization in the Coq proof assistant.
The definitions and theorems are labelled with two numbers. The first is
a global identifier.  The second is the corresponding theorem in
Rotman \cite{Rotman:GaloisTheory}.

%-------------------------------------------------------------------------------
% Rings
%-------------------------------------------------------------------------------

\section{Rings}

\begin{Definition}[1a] A \textbf{commutative ring with 1}, or just a \textbf{ring}, is a set $R$
equipped with two binary operations, $a + b,ab$ such that
\begin{enumerate}[(i)]
\item $R$ is an abelian group under addition
\item multiplication is commutation and associative
\item $1 r = r$
\item $r(s+t) = rs+rt$
\end{enumerate}
\end{Definition}

\subsection{Polynomials}

\begin{Definition}[1b] A \textbf{polynomial over} $R$ is a sequence
$$f(x) = (r_0,r_1,\ldots,r_n,0,0,\ldots)$$
with $r_i\in R$ and $r_i=0$ for $i>n$.  Equality and addition are
defined componentwise.  The product is defined by the usual
polynomial multiplication.
Denote the set of such polynomials by $R[x]$.
\end{Definition}

\begin{Theorem}[1c]
  If $R$ is a ring, then so is $R[x]$.
\end{Theorem}

\begin{Definition}[1d]
  The \textbf{leading coefficient} of a polynomial
$(r_0,r_1,\ldots,r_n,0,0,\ldots)$ is $r_n$ where
  $n$ is the largest index with nonzero coefficent.
\end{Definition}

\begin{Definition}
  The largest index of a nonzero coefficient is called the \textbf{degree} of the polynomial.
\end{Definition}

\begin{Definition}
  A polynomial is \textbf{monic} if its leading coefficient is 1.
\end{Definition}

\begin{Definition}
  A polynomial is \textbf{constant} if it is either the 0 polynomial, or has degree 0.
\end{Definition}

%-------------------------------------------------------------------------------
% Galois Theory
%-------------------------------------------------------------------------------

\section{Galois Theory}

\begin{Theorem}[63 (Fundamental Theorem of Galois Theory)]
  Let $\Extension{E}$ be a Galois extension with

\end{Theorem}

\bibliographystyle{abbrv}

\bibliography{all}

\end{document}


% Created 2021-07-01 Thu 12:10
% Intended LaTeX compiler: pdflatex
\documentclass[presentation,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{khpreamble}
\usepackage{amssymb}
\DeclareMathOperator{\shift}{q}
\DeclareMathOperator{\diff}{p}
\usepackage{tcolorbox}
\usetheme{default}
\author{Kjartan Halvorsen}
\date{2021-07-01}
\title{From analog to discrete-time systems}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={From analog to discrete-time systems},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.4.6)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Intro}
\label{sec:org580daf5}

\section{Discrete-time system example}
\label{sec:org4253ff0}

\section{Zero-order hold, or step-invariant sampling preview}
\label{sec:org5d042aa}

\begin{frame}[label={sec:org9b252d5}]{The world according to the discrete-time controller}
\begin{center}
\includegraphics[width=0.6\linewidth]{../../figures/fig1-1-schematic.png} Source: Åström and Wittenmark "Computer-controlled systems".
\end{center}
\end{frame}

\begin{frame}[label={sec:org5fb7d0a}]{Discrete-time Linear Shift-Invariant systems}
\begin{center}
\begin{tikzpicture}[node distance=20mm, anchor=north]
\node[coordinate] (input) {};
\node[rectangle, draw, right of=input, inner sep=3mm] (lti) {g};
\node[coordinate, right of=lti] (output) {};
\draw[->] (input) -- node[near start, above] {$u(k)$}  (lti);
\draw[->] (lti) -- node[near end, above] {$y(k)$} (output);
\end{tikzpicture}
\end{center}
\begin{block}{General case (non-causal)}
\[ y(k) = g \ast u = \sum_{n=-\infty}^\infty g(n) u(k-n) \]
\end{block}

\begin{block}{Causal case}
\[ y(k) = g \ast u = \sum_{n=0}^\infty g(n) u(k-n) \]

\(g(k)\) is called the \alert{weighting sequence}.
\end{block}
\end{frame}

\begin{frame}[label={sec:org5d0952b}]{Discrete-time LSI systems}
\begin{block}{Impulse response}
If the input signal is a unit discrete impulse

\begin{center}
\begin{tikzpicture}
\begin{axis}[
  width=14cm,
  height=2.5cm,
  xlabel={$k$},
  ylabel={$u(k)$},
  xmin=-2.5,
  xmax=10.5,
]

\addplot+[black, ycomb, domain=-2:10, samples=13,variable=k] { (k==0)};

\end{axis}
\end{tikzpicture}
\end{center}

\pause

\[ y(k) = \sum_{n=0}^\infty g(n) \delta(k-n) = g(k) \]

\alert{The weighting sequence is equal to the impulse response of the system.}
\end{block}
\end{frame}

\begin{frame}[label={sec:orge55d96a}]{The response of a discrete LSI system is a weighted sum of the current and previous values of the input}
\[ y(k) = g \ast u = \sum_{n=0}^\infty g(n) u(k-n) \]

\alert{Activity} What is the response of a system to the input signal \(u(k)\) if its impulse response (weighting sequence) is the one below?

\begin{center}
\begin{tikzpicture}
\small
\begin{axis}[
  width=14cm,
  height=3.5cm,
  xlabel={$k$},
  ylabel={$g(k)$},
  xmin=-0.5,
  xmax=10.5,
  ytick = {0, 1},
]

\addplot+[black, ycomb, domain=-2:10, samples=13,variable=k] { (k==4)};

\end{axis}
\end{tikzpicture}
\end{center}

\[y(k) = \]
\end{frame}

\section{The z-transform}
\label{sec:org7595ced}
\begin{frame}[label={sec:org6edb4b7}]{The Laplace transform}
\begin{block}{Definition}
\[ F(s) = \laplace{f(t)} = \int_0^\infty f(t) \mathrm{e}^{-st} dt\]
\end{block}
\begin{block}{Inverse transform}
\[ f(t) = \mathcal{L}^{-1}\{F(s)\} = \frac{1}{2\pi i} \int_{\gamma - i\infty}^{\gamma + i\infty} F(s)\mathrm{e}^{st} \, ds \]

Note that in control engineering, the one-sided transform is used.
\end{block}
\end{frame}

\begin{frame}[label={sec:org50832cf}]{The z-transform}
\begin{block}{Definition}
\[ F(z) = \ztrf{f(kh)} = \sum_{k=0}^{\infty} f(kh)z^{-k} \]
\end{block}

\begin{block}{Inverse transform}
\[ f(kh) = \frac{1}{2\pi i} \oint_r F(z) z^{k-1} \, dz \]

Note that in control engineering, the one-sided transform is used.
\end{block}
\end{frame}

\begin{frame}[label={sec:orgd9b9d0a}]{The Laplace transform of a sampled signal}
Assume right-sided signal \(f(t)\), meaning it is zero for negative times \(t<0\).
\[f_s(t) = f(t)m(t) = f(t) \sum_{k=0}^{\infty} \delta(t-kh) = \sum_{k=0}^{\infty} f(t)\delta(t-kh) = \sum_{k=0}^{\infty} f(kh) \delta(t-kh) \]

\pause

\begin{align*}
F_s(s) &= \laplace{f_s(t)} = \int_0^\infty \left(\sum_{k=0}^{\infty} f(kh) \delta(t-kh)\right)\mathrm{e}^{-st}\, dt \\
&= \sum_{k=0}^{\infty} \int_0^\infty  f(kh) \delta(t-kh) \mathrm{e}^{-st}\, dt = \sum_{k=0}^{\infty} f(kh) \mathrm{e}^{-skh}\\
&= \sum_{k=0}^{\infty} f(kh) \left(\mathrm{e}^{sh}\right)^{-k}
\end{align*}
\end{frame}

\begin{frame}[label={sec:orgb66f656}]{The Laplace transform of a sampled signal}
Nota:
\begin{align*}
F_s(s) &=  \sum_{k=0}^{\infty} f(kh) \left(\mathrm{e}^{sh}\right)^{-k}\quad \text{Laplace transform}\\
F(z) &= \sum_{k=0}^{\infty} f(kh) z^{-k} \quad \text{z-transform}
\end{align*}

\begin{tcolorbox}
 The z-transform of a sampled signal corresponds to its Laplace transform with the following relationship between the s-plane of the Laplace transform and the z-plane of the z-plane of the z-transform.
\[ z = \mathrm{e}^{sh}\]
\end{tcolorbox}
\end{frame}

\begin{frame}[label={sec:org09c424b}]{One of the most important transform pairs}
\[f(kh) = \alpha^{kh}, \quad \alpha \in \mathbb{C}\]

\pause

\begin{align*}
   F(z) &= \ztrf{f(kh)} = \sum_{k=0}^{\infty} f(kh)z^{-k}
   =  \sum_{k=0}^{\infty} \alpha^{kh}z^{-k} =  \sum_{k=0}^{\infty} \left(\alpha^{h}\right)^kz^{-k}\\
   &=  \sum_{k=0}^{\infty} \left(\frac{\alpha^{h}}{z}\right)^{k}
   =  \frac{1}{1 - \frac{\alpha^h}{z}} = \frac{z}{z-\alpha^{h}}, \quad |\frac{\alpha^h}{z}| < 1
\end{align*}

\pause

\begin{tcolorbox}
\[ \alpha^{kh} \quad  \overset{\mathcal{Z}}{\longleftrightarrow} \quad \frac{z}{z-\alpha^h} \]
\end{tcolorbox}
\end{frame}

\begin{frame}[label={sec:org8287211}]{Step-invariant sampling (a.k.a ZOH sampling)}
\end{frame}

\begin{frame}[label={sec:org95fe03b}]{Step-invariant sampling (a.k.a ZOH sampling)}
\begin{center}
\includegraphics[width=0.9\linewidth]{../../figures/invariant-sampling-white.pdf}
\end{center}

\pause
Step-invariant sampling (zero order hold): \(u(kh) = \begin{cases} 1, & k \ge 0\\0, & k<0 \end{cases}\)
\end{frame}

\begin{frame}[label={sec:orgb579e5e}]{Step-invariant sampling (a.k.a ZOH sampling)}
The idea is to sample the continuous-time system's response to a step input, in order to obtain a discrete approximation which is \alert{exact} (at the sampling instants) for such an input signal.

\begin{center}
\includegraphics[width=0.9\linewidth]{../../figures/invariant-sampling.pdf}
\end{center}

Step-invariant sampling (zero order hold): \(u(kh) = \begin{cases} 1, & k \ge 0\\0, & k<0 \end{cases}\)
\end{frame}

\begin{frame}[label={sec:org718c266}]{Why is step-invariant sampling a good idea?}
A piecewise constant (stair-case shaped) function can be written as a sum of delayed step-responses!
\begin{center}
  \begin{tikzpicture}
    \small
    \begin{axis}[
      clip = false,
      width=7cm,
      height=2.5cm,
      yshift=1.5cm,
      xlabel={$t$},
      ylabel={},
      xmax=10.5,
      ]
      \addplot+[black, no marks] coordinates {(-1,0) (0,0) (0,1) (10,1) } node[pos=0.7,coordinate, pin=40:$u_H(t)$] {};
    \end{axis}
    \begin{axis}[
      clip=false,
      width=7cm,
      height=2.5cm,
      yshift=-1.5cm,
      xlabel={$t$},
      ylabel={},
      xmax=10.5,
      ]
      \addplot+[black, no marks] coordinates {(-1,0) (1,0) (1,2) (10,2) } node[pos=0.7,coordinate, pin=40:$2u_H(t-1)$] {};;
    \end{axis}
    \begin{axis}[
      clip=false,
      width=7cm,
      height=2.5cm,
      xshift=7.5cm,
      xlabel={$t$},
      ylabel={},
      xmax=10.5,
      ]
      \addplot+[black, no marks] coordinates {(-1,0) (0,0) (0,1) (1,1) (1,3) (10,3) }  node[pos=0.7,coordinate, pin=40:$u(t) + 2u_H(t-1)$] {};;
    \end{axis}

    \node at (2.2,0.2) {\huge  +};
    \node at (6.5,0.2) {\huge  =};

  \end{tikzpicture}
\end{center}
\end{frame}

\section{The z-transform}
\label{sec:org7643698}

\section{Zero-order hold sampling procedure}
\label{sec:orga9007bd}
\begin{frame}[label={sec:org39bc2f4}]{Step-invariant sampling, or zero-order-hold sampling}
Let the input to the continuous-time system be a unit step \(u(t)=u_H(t),\) which has Laplace transform \(U(s)=\frac{1}{s}.\) In the Laplace-domain we get
\[Y(s) = G(s)\frac{1}{s}\]
\begin{enumerate}
\item Obtain the time-response by inverse Laplace: \(y(t)=\laplaceinv{Y(s)}\)
\item Sample the time-response to obtain the sequence \(y(kh)\) and apply  the z-transform to obtain \(Y(z) = \ztrf{y(kh)}\)
\item Calculate the pulse-transfer function by dividing with the z-transform of the input signal \(U(z) = \frac{z}{z-1}.\) \[H(z) = \frac{Y(z)}{U(z)} = \frac{z-1}{z}Y(z) \]
\end{enumerate}
\end{frame}
\end{document}