% Chapter Template

\chapter{Background Information and Theory} % Main chapter title

\label{Chapter2} % Change X to a consecutive number; for referencing this chapter elsewhere, use \ref{ChapterX}

%----------------------------------------------------------------------------------------
%	SECTION 1
%----------------------------------------------------------------------------------------

\section{Machine Learning}
% \begin{note}
%   \begin{itemize}
%     \item Una definición rápida
%     \item Clasificación y regresión
%     \item Cross-validation
%     \item Qué son los datos de train y test, y por qué se hace esa partición
%     \item Qué es el sobre-ajuste
%   \end{itemize}
% \end{note}

\begin{pre-delivery}
  % Machine Learning uses statistical and mathematical models to give answers to
  % problems when there is no known formula of procedure to compute the answer.

  Machine Learning uses statistical and mathematical models to give
  computational answers based on data to problems when there is no known
  formula of procedure.

  In the subfield of Supervised Learning, the objective is to predict a numerical
  or categorical variable in response to some input data, and the way of doing
  it is to feed the model with lots of different examples for which we already
  know the correct answer, and we expect the models to be able to predict
  the correct answer to instances that it hasn't seen before. When it does,
  we say that the model is able to generalize.

  When a model is trained with some data, there is always a risk of overfitting
  \cite{hawkins2004problem}.
  For a model to overfit means that it adjusts very well to the data that is
  has seen, but can't predict the correct answer to new, unseen data.
  % This
  % happens because it has not only from the relevant information, but also
  % from the random noise that the data had, and so it can only memorise, but
  % not generalize.
  This happens because it has not only fitted the relevant information,
  but also random noise present in the data sample, and thus it generalizes
  poorly. In the extreme case of ovefitting, the model tends to memorise the
  data sample.

  For this reason, when a Machine Learning algorithm is trained the data
  is split in two subsets, a \textit{Training  dataset} and a \textit{Testing
  dataset}. The training dataset will be used to train the model, while the
  Testing datasets will be used only to check it. If a model has generalized
  well, it will achieve a good accuracy score on both the training and the
  testing dataset, but if it has overfitted it will show good results in the
  training dataset and bad ones in the testing dataset.

  Many models need some parameters to tune the behaviour of the algorithm. For
  example, some of them are used to adjust how much a model will fit to the data.
  We usually call these ``hyperparameters''. The correct value for them is not
  straightforward, and it is normally chosen with a resampling process called
  ``cross-validation''\cite{geisser2017predictive}. This process consists of
  splitting the training dataset
  in many subsets and check many possible values for the hyperparameters in order
  to see which one gets a higher accuracy with unseen data.
\end{pre-delivery}

\section{Some currently used Machine Learning models}

% \begin{note}
  % \section{Review de los principales modelos que existen}
% \end{note}
  \subsection{Decision Tree}
  \label{sec:dec-tree}
  % \begin{note}
  %   \begin{itemize}
  %     \item No se basa en productos escalares
  %     \item Es extremadamente rápido
  %     \item Es más fácil de interpretar que otros modelos
  %     \item Es extremadamente inestable
  %     \item Cuando se hace un Random Forest, se randommiza un poco, de modo que
  %     árboles distintos entrenados con los mismos datos pueden ser destintos
  %     \item Es un modelo no lineal
  %   \end{itemize}
  % \end{note}
  \begin{pre-delivery}
    Decision Tree\cite{breiman2017classification}\cite{lewis2000introduction}
    is a predictive model which uses
    the training data to build
    a tree where each node splits the data in two sets according to some
    feature, and the leafs contain the set of instances that belong to some class
    (in classification problems) or that has a similar numerical response variable
    (for regression problems).

    To predict the answer to a new instance, it uses the features to ``decide''
    the nodes to cross until it reaches a leaf. The response given is the
    most prevailing class in the leaf for classification problems, or the mean
    of the values of the rest of the instances in the leaf.

    To decide what feature to use to split a node in two subsets, it uses
    the Gini impurity: it will pick the feature that minimises the sum of
    the Gini impurity of the two child nodes. Given a node with instances
    belonging to $k$ classes, if $p_i$ is the proportion of instances that
    belong to class $i$, the Gini impurity of the node is
    $1 - \sum_{i = 1}^k p_i^2$.

    Decision Trees have the advantages that it is easy to interpret the
    tree produced and that it is very fast to build the tree. The way to avoid
    overfitting is to limit its growth.

    These models are very unstable. This means that small differences in the
    training data can produce very different Decision Trees. This property
    is very useful to build an ensemble of estimators to produce better answers.
    Random Forest is an algorithm that trains many Decision Trees with some sort
    of randomization.
  \end{pre-delivery}
  \subsection{Logistic Regression}
  \label{ssec:log-reg}
  \begin{pre-delivery}
    Logistic Regression\cite{cox1958regression} models the probability that an instance belongs to
    a class, and predicts the class with a higher probability. To do so it
    uses the \textit{logistic sigmoid function}\cite{han1995influence}, defined by:

    \begin{equation}
      \sigma(a) = \frac{1}{1 + exp(-a)}
    \end{equation}

     Once the vector $w \in \reals^d$ has been found, the predicted probability
     that an instance $\vx \in \reals^d$ belongs to a class is
     $y(\vx) = \sigma(w^\transp\vx)$.

     % To find a suitable $w$ it solves an optimization problem of finding $w$
     % that maximizes the likelihood that each of the instances belongs to the
     % specified class

     Given
     $D = \{\bm{\chi}, \bm{t}\}$
     , where
     $\bm{\chi} = \{\bm{x}_1, \ldots \bm{x}_n\}$, $\bm{x}_i \in \reals^d$, $\bm{t} = \{0, 1\}^n$
     it tries to maximize a likelihood function that can be written

     \begin{equation}
       p(\bm{t} | w) = \prod_{i = 1}^n y_i^{\bm{t}_i} (1 - y_i)^{1 - \bm{t}_i}
     \end{equation}

     where $y_i = \sigma(w^\transp\vx_i + w_0)$.
     % , the problem is to find $w \in \reals^d$ and $c \in \reals$ that minimizes

     % \begin{equation}
     %   \frac{1}{2}w^\transp w + \sum_{i = 1}^n log\left( exp\left( -y_i(\vx_i^\transp w + c) \right) + 1\right)
     % \end{equation}
  \end{pre-delivery}
  \subsection{Support Vector Machines}
  \label{ssec:svm}
  % \begin{note}
  %   \begin{itemize}
  %     \item Inicialmente pensadas para clasificación en 2 clases
  %     \item Pero se puede más clases con \eng{one-vs-rest} y también hay
  %     formas de hacer regresión
  %     \item Se basa únicamente en el producto escalar de sus entradas
  %     \item Intenta separar los datos con un híper-plano
  %     \item Actualmente es poco eficiente usarlas porque su coste s cúbico
  %     con la cantidad de entradas.
  %     \item Las fórmulas que quiere optimizar
  %   \end{itemize}
  % \end{note}

  \begin{pre-delivery}
    Support Vector Machine\cite{Cortes1995} (SVM) is a model that finds in hyperplane that
    divides the data in two sets. In two-class classification problems, each
    side of the hyperplane contains the instances of each of the classes.
    It does so by converting the problem to an optimization one.

    Given some data
    $D = \{\bm{\chi}, \bm{y}\}$
    , where
    $\bm{\chi} = \{\bm{x}_1, \ldots \bm{x}_n\}$, $\bm{x}_i \in \reals^d$, $\bm{y} = \{-1, +1\}^n$
    , the optimization problem consists on finding $\bm{\alpha} \in \reals^n$
    the maximises

\begin{equation}
  L = \sum_{i = 1}^n\alpha_i -\frac{1}{2}\sum_{i = 1}^n\sum_{j = 1}^n\alpha_i\alpha_jy_iy_j\vx_i^\transp\vx_j
\end{equation}

subject to

\begin{align}
  0 \leq \alpha_i \leq C; \forall i\\
  \sum_{i = 1}^n \alpha_iy_i = 0
\end{align}

% $C$ is an hyperparameter to tune the amount of penalization for miss-classified
% instances.

$C$ is an hyper-parameter to tune the amount of penalization for missclassified
instances or instances located within the margin zone.

If we compute
\begin{equation}
  w = \sum_{i = 1}^n\alpha_iy_i\vx_i
\end{equation}
and

\begin{equation}
  b = y_i - w\cdot\vx_i
\end{equation}
for any $i$ so that $\alpha_i \neq 0$, we can compute the class of $\vx_0$ with

\begin{equation}
sign(w\cdot\vx_0 + b)
\end{equation}

Note that this algorithm just uses the dot product of the input data, not the
data itself. This property allows us to use the Kernel Trick with them.
See \ref{sec:kern-trick}

\end{pre-delivery}

\section{Ensemble Methods}
\label{sec:ens-meth}
  % \subsection{Bagging}
    % \begin{note}
    %   \begin{itemize}
    %     \item Bagging
    %     \begin{itemize}
    %       \item Inventado por Leo Breiman (referencia)
    %       \item Pretende reducir el sesgo
    %       \item Wikipedia dice que pretende reducir la varianza
    %       \item Es el boosting el que pretende reducir el sesgo
    %       \item Entrenamiento de los estimadores es independiente, se podría
    %       hacer en paralelo
    %       \item Actualmente casi solo se usa con DT, debido a su inestabilidad
    %     \end{itemize}
    %     \item Bootstrap
    %     \begin{itemize}
    %       \item Intenta solucionar el problema de que para bagging es bueno
    %       que los estimadores sean distintos
    %       \item Idealmente usaríamos un dataset distinto para cada estimador
    %       \item Consiste en hacer un resalmpling con repetición
    %       \item Si la cantidad de instancias del original es la misma que la de cada uno
    %       de los subconjuntos, se espera que la proporción de elementos úncos sea de
    %       $1 - \frac{1}{e} \approx 0.632$.
    %       \item Si el conjunto original tiene $n$ elementos, y tu haces un subconjunto
    %       de tamaño $r$, puedes esperar que la proporción de elementos del original que
    %       sí tienen presencia en el nuevo sea de $1 - e^{-\frac{r}{n}}$
    %     \end{itemize}
    %     \item Random Forest
    %   \end{itemize}
    % \end{note}

  \begin{pre-delivery}
    Ensemble methods\cite{polikar2006ensemble} are a technique used in Machine
    Learning to reduce the
    overall accuracy error of a basic classification or regression model. The
    idea is that a commimtee of models is expected to learn better than a single
    one.

    Some ensemble methods are focused on decreasing the error caused by the
    variance of the model. One example is \textit{Bagging}\cite{breiman1996bagging}. Others are focused
    on decreasing the bias error, like \textit{Boosting}\cite{freund1997decision}.

    In Bagging, every model in the ensemble vote with equal weight. Thus, it is
    important to promote the variance among each of the models, since not doing
    it would be equivalent to training just one model. Ideally, one would train
    each of the models with totally different datasets, with no correlation
    among them. But in practice this is not always possible, because of a
    limited number of instances to train. One alternative is to use a
    technique called \textit{Bootstrap}\cite{efron1994introduction}. Bootstrap allows to generate
    many different instances of a dataset by performing a resampling.

    Given a dataset $D$ of size $n$, Bootstrap generates $m$ new datasets
    $D_i$ of size $n$ by sampling instances from $D$ uniformly and with
    replacement. This means that some of the instances in $D$ may be repeated
    in $D_i$, and others may not appear at all. With a large $n$, it is expected
    that each dataset $D_i$ will contain $63.2 \% $ of the instances in $D$.

    Theoretically Bagging could be used with any kind of method. However, for
    most of them Bootstrap is not enough to decorrelate the estimators.
    In practice, Bagging is mostly used with Decision Tree, given that this
    method produces very different trees with a small variation in the data.
    Random Forest\cite{Breiman2001} is an algorithm that trains many Decision Trees with a
    Bagging. Instead of building the tree in a deterministic way, in each
    split it chooses a random subset of features on which to perform the
    separation. Besides, it lets the estimators overfit, since it has a positive
    impact in reducing the overall variance of the Forest.
  \end{pre-delivery}

\section{The kernel trick}
\label{sec:kern-trick}

% \begin{note}
%   \begin{itemize}
%     \item Teorema de Bochner
%     \item El kernel RBF
%     \begin{itemize}
%       \item Su fórmula es \ldots
%       \item Equivalencia entre $\gamma$ y $\sigma$
%       \item La noción de similitud que tiene
%       \item \Hspace\ es de dimensionalidad infinita
%       \item Permite ajustarse infinitamente a los datos, tuneando el
%       híper-parámetro
%       \item $\sigma$ más pequeño, más sobreajuste
%       \item $\gamma$ más grande, más sobreajuste
%     \end{itemize}
%   \end{itemize}
% \end{note}

\begin{pre-delivery}
  A Kernel\cite{bergman1970kernel} is a function that equals to the inner product of inputs mapped into
  some Hilbert Space
  \footnote{A Hilbert space is a generalization of the Euclidean Space which contains
  the structure of an inner product that allows length and angle to be
  measured.}
  , i.e:
  \begin{equation}
  \kernel(x,y) = \phi(x)\cdot\phi(y)
\end{equation}
% A Hilbert space is just a generalization of the Euclidean Space which contains
% the structure of an inner product that allows length and angle to be measured.

They are interesting in Machine Learning because we don't need to know the
explicit function $\phi(\cdot)$. In fact, $\phi(\cdot)$ could map the data to
a Hilbert Space with infinite dimensions, and we could still compute
$\phi(\vx)\cdot\phi(\vy)$ through the kernel $\kernel$

Support Vector Machines (explained in \ref{ssec:svm}) can benefit a lot of
Kernel Functions. SVMs solve an
optimization problem to maximise

\begin{equation}
  L = \sum_{i = 1}^n\alpha_i -\frac{1}{2}\sum_{i = 1}^n\sum_{j = 1}^n\alpha_i\alpha_jy_iy_j\vx_i^\transp\vx_j
\end{equation}

in order to find an hyperplane that separates the data points in two classes.
But with some problems there may not exist such hyperplane, and so it would
be needed to map the data to a different feature space. If we did that, then
the function to maximise would be

\begin{equation}
  L = \sum_{i = 1}^n\alpha_i -\frac{1}{2}\sum_{i = 1}^n\sum_{j = 1}^n\alpha_i\alpha_jy_iy_j\phi(\vx_i)^\transp\phi(\vx_j)
\end{equation}

As we said previously, SVMs don't work with the data points alone, but just with
their inner products. Thus, a Kernel could be used to define the optimization
problem as

\begin{equation}
  L = \sum_{i = 1}^n\alpha_i -\frac{1}{2}\sum_{i = 1}^n\sum_{j = 1}^n\alpha_i\alpha_jy_iy_j\kernel(\vx_i, \vx_j)
\end{equation}

This approach has one big advantage:
% first, we don't need to explicitly
% compute $\phi(\vx)^\transp\phi(\vy)$, which could have a high cost if the
% new dimensionality was too big.
as long as the learning technique relies
only on the inner product of the input, the underlying mapping $\phi(\cdot)$
does not need to be explicitly calculated and can, in fact, be unknown\cite{burges1998tutorial}.

Kernel functions can be characterised with the Mercer's condition
\cite{mercer1909functions}. It says that given a function $\kernel(\vx, \vy)$,
there exists a mapping $\phi(\cdot)$ so that
$\kernel(\vx, \vy) = \phi(\vx)\cdot\phi(\vy)$
if and only if for any $g(\vx)$ such that $\int g(\vx)^2 d\vx$ is finite then
$\int \kernel(\vx, \vy)g(\vx)g(\vy) \geq 0$.

There are many known Kernels. One that is very popular is the Radial Basis
Function Kernel\cite{vert2004primer}, RBF. This kernel is defined as:
\begin{equation}
\kernel(\vx,\vy) = \semiRbf
\end{equation}
where $\gamma > 0$ is a free parameter. The value of this Kernel decreases with the
euclidean distance of the parameters, so it can be interpreted as a measure
of similarity. The feature space of this kernel has infinite number of
dimensions.

When a kernel is used with an SVM, the answer can be computed with

\begin{equation}
sign\left(\sum_{i = 1}^n \alpha_iy_i\kernel(\vx_i, \vx)\right)
\end{equation}
SVMs using the RBF kernel have a huge ability to fit to the data, and is able
to separate classes for very difficult problems. The problem is that the
optimization of the function

\end{pre-delivery}
  % \subsection{The RBF kernel}

\section{Random Fourier Features}

% \begin{note}
%   \begin{itemize}
%     \item Teorema de bochner
%     \item Tiene que ser un shift invariant kernel
%     \item Es más, tiene que ser un positive definite shift-invariant kernel
%     \item Converge bounds for ability to approximate
%     \item Instead, we propose to factor the kernel function itself
%     \item La factorización no depende de los datos
%     \item we propose explicitly mapping
% the data to a low-dimensional Euclidean inner product space using a randomized feature map z :
% Rd --> RD so that the inner product between a pair of transformed points approximates their kernel
% evaluation
% \item Puesto que los valores están entre -1 y 1, hay un teorema que asegura
% la convergencia exponencial hacia el kernel real
%   \end{itemize}
% \end{note}

\begin{pre-delivery}
  A kernel function
  $\kernel(\vx, \vy)$ with $\vx, \vy \in \reals^d$
  equals the inner product of inputs mapped with some function $\phi(\cdot)$,
  so that
  $\phi(\vx)^\transp\cdot\phi(\vy) = \kernel(\vx, \vy)$.
  But $\phi(\cdot)$ could be a mapping to an infinitely-dimensional space, so
  calculating $\phi(\vx)$ is not possible for some kernels.

  Random Fourier Features\cite{rahimi2008random} provide a way to, given a
  kernel $\kernel(\vx, \vy)$,
  explicitly map the data to a
  low-dimensional Euclidean inner product space using a randomized feature
  map $z: \reals^d \mapsto \reals^D$ so that the inner product between a pair
  of transformed points approximates their kernel evaluation, i.e:

  \begin{equation}
    \kernel(\vx, \vy) = \phi(\vx)^\transp\cdot\phi(\vy) \approx z(\vx)^\transp\cdot z(\vy)
  \end{equation}

  To approximate the RBF kernel, it uses the Bochner's Theorem, which says:

\newtheorem{theorem}{Theorem}
  \begin{theorem}
    \cite{rudin1962fourier}
    A continuous kernel $\kernel(x, y) = \kernel(x - y)$ on $\reals^D$  is
    positive definite if and only if $k(\delta)$ is the
    Fourier Transform of a non-negative measure.
  \end{theorem}

  % Since RBF is defined as $\kernel(\vx, \vy) = e^{-\gamma\norm{\vx - \vy}^2}$

  Since it is known that RBF is shift-invariant and positive definite, then
  its Fourier transform is a proper probability distribution, and so

  \begin{equation}
    \kernel(\vx - \vy) = \int_{\reals^D} p(w)e^{iw^\transp (\vx-\vy)}
    = \int_{\reals^D} p(w)cos\left(w^\transp (\vx-\vy)\right)
  \end{equation}

  A random feature can be obtained by picking $w \sim \{\mathcal{N}(0, 2\gamma)\}^d$
  and $b \sim \mathcal{U}(0, 2\pi)$
  and computing $\sqrt{2}cos(w^\transp\vx + b)$. To generate a lower variance
  approximation of $\phi(\vx)$ with $D$ features we can concatenate $D$ randomly
  chosen features $(f_1, \ldots, f_D)$ into a column vector and normalize each
  component by $\sqrt{D}$.

  It is guaranteed an exponentially fast convergence in $D$ between
  $z(\vx)^\transp z(\vy)$ and $\kernel(\vx, \vy)$.

\end{pre-delivery}

\section{\Nys}

\begin{pre-delivery}
  The \Nys\cite{NIPS2000_1866} method is a general method for low-rank approximations of
  kernels. It achieves this by subsampling the data on which the kernel
  is evaluated.

  In kernel methods the data can be represented in a kernel matrix $K$, where
  $K_{i,j} = \kernel(\vx_i, \vx_j)$. The problem of these methods is their
  high computational cost associated with the kernel matrix: with non-linear
  kernels, the cost of training the model is cubic with the number of
  instances, something unacceptable for large-scale problems.

  The \Nys\ method consists on generating an approximation of the kernel matrix of
  ranq $q$, where $q$ can be a lot smaller than the number of instances, without
  any significant decrease in the accuracy of the solution. This way, if there
  are $n$ instances in a dataset, the complexity can be reduced from
  $\mathcal{O}(n^3)$ to $\mathcal{O}(nq^2)$.

  With \Nys, given a kernel $\kernel(\vx, \vy) = \phi(\vx)\cdot\phi(\vy)$, one can
  construct a mapping $z: \reals^d \mapsto \reals^q$ so that
  $z(\vx) \approx \phi(\vx)$. This function defines each component $j$ as
  $z_j(\vy) = \frac{1}{q}\sum_{i = 1}^q \kernel(\vy, \vx_i)g_i(\vx_i)$,
  where $\vx_1, \ldots, \vx_q$ are some chosen instances and
  $g_i(\cdot)$ comes from a column from the Singular Value Decomposition
  of the approximated kernel matrix.
\end{pre-delivery}


\documentclass[11pt, a4wide]{article}   	% use "amsart" instead of "article" for AMSLaTeX format
\usepackage{geometry}                		% See geometry.pdf to learn the layout options. There are lots.
\geometry{a4paper}                   		% ... or a4paper or a5paper or ...

%Custom commands
\newcommand{\data}{\text{data}}
\newcommand{\sessions}{s}
\newcommand{\purchases}{p}
\newcommand{\sepu}{\sessions, \purchases}
\newcommand{\convr}{R}
\newcommand{\ushi}{\text{UH}}

%Packages
\usepackage{graphicx}
\usepackage{amssymb, amsmath}

\usepackage{a4wide}
\usepackage{listings,url}
\usepackage[retainorgcmds]{IEEEtrantools}

\title{A Bayesian Approach to AB Testing [Draft]\\{\small Introducing a Python Engine for Testing}}
\author{Hugo Pibernat and Nuria Duran}
%\date{}

\begin{document}
\maketitle
\section{Introduction}
The main goal of our AB-Test is to decide which of two different versions of the same product performs better on a specific set of metrics. In this report we present a Bayesian approach to tackle this objective.

A Bayesian model includes a prior distribution which enables us to provide information about the parameters $KPI_A$ and $KPI_B$ governing our data \emph{before} observing the data. The information included in the data is taken into account by means of the likelihood function. Combining both the prior and the likelihood function results in a posteriori distribution, a probability distribution of the parameters (the metric of each group, in this case) which will help us to decide which of the version performs better.

The (joint) posterior probability distribution is defined as follows:

\begin{equation}
P(\text{KPI}_A,\text{KPI}_B|data)
\end{equation}

In order to compute this distribution, we will need the Bayes theorem, which states that

\begin{equation}
P(X|Y) = \frac{P(Y|X)\cdot P(X)}{P(Y)}
\end{equation}

%\section{Bayesian models}
\section{Binomial Variables: Modelling Conversion Rate}

The history of a user (UH) looks as follows:
\begin{equation}
<0,0,1,0,0,0,1,0,0,1,0,1,1,0,1>
\end{equation}
where 0 represents a non-converted visit and 1 represents a converted visit. Hence,
\begin{equation}
\ushi\sim Bin(n,\theta)
\end{equation}
where $n\in \mathbb{Z}^{+}$ and $\theta\in[0,1]$ is the parameter we want to model with the posteriori distribution once given the priori distribution $P(\theta)$ and the likelihood function. We usually call this parameter Conversion Rate (CR).

\subsection{Choosing the prior distribution}
The prior distribution $P(\theta)$ enables us to include some knowledge in the final distribution of the KPI that we are modelling. There are several considerations when choosing the prior. However, for the sake of simplicity, in this case we will use the Beta distribution, which has a support of $[0,1]$, exactly the range of values of CR:

\begin{equation}
Beta_{\alpha,\beta}(\theta) = \frac{\theta^{\alpha-1}(1-\theta)^{\beta-1}}{B(\alpha,\beta)}
\end{equation}

where $\alpha$ and $\beta$ are the parameters of the distribution and B is the beta function, defined as follows:

\begin{equation}
B(\alpha,\beta) = \int_0^1t^{\alpha-1}(1-t)^{\beta-1}{B(\alpha,\beta)}
\end{equation}

 A particularly useful set of parameters is $\alpha=\beta=1$ which results in the uniform distribution. That is, a priori it is considered that all the values a parameter can take are equally likely to be taken.

\subsection{Obtaining the posteriori distribution}

To compute the posteriori distribution, apart from the priori distribution, we need the likelihood function, the probability mass function of the Binomial distribution in our case:
\begin{equation}
P(X=k)=\binom{n}{k}\theta^k(1-\theta)^{n-k}
\end{equation}

In order to compute the posteriori distribution, we will use the Bayes theorem:

\begin{equation}
\label{eq:bayesconvr}
P(\theta|n,k) = \frac{P(n,k|\theta)\cdot P(\theta)}{P(n,k)}
\end{equation}

We therefore need to obtain $P(n,k|\theta)$ and $P(\theta)$. As it will be seen later, the value of P(n,k) does not actually need to be computed.

First, we have
\begin{equation}
P(n,k|\theta) = \binom{n}{k}\theta^k(1-\theta)^{n-k}
\end{equation}

Back to Equation~\ref{eq:bayesconvr} we now have that

\begin{equation}
\begin{array}{c}
P(\theta|n,k) = \\
\\
=  \frac{P(n,k|\theta)\cdot P(\theta)}{P(n,k)}  = \\
\\
=  \frac{\binom{n}{k}\theta^k(1-\theta)^{n-k} \cdot \frac{\theta^{\alpha-1}(1-\theta)^{\beta-1}}{B(\alpha,\beta)}}{P(n,k)} = \\
\\
=  \frac{\binom{n}{k}}{P(n,k)B(\alpha,\beta)} \cdot \theta^{k + \alpha-1}(1-\theta)^{n-k+\beta-1}
\end{array}
\end{equation}

We can rewrite the previous equation as:
\begin{equation}
P(\theta|n,k) = \lambda \cdot \theta^{k + \alpha-1}(1-\theta)^{n-k+\beta-1}
\end{equation}

where
\begin{equation}
\lambda = \binom{n}{k}\frac{1}{P(n,k)B(\alpha,\beta)}
\end{equation}

Hence, our posteriori distribution for the $\theta$ is:

\begin{equation}
P(\theta|n,k) = Beta_{n+\alpha,n-k+\beta}(\theta)
\end{equation}

Or, in terms of sessions $\sessions$, purchases $\purchases$, and conversion rate $\convr$:

\begin{equation}
P(\convr|\sepu) = Beta_{\sessions+\alpha,\sessions-\purchases+\beta}(\theta)
\end{equation}

\subsection{Running an AB-Test on Conversion Rate}

Let's focus now on obtaining the probability that one conversion rate group is higher than the other (although the result is easily generalisable to $n$ test groups). That is:

\begin{equation}
\label{eq:integral-binomial}
P(\convr_A > \convr_B|\data) = \iint\limits_{\convr_A>\convr_B} P(\convr_A,\convr_B|\data)\; d\convr_A\; d\convr_B
\end{equation}

where $P(\convr_A,\convr_B|\data)$ is the joint distribution of both conversion rates. If we can assume independence of our tests groups\footnote{Although it is quite difficult that with our sparse social graph any two events are really independent, this is still quite a general assumption in most of our tests.}, then the following holds:

\begin{equation}
\begin{array}{c}
P(\convr_A,\convr_B|\data) = \\
\\
= P(\convr_A|\data_A)\cdot P(\convr_B|\data_B) = \\
\\
= \frac{P(\data_A|\convr_A)P(\convr_A)}{P(\data_A)} \cdot \frac{P(\data_B|\convr_B)P(\convr_B)}{P(\data_B)} = \\
\\
= \frac{P(\data_A|\convr_A)P(\convr_A)P(\data_B|\convr_B)P(\convr_B)}{P(\data_A)P(\data_B)}
\end{array}
\end{equation}

where $P(\data_A|\convr_A)$, $P(\convr_A)$, on the one hand, and $P(\data_B|\convr_B)$, $P(\convr_B)$, on the other hand, stand for the \emph{likelihood function} and the \emph{prior distribution}, and they can be obtained as explained above. Look in the \emph{BayesianABTest.py} file for an implementation of this and the other methods explained in this paper. Or run some of the examples in \emph{examples\_ABTests.py} to see  the method in action.

\section{Gaussian Variables: Modelling Attempts per Level}

When our data follow a Gaussian Distribution

$$X\sim N(\mu,\sigma^2)$$

the prior distribution becomes slightly more complicated. In this case we need to obtain a joint distribution of the form:

\begin{equation}
P(\mu,\sigma^2)
\end{equation}

From Gelman et al.~[4], we know that it can be described as follows:

\begin{equation}
P(\mu,\sigma^2) = P(\sigma^2)P(\mu|\sigma^2)
\end{equation}

where $P(\sigma^2)$ is an Inverse Gamma Distribution and $P(\mu|\sigma^2)$ is also a Gaussian Distribution.

Following Gelman's indications we can sample the posteriori distribution and then run the AB-Test using the code from \emph{BayesianABTest.py}.

\section{Log-Normal Variables: Modelling Spend}

When, for instance, we are modelling user spend, data are positive. Therefore, a Gaussian (or Normal) distribution is not appropriate due to its $(-\infty,+\infty)$ support.

In those situations, a Log-Normal distribution might fit our data more accurately. A Log-Normal Distribution is such that if $X$ is distributed log-normally, then $ln(X)$ is distributed normally:

\begin{equation}
\text{if}\;\; X\sim Log-N(\mu,\sigma^2) \text{   then   } ln(X) \sim N(\mu,\sigma^2)
\end{equation}

In a Log-Normal distribution the $\mu$ and $\sigma^2$ are not the mean and the variance. The mean, variance, median, and mode from a Log-Normal distribution are computed as follows:

\begin{itemize}
\item Mean: $e^{\mu+\frac{\sigma^2}{2}}$
\item Variance: $(e^{\sigma^2}-1)e^{2\mu+\sigma^2}$
\item Median: $e^{\mu}$
\item Mode: $e^{\mu-\sigma^2}$
\end{itemize}

An examplary code to sample the posteriori distribution can be found in \emph{BayesianABTest.py}.

\section{Modelling Spend per Session with a Two-Part Model Approach}
When our data contain a substantial amount of zeroes, a log-normal distribution is not appropriate, as its support is strictly positive. In this situation, we can first model the zeros vs. not zeros with a Binomial distribution, then the non-zeroes with a Log normal distribution, and obtain our final posteriori distribution as the product of these two.

The following code runs an AB-Test on such a combination of variables: each set of variable values is randomly sampled, and then each random value from the first set is multiplied by one random value from the second set. Look at the file \emph{BayesianABTest.py} for a possible implementation of this method.

\section{References}
\begin{enumerate}
\item Evan Miller. How Not To Run An A/B Test.
Available at: \url{http://www.evanmiller.org/how-not-to-run-an-ab-test.html}
\item Chris Stucchio. \emph{Analyzing conversion rates with Bayes Rule (Bayesian statistics tutorial)}. Available at: \url{http://www.chrisstucchio.com/blog/2013/bayesian_analysis_conversion_rates.html}
\item Sergey Feldman. \emph{Rich Relevance, Collection on Bayesian Testing}. Available at: \url{http://engineering.richrelevance.com/category/bayesian/}
\item Gelman,~A. et al. \emph{Bayesian Data Analysis}, 3rd Edition. CRC Press, 2013.
\end{enumerate}

\end{document}


\documentclass[rnaas]{aastex62}

\pdfoutput=1

\usepackage{lmodern}
\usepackage{microtype}
\usepackage{url}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{natbib}
\usepackage{multirow}
\usepackage{graphicx}
\bibliographystyle{aasjournal}

% ------------------ %
% end of AASTeX mods %
% ------------------ %

% Projects:
\newcommand{\project}[1]{\textsf{#1}}
\newcommand{\kepler}{\project{Kepler}}
\newcommand{\ktwo}{\project{K2}}

% references to text content
\newcommand{\documentname}{\textsl{Note}}
\newcommand{\figureref}[1]{\ref{fig:#1}}
\newcommand{\Figure}[1]{Figure~\figureref{#1}}
\newcommand{\figurelabel}[1]{\label{fig:#1}}
\renewcommand{\eqref}[1]{\ref{eq:#1}}
\newcommand{\Eq}[1]{Equation~(\eqref{#1})}
\newcommand{\eq}[1]{\Eq{#1}}
\newcommand{\eqalt}[1]{Equation~\eqref{#1}}
\newcommand{\eqlabel}[1]{\label{eq:#1}}

% TODOs
\newcommand{\todo}[3]{{\color{#2}\emph{#1}: #3}}
\newcommand{\dfmtodo}[1]{\todo{DFM}{red}{#1}}
\newcommand{\alltodo}[1]{\todo{TEAM}{red}{#1}}
\newcommand{\citeme}{{\color{red}(citation needed)}}

% math
\newcommand{\T}{\ensuremath{\mathrm{T}}}
\newcommand{\dd}{\ensuremath{ \mathrm{d}}}
\newcommand{\unit}[1]{{\ensuremath{ \mathrm{#1}}}}
\newcommand{\bvec}[1]{{\ensuremath{\boldsymbol{#1}}}}
\newcommand{\Gaussian}[3]{\ensuremath{\frac{1}{|2\pi #2|^\frac{1}{2}}
            \exp\left[ -\frac{1}{2}#1^\top #2^{-1} #1 \right]}}

% VECTORS AND MATRICES USED IN THIS PAPER
\newcommand{\Normal}{\ensuremath{\mathcal{N}}}
\newcommand{\mA}{\ensuremath{\bvec{A}}}
\newcommand{\mC}{\ensuremath{\bvec{C}}}
\newcommand{\mS}{\ensuremath{\bvec{\Sigma}}}
\newcommand{\mL}{\ensuremath{\bvec{\Lambda}}}
\newcommand{\vw}{\ensuremath{\bvec{w}}}
\newcommand{\vy}{\ensuremath{\bvec{y}}}
\newcommand{\vt}{\ensuremath{\bvec{\theta}}}
\newcommand{\vm}{\ensuremath{\bvec{\mu}(\bvec{\theta})}}
\newcommand{\vre}{\ensuremath{\bvec{r}}}
\newcommand{\vh}{\ensuremath{\bvec{h}}}
\newcommand{\vk}{\ensuremath{\bvec{k}}}

% typography obsessions
\setlength{\parindent}{3.0ex}

\begin{document}\raggedbottom\sloppy\sloppypar\frenchspacing

\title{%
    Scalable backpropagation for Gaussian Processes using celerite
}

\author[0000-0002-9328-5652]{Daniel Foreman-Mackey}
\affil{Center for Computational Astrophysics, Flatiron Institute, New York, NY}

\keywords{%
methods: data analysis ---
methods: statistical
}

\section{Introduction}

This research note presents a derivation and implementation of efficient and
scalable gradient computations using the \emph{celerite} algorithm for Gaussian
Process (GP) modeling.
The algorithms are derived in a ``reverse accumulation'' or
``backpropagation'' framework and they can be easily integrated into existing
automatic differentiation frameworks to provide a scalable method for
evaluating the gradients of the GP likelihood with respect to all input
parameters.
The algorithm derived in this note uses less memory and is more efficient than
versions using automatic differentiation and the computational cost scales
linearly with the number of data points.

GPs \citep{Rasmussen:2006} are a class of models used
extensively in the astrophysical literature to model stochastic processes.
The applications are broad-ranging and some examples include the time domain
variability of astronomical sources \citep{Brewer:2009, Kelly:2014,
Haywood:2014, Rajpaul:2015, Foreman-Mackey:2017, Angus:2018}, data-driven
models of light curves or stellar spectra \citep{Wang:2012, Luger:2016,
Czekala:2017}, and the cosmic microwave background
\citep{Bond:1987,Wandelt:2003}.
In all of these applications, the calculation and optimization of the GP
marginalized likelihood function (here we follow the notation of
\citealt{Foreman-Mackey:2017})
\begin{eqnarray}\eqlabel{loglike}
\log \mathcal{L}(\bvec{\theta},\,\bvec{\alpha}) &=&
    -\frac{1}{2}\,\left[\bvec{y} - \bvec{\mu}_\bvec{\theta}\right]^\T\,
        {K_\bvec{\alpha}}^{-1}\,\left[\bvec{y}-\bvec{\mu}_\bvec{\theta}\right]
    -\frac{1}{2}\,\log\det K_\bvec{\alpha} + \mathrm{constant}
\end{eqnarray}
is generally the computational bottleneck.
The details of these models are omitted here (see \citealt{Rasmussen:2006} and
\citealt{Foreman-Mackey:2017} for details), but the key point is that, for a
dataset with $N$ data points, every evaluation of a GP model requires
computing the log-determinant and multiplying a vector by the inverse of
%\footnote{Or, more specifically, a linear operator
%that can left multiply a vector or a matrix by the inverse of this matrix.}
the $N \times N$ covariance matrix $K_\bvec{\alpha}$.
The computational cost of these operations scales as $\mathcal{O}(N^3)$ in the
general case, but the \emph{celerite} method was recently introduced in the
astronomical literature to compute the GP likelihood for a class of
one-dimensional models with $\mathcal{O}(N)$ scaling
\citep{Ambikasaran:2015, Foreman-Mackey:2017}.

The details of the \emph{celerite} method can be found in
\citet{Foreman-Mackey:2017} and I will not repeat them here.
The only difference in notation is that all the matrices in what follows are
the ``pre-conditioned'' matrices that are indicated with a tilde by
\citet{Foreman-Mackey:2017}.
The tilde is not included here for simplicity and to improve the readability
of the algorithms.
I will also use the symbol $P$ for the $(N-1) \times J$ pre-conditioning matrix
that was called $\phi$ by \citet{Foreman-Mackey:2017}.
The Cholesky factorization algorithm derived by \citet[][their
Equation~46]{Foreman-Mackey:2017} is as follows:

\medskip
\begin{minipage}{\linewidth}
\textbf{function} \texttt{celerite\_factor}($U$, $P$, $\bvec{d}$, $W$) \\
\hspace*{2em}\textsf{\# At input $\bvec{d} = \bvec{a}$ and $W = V$} \\
\hspace*{2em}$S \gets$ \texttt{zeros}($J$, $J$) \\
    \hspace*{2em}$\bvec{w}_1 \gets \bvec{w}_1 / d_{1}$ \\
\hspace*{2em}\textbf{for} $n = 2,\ldots,N$:\\
\hspace*{2em}\hspace*{2em}$S \gets \texttt{diag}(\bvec{p}_{n-1})\,[
    S + d_{n-1}\,{\bvec{w}_{n-1}}^\T\,{\bvec{w}_{n-1}}
]\,\texttt{diag}(\bvec{p}_{n-1})$ \\
\hspace*{2em}\hspace*{2em}$d_{n} \gets d_{n} - \bvec{u}_n\,S\,{\bvec{u}_n}^\T$\\
\hspace*{2em}\hspace*{2em}$\bvec{w}_n \gets \left[\bvec{w}_n -
    \bvec{u}_n\,S \right] / d_{n}$\\
    \hspace*{2em}\textbf{return} $\bvec{d}$, $W$, $S$
\end{minipage}
\medskip

\noindent In this algorithm, the \texttt{zeros}$(J,\,K)$ function creates a $J
\times K$ matrix of zeros, the \texttt{diag} function creates a diagonal
matrix from a vector, and $\bvec{x}_n$ indicates a row vector made from the
$n$-th row of the matrix $X$.
The computational cost of this algorithm scales as $\mathcal{O}(N\,J^2)$.
Using this factorization, the log-determinant of $K$ is
\begin{eqnarray}
\log \det K &=& \sum_{n=1}^N \log d_{n} \quad.
\end{eqnarray}
Similarly, \citet{Foreman-Mackey:2017} derived a $\mathcal{O}(N\,J)$ algorithm
to apply the inverse of $K$ (i.e.\ compute $Z = K^{-1}\,Y$) as follows
(Equations~47 and~48 in \citealt{Foreman-Mackey:2017}):

\medskip
\begin{minipage}{\linewidth}
\textbf{function} \texttt{celerite\_solve}($U$, $P$, $\bvec{d}$, $W$, $Z$) \\
\hspace*{2em}\textsf{\# At input $Z = Y$} \\
\hspace*{2em}$F \gets$ \texttt{zeros}($J$, $N_\mathrm{rhs}$) \\
\hspace*{2em}\textbf{for} $n = 2,\ldots,N$:\\
\hspace*{2em}\hspace*{2em}$F \gets \texttt{diag}(\bvec{p}_{n-1})\,[F +
    {\bvec{w}_{n-1}}^\T\,\bvec{z}_{n-1}]$\\
\hspace*{2em}\hspace*{2em}$\bvec{z}_n \gets \bvec{z}_n - \bvec{u}_n\,F$\\
\hspace*{2em}\textbf{for} $n = 1,\ldots,N$:\\
\hspace*{2em}\hspace*{2em}$\bvec{z}_n \gets \bvec{z}_n / d_{n}$\\
\hspace*{2em}$G \gets$ \texttt{zeros}($J$, $N_\mathrm{rhs}$) \\
\hspace*{2em}\textbf{for} $n = N-1,\ldots,1$:\\
\hspace*{2em}\hspace*{2em}$G \gets \texttt{diag}(\bvec{p}_{n})\,[G +
    {\bvec{u}_{n+1}}^\T\,\bvec{z}_{n+1}]$\\
    \hspace*{2em}\hspace*{2em}$\bvec{z}_n \gets \bvec{z}_n - \bvec{w}_n\,G$\\
\hspace*{2em}\textbf{return} $Z$, $F$, $G$
\end{minipage}
\medskip

\noindent The empirical scaling of these algorithms is shown in
\Figure{figure}.

\section{Gradients of GP models using celerite}

It is standard practice to make inferences using \Eq{loglike} by optimizing or
choosing a prior and sampling with respect to $\bvec{\theta}$ and
$\bvec{\alpha}$.
Many numerical inference methods (like non-linear optimization or Hamiltonian
Monte Carlo) can benefit from efficient calculation of the gradient of
\Eq{loglike} with respect to the parameters.
The standard method of computing these gradients uses the identity
\citep{Rasmussen:2006}
\begin{eqnarray}\eqlabel{naive-grad}
\frac{\dd \log \mathcal{L}}{\dd \alpha_k} &=&
    \frac{1}{2}\,\mathrm{Tr}\left[
        \left[
        \bvec{\tilde{r}}\,\bvec{\tilde{r}}^\T - {K_\bvec{\alpha}^{-1}}
        \right]
        \,\frac{\dd K}{\dd \alpha_k}
    \right]
\end{eqnarray}
where
\begin{eqnarray}
    \bvec{\tilde{r}} &=&
        {K_\bvec{\alpha}}^{-1}\,\left[\bvec{y}-\bvec{\mu}_\bvec{\theta}\right]
    \quad.
\end{eqnarray}
Similar equations exist for the parameters $\bvec{\theta}$.
Even with a scalable method of applying ${K_\bvec{\alpha}}^{-1}$, the
computational cost of \Eq{naive-grad} scales as $\mathcal{O}(N^2)$.
This scaling is prohibitive when applying the \emph{celerite} method to large
datasets and I have not found a simple analytic method of improving this
scaling for semi-separable matrices.
However, it was recently demonstrated that substantial computational gains can
be made by directly differentiating Cholesky factorization algorithms even in
the general case \citep{Murray:2016}.

Following this reasoning and using the notation from an excellent review of
matrix gradients \citep{Giles:2008}, I present the reverse-mode gradients of
the \emph{celerite} method.
While not yet popular within astrophysics, ``reverse accumulation'' of
gradients (also known as ``backpropagation'') has recently revolutionized the
field of machine learning \citep[see][for example]{LeCun:2015} by enabling the
non-linear optimization of models with large numbers of parameters.
The review \citep{Giles:2008} provides a thorough overview of these methods
and the interested reader is directed to that discussion for details and for
an explanation of the notation.

Using the notation from \citet{Giles:2008}, after some tedious
algebra, the reverse accumulation function corresponding to
\texttt{celerite\_factor} is found to be:

\medskip
\begin{minipage}{\linewidth}
\textbf{function} \texttt{celerite\_factor\_rev}($U$, $P$, $\bvec{d}$, $W$,
    $S$, $\bar{S}$, $\bar{\bvec{a}}$, $\bar{V}$) \\
\hspace*{2em}\textsf{\# At input $\bar{\bvec{a}} = \bar{\bvec{d}}$ and
    $\bar{V} = \bar{W}$}\\
\hspace*{2em}$\bar{U} \gets \mathrm{zeros}(N,\,J)$\\
\hspace*{2em}$\bar{P} \gets \mathrm{zeros}(N-1,\,J)$\\
\hspace*{2em}$\bar{\bvec{v}}_N \gets \bar{\bvec{v}}_N / d_N$\\
\hspace*{2em}\textbf{for} $n = N,\ldots,2$:\\
\hspace*{2em}\hspace*{2em}$\bar{a}_n \gets \bar{a}_n -
    \bvec{w}_n\,{\bar{\bvec{v}}_n}^\T$\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{u}}_n \gets - [\bar{\bvec{v}}_n +
    2\,\bar{a}_n\,\bvec{u}_n]\,S$\\
\hspace*{2em}\hspace*{2em}$\bar{S} \gets \bar{S} -
    {\bvec{u}_n}^\T\,[\bar{\bvec{v}}_n + \bar{a}_n\,\bvec{u}_n]$\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{p}}_{n-1} \gets \mathrm{diag}(\bar{S}\,S
    \,\mathrm{diag}(\bvec{p}_{n-1})^{-1} + \mathrm{diag}(\bvec{p}_{n-1})^{-1}\,
    S\,\bar{S})$\\
\hspace*{2em}\hspace*{2em}$\bar{S} \gets \mathrm{diag}(\bvec{p}_{n-1})\,
    \bar{S}\,\mathrm{diag}(\bvec{p}_{n-1})$\\
\hspace*{2em}\hspace*{2em}$\bar{d}_{n-1} \gets \bar{d}_{n-1} +
    \bvec{w}_{n-1}\,\bar{S}\,{\bvec{w}_{n-1}}^\T$\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{v}}_{n-1} \gets \bar{\bvec{v}}_{n-1}
    / d_{n-1} + \bvec{w}_{n-1}\,[\bar{S} + \bar{S}^\T]$\\
\hspace*{2em}\hspace*{2em}$S \gets \mathrm{diag}(\bvec{p}_{n-1})^{-1}\,S\,
    \mathrm{diag}(\bvec{p}_{n-1})^{-1}
    - d_{n-1}\,{\bvec{w}_{n-1}}^\T\,\bvec{w}_{n-1}$\\
\hspace*{2em}$\bar{a}_1 \gets \bar{a}_1 -
    \bar{\bvec{v}}_1\,{\bvec{w}_1}^\T$\\
\hspace*{2em}\textbf{return} $\bar{U}$, $\bar{P}$, $\bar{\bvec{a}}$, $\bar{V}$
\end{minipage}
\medskip

\noindent Similarly, the reverse accumulation function for to
\texttt{celerite\_solve} is:

\medskip
\begin{minipage}{\linewidth}
\textbf{function} \texttt{celerite\_solve\_rev}($U$, $P$, $\bvec{d}$, $W$,
    $Z$, $F$, $G$, $\bar{F}$, $\bar{G}$, $\bar{Y}$) \\
\hspace*{2em}\textsf{\# At input $\bar{Y} = \bar{Z}$}\\
\hspace*{2em}$\bar{U} \gets \mathrm{zeros}(N,\,J)$\\
\hspace*{2em}$\bar{P} \gets \mathrm{zeros}(N-1,\,J)$\\
\hspace*{2em}$\bar{\bvec{d}} \gets \mathrm{zeros}(N)$\\
\hspace*{2em}$\bar{W} \gets \mathrm{zeros}(N,\,J)$\\
\hspace*{2em}\textbf{for} $n = 1,\ldots,N-1$:\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{w}}_n \gets
    - \bar{\bvec{y}}_n\,G^\T$\\
\hspace*{2em}\hspace*{2em}$\bar{G} \gets \bar{G} - {\bvec{w}_n}^\T \,
    \bar{\bvec{y}}_n$\\
\hspace*{2em}\hspace*{2em}$\bvec{z}_n \gets \bvec{z}_n + \bvec{w}_n\,G$\\
\hspace*{2em}\hspace*{2em}$G \gets \mathrm{diag}(\bvec{p}_n)^{-1}\,G$\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{p}}_n \gets \mathrm{diag}(
    \bar{G}\,G^\T)$\\
\hspace*{2em}\hspace*{2em}$\bar{G} \gets
    \mathrm{diag}(\bvec{p}_n)\,\bar{G}$\\
\hspace*{2em}\hspace*{2em}$G \gets G - {\bvec{u}_{n+1}}^\T\,\bvec{z}_{n+1}$\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{u}}_{n+1} \gets
    \bvec{z}_{n+1}\,\bar{G}^\T$\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{y}}_{n+1} \gets
    \bvec{u}_{n+1}\,\bar{G}$\\
\hspace*{2em}\textbf{for} $n = 1,\ldots,N$:\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{y}}_n \gets \bar{\bvec{y}}_n/d_n$\\
\hspace*{2em}\hspace*{2em}$\bar{d}_n \gets -
    \bvec{z}_n\,{\bar{\bvec{y}}_n}^\T$\\
\hspace*{2em}\textbf{for} $n = N,\ldots,2$:\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{u}}_n \gets \bar{\bvec{u}}_n
    - \bar{\bvec{y}}_n\,F^\T$\\
\hspace*{2em}\hspace*{2em}$\bar{F} \gets \bar{F} - {\bvec{u}_n}^\T \,
    \bar{\bvec{y}}_n$\\
\hspace*{2em}\hspace*{2em}$F \gets \mathrm{diag}(\bvec{p}_{n-1})^{-1}\,F$\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{p}}_{n-1} \gets \bar{\bvec{p}}_{n-1} +
    \mathrm{diag}(\bar{F}\,F^\T)$\\
\hspace*{2em}\hspace*{2em}$\bar{F} \gets
    \mathrm{diag}(\bvec{p}_{n-1})\,\bar{F}$\\
\hspace*{2em}\hspace*{2em}$F \gets F - {\bvec{w}_{n-1}}^\T\,\bvec{z}_{n-1}$\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{w}}_{n-1} \gets \bar{\bvec{w}}_{n-1} +
    \bvec{z}_{n-1}\,\bar{F}^\T$\\
\hspace*{2em}\hspace*{2em}$\bar{\bvec{y}}_{n-1} \gets \bar{\bvec{y}}_{n-1} +
    \bvec{w}_{n-1}\,\bar{F}$\\
\hspace*{2em}\textbf{return} $\bar{U}$, $\bar{P}$, $\bar{\bvec{d}}$,
    $\bar{W}$, $\bar{Y}$
\end{minipage}
\medskip

\noindent A reference C++ implementation of this algorithm can be found online
\citep{Foreman-Mackey:2018} and \Figure{figure} shows the performance of this
implementation.

\newpage
\section{Discussion}

This research note presents the algorithms needed to efficiently compute
gradients of GP models applied to large datasets using the \emph{celerite}
method.
These developments increase the performance of inference methods based on
\emph{celerite} and improve the convergence properties of non-linear
optimization routines.
Furthermore, the derivation of reverse accumulation algorithms for
\emph{celerite} allow its integration into popular model building and
automatic differentiation libraries like Stan \citep{Carpenter:2015},
TensorFlow \citep{Abadi:2016}, and others.

\acknowledgments
It is a pleasure to thank
Eric Agol,
Sivaram Ambikasaran, and
Victor Minden
for conversations that inspired this work.
A reference implementation of these algorithms, benchmarks, and tests can be
found at \url{https://github.com/dfm/celerite-grad} and
\citet{Foreman-Mackey:2018}.

\begin{figure}[htbp]
\begin{center}
\includegraphics[width=0.8\textwidth]{figure.pdf}
\caption{%
    The empirical computational scaling of the algorithms presented in this
    note.
    \emph{(top row)}: The cost of computing \eq{loglike} as a
    function of the number of data points ($N$; left) and the model complexity
    ($J$; right).
    \emph{(bottom row)}: The cost of computing the gradient of \eq{loglike}
    with respect to the vector $\bvec{a}$ and the matrices $U$, $V$, and $P$.
    In the left panels, each line corresponds to a different value of $J$ as
    indicated in the legend (with $J$ increasing from bottom to top).
    Similarly, in the right panels, the lines correspond to different values
    of $N$ increasing from bottom to top.
    In both cases, the theoretical scaling is $\mathcal{O}(N,J^2)$.
\figurelabel{figure}}
\end{center}
\end{figure}

\bibliography{celerite}

\end{document}


\documentclass[main.tex]{subfiles}
\begin{document}

\chapter{Free field theories}

\section{Lagrangian and Hamiltonian formalisms}

\marginpar{Saturday\\ 2020-5-2, \\ compiled \\ \today}

The Lagrangian and Hamiltonian formalism can aid in the description of systems with either a finite of infinite number of degree of freedom.

\subsection{Classical system with finite DoF: Lagrangian formalism}

The usual example is a system of particles labelled by the index \(i\) with masses \(m_i\), positions \(q_i (t)\) and velocities \(\dot{q}_{i}(t)\).

If the forces acting on the particles are \textbf{conservative}, we can express them in terms of a potential, which we assume not to depend on the velocities nor on time: \(V(q_{i})\).
From now on when we write \(q\) or \(\dot{q}\) we will mean the full vector of the positions or velocities.

If this is the case, the motion of the particles is described by Newton's equation:
%
\begin{align}
m_i \ddot{q}_{i} = - \pdv{V}{q_{i}}
\,.
\end{align}

This can alternatively be described in terms of a function called the Lagrangian:
%
\begin{align}
L(q, \dot{q}, t)
\,,
\end{align}
%
which could depend on time, but we usually assume it to be independent of time explicitly, that is, \(\partial_{t} L = 0\) (although the \emph{total} derivative of the Lagrangian may be nonzero!).

The condition of the Lagrangian being explicitly time-independent is equivalent to the system of forces being conservative.

The Lagrangian can be a generic function of the positions and velocities of the particles, but in order to reproduce Newton's law we need it to be in the form
%
\begin{align}
L = T-V = \frac{1}{2} m_i \dot{q}_i^2 - V(q_i)
\,,
\end{align}
%
where a sum over the particles is implied in the kinetic energy term.

Typically problems in Lagrangian mechanics are given by fixing the boundary conditions as the initial and final position, as opposed to writing the initial values of position and velocity.

We want to find the physical trajectory the particle(s) will take corresponding to those initial and final conditions.\footnote{There can be issues arising from this approach: for instance, the process for finding trajectories may find many equivalent ones, for example in situations with some symmetry or nontrivial topology.
In the gravitational two-body problem we can ask what is the stationary action path needed in order to reach the antipodal point to the current one: there are infinite ones, corresponding to the choice of azimuthal angle of the initial velocity.
However, this is not really a problem, since the approach we are about to introduce --- Hamilton's principle of stationary action --- just provides a formulation which eventually yields the same differential equations as the initial value problem; physically the velocity as well as the position of the particles is well determined in the initial moment.}

This can be accomplished using Hamilton's principle of stationary action. We start by introducing the action:
%
\begin{align}
S[q(t), t _{\text{in}}, t _{\text{fin}}] = \int_{t _{\text{in}}}^{t _{\text{fin}}} L(q(t), \dot{q}(t)) \dd{t}
\,,
\end{align}
%
which depends on the full \emph{path} \(q(t)\): the integral is computed by evaluating the Lagrangian along it.
Because of this the action \(S\) is called a \emph{functional}, since it is a function of a function. The Lagrangian, on the other hand, is  a regular function.

We then state the following:
\begin{claim}
The path taken physically by the system satisfies the stationary action principle:
%
\begin{align}
\delta S = 0
\,.
\end{align}

The variation is the functional derivative of \(S\), meant to be a variation in the infinite-dimensional space of possible curves.
\end{claim}

This is not a proven principle, but rather an axiom to be taken, analyzed and confronted with experiment.

The action principle, as stated, only guarantees that the action should be stationary: it could be a minimum or a maximum.
There are considerations to be made about stability: a maximum-action path will solve the equations of motion and thus be physical, but it will be unstable. This is why the principle is often referred to as the minimum-action principle.

Let us calculate what the variation of the action means physically. We start by considering two nearby paths, \(\gamma \) and \(\gamma '\), such that
%
\begin{align}
\gamma ' = \gamma + \delta_0 \gamma
\,,
\end{align}
%
so that the coordinates \(q\) at a time \(t\) are given by
%
\begin{align}
q_{\gamma } (t)
\qquad \text{and} \qquad
q_{\gamma '} (t) = q_{\gamma }(t) + \delta_0 q_{\gamma }(t)
\,,
\end{align}
%
and we impose that the variation of the path, \(\delta_0 q_{\gamma } (t)\), is zero at the initial and final time, so that the boundary conditions are satisfied: \(\delta_0 q_{\gamma }(t _{\text{in}}) = 0 = \delta_0 q_{\gamma }(t _{\text{fin}})\).

The reason for the subscript \(0\) for the variation \(\delta \) is that we consider \emph{synchronous} variations: the difference between the perturbed and unperturbed trajectory, at a \emph{fixed time}.
Also, we take the synchronous variation of the action:
%
\begin{subequations}
\begin{align}
\delta_0 S [q_{\gamma }] &= S [q_{\gamma } + \delta_0 q_{\gamma }] - S[q_{\gamma }]  \\
&= \int_{t _{\text{in}}}^{t _{\text{fin}}} \delta_0 L (q_{\gamma }, \dot{q}_{\gamma }) \dd{t}  \\
&= \int_{t _{\text{in}}}^{t _{\text{fin}}} \qty[ \pdv{L}{q} \cdot \delta_0 q + \pdv{L}{\dot{q}} \cdot \dot{q}] \dd{t}  \\
&= \int_{t _{\text{in}}}^{t _{\text{fin}}} \qty[\pdv{L}{q}  - \dv{}{t} \pdv{L}{\dot{q}}] \cdot \delta_0 q \dd{t}
\,,
\end{align}
\end{subequations}
%
since, as we are considering synchronous variations, \(\delta_0 \) and \(\partial_{t}\) commute; also we neglected boundary terms in the integration by parts as they are set to zero by the fact that the variation of the path,  \(\delta_0 q_{\gamma }\), is zero at the boundaries.

Since this must hold for any variation of the path, by the fundamental lemma of the calculus of variation the integrand must vanish: this yields the Euler-Lagrange equations
%
\begin{align}
\pdv{L}{q} - \dv[]{}{t} \pdv{L}{\dot{q}} = 0
\,.
\end{align}

Note that there is \textbf{gauge freedom} in the choice of Lagrangian: an easy symmetry to see is the scaling one; the Lagrange equations of \(L\) and \(cL\) for \(c \in \mathbb{R}\) are the same.
Also, the equations for \(L\) and \(L + \dv*{F}{t}\) are the same: the action is the integral in time of the Lagrangian, therefore adding a total derivative to it shifts the action by a constant \(\eval{\Delta F}_{\text{in}}^{\text{fin}}\), which vanishes when taking the variation.

\subsection{Classical system with finite DoF: Hamiltonian formalism}

We start off by defining the momenta: in component form, they are
%
\begin{align}
p_{i} = \pdv{L}{\dot{q}^{i}}
\,.
\end{align}

As long as the Lagrangian is well behaved\footnote{The formal requirement is that the matrix
%
\begin{align}
\pdv[2]{L}{\dot{q}}{\dot{q}}
\,
\end{align}
%
should be invertible.}
we can move between momenta and velocities, by using the inverse relation \(\dot{q} = \dot{q} (q, p)\).

The Hamiltonian is then defined as
%
\begin{align}
H(q, p) = \dot{q}(q, p) \cdot p - L (q, \dot{q}(q, p))
\,,
\end{align}
%
and this procedure is called a \emph{Legendre transform}.
Note that since we assumed the Lagrangian to have no explicit dependence of time the Hamiltonian will not depend on time either. This is tied to the conservation of energy: if the Lagrangian is given by \(T - V\) the Hamiltonian is given by \(T + V\), and thus represents the total energy of the system. A theorem in Hamiltonian mechanics is
%
\begin{align}
\dv{H}{t} = \pdv{H}{t}
\,,
\end{align}
%
so the variation of the energy in the evolution of the system (total derivative) is equal to the partial derivative of the Hamiltonian with respect to time.

Let us take the functional derivative of the Hamiltonian: using indices for the clarity of the tensorial structure, we find
%
\begin{subequations}
\begin{align}
\delta H &= \pdv{H}{p_{i}}  \delta p_{i} + \pdv{H}{q^{i}} \delta q^{i}  \\
&= \qty[\dot{q}^{i} + \pdv{\dot{q}^{j}}{p_{i}} p_{j} - \pdv{L}{\dot{q}^{j}}  \pdv{\dot{q}^{j}}{p_{i}}] \delta p_{i}
+ \qty[\pdv{\dot{q}^{j}}{q^{i}} p_{j}
- \pdv{L}{q^{i}} - \pdv{L}{\dot{q}^{j}} \pdv{\dot{q}^{j}}{q^{i}}] \delta q^{i}  \\
&= \dot{q}^{i} \delta p_{i} - \pdv{L}{q^{i}} \delta q^{i}
\marginnote{Used the fact that \(p_{i} = \pdv*{L}{\dot{q}^{i}}\).}  \\
&= \dot{q}^{i} \delta p_{i} - \dot{p}_{i} \delta q^{i}
\marginnote{Used the Euler-Lagrange equations.}
\,,
\end{align}
\end{subequations}
%
so we can equate the first and last equations to find the coupled equations
%
\boxalign{
\begin{align}
\dot{q}^{i} = \pdv{H}{p_{i}} \qquad \text{and} \qquad
\dot{p}_{i} = -\pdv{H}{q^{i}}
\,.
\end{align}}
%

These are known as \textbf{Hamilton's equations of motion}, they are an equivalent formulation of Lagrange's ones but they may be more useful in certain contexts such as when performing numerical integration.

In the Hamiltonian contexts it is useful to define the Poisson bracket.
If we define the operator \(\nabla\) as the derivative operator on the \(2N\)-dimensional \(q, p\) phase space, then the Poisson bracket is defined as
%
\begin{subequations}
\begin{align}
\qty{f, g} &= \qty(\nabla_{i} f) \mathbb{J}_{ij} \qty(\nabla_{j} g)  \\
&= \pdv{f}{q^{i}} \pdv{g}{p_{i}}
- \pdv{f}{p_{i}} \pdv{g}{q^{i}}
\,,
\end{align}
\end{subequations}
%
where we defined the \emph{symplectic unity} tensor
%
\begin{subequations}
\begin{align}
\mathbb{J} = \left[\begin{array}{cc}
0 & \mathbb{1}_{N} \\
-\mathbb{1}_{N} & 0
\end{array}\right]
\,.
\end{align}
\end{subequations}

With the aid of these, and defining the vector \(X = \qty[q, p]^{\top}\) we can write Hamilton's equations as
%
\begin{align}
\dv{X}{t} = \qty{X, H}
\,,
\end{align}
%
or, more explicitly,
%
\begin{align}
\dot{q} = \qty{q, H}
\qquad \text{and} \qquad
\dot{p} = \qty{p, H}
\,.
\end{align}

The Poisson brackets between the coordinates in phase space are given by
%
\begin{align}
\qty{q^{i}, q^{j}} = 0
 \qquad
\qty{p_{i}, p_{j}} = 0
\qquad
\qty{q^{i}, p_{j}} = \delta^{i}_{j}
\,.
\end{align}

A generic function's variation in time is given by
%
\begin{align}
\dv{f}{t} = \pdv{f}{t} + \qty{f, H}
\,.
\end{align}

There is a clear analogy between the classical Poisson bracket and the commutator in quantum mechanics.

\end{document}


\documentclass[12pt]{cdblatex}
\usepackage{exercises}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% --------------------------------------------------------------------------------------------
\section*{Exercise 4.2 Inconsistent free indices}

\begin{cadabra}
   {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w#}::Indices(position=independent).

   def deriv (poly):

       \delta^{a}::Weight(label=\epsilon).

       bah := @(poly).

       substitute     (bah,$x^{a} -> x^{a} + \delta^{a}$)
       distribute     (bah)

       foo := @(bah) - @(poly).

       keep_weight    (foo, $\epsilon = 1$)
       substitute     (foo, $\delta^{a} -> 1$)

       return foo

   # ---------------------------------------------------------------

   poly := c^{a}
         + c^{a}{}_{b} x^b
         + c^{a}{}_{b c} x^b x^c.    # cdb (ex-0402.100,poly)

   dpoly = deriv (poly)              # cdb (ex-0402.101,dpoly)

\end{cadabra}

\begin{dgroup*}
   \Dmath*{  p = \Cdb*{ex-0402.100} }
   \Dmath*{ dp = \Cdb*{ex-0402.101} }
\end{dgroup*}

\end{document}


  \subsection{House size crowds out investment}

  However, when comparing households on a total wealth basis, i.e.\ their liquid assets plus expected house liquidation, we can see that house size crowds out investment for households with low liquid wealth. In figure~\ref{fig:liquidAssetsByEquity}, we can consider a household whose house size is equal to 5 (5 times their yearly net income) and liquid assets are 0, so their total expected wealth is 5. As this household becomes wealthier, they invest all of their liquid assets in the stock market (such that their risky share is 100 percent), up to the point where they start rebalancing their portfolio between the risky and the safe asset. In this region, they are constrained from investing in the stock market by their low liquid wealth, as they surely would like to invest more in the market. This point becomes clearer by comparing the household to an equally wealthy peer with a smaller house.  At the point where the household with house size of 5 has liquid wealth of 1 (1 times their yearly net income), they are investing less into the stock market in absolute terms than an equally wealthy household whose house size is equal to 2 and liquid assets are equal to 4. The total expected wealth of both these households is 6, but the household with the larger house is investing fewer assets in the stock market than the household with the smaller house. As their total wealth increases, however, both households are unconstrained by their house size and end up investing about the same amount into the stock market in absolute terms.

  \providecommand{\figName}{}
  \renewcommand{\figName}{liquidAssetsByEquity}
  \providecommand{\figFile}{}
  \renewcommand{\figFile}{\figName}
  \input{\FigDir/\figName} % Read in the tex to generate the figure


\documentclass[utf8x,hyperref={pdfpagelabels=false}]{beamer}

\usepackage[utf8x]{inputenc}
\usepackage[OT1]{fontenc}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{tikz}
\usetikzlibrary{shapes.arrows}
%\logo{\includegraphics[width=.8in]{UdeM_NoirBleu_logo_Marie_crop}}

\usetheme{Malmoe}  % Now it's a beamer presentation with the lisa theme!
\usecolortheme{beaver}
\setbeamertemplate{footline}[page number]
\setbeamertemplate{navigation symbols}{}

\lstloadlanguages{Python}

\definecolor{darkgreen}{RGB}{0,93,21}
\definecolor{greenblue}{RGB}{40,110,126}
\definecolor{lightgray}{RGB}{246,246,246}
\definecolor{bordergray}{RGB}{193,193,193}
\definecolor{lightblue}{RGB}{0,114,168}
\definecolor{methblue}{RGB}{0,31,108}

\newcommand{\superscript}[1]{\ensuremath{^{\textrm{#1}}}}

\mode<presentation>

\title{Introduction to Theano}
\author{%
\footnotesize
Arnaud Bergeron \newline
(slides adapted by Frédéric Bastien from slides by Ian G.) \newline
(further adapted by Arnaud Bergeron)
}
\date{February 26, 2015}

\lstdefinestyle{theano}{
language=Python,
basicstyle=\fontfamily{pcr}\selectfont\footnotesize,
keywordstyle=\color{darkgreen}\bfseries,
commentstyle=\color{greenblue}\itshape,
%commentstyle=\color{blue}\itshape,
stringstyle=\color{violet},
showstringspaces=false,
tabsize=4,
backgroundcolor=\color{lightgray},
frame=single,
emph={[2]__init__,make_node,perform,infer_shape,c_code,make_thunk,grad,R_op},emphstyle={[2]\color{methblue}},
emph={[3]self},emphstyle={[3]\color{darkgreen}},
moredelim=**[is][{\color{red}}]{`}{`}
}

% We don't have code till the end of the file.
\lstdefinestyle{output}{
language={},
basicstyle=\ttfamily\footnotesize,
backgroundcolor=\color{white},
frame={},
breaklines=true,
emph={[2]},
emph={[3]},
}

\lstset{style=theano}

\newcommand{\code}[1]{\lstinline[emph={[2]}]|#1|}

\begin{document}

\begin{frame}[plain]
 \titlepage
% \vspace{-5em}
% \includegraphics[width=1in]{../hpcs2011_tutorial/pics/lisabook_logo_text_3.png}
% \hfill
% \includegraphics[width=.8in]{../hpcs2011_tutorial/pics/UdeM_NoirBleu_logo_Marie_crop}
\end{frame}

\section{Outline}
\begin{frame}{High level}\setcounter{page}{1}
  \begin{itemize}
  \item Overview of library (3 min)
  \item Building expressions (30 min)
  \item Compiling and running expressions (30 min)
  \item Modifying expressions (25 min)
  \item Debugging (30 min)
  \item Citing Theano (2 min)
  \end{itemize}
\end{frame}

\begin{frame}{Overview of Library}
  Theano is many things
  \begin{itemize}
  \item Language
  \item Compiler
  \item Python library
  \end{itemize}
\end{frame}

\begin{frame}{Overview}
  Theano language:
  \begin{itemize}
  \item Operations on scalar, vector, matrix, tensor, and sparse variables
  \item Linear algebra
  \item Element-wise nonlinearities
  \item Convolution
  \item Extensible
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Overview}
  Using Theano:
  \begin{itemize}
  \item define expression $f(x,y) = x + y$
\begin{lstlisting}
>>> z = x + y
\end{lstlisting}
  \item compile expression
\begin{lstlisting}
>>> f = theano.function([x, y], z)
\end{lstlisting}
  \item execute expression
\begin{lstlisting}
>>> f(1, 2)
3
\end{lstlisting}
  \end{itemize}
\end{frame}

\section{Building}

\begin{frame}{Building expressions}
  \begin{itemize}
  \item Scalars
  \item Vectors
  \item Matrices
  \item Tensors
  \item Broadcasting
  \item Reduction
  \item Dimshuffle
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Scalar math}
\begin{lstlisting}
from theano import tensor as T
x = T.scalar()
y = T.scalar()
z = x+y
w = z*x
a = T.sqrt(w)
b = T.exp(a)
c = a ** b
d = T.log(c)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Vector math}
\begin{lstlisting}
from theano import tensor as T
x = T.vector()
y = T.vector()
# Scalar math applied elementwise
a = x * y
# Vector dot product
b = T.dot(x, y)
# Broadcasting
c = a + b
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Matrix math}
\begin{lstlisting}
from theano import tensor as T
x = T.matrix()
y = T.matrix()
a = T.vector()
# Matrix-matrix product
b = T.dot(x, y)
# Matrix-vector product
c = T.dot(x, a)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Tensors}
   \begin{itemize}
    \item Dimensionality defined by length of ``broadcastable'' argument
    \item Can add (or do other elemwise op) on two
      tensors with same dimensionality
    \item Duplicate tensors along broadcastable axes to
      make size match
  \end{itemize}
\begin{lstlisting}
from theano import tensor as T
tensor3 = T.TensorType(
    broadcastable=(False, False, False),
    dtype='float32')
x = tensor3()
\end{lstlisting}
\end{frame}

\begin{frame}{Broadcasting}
\begin{tabular}{lcccccccl}
    &
    \begin{tabular}{cc}
        1 & 2 \\
        3 & 4 \\
        5 & 6 \\
    \end{tabular} &
    + &
    \begin{tabular}{cc}
        1 & 2 \\
    \end{tabular} &
    = &
    \begin{tabular}{cc}
        1 & 2 \\
        3 & 4 \\
        5 & 6 \\
    \end{tabular} &
    + &
    \begin{tabular}{cc}
        1 & 2 \\
        \color{blue} 1 & \color{blue} 2 \\
        \color{blue} 1 & \color{blue} 2 \\
    \end{tabular} &
    \hspace{-1.3em}
    \tikz[baseline={([yshift=-.5ex]current bounding box.center)}]{
        \draw [->, very thick] (0,0) -- (0,-1.2);
    } \\[1.5em]
    shape: & (3, 2) & & (2,) & & (3, 2) & & ({\color{blue}3}, 2) &
\end{tabular}
\vfill
\begin{itemize}
    \item Pad shape with 1s on the left : $(2,) \equiv (1,2)$
    \item Two dimensions are compatible when they have the same length or one of them is broadcastable
    \item broadcastable dimensions must have a length of 1
    \item Adding tensors of shape (8, 1, 6, 1) and (7, 1, 5) gives a tensor of shape (8, 7, 6, 5)
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Reductions}
\begin{lstlisting}
from theano import tensor as T
tensor3 = T.TensorType(
    broadcastable=(False, False, False),
    dtype='float32')
x = tensor3()
total = x.sum()
marginals = x.sum(axis=(0, 2))
mx = x.max(axis=1)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Dimshuffle}
\begin{lstlisting}
from theano import tensor as T
tensor3 = T.TensorType(
    broadcastable=(False, False, False),
    dtype='float32')
x = tensor3()
y = x.dimshuffle((2, 1, 0))
a = T.matrix()
b = a.T
# Same as b
c = a.dimshuffle((0, 1))
# Adding to larger tensor
d = a.dimshuffle((0, 1, 'x'))
e = a + d
\end{lstlisting}
\end{frame}

\begin{frame}{Exercices}
Work through the "Building Expressions" section of the ipython notebook.
\end{frame}

\section{Compiling/Running}
\begin{frame}{Compiling and running expression}
  \begin{itemize}
  \item \code{theano.function}
  \item shared variables and updates
  \item compilation modes
  \item compilation for GPU
  \item optimizations
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{\code{theano.function}}

\begin{lstlisting}
>>> from theano import tensor as T
>>> x = T.scalar()
>>> y = T.scalar()
>>> from theano import function
>>> # first arg is list of symbolic inputs
>>> # second arg is symbolic output
>>> f = function([x, y], x + y)
>>> # Call it with numerical values
>>> # Get a numerical output
>>> f(1., 2.)
array(3.0)
\end{lstlisting}
\end{frame}

\begin{frame}{Shared variables}
  \begin{itemize}
  \item It’s hard to do much with purely functional programming
  \item \emph{shared variables} add just a little bit of imperative programming
  \item A \emph{shared variable} is a buffer that stores a numerical value for a Theano variable
  \item Can write to as many shared variables as you want, once each, at the end of the function
  \item  Modify outside Theano function with \code{get_value()} and \code{set_value()} methods.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Shared variable example}
\begin{lstlisting}
>>> from theano import shared
>>> x = shared(0.)
# Can also use a dict for more complex code
>>> updates = [(x,  x + 1)]
>>> f = function([], updates=updates)
>>> f()
>>> x.get_value()
1.0
>>> x.set_value(100.)
>>> f()
>>> x.get_value()
101.0
\end{lstlisting}
\end{frame}

\begin{frame}{Which dict?}
  \begin{itemize}
  \item Use theano.compat.python2x.OrderedDict
  \item Not collections.OrderedDict
  \begin{itemize}
  \item This isn’t available in older versions of python, and will limit the portability of your code.
  \end{itemize}
  \item Not \code{\{\}} aka dict
  \begin{itemize}
  \item The iteration order of this built-in class is not deterministic so if Theano accepted this, the same script could compile different C programs each time you run it.
  \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Compilation modes}
  \begin{itemize}
  \item Can compile in different modes to get different kinds of programs
  \item Can specify these modes very precisely with arguments to \code{theano.function()}
  \item Can use a few quick presets with environment variable flags
  \end{itemize}
\end{frame}

\begin{frame}{Example preset compilation modes}
  \begin{description}[FAST\_RUN]
  \item[FAST\_RUN] Default. Spends a lot of time on
compilation to get an executable that runs
fast.
  \item[FAST\_COMPILE] Doesn’t spend much time compiling.
Executable usually uses python
instead of compiled C code. Runs slow.
  \item[DEBUG\_MODE] Adds lots of checks.
Raises error messages in situations other modes don't check for.
  \end{description}
\end{frame}

\begin{frame}{Compilation for GPU}
  \begin{itemize}
  \item Theano's current back-end only supports 32 bit on GPU
  \item CUDA supports 64 bit, but is slow in gamer card
  \item \code{T.fscalar}, \code{T.fvector}, \code{T.fmatrix} are all 32 bit
  \item \code{T.scalar}, \code{T.vector}, \code{T.matrix} resolve to 32 or 64 bit depending on theano’s floatX flag
  \item floatX is float64 by default, set it to float32
  \item Set the device flag to gpu (or a specific gpu, like gpu0)
  \item Optional: warn\_float64=\{'ignore', 'warn', 'raise', 'pdb'\}
  \end{itemize}
\end{frame}

\begin{frame}{Optimizations}
  \begin{itemize}
  \item Theano changes the symbolic expressions
    you write before converting them to C code
  \item It makes them faster
  \begin{itemize}
  \item $(x+y)+(x+y) \to 2\times(x + y)$
  \end{itemize}
  \item It makes them more stable
  \begin{itemize}
  \item $\exp(a)/\sum{\exp(a)} \to \operatorname{softmax}(a)$
  \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Optimizations (2)}
Sometimes optimizations discard error checking and produce incorrect output rather than an exception.
\begin{lstlisting}
>>> x = T.scalar()
>>> f = function([x], x/x)
>>> f(0.)
array(1.0)
\end{lstlisting}
\end{frame}

\begin{frame}{Exercises}
Work through the "Compiling and Running" section of the ipython notebook.
\end{frame}

\section{Modifying expressions}
\begin{frame}{Modifying expressions}
  \begin{itemize}
  \item The \code{grad()} method
  \item Variable nodes
  \item Types
  \item Ops
  \item Apply nodes
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{The \code{grad()} method}
\begin{lstlisting}
>>> x = T.scalar('x')
>>> y = 2. * x
>>> g = T.grad(y, x)
>>> from theano.printing import min_informative_str
# Print the unoptimized graph
>>> print min_informative_str(g)
A. Elemwise{mul}
 B. Elemwise{second,no_inplace}
  C. Elemwise{mul,no_inplace}
   D. TensorConstant{2.0}
   E. x
  F. TensorConstant{1.0}
 <D>
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{The \code{grad()} method}
\begin{lstlisting}
>>> x = T.scalar('x')
>>> y = 2. * x
>>> g = T.grad(y, x)
>>> from theano.printing import min_informative_str
# Print the optimized graph
>>> f = theano.function([x], g)
>>> theano.printing.debugprint(f)
DeepCopyOp [@A] ''   0
 |TensorConstant{2.0} [@B]
\end{lstlisting}
\end{frame}

\begin{frame}{Theano variables}
  \begin{itemize}
  \item A \emph{variable} is a theano expression.
  \item Can come from \code{T.scalar()}, \code{T.matrix()}, etc.
  \item Can come from doing operations on other variables.
  \item Every variable has a type field, identifying its \emph{type}, such as \code{TensorType((True, False), 'float32')}
  \item Variables can be thought of as nodes in a graph
  \end{itemize}
\end{frame}

\begin{frame}{Ops}
  \begin{itemize}
  \item  An Op is any class that describes a function operating on some variables
  \item Can call the op on some variables to get a
new variable or variables
  \item An Op class can supply other forms of
information about the function, such as its
derivative
  \end{itemize}
\end{frame}

\begin{frame}{Apply nodes}
  \begin{itemize}
  \item The Apply class is a specific instance of an application of an Op.
  \item Notable fields:
    \begin{description}[\texttt{outputs}]
    \item[\texttt{op}] The Op to be applied
    \item[\texttt{inputs}] The Variables to be used as input
    \item[\texttt{outputs}] The Variables produced
    \end{description}
  \item The \code{owner} field on variables identifies the Apply that created it.
  \item Variable and Apply instances are nodes and owner/
    inputs/outputs identify edges in a Theano graph.
  \end{itemize}
\end{frame}

\begin{frame}{Exercises}
Work through the "Modifying" section in the ipython notebook.
\end{frame}

\section{Debugging}
\begin{frame}{Debugging}
  \begin{itemize}
  \item DEBUG\_MODE
  \item Error message
  \item \code{theano.printing.debugprint()}
  \item \code{min_informative_str()}
  \item compute\_test\_value
  \item Accessing the FunctionGraph
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Error message: code}
\begin{lstlisting}
import numpy as np
import theano
import theano.tensor as T
x = T.vector()
y = T.vector()
z = x + x
z = z + y
f = theano.function([x, y], z)
f(np.ones((2,)), np.ones((3,)))
\end{lstlisting}
\end{frame}

\begin{frame}[fragile,allowframebreaks]{Error message}
\vspace{1em}
\begin{lstlisting}[style=output]
Traceback (most recent call last):
  File "test.py", line 9, in <module>
    f(np.ones((2,)), np.ones((3,)))
  File "/Users/anakha/Library/Python/2.7/site-packages/theano/compile/function_module.py", line 606, in __call__
    storage_map=self.fn.storage_map)
  File "/Users/anakha/Library/Python/2.7/site-packages/theano/compile/function_module.py", line 595, in __call__
    outputs = self.fn()
ValueError: Input dimension mis-match. (input[0].shape[0] = 3, input[1].shape[0] = 2)
Apply node that caused the error: Elemwise{add,no_inplace}(<TensorType(float64, vector)>, <TensorType(float64, vector)>, <TensorType(float64, vector)>)
Inputs types: [TensorType(float64, vector), TensorType(float64, vector), TensorType(float64, vector)]
Inputs shapes: [(3,), (2,), (2,)]
Inputs strides: [(8,), (8,), (8,)]
Inputs values: [array([ 1.,  1.,  1.]), array([ 1.,  1.]), array([ 1.,  1.])]

HINT: Re-running with most Theano optimization disabled could give you a back-trace of when this node was created. This can be done with by setting the Theano flag  'optimizer=fast_compile'. If that does not work, Theano  optimizations can be disabled with 'optimizer=None'.
HINT: Use the Theano flag 'exception_verbosity=high'  for a debugprint and storage map footprint of this apply node.
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Error message: exception\_verbosity=high}
\begin{lstlisting}[style=output]
Debugprint of the apply node:
Elemwise{add,no_inplace} [@A] <TensorType(float64, vector)> ''
 |<TensorType(float64, vector)> [@B] <TensorType(float64, vector)>
 |<TensorType(float64, vector)> [@C] <TensorType(float64, vector)>
 |<TensorType(float64, vector)> [@C] <TensorType(float64, vector)>

Storage map footprint:
 - <TensorType(float64, vector)>, Shape: (3,), ElemSize: 8 Byte(s), TotalSize: 24 Byte(s)
 - <TensorType(float64, vector)>, Shape: (2,), ElemSize: 8 Byte(s), TotalSize: 16 Byte(s)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Error message: optimizer=fast\_compile}
\begin{lstlisting}[style=output]
Backtrace when the node is created:
  File "test.py", line 7, in <module>
    z = z + y
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{debugprint}
\begin{lstlisting}
>>> from theano.printing import debugprint
>>> debugprint(a)
Elemwise{mul,no_inplace} [@A] ''
 |TensorConstant{2.0} [@B]
 |Elemwise{add,no_inplace} [@C] 'z'
   |<TensorType(float64, scalar)> [@D]
   |<TensorType(float64, scalar)> [@E]
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{min\_informative\_str}
\begin{lstlisting}
>>> x = T.scalar()
>>> y = T.scalar()
>>> z = x + y
>>> z.name = 'z'
>>> a = 2. * z
>>> from theano.printing import min_informative_str
>>> print min_informative_str(a)
A. Elemwise{mul,no_inplace}
 B. TensorConstant{2.0}
 C. z
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{compute\_test\_value}
\begin{lstlisting}
>>> from theano import config
>>> config.compute_test_value = 'raise'
>>> x = T.vector()
>>> import numpy as np
>>> x.tag.test_value = np.ones((2,))
>>> y = T.vector()
>>> y.tag.test_value = np.ones((3,))
>>> x + y
...
ValueError: Input dimension mis-match.
(input[0].shape[0] = 2, input[1].shape[0] = 3)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Accessing a function’s fgraph}
\begin{lstlisting}
>>> x = T.scalar()
>>> y = x / x
>>> f = function([x], y)
>>> debugprint(f.maker.fgraph.outputs[0])
DeepCopyOp [@A] ''
 |TensorConstant{1.0} [@B]
\end{lstlisting}
\end{frame}

\begin{frame}{Exercises}
Work through the "Debugging" section of the ipython notebook.
\end{frame}

\section*{}
\begin{frame}{Citing Theano}
Please cite both of the following papers in all work that uses Theano:
  \begin{itemize}
  \item Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and
     Bengio,Yoshua. Theano: new features and speed improvements. Deep Learning and Unsupervised Feature Learning NIPS 2012
    Workshop, 2012.
  \item Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-
     Farley, David, and Bengio,Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific
      Computing Conference (SciPy), June 2010. Oral Presentation.
  \end{itemize}
\end{frame}

\begin{frame}{Example acknowledgments}
We would like to thank the developers of Theano \textbackslash citep\{bergstra+al:2010-scipy,Bastien-Theano-2012\}.
We would also like to thank NSERC, Compute Canada, and Calcul Québec for providing computational resources.
\end{frame}

\begin{frame}
\begin{center}
\bibliography{strings,strings-short,ml,aigaion-shorter}
\Huge
Questions?
\end{center}
\end{frame}

\end{document}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}

\title{Modeling with Python and SMAP}
\author{Sarah Koehler}
\date{April 2015}

\begin{document}

\maketitle

\section{Modeling}
Many say modeling is an art. Let's look at the simplest forms of this art.

Linear Model:
\begin{align}
x(k+1) &= Ax(k) + Bu(k)  \\
y(k) &= Cx(k) + Du(k)
\end{align}
\noindent Let's assume $y(k) = x(k)$.

In the most general case, define $x$ to be all physical \textit{outputs} and $u$ to be all \textit{controllable inputs}. You can also add an additive $ + Ed(k)$ where $d$ are all \textit{uncontrollable inputs}.

A form that works well for Temperature dynamics is a bilinear form. Specifically:
\begin{align}
T(k+1) = AT(k) + ...
\end{align}
(TO DO: To be copied from ME 231 write ups)

\pagebreak
\section{Identification Method}
Two methods are Least Squares and Lasso. An oversimplified explanation is that Least Squares considers all data streams equally important, while Lasso selects the important streams (i.e. feature selection). Both select the corresponding weight for that stream.

Least Squares from Python scikit-learn:
\begin{figure}[h]
\centering
\includegraphics[width=\columnwidth]{LeastSquaresPython.pdf}
\caption{Documentation on Least Squares}
\label{fig:lasso}
\end{figure}

\pagebreak
Lasso from Python scikit-learn:
\begin{figure}[h]
\centering
\includegraphics[width=\columnwidth]{LassoPython.pdf}
\caption{Documentation on Lasso}
\label{fig:lasso}
\end{figure}

\section{Data Manipulation}
This is the most annoying part and it would be great to never ever have to do this again.

TO DO: Copy python code when it's done and/or write the math here.

\section{Validation}
Cross-validation is important! The model usually fits the day it was identified on.

TO DO: Show some plots of same day verification and cross day verification.

\section{Matlab code}
\begin{verbatim}
% % Interpolate data
% interp_start = PDTtoUTC(datenum(2014,9,4,4,0,0));
% interp_end = PDTtoUTC(datenum(2014,9,4,16,0,0));
interp_start = datenum(2014,8,30,10,1,0);
interp_end = datenum(2014,8,30,16,0,0);
interp_times = (interp_start:1/24/60:interp_end)';

fan_interp = interp1(fan_data(:,1),fan_data(:,2),interp_times);
temp_interp = interp1(temp_data(:,1),temp_data(:,2),interp_times);
supplytemp_interp = interp1(supplytemp_data(:,1),supplytemp_data(:,2),interp_times);

% Solve least squares using quadprog
N = length(interp_times)-2;
H = 2*(blkdiag(zeros(2,2),eye(N)));
f = zeros(N+2,1);
Aeq = [zeros(N,2),-eye(N)];
beq = zeros(N,1);
for k = 1:N
   f(k+2) = -2*temp_interp(k+1);
   Aeq(k,1:2) = [-fan_interp(k)*temp_interp(k),fan_interp(k)*supplytemp_interp(k)];
   beq(k) = -temp_interp(k);
end

% options = optimset('Algorithm','interior-point-convex');
% [x,cost] = quadprog(H,f,[],[],Aeq,beq,[],[],[],options);
[x,cost] = quadprog(H,f,[],[],Aeq,beq);

a = x(1);
b = x(2);

%% Least Squares via A\b
A = Aeq(:,1:2);
b = zeros(N,1);
for k = 1:N
   b(k) = -temp_interp(k)+temp_interp(k+1);
end

x = A\b;
a = x(1);
b = x(2);

\end{verbatim}

\section{Python code}
At this point can open up Python code like Arka did and expose the issues we have with its structure.

TO DO: Can also make a list of issues here:
\begin{itemize}
\item Cannot choose system ID procedure without digging into code
\item Need a library of models to choose from? Not everything is linear or bilinear...
\end{itemize}

\end{document}


%%==========================
%% Chapter 4.0: Introduction
%%==========================

\documentclass[../dissertation.tex]{subfiles}

\begin{document}
% \setcounter{section}{-1}
\section{Introduction}\label{sec4.0:Intro}

Following our deep dive into the properties of the Green's functions, let us take
a brief respite to see where we are in the process of understanding the direct
map for the Intermediate Long Wave equation Inverse Scattering Transform. In Section
\ref{sec0:DM} we introduced the notion of a Jost Solution, which we now repeat for
reference:
\begin{defn}[Jost solutions]\label{defn4:jost}
	Recall the linear spectral problem
	\begin{align}\label{eq4:SpecProb}
		L_\delta (\Psi)
			:= \frac{1}{i} \frac{\partial}{\partial x} \Psi^+
				- \zeta \left(\Psi^+ - \Psi^-\right) = u \Psi^+,
	\end{align}
	The Jost solutions $M_1$, $M_e$, $N_1$, $N_e$ are solutions to the linear
	spectral problem \eqref{eq4:SpecProb} whose lower boundary values
	$M_1^+$, $M_e^+$, $N_1^+$, $N_e^+$ as defined in \eqref{eq0:bndryvaluedefn}
	obey the following asymptotic conditions
	\begin{subequations}\label{eq4:JostDEasymp}
		\begin{align}
			\lim_{x\to -\infty}
					\inn{x}
					\left(
						M_1^+(x; \lambda, \delta) - 1
					\right)
				&= \lim_{x\to \infty}
						\inn{x}
						\left(
							N_1^+(x; \lambda, \delta) - 1
						\right)
				= 0 \\
			\lim_{x\to -\infty}
					\inn{x} \left(
						M_e^+(x; \lambda, \delta) - e^{i\lambda x}
					\right)
				&= \lim_{x\to \infty}
						\inn{x}
						\left(
							N_e^+(x; \lambda, \delta) - e^{i\lambda x}
						\right)
				= 0
		\end{align}.
	\end{subequations}

	Additionally, we require the upper boundary values $M_{(\dotarg)}^-$, $N_{(\dotarg)}^-$
	(where $(\dotarg)$ represents either the subscript $1$ or $e$) of
	$M_{(\dotarg)}$, $N_{(\dotarg)}$ to have a decomposition
	\begin{align*}
		M_1^- - 1 &= M_1^{(1)} + M_1^{(2)} \qquad &\text{and}& \qquad
		&N_1^- - 1 &= N_1^{(1)} + N_1^{(2)} \\
		M_e^- - e^{i\lambda x}\,e^{-2\delta\lambda} &= M_e^{(1)} + M_e^{(2)} \qquad &\text{and}& \qquad
		&N_e^- - e^{i\lambda x}\,e^{-2\delta\lambda} &= N_e^{(1)} + N_e^{(2)} \\
	\end{align*}
	satisfying
	\begin{align*}
		\inn{x}^{1+\upsilon} \left|M_{(\dotarg)}^{(1)}(x)\right| \lesssim 1
			\quad (\text{for } x \ll -1), \qquad
		\inn{x}^{1+\upsilon} \left|N_{(\dotarg)}^{(1)}(x)\right| \lesssim 1
			\quad (\text{for } x \gg 1),
	\end{align*}
	and
	\begin{align*}
		\inn{\dotarg}^\tau M_{(\dotarg)}^{(2)},
			~\inn{\dotarg}^\tau N_{(\dotarg)}^{(2)} \in L^2(\mathbb R)
	\end{align*}
	for any $\upsilon \in \left(0,\frac{1}{2}\right)$ and $\tau \in [0,1)$.
\end{defn}

The direct scattering map
$\mathscr D$ maps $u$ to the reflection coefficient
$r(\lambda) := b(\lambda) / a(\lambda)$ where $a$ and $b$ are determined by
the following formulas involving the boundary value $M_1^+$ of the Jost solution
$M_1$
\begin{subequations}
	\label{eq3:ScatData}
	\begin{align}
		\label{eq3:ScatDataA}
		a(\lambda)
			&= 1 + i \alpha(\lambda)
				\int_{\mathbb R}
					u(x) \, M_1^+(x;\lambda, \delta, u)
				\, \mathrm{d}x \\
		\label{eq3:ScatDataB}
		b(\lambda)
			&= i \beta(\lambda)
				\int_{\mathbb R}
					e^{-i\lambda x} \, u(x) \, M_1^+(x; \lambda, \delta, u)
				\, \mathrm{d}x,
	\end{align}
\end{subequations}
where we have written the Jost solutions as functions of $u$ in order to
explicitly highlight the dependence of the Jost solutions on the eigenfunction $u$.
The goal of this chapter is to both prove that $\mathscr D$ is well-defined and
Lipschitz continuous for $|\lambda| > 1$.

As we see from \eqref{eq3:ScatData}, proving that $\mathscr D$ is well defined
requires the Jost solutions to exist and be unique. Further, in order to prove
that $\mathscr D$ is Lipschitz continuous, we need to establish that
the four maps from $u$ to each of the four Jost solutions $M_1$, $M_e$, $N_1$,
and $N_e$ are themselves Lipschitz as maps from $B_X(0, c_0)$ to
$\inn{\dotarg}L^\infty(\mathbb R)$. As is common practice in the inverse
scattering world, to prove these desired results we seek to reformulate
\eqref{eq4:SpecProb} with asymptotic conditions \eqref{eq4:JostDEasymp} as a set
of integral equations that are easier to analyze. This approach is relevant in
our case given the limited theory about partial differential equations
involving functions analytic in a complex strip and the functions' lower and
upper boundary values along the corresponding boundary values of the strip.

Following our analysis of $G_L$ and $G_R$ in Chapters \ref{cptr01:GF} and
\ref{cptr03:xContin} we are almost now in a position to prove (under the right hypotheses)
the equivalence of the Jost solutions and solutions to the following integral
equations
\begin{subequations}
	\label{eq4:JostIE}
	\begin{align}
		\label{eq4:JostIEleft}
		\begin{pmatrix}
			M_1^+(x; \zeta, \delta) \\
			M_e^+(x; \zeta, \delta)
		\end{pmatrix}
			&=
				\begin{pmatrix}
					1 \\
					e^{i\lambda x}
				\end{pmatrix}
				+ \int_{\mathbb R} G_L(x - x'; \zeta, \delta)
					u(x')
					\begin{pmatrix}
						M_1^+(x'; \zeta, \delta) \\
						M_e^+(x'; \zeta, \delta)
					\end{pmatrix}
					\, \mathrm{d}x' \\[0.3\baselineskip]
		\label{eq4:JostIEright}
		\begin{pmatrix}
			N_1^+(x; \zeta, \delta) \\
			N_e^+(x; \zeta, \delta)
		\end{pmatrix}
			&=
				\begin{pmatrix}
					1 \\
					e^{i\lambda x}
				\end{pmatrix}
				+ \int_{\mathbb R} G_R(x - x'; \zeta, \delta)
					u(x')
					\begin{pmatrix}
						N_1^+(x'; \zeta, \lambda) \\
						N_e^+(x'; \zeta, \lambda)
					\end{pmatrix}
					\, \mathrm{d}x'.
	\end{align}
\end{subequations}
In Section \ref{sec4:equiv} we prove this equivalence.
We begin by presenting the framework we use to prove this equivalence,
followed by proving in Subsection \ref{subsec4:DEtoIE} that Jost solutions solve the
corresponding integral equations \eqref{eq4:JostIE}. We prove solutions to
\eqref{eq4:JostIE} are Jost solutions in the following subsection, Section
\ref{subsec4:IEtoDE}.
However, for reasons that are made obvious in
Section \ref{sec4:equiv}, prior to proving
the equivalence of Jost solutions and solutions to \eqref{eq4:JostIE}, we do need
several results about both the existence of solutions of solutions to \eqref{eq4:JostIE}
and the continuity of maps from $u$ to \eqref{eq4:JostIE} solutions. For this reason,
we begin this chapter by studying the \eqref{eq4:JostIE} solutions in Section
\ref{sec4:Exist}, and note that because we prove in Section \ref{sec4:equiv} the
equivalence of Jost solutions and the solutions to the integral equations \eqref{eq4:JostIE},
the results proven in Section \ref{sec4:Exist} about the existence and uniqueness
\eqref{eq4:JostIE} solutions applies to Jost solutions.

The final section of this chapter, Section \ref{sec4:DM}, is the
\textit{rasion d'\^etre} of this dissertation, in that after
\pageref{lastpagePenultimateSection} pages of diligent mathematical exploration,
we arrive at our study of the ILW direct scattering map. We begin Section
\ref{sec4:DM} by verifying the so-called ``scattering equations'' which are
instrumental in the formulation of the ILW inverse scattering map. We then prove
that as a map from $B_X(0, c_0)$ to $L_\lambda^\infty(\mathbb R)$ the direct
scattering map $\mathscr D$ is well-defined (Theorem \ref{thm4:Dwelldefined}),
where  $B_X(0, c_0)$ in the space $X$ of radius $c_0$ and $c_0$ is chosen
according to our work in Section \ref{sec4:Exist} to ensure the existence of the
Jost solutions. In the remainder of Section \ref{sec4:DM}, we turn our attention
towards understanding the Lipschitz continuity properties of $\mathscr D$. Specifically,
we prove that as a map from $B_X(0, c_0)$ to
$L_\lambda^\infty\big((-\infty,k]\cap[k, \infty) \big)$, the direct scattering map
$\mathscr D$ is Lipschitz continous for all $k>0$ (Theorem \ref{thm4:DlipR}). As
an almost immediate consequence of our proof of Theorem \ref{thm4:DlipR}, we
also prove Corollary \ref{cor4:Lip}, which holds that for every
$u \in B_X(0,c_0)$ with the property that
\[
	\int_{\mathbb R} u \, M_1^+(x; \lambda = 0) \, \mathrm{d}x \ne 0,
\]
there is a neighborhood $\mathcal N(u)$ in $B_X(0, c_0)$ about $u$ for which
the map $\mathscr D: \mathcal N(u) \to L^\infty_\lambda(\mathbb R)$ is
Lipschitz continuous.

While we have not yet found a proof that $\mathscr D$ is Lipschitz continuous
uniformly in the parameter $\lambda$ for all real $\lambda$ (which is necessary
as the scattering data are functions of $\lambda$), we do discuss the regimes
under which we are currently able to prove that $\mathscr D$ is Lipschitz
continuous.

A final remark as we set off on the ultimate leg of our mathematical
peregrination within this dissertation:
given the $\delta$-dilation property satisfied by both $G_L$ and $G_R$, throughout
the remainder of this Chapter, we again take $\delta = 1$ noting that the more general
case of $\delta>0$ arbitrary follows from this dilation property and the results
contained within this chapter.

\end{document}