% this file is called up by thesis.tex
% content in this file will be fed into the main document

\chapter{Compression model}
\label{ch:exprlang}

% ----------------------- paths to graphics ------------------------

\graphicspath{{4_expression_language/images/}}

% ----------------------- contents from here ------------------------
%

\textit{Whitebox compression} is a compression model for columnar database systems. Its purpose is to represent data more compactly through elementary operator expressions. These operators---chained together into expression trees---form the expression language used for data representation. This chapter describes the \textit{whitebox compression} model, its expression language and operators, the structure of expression trees and their evaluation.

\section{Expression language}
\label{sec:exprlang:exprlang}

The \textit{whitebox} compression model represents logical columns as composite functions of physical columns. We refer to these functions as \textit{operators}. With respect to databases, logical columns are columns as seen by the database user, containing the data in its original format. Physical columns contain the physical representation of the data as it is stored on disk, in a different format.

Formally, we define an operator as a function \(o\) that takes as input zero or more columns and optional metadata information and outputs a column:
\begin{equation}
\label{eq:exprlang:exprlang:operator}
    o \colon [C \times C \times ...] \times [M] \to C
\end{equation}

The domain of \(o\) is composed of columns and metadata and the codomain is columns. A column is defined by its datatype \(d\) and the values that it contains \(V\). The metadata is structured information of any type.

We defined our expression language based on a set of elementary column representation types, each having an associated operator. They are listed in Table~\ref{tab:exprlang:exprlang_1}.

\input{4_expression_language/expression_language-table_1.tex}

The concatenation of two string columns \(c_a\) and \(c_b\) is the concatenation of each pair of values \(v_a\) and \(v_b\). E.g. \(\verb|"123abc"| = \mathit{concat}(\verb|"123"|, \verb|"abc"|)\), where \(v_a = \verb|"123"|\) and \(v_b = \verb|"abc"|\). The representation of a string column \(c_a\) as a formatted non-string column \(c_b\) consists of the individual values \(v_b\) formatted as strings based on the format string metadata \(m_{\mathit{format}}\). This can be seen as datatype change. E.g.
\(\verb|"-12000"| = \mathit{format}(\verb|-12000|, \verb|"%d"|)\), where \(v_b = \verb|-12000|\)
and
\(m_{\mathit{format}} = \verb|"%d"|\)
. The direct mapping representation of a column \(c_a\) as a column \(c_b\) through the mapping \(m_{map}\) is a key-value lookup in a dictionary-like data structure. E.g. \(\verb|"valueoncolumnA"| = dict[\verb|"valueoncolumnB"|]\), where \verb|valueoncolumnB| is the key and \verb|"valueoncolumnA"| is the value in the dictionary \verb|dict|. The constant representation of a column indicates that all its values are equal to the constant value \(m_{\mathit{const}}\). The \(\mathit{const}\) operator just returns \(m_{\mathit{const}}\).

These operators and transformations can be composed, resulting in operator expressions. For example, the logical columns \(A\) and \(B\) in Table~\ref{tab:exprlang:exprlang_2}, can be represented as composite functions of the physical columns in Table~\ref{tab:exprlang:exprlang_2}, through the following expressions:
\begin{equation}
\label{eq:exprlang:exprlang:example}
\begin{array}{ll}
    A &= \mathit{concat}(\mathit{map}(P, {dict_{AP}}), \mathit{const}(\verb|"_"|), \mathit{format}(Q, \verb|"%d"|))\\
    B &= \mathit{map}(P, {dict_{BP}})
\end{array}
\end{equation}

\input{4_expression_language/expression_language-table_2_3.tex}

We observe that column \(A\) has the following structure: a dictionary compressible prefix and a numeric suffix, separated by a the '\verb|_|' character. If we store these logical parts separated into 3 columns \(C_{\mathit{prefix}}\), \(C_{\mathit{delim}}\), \(C_{\mathit{suffix}}\), we can represent column \(A\) as their concatenation. Since \(C_{\mathit{prefix}}\) has repeated values, we can represent it more compactly as the mapping of column \(P\)---containing dictionary keys---and the dictionary \(dict_{AP}\)---presented in Table~\ref{tab:exprlang:exprlang_3}. We can represent \(C_{\mathit{delim}}\) through the \textit{const} operator since all its values are equal to '\verb|_|'. \(C_{\mathit{suffix}}\) contains numbers stored in strings. We can store these values more compactly as numbers, by changing the column datatype. Therefore, we represent \(C_{\mathit(suffix)}\) based on the numeric column \(Q\), through the \(format\) operator, with the format string \verb|"%d"|. We move our attention to column \(B\) and observe that it is correlated with column \(C_{\mathit{prefix}}\)---and implicitly also to column \(P\). We can therefore represent \(B\) as the mapping of column \(P\) and the dictionary \(dict_{BP}\)---presented in Table~\ref{tab:exprlang:exprlang_3}. In the end, we store only the physical columns \(P\) and \(Q\) and the metadata: \(dict_{AP}\), \(dict_{BP}\) and the constant string \verb|"_"|. The original values on the logical columns \(A\) and \(B\) can be reconstructed by evaluating the expressions in Equation~\ref{eq:exprlang:exprlang:example}.

So far, we described 4 column representation types and their associated operators: \(concat\), \(format\), \(map\) and \(const\). The \textit{whitebox compression} model does not limit itself to these representation types. It is a generic model and supports any type of column operators (e.g. mathematical operators like addition or multiplication). A practical example is the \textit{whitebox} version of the Frame of Reference compression method: \(const(\mathit{reference}) + C_{\mathit{diff}}\), where \(+\) is the numeric addition/sum operator, \(\mathit{reference}\) is the reference value and \(C_{\mathit{diff}}\) is the physical column containing the differences between the original values and \(\mathit{reference}\).

The purpose of \textit{whitebox compression} is to represent data more compactly through elementary operator expressions similar to the ones presented above. However, there are a multitude of different possible representations of the same logical columns, each one giving a different result. We will describe the optimization problem of finding the best representation for a set of columns in \ref{subsec:learningprocess:optimizationproblem}~\nameref{subsec:learningprocess:optimizationproblem}.

These operator expressions create more compact representations of logical columns, exploiting the underlying compression opportunities in the data. We showed how we can remove redundancy from data by representing columns as functions of other columns through the \(map\) operator and how we can store numeric values in more suitable datatypes through the \(format\) operator. We are able to decompose string columns into subcolumns with values from different distributions, thus enabling independent representations. Finally, the key factor of the \textit{whitebox} model is that it allows recursive representation of columns, ultimately leading to improved compression ratios.

\section{Expression tree}
\label{sec:exprlang:exprtree}

The operators presented so far are useful for describing the data representation and for transforming the physical data into its original logical format. We call this process \textit{decompression}. In practice, we need to transform the logical data into its physical format first---\textit{compression}. The compression process requires a different expression, one that represents the physical columns as functions of the logical columns, through the inverse operators of the ones presented until now. Table~\ref{tab:exprlang:exprlang_4} presents the compression operators types.

\input{4_expression_language/expression_language-table_4.tex}

All these operator types and their practical implementations will be discussed in detail in \ref{sec:pd}~\nameref{sec:pd}. For now, we are interested in their definition. We notice how the formal definition of the compression operator is different from the one of the compression operators:
\begin{equation}
\label{eq:exprlang:exprtree:operator:comp}
    o \colon C \times [M] \to [C \times C \times ...]
\end{equation}

The compression operators take as input a single column and compression metadata information and output 0 or more columns. Because of the multiple column output, representing physical columns as composite functions of logical columns is not straightforward. Therefore, we introduce the concept of \textit{expression trees}, as an alternative representation instead of the nested operator expressions.

\textit{Expression trees} are tree-like structures with 2 types of nodes: \textit{column nodes} and \textit{operator nodes}. We also use the term \textit{expression node} to refer to the \textit{operator nodes}---they are interchangeable. An \textit{expression tree} is composed of alternating levels of \textit{column} and \textit{operator} nodes. Root nodes are always \textit{column nodes}. Leaf nodes can be either \textit{column nodes} or \textit{operator nodes}---in the case of operators that do not output any column. An \textit{operator node} in an \textit{expression tree} is the equivalent of and operator in an operator expression: it has input columns---connected through incoming edges---and output columns---connected through outgoing edges. \textit{Expression trees} are used in the compression and decompression processes as more practical alternatives to the operator expressions. To better understand the similarities between the two, we created the equivalent \textit{expression tree} of the operator expressions for columns \(A\) and \(B\) in Equation~\ref{eq:exprlang:exprlang:example}. Recall the expressions: \(A = \mathit{concat}(\mathit{map}(P, {dict_{AP}}), \mathit{const}(\verb|"_"|), \mathit{format}(Q, \verb|"%d"|))\) and \(B = \mathit{map}(P, {dict_{BP}})\). The equivalent \textit{expression tree} is presented in Figure~\ref{fig:exprlang:exprtree:tree_1}.

\begin{figure}[h]
  \centering
  \includegraphics[width={0.9\linewidth}]{expression_language-tree_1_1.pdf}
  \caption{Expression tree representation}
  \label{fig:exprlang:exprtree:tree_1}
\end{figure}

The first thing to notice is that the \textit{expression tree} is not actually a tree, but a directed acyclic graph (DAG) with 2 root nodes. However, we chose to stick to the term \textit{expression tree} instead of graph, since it is more intuitive. In this case the graph is connected, but in other cases it can have multiple connected components. For example, imagine that in our example we had an additional logical column \(C\) that is represented as a function of a physical column \(S\), without any connection with the columns or operators used to represent columns \(A\) and \(B\). Then, our graph will have 2 connected components.

Besides the graph-like structure, the \textit{expression tree} is an equivalent representation of the operator expressions. We notice the similarities between the operator nodes and the operators in the nested expressions and the alternating levels of columns and operators. A noticeable difference is the additional columns \(A_{\mathit{prefix}}\), \(A_{\mathit{delim}}\) and \(A_{\mathit{suffix}}\). These are non-materialized internal columns that make the recursive representation possible. In terms of representation type, this is a \textit{decompression tree}, since the root nodes are the physical columns \(P\) and \(Q\) and the expression nodes are decompression operators. The \textit{compression tree} will have the same structure, only that the root nodes will be the logical columns \(A\) and \(B\) and the expression nodes will be compression operators---the inverse functions of the decompression operators. The metadata will also differ. The \textit{compression tree} can be derived from the \textit{decompression tree} and vice versa, by using the inverse operators and transforming the metadata where it is necessary.

There is the case that different subsets of the values on a column come from different distributions and cannot be represented through the same expression. E.g. a string column where odd rows contain the same constant value and even rows are numbers. These situations are common in real data, as we have seen in the Public BI benchmark, where there are not many columns for which a single representation perfectly fits all the values. One option to accommodate these cases is to allow a single column to have multiple representations. The representation of the column is then the union of its multiple representations. In terms of \textit{decompression trees}, the column will have multiple incoming edges, each one from a different operator. These multi-representation structures need to be explicitly handled in the \textit{expression tree} evaluation process (described in \ref{sec:exprlang:compdecomp}~\nameref{sec:exprlang:compdecomp}). The second option of handling these cases is through recursive representation of \textit{exception columns}, which is discussed in \ref{sec:exprlang:exceptions}~\nameref{sec:exprlang:exceptions}.

\section{Exception handling}
\label{sec:exprlang:exceptions}

While analyzing the Public BI benchmark in search for patterns and \textit{whitebox compression} opportunities we noticed that columns where all values perfectly fit the same pattern/representation are not very common. Instead there is a smaller or larger subset of values that do not fit the dominant pattern of the column. Let us take for example the data in Table~\ref{tab:exprlang:exprlang_2}. Image that a few values on column \(A\) did not have the \verb|prefix-delim-suffix| structure and instead they were just arbitrary strings. Then, the representation in Figure~\ref{fig:exprlang:exprtree:tree_1} could not be applied on the entire column. We call these values that do not match the representation: \textit{exceptions}. In the \textit{whitebox compression} model \textit{exceptions} are stored on separate \textit{exception columns}. These are nullable columns that contain \verb|null| on positions where the value was not an exception and the original values otherwise. Conversely, the non-exception columns are also nullable and contain null on the positions of exceptions.

We defined 2 ways of handling \textit{exceptions}: 1) through the multi-representation approach mentioned in \ref{sec:exprlang:exprlang}~\nameref{sec:exprlang:exprlang}; 2) through recursive representation of \textit{exception columns}. The first option implies having an operator expression for each subset of values that requires a separate representation. The second option implies choosing a single representation, storing \textit{exceptions} on a separate \textit{exception column} and then recursively applying the same process on the \textit{exceptions}. The two options are equivalent from the physical data perspective. Only the shape of the tree differs: flat and wide trees in the first case and deeper trees in the second case. An additional difference between the two options is the number of \textit{exception columns}: the first option requires at most one \textit{exception column} for every logical column (to store values that do not fit any pattern), while the second option implies having a separate \textit{exception column} for each operator node in the expression tree. The two approaches are illustrated in Figure~\ref{fig:exprlang:exceptions}.

\begin{figure}[h]
\centering
\makebox[\textwidth][c]{
\begin{subfigure}[t]{0.49\linewidth}
  \centering
  \includegraphics[width={1.0\linewidth}]{4_expression_language/images/expression_language-exceptions_1_option_1.pdf}
  \caption{Multi-representation}
  \label{fig:exprlang:exceptions:option1}
\end{subfigure}
\hspace{3em}
\begin{subfigure}[t]{0.31\linewidth}
  \centering
  \includegraphics[width={1.0\linewidth}]{4_expression_language/images/expression_language-exceptions_1_option_2.pdf}
  \caption{Recursive exception representation}
  \label{fig:exprlang:exceptions:option2}
\end{subfigure}
}
\caption{Representation options \& exception handling}
\label{fig:exprlang:exceptions}
\end{figure}

Figure~\ref{fig:exprlang:exceptions} shows the 2 equivalent representations of a string column \(A\), which has 2 major subsets of values: one containing numeric values and the other one a constant value. The figure on the left shows the multi-representation approach: the 2 operators are on the same level of the tree and the exceptions---i.e. the values that do not fit any of the 2 representations---are stored separately in the \(A_{\mathit{exception}}\) column. \(A_{\mathit{numeric}}\) and \(A_{\mathit{exception}}\) are both physical columns. The figure on the right shows the recursive exception representation: the subset of numeric values are represented through the \(format\) operator and the rest are rejected to the \textit{exception column} \(X_{\mathit{format}}\). \(X_{\mathit{format}}\) now contains the subset of constant values, and is represented through the \(const\) operator. The remaining values which are not constant are stored in the \(X_{\mathit{const}}\) \textit{exception column}. The physical columns in this case are \(A_{\mathit{numeric}}\) and \(X_{\mathit{const}}\), while \(A_{\mathit{format}}\) is just an intermediate non-materialized \textit{exception column}. The 2 representations are equivalent in terms of the physical data structure: the numeric values are stored in \(A_{\mathit{numeric}}\) and the exceptions---values that are neither numeric nor constant---are stored in the \(A_{\mathit{exception}}\) respectively \(X_{\mathit{const}}\) column.

In our implementation we used a combination of the 2 approaches: select the dominant patterns in the data and represent each column through multi-representation expressions and then store the rest of the values---which do not fit the representations---in \textit{exception columns}. If there are more opportunities left in the exceptions, recursive representation of the \textit{exception columns} is implicitly performed by the compression learning algorithm, since they are treated as normal columns.

\iffalse
- exceptions can also contain patterns; can be recursively compressed with other methods
- 2 options: 1) a single exception column per logical column (unable to recursively compress exceptions, thus we need multi-representation structures in the tree); 2) an exception column for each operator: allowing recursive compression of exceptions and implicitly used multiple representations for the same column
\fi

\section{Compression and decompression}
\label{sec:exprlang:compdecomp}

The evaluation of a \textit{compression tree} on a set of logical columns---i.e. \textit{compression}---means applying the operators on the logical values in order to generate the physical values that will be stored in the physical columns. Conversely, the evaluation of a \textit{decompression tree} on a set of physical columns---i.e. \textit{decompression}---is the process of applying the operators on the physical values to obtain the original data. The 2 process are similar and we will further discuss only \textit{decompression}.

Given a table with multiple logical columns, its expression tree (graph) will have 1 or more connected components. Each component can be evaluated independently from the other components. The decompression process starts from the root nodes and evaluates the operators on the path to the target logical/physical column, in topological order. In the case of \textit{decompression}, exceptions are handled by checking for \verb|null| values on the \textit{exception column}. If the value at a given position on the \textit{exception column} is not \verb|null| then it was an exception, otherwise the operator needs to be evaluated. A special case is when a logical data value is \verb|null| and also an exception. For this case we use a bitmap indicating which values were \verb|null| in the first place. In the case of \textit{compression}, the operators are responsible for deciding which values are exceptions and which are not: if an operator throws an exception then the value is stored on the \textit{exception column}. The \textit{compression} and \textit{decompression} processes are similar for the multi-representation structures defined in the previous sections. For \textit{compression}, the decision upon which representation fits a given value is determined by the (first) operator that does not raise an exception. If all operators raise an exception then the value is stored on the \textit{exception column}. For \textit{decompression}, the physical columns that do not contain \verb|null| values indicate the representation of each value. This process is similar to the SQL \verb|COALESCE| function \cite{sqlcoalesce}, which returns the first non-null value in a list of expressions.

The process of evaluating \textit{expression trees} in topological order is suitable for vectorized execution \cite{kersten2018everything} and SIMD instructions since the elementary operators can be evaluated in the same order for blocks of data, generating intermediate results which fit in the cache. Alternatively, JIT compilation \cite{kersten2018everything} can be used to generate compiled code for each component of the expression tree during compression time, which is then executed for each query. However, the scope of this thesis does not cover fast evaluation of expression trees. We leave this topic for future work.

% \section{Representation example}
% \label{sec:exprlang:repexamples}

% TODO-1: give example from PBIB with compression ratio and compressed execution opportunity (the one in the presentation)

% \iffalse
% - expression tree examples & compression ratio calculation + compressed execution potential; also mention thesis focus (not compressed execution)
% \fi

% ---------------------------------------------------------------------------
% ----------------------- end of thesis sub-document ------------------------
% ---------------------------------------------------------------------------

\iffalse
where:
\begin{itemize}
    \item[] \(n\) = total number of expression nodes
    \item[] \(c_{in}\) = number of input columns
    \item[] \(p\) = number of pattern detectors
    \item[] \(n_{p}\) = number of expression nodes returned by a pattern detector
    \item[] \(b\) = branching factor of the compression tree
    \item[] \(h\) = height of the compression tree
    \item[] \(c_{out}\) = number of output columns of an expression node
\end{itemize}
\fi


\documentclass{scrartcl}
\usepackage[utf8]{inputenc}
\usepackage{natbib}
\usepackage{hyperref}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{color}
\usepackage{commath}
\usepackage{enumerate}
\usepackage{algpseudocode}

\bibliographystyle{plainnat}  % use the plainnat instead of plain

\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\minimize}{minimize}
\DeclareMathOperator*{\prox}{prox}
\DeclareMathOperator*{\loss}{loss}
\DeclareMathOperator*{\reg}{reg}
\def\RR{{\mathbb R}}
\newcommand{\blue}{\color{blue}}

\newtheorem{example}{Example}
\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{conjecture}[theorem]{Conjecture}
\newtheorem{axiom}[theorem]{Axiom}

\newcommand{\SAGA}{\textsc{Saga}}
\newcommand{\SAG}{\textsc{Sag}}
\newcommand{\LASSO}{\textsc{Lasso}}

\title{Large-scale optimization with the \textsc{SAGA} algorithm}
\author{Fabian Pedregosa \qquad Arnaud Rachez \qquad Mathieu Blondel \\\\
   % \emph{Chaire Havas-Dauphine  \'Economie des Nouvelles Donn\'ees} \\
   % \emph{Universit\'e Paris-Dauphine, PSL Research University} \\
   %  \emph{INRIA - Sierra project-team}
}
\date{\today} % {November 2015}

\begin{document}

\maketitle

\begin{abstract}
In this technical report we describe the implementation of the \SAGA\ algorithm in the lightning\footnote{\url{http://www.mblondel.org/lightning/}} python library. We discuss implementation details such as the use of lazy or lagged updates for squared $\ell_2$ and $\ell_1$ regularization.
\end{abstract}

\section{Introduction}

A large class of optimization problems in machine learning can be expressed as the minimization of a finite sum of the form
$$
\argmin_{x \in \RR^p} \left\{ \sum_{i=1}^n f_i(x) + \lambda \norm{x}^2 + \mu \Omega(x)\right\} \quad,
$$
where $\mu, \lambda$ are real constants, each $f_i: \RR^p \to \RR$ is convex and has Lipschitz continuous derivatives with constant $L$. $\Omega: \RR^p \to \RR$ is also assumed to be convex but potentially non-differentiable. We will further assume that we have acces to its proximal operator, denoted $\text{Prox}_{\mu}$.

These problems arise often in machine learning, where the cost function is a misfit over a large number of data points. The \SAGA\ algorithm~\citep{defazio2014saga} is a recently proposed algorithm to solve optimization problems of this type. As other iterative methods, this algorithm creates a sequence of iterates $w_1, w_2, \ldots$ that converge towards the desired solution $x^*$. The updates of \SAGA\ take the form
\begin{equation} \label{eq:update_rule}
x_{k+1} = \text{Prox}_{\gamma \mu}((1 - \gamma \lambda)x_k - \gamma (\nabla f_i(x_k) - {\alpha}_i + \bar{\alpha})) \quad,
\end{equation}
where $\bar{\alpha} = \sum_{i=1}^n \alpha_i$ and $\alpha_i$ is a table (memory) of historical gradients, $\gamma$ is the step size and $i$ is an index selected uniformly at random. The update rule for the historical gradients is $\alpha_i = f_i(x)$ for the selected $i$. The corrections will be used the next time the same index $i$ gets sampled. The \SAGA\ algorithm is further described in~\citep{defazio2014saga,hofmann2015variance}.

We will focus in the case in which the smooth term is linearly-parametrized, that is, $f_i(x)$ is of the form $g(a_i^T x)$ for all $i$. In this setting $\nabla f_i(x) = a_i g'(a_i^T x)$, so since each $a_i$ is constant we only need to store the scalar $g'(a_i^T x)$. This reduces the storage cost from $\mathcal{O}(n p)$ down to $\mathcal{O}(n)$.

The values of $\Omega$ that we will examine are the cases of $\ell_1$-regularization and $\ell_1/\ell_2$-regularization. $\ell_1$ regularization, known as \LASSO\ when the smooth term is a least squares loss, is a popular penalty that is commonly used to promote sparsity within the vector of coefficients. It is denoted $\norm{\cdot}_1$ and defined as the absolute sum of its components,
$
\norm{x}_1 = \sum_{i=1}^p |x_i|
$. Its proximal operator is the \emph{soft-thresholding} operator, and is defined component-wise as
\begin{equation}\label{eq:soft_thresholding}
\left[\text{Prox}_{\mu \norm{\cdot}_1}(x)\right]_j = \left(1 - \frac{\mu}{|x_j|}\right)_+ x_j =
\begin{cases}
x_j - \mu \text{ if } \mu \leq x_j \\
0 \text{ if } -\mu \leq x_j \leq \mu \\
\mu - x_j \text{ if }  x_j \leq - \mu \\
\end{cases}
\end{equation}

A practical implementation of \SAGA\ relies on a number of non-trivial techniques that need to be used. In this technical report we detail the techniques that we found where essential to obtain a practical implementation of this algorithm. These are:
\begin{itemize}
% \item {\bf Adaptive step size}. The SAGA algorithm relies on a particular choice of the step size $\gamma$. While in theory this step size can be estimated from the data, we found that much faster convergence can be achieved by choosing an \emph{adaptive} step size. We will detail this choice in Section~\ref{scs:step_size}.
\item {\bf Lagged updates}. Even if the gradients $\nabla f_i(x)$ are sparse, the vector $\bar{\alpha}$ is typically dense and so update rule~\eqref{eq:update_rule} involves dense vector operations. However, a technique known as implicit updates, lagged updates~\citep{defazio2014saga} or just-in-time updates~\citep{schmidt2013minimizing} can be used in order to reduce the vector update to sparse operations.
We extend this technique to the case of an $\ell_1$ penalty term.
\item {\bf Step size}. The theory of \SAGA\ is developed for a step-size that is sub-optimal. In particular,
\end{itemize}

\section{Impicit updates}

When the vectors $a_i$ are sparse (recall that we assume $f_i$ is of the form $f_i(x) = g(a_i^T x)$, an individual gradient $\nabla f_i$ will inherit the sparsity pattern of the corresponding $a_i$. However, the update rule~\eqref{eq:update_rule} appears unappealing since in general $\bar{\alpha}$ will be dense. Nevertheless, we can use the following technique to obtain iterations with a cost that is proportional to the number of non-zeros in $a_i$.

This technique consists in not explicitly storing the full vector $x_k$ after each iteration. Instead, on each iteration we only compute the elements corresponding to non-zero elements of $a_i$ and at the same time storing a record of when was the last time that a given feature or coordinate was updated. We denote the vector that keeps track of this $m$, such that $m_i$ returns the last iteration in which $x_i$ was updated.

We will first present the technique in its general form and then see how this simplifies for some common choice of the regularizers such as $\ell_2$ or $\ell_1$. The technique can be defined recursively as follows (assumes the prox is computed element-wise):

\begin{enumerate}[(i)]
\item In the first iteration, perform the update rule only on the support of $a_1$, which we denote $J$, and we updated the vector $m$.
$$
\begin{aligned}
(x^1)_J &\leftarrow \text{Prox}_{\gamma \mu}((1 - \gamma^k \lambda)x_k - \gamma^k (\nabla f_i(x_k) - {\alpha}_i + \bar{\alpha}))_J \\
m_J &\leftarrow 1
\end{aligned}
$$
\item At iteration $k$, we update the vector of coefficients $x_k$ to ensure that it is up-to-date on the support of $a_k$. That is, for all indices $j$ in the support of $a_i$:
\begin{algorithmic}
\For{$j \text{ in the support of }a_i$}
\For{$l \gets m_j,k$ }
\State $x_j \gets \text{Prox}_{\gamma^k \mu}((1 - \gamma^k \lambda)x_j - \gamma^k \bar{\alpha}_j)$
\EndFor
\State $m_j \gets k$ \Comment{ Mark the $j$-th variable as updated }
\EndFor
\end{algorithmic}
\item Perform the $k$-th update on the support of $a_i$:
\begin{algorithmic}
\For{$j \in \text{ support of }a_i$}
\State $(x_k)_j \gets \text{Prox}_{\gamma^k \mu}((1 - \gamma^k \lambda)x_k - \gamma^k (\nabla f_i(x_k) - {\alpha}_i + \bar{\alpha}))_j$
\EndFor
\end{algorithmic}
% $$
% \begin{aligned}
% (x_k)_j &\leftarrow \text{Prox}_{\gamma \mu}((1 - \gamma \lambda)x_k - \gamma (\nabla f_i(x_k) - {\alpha}_i + \bar{\alpha}))_j \\
% \alpha_i &\leftarrow \nabla f_i(x_k) (XXX)
% \end{aligned}
% $$
\end{enumerate}

The technique of implicit (or lagged) updates consists in finding a shortcut for the update in step $(ii)$. As we will see, this can be done for the case of $\ell_2$ regularization and (to a certain extend) for the case of $\ell_1$ regularization. For simplicity we define the following function $P$ which represents the update of a single coordinate. Given arbitrary real numbers $s, t$ this is defined as follows:
$$
U_t(s) = \text{{Prox}}_{\gamma \mu}((1 - \gamma \lambda)s - \gamma t) \quad,
$$
The update in step $(ii)$ can be written in this notation as $x \leftarrow U_{\bar{\alpha_j}}(x_j)$. The for loop in this step can be interpreted as the composition of $U$ with itself. Hence, the efficiency of the lagged update will depend on how fast we are able to compute $U^m$, where the exponent denotes composition with itself.

\subsection{Implicit updates with $\ell_2$ regularization}

We start with the simple case in which there is no non-smooth penalty term, i.e. $\Omega$ is zero. This setting is described in~\citep{defazio2014saga} and in ~\citep{schmidt2013minimizing} (in this case of the \SAG\ algorithm instead of \SAGA).

In this case, the updates in step (ii) take a simple form. Supposing $k-m_j = 1$ then the update for coordinate $j$ is simply
$$
x^+_j \gets (1 - \gamma \lambda)x_j - \gamma \bar{\alpha}_j \quad.
$$
If $k-m_j=2$ then the update becomes
$$
\begin{aligned}
x^+_j &\gets (1 - \gamma^{k} \lambda)((1 - \gamma^{k-1} \lambda)x_j - \gamma^{k-1} \bar{\alpha}_j) - \gamma^k \bar{\alpha}_j \quad. \\
&= (1 - \gamma^{k} \lambda)(1 - \gamma^{k-1} \lambda) x_j - \gamma^k \bar{\alpha} (1 + (1 - \gamma^{k-1} \lambda))
\end{aligned}
$$

From this it is easy to generalize to an arbitrary value of $k-m_j$. The full update in this case becomes:
\begin{algorithmic}
\For{$j \in \text{ support of }a_i$}
\State $x_j \gets \left(\prod_{i=1}^{k-m_j} (1 - \gamma^{k-i+1} \lambda )\right) x_j + \gamma^k \bar{\alpha} \left(1 + \sum_{i=1}^{k-m_j} \prod_{j=1}^i (1 - \gamma^{k-j} \lambda) \right)$
\State $m_j \gets k$
\EndFor
\end{algorithmic}

That this is indeed equivalent to the update rule in in (ii) can be easily proven by induction on $k - m_j$.

\subsection{Implicit updates for $\ell_2 + \ell_1$ regularization}

Through this section $\text{Prox}$ denotes the proximal operator for the $\ell_1$ norm, defined in Eq.~\eqref{eq:soft_thresholding}. Our update scheme is based upon the following lemma.

\begin{lemma}\label{lemma:update_l1}
Let $s, t$ be arbitrary real numbers and $P$ be a function defined as follows:
where $j$ is an arbitrary index. Let $P^m$ denote the composition of $P$ with itself $m$ times. Then, if $|\bar{\alpha}_j| \leq \mu$ or if $\bar{\alpha}_j x_j \leq 0$, then it is verified that
$$
P^m(s, t) =
\text{\emph{Prox}}_{m \gamma \mu}\left((1 - \gamma \lambda)^{m} x_j + \gamma \bar{\alpha}_j \sum_{i=1}^{m} (1 - \gamma \lambda)^{i-1}\right)
$$
\end{lemma}
\begin{proof}
% Take first the case of $k - m_j = 1$. The update in (ii) takes the following form
% $$
% x_j \gets \text{Prox}_{\gamma \mu}((1 - \gamma \lambda)x_j - \gamma \bar{\alpha}_j)
% $$
We first consider the case in which $\abs{\bar{\alpha}_j} \leq \mu$. Make make the following distinction of cases:
\begin{itemize}
\item $x_j = 0$. In this case $P(x_j) = \text{{Prox}}_{\gamma \mu}(- \gamma \bar{\alpha}_j)$ and the assumption $\abs{\bar{\alpha}_j} \leq \mu$ implies that $T(x_j) = \text{{Prox}}_{\gamma \mu}(- \gamma \bar{\alpha}_j) = 0$.
\end{itemize}
% and $(1 - \gamma \lambda)x_j - \gamma \bar{\alpha}_j \geq \gamma \mu$
. Then by the definition of soft-thresholding we have the update
$$
T(x) =
$$
XXX
$$
x_j^+ \gets (1 - \gamma \lambda)x_j - \gamma \bar{\alpha}_j - \gamma \mu
$$
\end{proof}

This proximal function also features a homogeneous property, namely that if $x = \tau w$, then
$$
\text{Prox}_\lambda(x) = \tau \text{Prox}_{\lambda /\tau}(w_k) \quad.
$$

This, together with Lemma~\ref{lemma:update_l1}, yield the following lagged update rule for the $\ell_1$ regularization.

% \section{Adaptive step size} \label{scs:step_size}

% Following~\citep{schmidt2013minimizing,schmidt2015non}, we take the step size to be $\gamma = 1 / L$, where $L$ is an approximation to the maximum Lipschitz constant of the gradients. This is the smallest number such that
% $$
% \norm{\nabla f_i(w) - \nabla f_i(v)} \leq L \norm{w - v}\quad,
% $$
% for all $i, w$ and $v$. A classical result on quadratic upper bounds for $L$-smooth functions (see e.g.~\citep[Lemma 1.2.3]{nesterov2004introductory}) states that the following inequality is verified:
% \begin{equation*}
% f_{i}(x_k - \frac{1}{L}\nabla f_{i}(x_k)) \leq f_{i}(x_k) - \frac{1}{2 L} \norm{\nabla f_{i}(x_k)}^2
% \end{equation*}
% The adaptive step size strategy is the following: at the current iterate, compute the above inequality for the current value of $i$ and $k$. If the inequality is satisfied, use the step size $1 / L$. Otherwise, double the value of $L$ until the inequality is verified.

\subsection{Experiments}

Some convergence plots.

\section{Choice of step size}

\section{Feedback}

\bibliography{biblio}{}
\end{document}


\documentclass[12pt]{cdblatex}
\usepackage{exercises}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% --------------------------------------------------------------------------------------------
\section*{Exercise 4.3 Polynomial products}

\begin{cadabra}

   {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w#}::Indices(position=independent).

   def get_term (poly,n):

       x^{a}::Weight(label=xnum).     # assign weights to x^{a}

       foo := @(poly).                # make a copy of poly
       bah  = Ex("xnum = " + str(n))  # choose a target
       keep_weight (foo,bah)          # extract the target

       return foo

   def poly_product (p,q,n):

       pq = Ex("0")

       for i in range (0,n+1):
          for j in range (0,i+1):
             termA = get_term (p,j)
             termB = get_term (q,i-j)
             termAB := @(termA) @(termB).
             pq = pq + termAB

       sort_product   (pq)
       rename_dummies (pq)
       factor_out     (pq,$x^{a?}$)

       return pq

   # ---------------------------------------------------------------

   # two polynomials

   polyA := c^{a}
          + c^{a}_{b} x^b
          + c^{a}_{b c} x^b x^c
          + c^{a}_{b c d} x^b x^c x^d
          + c^{a}_{b c d e} x^b x^c x^d x^e.    # cdb(ex-0403.100,polyA)

   polyB := d^{f}
          + d^{f}_{b} x^b
          + d^{f}_{b c} x^b x^c
          + d^{f}_{b c d} x^b x^c x^d
          + d^{f}_{b c d e} x^b x^c x^d x^e.    # cdb(ex-0403.101,polyB)

   # multiply polynomials and truncate

   polyAB = poly_product (polyA,polyB,3)        # cdb(ex-0403.102,polyAB)

\end{cadabra}

\begin{dgroup*}
   \Dmath*{  p = \Cdb*{ex-0403.100} }
   \Dmath*{  q = \Cdb*{ex-0403.101} }
   \Dmath*{ pq = \Cdb*[\hskip1cm\hfill]{ex-0403.102} }
\end{dgroup*}

\end{document}


\chapter{Introduction}

\section{Knowledge Discovery in Database, KDD}

The overall process of non-trivial extraction of implicit, previously unknown and potentially useful knowledge from large amounts of data

\begin{align*}
    Database &\xrightarrow{Data\ Cleaning} Data Warehouse \\
    &\xrightarrow{Selection\ Transformation} Task relevant\ Data \\
    &\xrightarrow{Data\ Mining} Pattern \\
    &\xrightarrow{Pattern\ Evaluation\ and\ Visualisation} Knowledge
\end{align*}

\section{Data Mining}

\begin{description}
  \item[Prediction\ Methods] Classification, Outlier Detection, Regression
  \item[Description Methods] Clustering, Association Rule, Sequence Pattern
\end{description}


\input{../tex/headerfile}
\input{../tex/mathdefs}
\setcounter{MaxMatrixCols}{20}
\usepackage{enumerate}
\usepackage{Sweave}
\begin{document}
\input{basic-matrix-math-key-concordance}

\chapter*{Solutions Chapter \ref{chap:basicmat}}
\addcontentsline{toc}{chapter}{Solutions Chapter \ref{chap:basicmat}}

%######################################
%# Basic Matrix Math
%# Homework Questions
%######################################

\begin{enumerate}
\item
\begin{Schunk}
\begin{Sinput}
 A=matrix(1:4,4,3)
\end{Sinput}
\end{Schunk}
\item
\begin{Schunk}
\begin{Sinput}
 A[1:2,1:2]
\end{Sinput}
\end{Schunk}
\item
\begin{Schunk}
\begin{Sinput}
 A=matrix(1:12,4,3, byrow=TRUE)
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 #end up with a vector
 A[3,]
 #end up with a matrix
 A[3,,drop=FALSE]
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 B=matrix(1,4,3)
 B[2,3]=2
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 t(B)
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 diag(1:4)
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 B=diag(1,5)
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 diag(B)=2
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 diag(1,4)+1
 #or
 B=matrix(1,4,4)
 diag(B)=2
 B
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 solve(B)
 #or this but only works because B is symmetric
 chol2inv(chol(B))
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 B=matrix(letters[1:9],3,3)
 B
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 diag(B)="cat"
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 A=matrix(1,4,3)
 B=matrix(2,3,4)
 A%*%B
 #or
 B%*%A
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 # A%*%A #throws an error
 A%*%t(A) #works
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 #this is an example where you use B to select values in A
 A=matrix(1:9,3,3)
 B=matrix(0,3,3)
 B[1,1]=1
 B[2,3]=1
 B[3,2]=1
 C=A%*%B
 diag(C)
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 #this shows one of the uses of diagonal matrices
 B=diag(2,3)
 C=A%*%B
 C
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 #this shows how to use a column vector (matrix with 1 col)
 #to compute row sums
 B=matrix(1,3,1)
 C=A%*%B
 C
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 #this shows how to use a row vector (matrix with one row)
 #to compute column sums
 B=matrix(1,1,3)
 C=B%*%A
 C
\end{Sinput}
\end{Schunk}

\item
\begin{Schunk}
\begin{Sinput}
 A=diag(1,3)+1
 C=matrix(3,3,1)
 #AB=C
 #B=inv(A)%*%C
 B=solve(A)%*%C
 B
\end{Sinput}
\end{Schunk}

\end{enumerate}

\bibliography{../tex/Fish507}

\end{document}

docs/Labs/Week 0 basic matrix/basic-matrix-math-key.tex