\documentclass[a4paper,11pt]{article}
\usepackage[english]{babel}
\usepackage{a4}
\usepackage[cm]{fullpage}
\usepackage{amsmath,amsfonts,amssymb}
\usepackage{amsthm}
\usepackage[T1]{fontenc}
\usepackage{lmodern} % Latin modern font family
\usepackage{enumitem}
\usepackage{fitch} % http://folk.uio.no/johanw/FitchSty.html

\newtheorem*{lemma}{Lemma}
\newtheorem*{theorem}{Stelling}

\newcommand{\dotmin}{\buildrel\textstyle.\over{\hbox{\vrule height3pt depth0pt width0pt}{\smash-}}}

% Corner quotes (from http://www.phil.cam.ac.uk/teaching_staff/Smith/LaTeX/resources/godelquotes.txt)
\newbox\gnBoxA
\newdimen\gnCornerHgt
\setbox\gnBoxA=\hbox{$\ulcorner$}
\global\gnCornerHgt=\ht\gnBoxA
\newdimen\gnArgHgt
\def\Godelnum #1{%
	\setbox\gnBoxA=\hbox{$#1$}%
	\gnArgHgt=\ht\gnBoxA%
	\ifnum \gnArgHgt<\gnCornerHgt
		\gnArgHgt=0pt%
	\else
		\advance \gnArgHgt by -\gnCornerHgt%
	\fi
	\raise\gnArgHgt\hbox{$\ulcorner$} \box\gnBoxA %
		\raise\gnArgHgt\hbox{$\urcorner$}}

% Sans-serif fonts
%\usepackage[T1experimental,lm]{sfmath} % http://dtrx.de/od/tex/sfmath.html
%\renewcommand{\familydefault}{\sfdefault}

% Some configuration for listings
\renewcommand{\labelenumi}{\arabic{enumi}.}
\renewcommand{\labelenumii}{(\alph{enumii})}

\newcounter{firstcounter}
\newcommand{\labelfirst}{(\roman{firstcounter})}
%\newcommand{\spacingfirst}{\usecounter{firstcounter}\setlength{\rightmargin}{\leftmargin}}
\newcommand{\spacingfirst}{\usecounter{firstcounter}}

\newcounter{secondcounter}
\newcommand{\labelsecond}{(\arabic{secondcounter})}
%\newcommand{\spacingsecond}{\usecounter{secondcounter}\setlength{\rightmargin}{\leftmargin}}
\newcommand{\spacingsecond}{\usecounter{secondcounter}}

\title{Recursion Theory (UvA autumn 2008)\\
\normalsize{Exercises Part 7 -- Martijn Vermaat (mvermaat@cs.vu.nl)}}

%\author{Martijn Vermaat (mvermaat@cs.vu.nl)}
%\date{Updated 5th December 2005}
\date{}

\begin{document}

\maketitle

\paragraph{Exercise 32:1}

Suppose $\psi(x_1, x_2)$ represents `$x_2$ codes a proof of the formula
with G\"odel number $Sub(x_1, x_1)$'. Let $n$ be the G\"odel number of
$\forall x_2 \neg \psi(x_1, x_2)$.

\begin{enumerate}[label=(\roman*)]

\item
Show that if $T$ is consistent, $T \not\vdash \forall x_2 \neg \psi(\underline{n}, x_2)$.

\begin{proof}
Suppose $T \vdash \forall x_2 \neg \psi(\underline{n}, x_2)$,
i.e. there is a proof of $\forall x_2 \neg \psi(\underline{n}, x_2)$ in $T$.
Let's say this proof has G\"odel number $p$.

The G\"odel number of $\forall x_2 \neg \psi(\underline{n}, x_2)$ is $Sub(n, n)$,
because $Sub(\Godelnum{\forall x_2 \neg \psi(x_2, x_2)}, n) = \Godelnum{\forall x_2 \neg \psi(\underline{n}, x_2)}$.

Reformulating, $p$ codes a proof of the formula with G\"odel number $Sub(n, n)$.
This is represented by $\psi(\underline{n}, \underline{p})$,
so $T \vdash \psi(\underline{n}, \underline{p})$.
This is in contradiction with $T$ being consistent.
\end{proof}

\item
Prove that if $T$ is also $\omega$-consistent, $\neg \forall x_2 \neg \psi(\underline{n}, x_2)$
is not a theorem of $T$ either.

\begin{proof}
We have $T \vdash \neg \psi(\underline{n}, \underline{p})$ for any $p$,
because $\psi(\underline{n}, \underline{p})$ represents that $p$ codes
a proof of the formula with G\"odel number $Sub(n, n)$ and there is no
such proof by (i).

By $T$ $\omega$-consistent, $T \not\vdash \neg \forall x_2 \neg \psi(\underline{n}, x_2)$.
\end{proof}

\end{enumerate}

\paragraph{Exercise 32:2}

Note that a real number is computable if and only if its absolute value is
computable. Therefore, we only have to consider positive real numbers.

\begin{enumerate}[label=(\alph*)]

\item
Show that every rational number is computable.

\begin{proof}
Consider the rational number $\frac{m}{n}$ with $m, n \in \omega$ and $n \neq 0$.
Take $f(x) = m$ and $g(x) = n$.
Both are obviously computable and $|\frac{m}{n}| = \frac{f(x)}{g(x)}$ for all $x > 0$.
\end{proof}

\item
Show that (i) $e$ and (ii) $\pi$ are computable.

\begin{proof}
\begin{enumerate}[label=(\roman*)]
\item Take $f(x) = \Sigma_{n=0}^{x+1} \frac{(x+1)!}{n!}$ and $g(x) = (x+1)!$.
Now we have $|e - \frac{f(x)}{g(x)}| = \Sigma_{n=x+2}^\infty \frac{1}{n!}$
which is smaller than $\Sigma_{n=1}^\infty \frac{1}{(x+1)^n} \leq \frac{1}{x}$ for $x > 0$.
\item Sorry, no time left.\qedhere
\end{enumerate}
\end{proof}

\item
Show that if $a$ and $b$ are computable, so are $a + b$, $a - b$, and $a \cdot b$; and if $b \neq 0$,
$\frac{a}{b}$ is computable.

\begin{proof}
There are computable $f_a$, $g_a$ for $a$ and $f_b$, $g_b$ for $b$ as in the definition of
computability of a real number.

\begin{enumerate}[label=(\roman*)]
\item
Take $f(x) = f_a(2x) \cdot g_b(2x) + f_b(2x) \cdot g_a(2x)$ and $g(x) = g_a(2x) \cdot g_b(2x)$.
Both $f$ and $g$ are computable and $\left|\left|a+b\right| - \frac{f(n)}{g(n)}\right| < \frac{1}{n}$ for $n > 0$.

(I'm hasty and not entirely sure this is correct, but it looks like we can get twice the inaccuracy
of $\frac{1}{n}$ in the worst-case (for both $a$ and $b$). Computing the parts for twice the
accuracy should correct this.)

There are some problems for $a$ or $b$ negative.
\item
Likewise.
\item
Take $f(x) = f_a(x) \cdot f_b(x)$ and $g(x) = g_a(x) \cdot g_b(x)$, but this is not precise enough.
\item \ldots \qedhere
\end{enumerate}
\end{proof}

\item
Sorry, no time left.

\item
Sorry, no time left.

\end{enumerate}

\paragraph{Exercise 32:3}

\begin{enumerate}[label=(\alph*)]

\item
Show:
\begin{equation*}
  T \text{ is axiomatizable } \quad \Longleftrightarrow \quad Thm_T \text{ is computably enumerable}
\end{equation*}

\begin{proof}
\begin{description}
\item[($\Rightarrow$)]
There is an axiomatized theory $T'$ with $Thm_T = Thm_{T'}$. Since the set of G\"odel
numbers of theorems of an axiomatized theory is computably enumerable, $Thm_T$ is
computably enumerable.

\item[($\Leftarrow$)]
By $Thm_T$ computably enumerable, there is a computable function $f(x)$ with $Thm_T$ as
its range. Define
\begin{equation*}
  A = \{ \bigwedge_{i=0}^n f(i) \text{ : } n \in \omega \} \text{ ,}
\end{equation*}
where we take $\bigwedge_{i=0}^n \varphi_i$ to be the G\"odel number of the conjunction
of the formulas coded by $\varphi_0 \ldots \varphi_n$. $A$ is computable, since its
characteristic function can decide for any input if it is the G\"odel number of a
conjunction of formulas $\varphi_0 \ldots \varphi_k$ such that $f(i) = \Godelnum{\varphi_i}$.

Consider the first-order theory $T'$ with axioms $A$. It is axiomatized and $Thm_T = Thm_{T'}$.\qedhere

\end{description}
\end{proof}

\item
The theory with as axioms the theorems of $N$ is axiomatizable (it is equivalent to
$N$ and $N$ is axiomatized), but not axiomatized ($Thm_N$ is not computable).

\end{enumerate}

\paragraph{Exercise 34}

Prove that $K$ is not an index set.

\begin{proof}
By Proposition 33.2, there exists an $n \in K$ such that $W_n = \{n\}$.
By the Padding Lemma, there is an $m > n$ with $\varphi_m = \varphi_n$.
If $K$ were an index set, $m$ would be in $K$.
But this implies $m \in W_m$ while $W_m = W_n = \{n\}$.
Contradiction.
\end{proof}

\end{document}


\documentclass{article}

\usepackage[squaren]{SIunits}
\usepackage{amsmath,amsfonts}
\usepackage{appendix}
\usepackage{enumerate}
\usepackage{listings}
\usepackage{url}

\newcommand{\cps}{\textnormal{cps}}
\newcommand{\braces}[1]{\ensuremath{\left\lbrace #1 \right\rbrace}}
\newcommand{\angles}[1]{\ensuremath{\left\langle #1 \right\rangle}}
\newcommand{\brackets}[1]{\ensuremath{\left[ #1 \right]}}
\newcommand{\parens}[1]{\ensuremath{\left( #1 \right)}}
\newcommand{\setbuilder}[2]{\ensuremath{\braces{#1 \left| #2 \right.}}}

\newcommand{\stdin}{\texttt{stdin}}
\newcommand{\stdout}{\texttt{stdout}}
\newcommand{\stderr}{\texttt{stderr}}

\newcommand{\program}[1]{\texttt{#1}}

\newcommand{\gn}[1]{\ensuremath{g^{(#1)}}}
\newcommand{\integers}{\ensuremath{\mathbb{Z}}}
\newcommand{\wholes}{\ensuremath{\mathbb{N}}}
\newcommand{\reals}{\ensuremath{\mathbb{R}}}
\newcommand{\rationals}{\ensuremath{\mathbb{Q}}}
\renewcommand{\vec}{\boldsymbol}
\newcommand{\abs}[1]{\ensuremath{\left|#1\right|}}
\newcommand{\eV}{\textnormal{eV}}
\newcommand{\channel}{\ensuremath{c}}
\newcommand{\channels}{\ensuremath{C}}
\newcommand{\Channel}{\ensuremath{\mathcal{C}}}
\newcommand{\Time}{\ensuremath{\mathcal{T}}}
\newcommand{\photon}{\ensuremath{\gamma}}
\newcommand{\photons}{\ensuremath{\Gamma}}
\newcommand{\Pulse}{\ensuremath{\mathcal{P}}}
\newcommand{\integrationtime}{\ensuremath{\Xi}}
\newcommand{\timewindow}{\ensuremath{\xi}}
\newcommand{\resolution}{\ensuremath{\epsilon}}
\newcommand{\Index}{\ensuremath{\mathcal{J}}}
\newcommand{\Histogram}{\ensuremath{\mathcal{H}}}
\newcommand{\ceil}[1]{\ensuremath{\left\lceil #1\right\rceil}}
\newcommand{\GN}{\texttt{gn}}
\newcommand{\tstart}{\integrationtime\upminus}
\newcommand{\tstop}{\integrationtime\upplus}
\newcommand{\correlationset}{\ensuremath{G}}
\newcommand{\upplus}{\ensuremath{^{(+)}}}
\newcommand{\upminus}{\ensuremath{^{(-)}}}
\newcommand{\nth}{\ensuremath{n}\textnormal{th}}
\newcommand{\closedopenrange}[2]{\ensuremath{\left[#1, #2\right)}}
\newcommand{\timedelay}{\ensuremath{\tau}}
\renewcommand{\time}{\ensuremath{t}}
\newcommand{\pulse}{\ensuremath{p}}
\newcommand{\pulsedelay}{\ensuremath{\rho}}
\newcommand{\bin}{\ensuremath{b}}
\newcommand{\counts}{\ensuremath{n}}
\newcommand{\reprate}{\ensuremath{f}}
\newcommand{\intensity}{\ensuremath{I}}
\newcommand{\heaviside}{\ensuremath{H}}

\title{Photon correlation: General-purpose tools for interacting with photon-arrival data, and for performing $n$th order correlations \\
v1.6}
\author{Thomas Bischof \\ \texttt{tbischof@mit.edu} \\ \url{http://tsbischof.dyndns.org}}
\date{\today}

\numberwithin{equation}{section}
\numberwithin{figure}{section}

\begin{document}

\maketitle
\tableofcontents
\newpage

\section{Introduction}
\subsection{The layout of this document}
This document is laid out in roughly three parts:
\begin{enumerate}
\item Overview of terminology and methods
\item Documentation for each program
\item Applications of the software to real problems
\end{enumerate}
In the chapters devoted to the various programs, the documentation is divided further:
\begin{enumerate}
\item Command-line syntax
\item Theoretical overview of the purpose of the program
\item Details of the implementation
\end{enumerate}
If you are familiar with the details of photon correlation methods, you can safely skip to the sections describing the different programs, and to the discussion of their applications.

\subsection{A (very) brief overview of photon-arrival timing}
In single-molecule spectroscopy, single-photon detectors are often used to perform time-resolved experiments. Detectors such as silicon-based avalanche photodiodes (APDs) can be used to detect the arrival time a photon with about 500\pico\second{} resolution, and can detect up to $10^{7}$ photons per second before saturation, abbreviated as 10\mega\cps{} hereafter (counts per second). Hardware capable of resolving these arrival times is of great use for revealing time-dependent structure to the photon stream, such as intensity fluctuations and bunching, so various hardware designs have been developed to permit such measurements.

One line of instruments is the timing hardware produced by Picoquant GmBH, such as the Timeharp, Picoharp, and Hydraharp. These modules are capable of detecting pulse arrivals on multiple input channels with a resolution of as little as 1\pico\second, and operate in a few distinct modes:
\begin{enumerate}
\item Interactive (histogram): one input channel is designated as a sync source, representing a clock-starting signal. On the other channels, pulse arrival times are recorded relative to this clock source, and the times binned into a histogram.
\item Time-tagged time-resolved (TTTR):
	\begin{enumerate}
	\item T2: all channels are treated equally, and all pulse arrival times relative to the start of the experiment are recorded.
	\item T3: this is similar to histogram mode, but instead of binning the arrival times, the sync event number and relative arrival time are recorded.
	\end{enumerate}
\end{enumerate}

In all cases, the times are discrete and represent some number of cycles of a clock, so all times must be treated as integers representing some number of picoseconds. This has important effects on the definition of time bins for histograms, and where appropriate some time will be devoted to discussing these factors.

Considering these distinct modes, it is important to spend some time discussing their uses, and how their data should be handled.

\subsection{Data collection modes}
\label{sec:modes}
\subsubsection{Interactive (histogram)}
A common experiment for studying fluorophores is to measure the time dependence of their response to an excitation. For example, a pulsed laser can be used to excite a sample, and the resulting fluorescence detected by an APD. Because single-photon detectors are limited to detection of a single photon at once, to reconstruct the decay curve it is necessary to average the result over many pulses of the laser, binning the arrival times of the emitted photons relative to that of the laser. If measuring this time-averaged behavior is sufficient, then the interactive histogram mode is used to perform all of this collection and binning on the hardware without any post-processing.

Consequently, interactive data consists of $N$ unique arrival time bins with boundaries $\parens{\bin_{j}\upminus, \bin_{j}\upplus}$ and the number of counts $\counts_{j}$ associated with that bin, so the data can be represented by the set:
\begin{equation}
\braces{
    \parens{
          \parens{\bin_{j}\upminus,
                  \bin_{j}\upplus},
          \counts_{j}}}
\end{equation}
The exact choice of where the boundaries lie has some effect on the resulting histogram, but this performed in the hardware and presumably represents $\time\in\closedopenrange{\bin_{j}\upminus}{\bin_{j}\upplus}$. This choice is not detailed in the manuals for the hardware, but it is of little practical importance, since all photons will be counted one way or the other.

\subsubsection{T2}
In T2 mode, all input channels are connected to photon detectors. At the start of the experiment, an internal clock is reset and started, providing a master timing reference. As a pulse arrives, the machine emits data encoding the channel and time of arrival, so the data follow the form:
\begin{equation}
\braces{\parens{\channel, \time}}
\end{equation}
for a channel \channel{} and arrival time \time{} of a photon.
%Typically, these data are used to examine time-dependent behavior, such as intensity fluctuations, particularly when the excitation source is continuous-wave. These data are also useful for calculation of correlation functions such as:
%\begin{equation}
%\gn{n}(\tau_{1}, \ldots \tau_{n-1}) = \frac
%	{\angles{I_{0}(t)\prod_{j=1}^{n-1}{I_{j}(t+\tau_{j})}}}
%	{\angles{I_{0}(t)}\prod_{j=1}^{n-1}{\angles{I_{j}(t+\tau_{j})}}}
%\end{equation}
%The details of this calculation will laid out later in section~\ref{sec:correlate}, but one use of the $g^{(2)}(t)$ is to determine the number of emitters present in a signal. For example, a single emitter emitting one photon at any time will exhibit so-called antibunching behavior, where $g^{(2)}(t)\rightarrow 0$ for $t\approx 0$, indicating a diminished probability of seeing two successive photon emissions ($g^{(n)}(\braces{t_{j}})=1$ indicates no correlation). Correlations of higher order ($n\ge 3$) have their own uses, but quickly become computationally expensive for reasons which will become clear later.

\subsubsection{T3}
\label{sec:t3}
T3 mode is closest in character to the interactive mode, except that, instead of binning the photon arrival times on the hardware, the arrivals are recorded directly for later examination, as in T2 mode. This gives a data set of the form:
\begin{equation}
\braces{\parens{\channel, \pulse, \time}}
\end{equation}
for channel \channel{}, sync pulse number \pulse{}, and relative arrival time \time{}. Indeed, by defining temporal bins and histogramming the \time{}, the result from the interactive mode can be reproduced exactly.

\paragraph{Mapping T3 onto T2 data}
\label{sec:t3_to_t2}
For signals produced by regular excitation it is possible to convert T3 data to T2 data. For example, consider a sample excited by a laser with repetition rate \reprate. The pulse number therefore defines a time $1/\reprate$, and \time{} acts as a correction to this time:
\begin{equation}
\parens{\channel, \pulse, \time} \rightarrow \parens{\channel, \frac{\pulse}{\reprate} + \time}
\end{equation}
Of course, this conversion is not necessarily perfect. For example, sync pulses may be missed or overcounted, introducing a cumulative error $\delta\pulse$, the frequency of the sync pulse might deviate with some jitter $\delta\reprate$, or the timing of the arrival might have some jitter $\delta\time$, such that the true expression is:
\begin{equation}
\parens{\channel, \pulse + \delta\pulse, \time + \delta\time}
      \rightarrow
\parens{\channel, \frac{\pulse+\delta\pulse}{\reprate+\delta\reprate} + \time+\delta\time}
\end{equation}
For a well-designed system these errors should not be very large, but if an application requires precision approaching that of the timing hardware they can be quite important.

% However, this mode also allows for studying time dependence and correlation in the data, as with T2 mode. In principle, any data collected in T3 mode can be transformed into T2 mode data if the sync source is regular, as the pulse number will represent some amount of time in the experiment, but there are subtle hardware and numerical issues (discussed later) which limit the practicality of this transformation.
%
%Uses of this mode include the study of the time-dependent fluoresence lifetime of single molecules, which can switch between distinct states under various conditions. Correlation methods can also reveal important behavior, but again this will be discussed later.

\subsection{Correlation techniques}
A number of techniques involving the correlation of one or more signals exist, and their use in fluorescence microscopy requires that we be able to calculate correlation functions of the following form:
\begin{equation}
\gn{n}(\timedelay_{1}, \ldots \timedelay_{n-1}) =
    \frac{\angles{\prod_{j=0}^{n-1}{\intensity(\time+\timedelay_{j})}}}
	     {\prod_{j=0}{\angles{\intensity(\time+\timedelay_{j})}}}
\end{equation}
where $\timedelay_{0}=0$ for notational simplicity. The details of calculating this function will be laid out in section~\ref{sec:math_background}. For now, note that the problem can be divided into three distinct parts:
\begin{enumerate}
\item Calculation of \angles{\intensity(\time+\timedelay_{j})}
\item Finding correlation events
\item Histogramming correlation events
\end{enumerate}

\subsection{General design principles}
Analysis of these timing data can be roughly described as follows:
\begin{enumerate}
\item Produce a stream of photon events
\item Condition the stream by correlation, removal of extraneous information, homogenization of the detection channels, etc.
\item Collect the result to form some histogram of events.
\end{enumerate}
As such, these three phases are handled by distinct programs, which act as filters of a data stream by reading in values and outputting the appropriate new values. These streams are ultimately streams of binary data, but for any program the stream may be defined by the standard data streams (\stdin, \stdout) or by some file containing the data.

For time-tagged modes, times are represented as integer multiples of 1\pico\second, although only the initial data streamer is actually aware of units; the conditioning and collection routines operate on time as an integer, without regard for its units.

All programs are designed to operate on a data stream until that stream terminates, so any division of a data stream should be handled by a separate program and the result fed into a distinct instance of the handler.

The main processing software and some helper routines are written in the C programming language, using the C99 standard. Where necessary, the definitions of data types have been defined to be of fixed width, but otherwise the definitions are those minimal for handling reasonable values. For example, in T2 mode time is defined as a signed 64-bit integer, representing $\approx 10^{19}\pico\second$, or $10^{7}\second$, or $106$ days. Pulses are counted as signed 64-bit integers, limiting their total to several thousand years of 10\mega\hertz{} pulsed laser excitation. With luck, your experiments will not exceed these values.

In addition to the C code, several analysis scripts written in Python have been provided in \texttt{scripts/}. These include routines to read in the outputs of most of the processing software, generate correlation functions, plot lifetime data, and so on.

All programs will display command-line syntax when the flag \texttt{-h} or \texttt{--help} is given. The C programs have been tested most thoroughly on 32-bit and 64-bit Linux systems with the GNU C compiler, but in principle should work on any system with a C99-compliant compiler. Non-standard libraries are not used, to limit portability issues. A Makefile is present in the \texttt{src} directory which should be sufficient for compiling the software, although it will require some modification for different compiler configurations.

The Python programs have been tested primarily with version 3.2 on the same systems described above.

\subsection{Brief description of each program and its purpose}
\paragraph{\program{bin\_intensity}}
Determines the exact intensity for each time bin for a correlation. For most purposes, \program{intensity} is sufficient.

\paragraph{\program{channels}}
Manipulates data arriving on channels. This tool can either suppress a channel entirely or add a fixed time or pulse to each photon arriving on a given channel.

\paragraph{\program{correlate}}
Produces a stream of correlation events from a stream of photons.

\paragraph{\program{correlate\_vector}}
Produces the correlation function of a signal represented by a vector.

\paragraph{\program{gn}}
Performs the calculation of a photon correlation function, including all normalization.

\paragraph{\program{histogram}}
Histograms correlation events to form the correlation function, without any normalization.

\paragraph{\program{intensity}}
Counts the number of photons arriving in fixed time windows.

\paragraph{\program{intensity\_to\_t2}}
Converts a single-channel intensity into T2 photons, such that a photon correlation function can be calculated directly for data where only photon number is reported.

\paragraph{\program{picoquant}}
Decodes binary data generated by Picoquant GmBH hardware. This produces data in the general formats as defined in section~\ref{sec:modes}.

\paragraph{\program{t3\_as\_t2}}
Given a stream of T3-mode photons, strips the arrival time information to leave just the arrival pulse. This is useful for treating T3 data as T2 data, if the arrival time is not important. If the arrival time is important, pass the flag \texttt{--to-t2} in \program{picoquant}, or use the formula of section~\ref{sec:t3_to_t2}.

\input{math.tex}
\input{programs.tex}
\input{applications.tex}
\input{appendix.tex}

\end{document}


\chapter{Atomic and Molecular Mass}

A proton and a neutron have about the same mass. An electron, on the
other hand, has much less mass: One neutron weighs about the same
amount as 2000 electrons. Thus, the mass of any object comes mostly
from the protons and neutrons in the nucleus of its atoms.\index{proton} \index{neutron}

We know how many protons an atom has by what element it is, but how do we know the number neutrons?

If you buy a balloon filled with helium, it will have two different
kinds of helium atoms: Most of the helium atoms will have 2 neutrons, but a
few will have only 1 neutron. We say that these are two different
\textit{isotopes} of helium. We call them helium-4 (or $^4He$) and
helium-3 (or $^3He$).  Isotopes are named for the sum of protons and
neutrons the atom has: helium-3 has 2 protons and 1 neutron.\index{isotopes}
% KA: https://www.khanacademy.org/science/chemistry/atomic-structure-and-properties/introduction-to-the-atom/v/atomic-number-mass-number-and-isotopes

A hydrogen atom nearly always has just 1 proton and no neutrons. A
helium atom nearly always has 2 protons and 2 neutrons. So, if you
have a 100 hydrogen atoms and 100 helium atoms, the helium will have
about 4 times more mass than the hydrogen. We say ``Hydrogen is about
1 atomic mass unit(amu), and helium-4 is about 4 atomic mass
units.''\index{atomic mass unit}

What, precisely, is an atomic mass unit? It is defined as 1/12 of
the mass of a carbon-12 atom. Scientists have measured the mass of
helium-4, and it is about 4.0026 atomic mass units. (By the way, an
atomic mass unit is also called a \textit{dalton}.)

\pagebreak

Now you are ready to take a good look at the periodic table of
elements. Here is the version from Wikipedia:\index{periodic table of elements}

\includegraphics[width=0.75\textwidth]{periodic.png}

% ADD: Periodic Trends, Periods, Collums, Atomic Radius, Electronegativity,

\pagebreak
There is a square for each element. In the middle, you see the atomic
symbol and the name of the element. In the upper right corner is the
atomic number -- the number of protons in the atom.

In the upper left corner is the atomic mass in atomic mass units.\index{atomic mass}

\includegraphics[width=0.8\textwidth]{Atomic_Carbon_Diagram.png}

Look at the atomic mass of boron. About 80\% of all boron atoms have
six neutrons. The other 20\% have only 5 neutrons. So most boron atoms
have a mass of about 11 atomic mass units, but some have a mass of
about 10 atomic mass units. The atomic mass of boron is equivalent to the average
mass of a boron atom: 10.811.
% ADD: Talk about mass spectroscopy
%\includegraphics[width=0.75\textwidth]{KA_Mass_Spectroscopy_Zr.png}

\begin{Exercise}[title={Mass of a Water Molecule}, label=water_mass]

Using the periodic table, what is the average mass of one water molecule in atomic mass units?

\end{Exercise}
\begin{Answer}[ref=water_mass]

  The average hydrogen atom has a mass of 1.00794 atomic mass units.

  The average oxygen atom has a mass of 15.9994.

  $2 \times 1.00794 + 15.9994 = 18.01528$ atomic mass units.

\end{Answer}

\section{Molar Mass}

An atomic mass unit is a very, very, very small unit; we would much
rather work in grams.  It turns out that $6.02214076 \times 10^{23}$
atoms equal 1 mole( a standard measure for chemistry). Scientists use this number so much
that they gave it a name: \textit{the Avogadro constant} or
\textit{Avogadro's number}.\index{Avogadro's number}
% KA: https://www.khanacademy.org/science/ap-chemistry-beta/x2eef969c74e0d802:atomic-structure-and-properties/x2eef969c74e0d802:moles-and-molar-mass/v/the-mole-and-avogadro-s-number

If you have 12 doughnuts, that's a dozen doughnuts.  If you have
$6.02214076 \times 10^{23}$ doughnuts, you have a \textit{mole} of
doughnuts. (Note: it isn't practical to measure doughnuts this
way: A mole of doughnuts would be about the size of the earth. We use
moles for small things like molecules.)\index{mole}

Let's say you want to know how much a mole of $NaCl$ weighs. From the
periodic table, you see that $Na$ has an atomic mass of 22.98976
atomic mass units. And $Cl$ has 35.453 atomic mass units.  One atom of
$NaCl$ has a mass of $22.98976 + 35.453 = 58.44276$ atomic mass units.
Then a mole of $NaCl$ has a mass of $58.44276$ grams. Handy, right?
% ADD: Conversions should probably come before this

\begin{Exercise}[title={Burning Methane}, label=burning_methane]

Natural gas is mostly methane ($CH_4$). When one molecule of methane
burns, two oxygen molecules ($O_2$) are consumed. One molecule of
$H_2O$ and one molecule of $CO_2$ are produced.
% ADD: Need to explain mole to mole ratios first, Law of Divine Proportion
% ADD: Include Significant Figures

If I need 200 grams of water, how many grams of methane do I need
to burn?

(This is how the hero in ``The Martian'' made water for his garden.)

\end{Exercise}
\begin{Answer}[ref=burning_methane]

From the last exercise, you know that 1 mole of water weighs 18.01528
grams. So 200 grams of water is about 11.1 moles. So you need to burn
11.1 moles of methane.

What does one mole of methane weigh? Using the periodic table:
$12.0107 + 4 \times 1.00794 = 16.04246$ grams.

$16.0424 \times 11.10 = 178.1$ grams of methane.

\end{Answer}

\section{Heavy atoms aren't stable}

When you look at the periodic table, there are a surprisingly large
number of elements. You might be told to ``Drink milk so that you can
get the calcium you need.'' However, no one has told you ``You should
eat kale so that you get enough copernicium in your diet.''

Copernicium, with 112 protons and 173 neutrons, has only been observed
 in a lab. It is highly radioactive and unstable(meaning it decays): a copernicium
atom usually lives for less than a minute before decaying.
% ADD: Half Life

The largest stable element is lead, which has 82 protons and between
122 and 126 neutrons. Elements with lower atomic numbers than lead,
have at least one stable isotope. Elements with higher atomic numbers
than lead don't.

Bismuth, with an atomic number of 83, is \textit{almost} stable. In fact, most
bismuth atoms will live for billions of years before decaying.


\chapter{Scalefit Results \label{chap:results}}

Here we give the results of running \scalefit\ on the full data sample.

\begin{center}
\begin{longtable}{cccccc}
%\caption{Median Values of Non-Degenerate Parameters For All Bursts \label{scalefit:tab:results}} \\
\caption{The median values of the posterior distributions for each of $\thO$, $\thobs/\thO$, and $p$.  Uncertainties are given at the $68\%$ level. The last column denotes whether the fit reported extrapolated in time outside the tabulated values of $\cfp$, $\cfm$, and $\cfc$. \label{scalefit:tab:results}} \\
\hline \hline
GRB & $\thO$ & $\thobs/\thO$ & $p$ & Min $\chi^2/\text{dof}$ & Extrapolation \\
\hline
\endfirsthead

\hline \hline
GRB & $\thO$ & $\thobs/\thO$ & $p$ & Min $\chi^2/\text{dof}$ & Extrapolation \\
\hline
\endhead

\hline
\multicolumn{6}{l}{\textsuperscript{a} Well-Fit in $\thO$ and $\thobs/\thO$.} \\
\multicolumn{6}{l}{\textsuperscript{b} Well-Fit in $p$.} \\
%\multicolumn{6}{p{0.9\textwidth}}{The median values of the posterior distributions for each of $\thO$, $\thobs/\thO$, and $p$.  Uncertainties are given at the $68\%$ level. The last column denotes whether the fit reported extrapolated in time outside the tabulated values of $\cfp$, $\cfm$, and $\cfc$.} \\
\endfoot

\hline
\multicolumn{6}{l}{\textsuperscript{a} Well-Fit in $\thO$ and $\thobs/\thO$.} \\
\multicolumn{6}{l}{\textsuperscript{b} Well-Fit in $p$.} \\
%\multicolumn{6}{p{0.9\textwidth}}{The median values of the posterior distributions for each of $\thO$, $\thobs/\thO$, and $p$.  Uncertainties are given at the $68\%$ level. The last column denotes whether the fit reported extrapolated in time outside the tabulated values of $\cfp$, $\cfm$, and $\cfc$.} \\
\endlastfoot

050126 & $0.365^{+0.095}_{-0.125}$ & $0.45^{+0.49}_{-0.31}$ & $2.26^{+0.10}_{-0.10}$ & $22.7/4$ & NoEx\\[2pt]
050219A\textsuperscript{b} & $0.28^{+0.15}_{-0.16}$ & $0.44^{+0.33}_{-0.30}$ & $2.042^{+0.063}_{-0.034}$ & $20.4/12$ & NoEx\\[2pt]
050315\textsuperscript{a,b} & $0.343^{+0.038}_{-0.035}$ & $0.176^{+0.081}_{-0.099}$ & $2.084^{+0.030}_{-0.038}$ & $145.8/177$ & NoEx\\[2pt]
050318 & $0.145^{+0.097}_{-0.073}$ & $0.338^{+0.214}_{-0.089}$ & $2.30^{+0.22}_{-0.18}$ & $64.7/71$ & Ex\\[2pt]
050319 & $0.0469^{+0.0052}_{-0.0014}$ & $0.627^{+0.054}_{-0.065}$ & $2.0103^{+0.0091}_{-0.0049}$ & $114.9/86$ & Ex\\[2pt]
050401 & $0.472^{+0.020}_{-0.044}$ & $0.49^{+0.26}_{-0.35}$ & $2.53^{+0.14}_{-0.14}$ & $382.3/316$ & NoEx\\[2pt]
050408\textsuperscript{b} & $0.29^{+0.14}_{-0.16}$ & $0.54^{+0.29}_{-0.36}$ & $2.183^{+0.085}_{-0.084}$ & $35.0/43$ & Ex\\[2pt]
050416A & $0.237^{+0.114}_{-0.059}$ & $0.34^{+0.24}_{-0.22}$ & $2.0058^{+0.0065}_{-0.0036}$ & $106.5/96$ & Ex\\[2pt]
050505\textsuperscript{b} & $0.216^{+0.090}_{-0.156}$ & $0.438^{+0.071}_{-0.097}$ & $2.34^{+0.12}_{-0.18}$ & $169.5/166$ & Ex\\[2pt]
050525A\textsuperscript{a,b} & $0.0551^{+0.0069}_{-0.0062}$ & $0.579^{+0.047}_{-0.052}$ & $2.044^{+0.041}_{-0.028}$ & $47.9/27$ & Ex\\[2pt]
050603 & $0.27^{+0.16}_{-0.14}$ & $0.53^{+0.31}_{-0.35}$ & $2.92^{+0.20}_{-0.22}$ & $50.0/48$ & NoEx\\[2pt]
050730\textsuperscript{b} & $0.108^{+0.301}_{-0.058}$ & $0.56^{+0.24}_{-0.41}$ & $4.39^{+0.15}_{-0.58}$ & $454.7/333$ & NoEx\\[2pt]
050801\textsuperscript{b} & $0.29^{+0.14}_{-0.12}$ & $0.53^{+0.29}_{-0.35}$ & $2.157^{+0.176}_{-0.091}$ & $23.8/11$ & NoEx\\[2pt]
050802 & $0.29^{+0.15}_{-0.15}$ & $0.55^{+0.32}_{-0.38}$ & $2.42^{+0.24}_{-0.20}$ & $23.2/24$ & Ex\\[2pt]
050814 & $0.0469^{+0.0074}_{-0.0015}$ & $0.557^{+0.094}_{-0.112}$ & $2.019^{+0.022}_{-0.011}$ & $86.3/27$ & Ex\\[2pt]
050820A\textsuperscript{a,b} & $0.151^{+0.057}_{-0.022}$ & $0.57^{+0.12}_{-0.12}$ & $2.089^{+0.036}_{-0.034}$ & $359.4/321$ & NoEx\\[2pt]
050822 & $0.125^{+0.224}_{-0.068}$ & $0.84^{+0.11}_{-0.25}$ & $2.064^{+0.051}_{-0.033}$ & $107.9/85$ & Ex\\[2pt]
050824\textsuperscript{b} & $0.140^{+0.229}_{-0.088}$ & $0.39^{+0.30}_{-0.26}$ & $2.0143^{+0.0157}_{-0.0080}$ & $69.4/34$ & Ex\\[2pt]
050826 & $0.35^{+0.10}_{-0.11}$ & $0.40^{+0.27}_{-0.26}$ & $2.098^{+0.172}_{-0.058}$ & $12.6/15$ & NoEx\\[2pt]
050904 & $0.148^{+0.275}_{-0.095}$ & $0.48^{+0.20}_{-0.31}$ & $2.63^{+1.71}_{-0.55}$ & $28.2/11$ & Ex\\[2pt]
050908\textsuperscript{b} & $0.26^{+0.14}_{-0.13}$ & $0.37^{+0.33}_{-0.25}$ & $2.168^{+0.116}_{-0.094}$ & $33.4/7$ & NoEx\\[2pt]
050915A\textsuperscript{b} & $0.24^{+0.18}_{-0.14}$ & $0.64^{+0.24}_{-0.41}$ & $2.19^{+0.10}_{-0.11}$ & $20.8/17$ & Ex\\[2pt]
050922C\textsuperscript{a,b} & $0.074^{+0.033}_{-0.011}$ & $0.673^{+0.064}_{-0.075}$ & $2.100^{+0.228}_{-0.040}$ & $140.5/137$ & Ex\\[2pt]
051001 & $0.27^{+0.16}_{-0.15}$ & $0.47^{+0.32}_{-0.32}$ & $2.084^{+0.081}_{-0.054}$ & $14.5/6$ & Ex\\[2pt]
051006\textsuperscript{b} & $0.29^{+0.15}_{-0.15}$ & $0.48^{+0.35}_{-0.33}$ & $2.78^{+0.34}_{-0.13}$ & $57.2/19$ & NoEx\\[2pt]
051016B\textsuperscript{b} & $0.35^{+0.11}_{-0.24}$ & $0.61^{+0.26}_{-0.20}$ & $2.085^{+0.108}_{-0.057}$ & $46.5/55$ & NoEx\\[2pt]
051022 & $0.22^{+0.14}_{-0.14}$ & $0.15^{+0.22}_{-0.10}$ & $2.136^{+0.123}_{-0.072}$ & $52.9/51$ & Ex\\[2pt]
051109A & $0.30^{+0.14}_{-0.14}$ & $0.52^{+0.33}_{-0.36}$ & $2.22^{+0.19}_{-0.11}$ & $171.6/158$ & NoEx\\[2pt]
051109B\textsuperscript{b} & $0.060^{+0.162}_{-0.012}$ & $0.63^{+0.23}_{-0.33}$ & $2.035^{+0.042}_{-0.025}$ & $47.5/18$ & NoEx\\[2pt]
051111\textsuperscript{b} & $0.28^{+0.15}_{-0.15}$ & $0.49^{+0.34}_{-0.33}$ & $2.78^{+0.35}_{-0.21}$ & $18.9/25$ & Ex\\[2pt]
051221A\textsuperscript{a} & $0.447^{+0.032}_{-0.040}$ & $0.449^{+0.051}_{-0.059}$ & $2.025^{+0.025}_{-0.014}$ & $55.0/44$ & NoEx\\[2pt]
060111A & $0.384^{+0.083}_{-0.123}$ & $0.34^{+0.30}_{-0.24}$ & $2.061^{+0.082}_{-0.039}$ & $29.4/35$ & NoEx\\[2pt]
060115 & $0.18^{+0.20}_{-0.11}$ & $0.76^{+0.16}_{-0.42}$ & $2.080^{+0.076}_{-0.053}$ & $31.5/21$ & Ex\\[2pt]
060123\textsuperscript{b} & $0.32^{+0.13}_{-0.15}$ & $0.54^{+0.32}_{-0.36}$ & $2.25^{+0.22}_{-0.17}$ & $18.0/12$ & Ex\\[2pt]
060124 & $0.26^{+0.16}_{-0.15}$ & $0.55^{+0.31}_{-0.37}$ & $2.58^{+0.13}_{-0.26}$ & $462.5/291$ & Ex\\[2pt]
060202\textsuperscript{b} & $0.28^{+0.15}_{-0.15}$ & $0.48^{+0.33}_{-0.33}$ & $3.58^{+0.56}_{-0.24}$ & $149.2/157$ & NoEx\\[2pt]
060206\textsuperscript{b} & $0.377^{+0.084}_{-0.111}$ & $0.32^{+0.32}_{-0.23}$ & $2.089^{+0.103}_{-0.060}$ & $23.5/13$ & Ex\\[2pt]
060210\textsuperscript{a} & $0.0604^{+0.0236}_{-0.0093}$ & $0.735^{+0.044}_{-0.117}$ & $2.111^{+0.047}_{-0.044}$ & $245.8/250$ & Ex\\[2pt]
060218\textsuperscript{b} & $0.33^{+0.12}_{-0.16}$ & $0.57^{+0.28}_{-0.38}$ & $2.25^{+0.17}_{-0.17}$ & $23.1/31$ & Ex\\[2pt]
060306 & $0.084^{+0.185}_{-0.029}$ & $0.78^{+0.13}_{-0.11}$ & $2.0141^{+0.0155}_{-0.0076}$ & $93.0/79$ & Ex\\[2pt]
060319\textsuperscript{b} & $0.32^{+0.12}_{-0.14}$ & $0.53^{+0.31}_{-0.37}$ & $2.137^{+0.150}_{-0.079}$ & $38.8/39$ & NoEx\\[2pt]
060418 & $0.31^{+0.13}_{-0.14}$ & $0.46^{+0.37}_{-0.32}$ & $2.70^{+0.13}_{-0.10}$ & $119.0/110$ & Ex\\[2pt]
060502A & $0.32^{+0.12}_{-0.13}$ & $0.45^{+0.27}_{-0.30}$ & $2.0168^{+0.0149}_{-0.0094}$ & $50.1/60$ & Ex\\[2pt]
060512 & $0.27^{+0.15}_{-0.13}$ & $0.49^{+0.30}_{-0.32}$ & $2.144^{+0.140}_{-0.089}$ & $9.1/12$ & NoEx\\[2pt]
060522\textsuperscript{b} & $0.30^{+0.13}_{-0.14}$ & $0.43^{+0.29}_{-0.28}$ & $2.124^{+0.173}_{-0.072}$ & $25.4/13$ & NoEx\\[2pt]
060526 & $0.0468^{+0.0058}_{-0.0014}$ & $0.122^{+0.448}_{-0.087}$ & $2.023^{+0.047}_{-0.013}$ & $54.0/22$ & Ex\\[2pt]
060604 & $0.063^{+0.165}_{-0.015}$ & $0.77^{+0.10}_{-0.14}$ & $2.051^{+0.042}_{-0.025}$ & $78.2/58$ & NoEx\\[2pt]
060605\textsuperscript{b} & $0.04614^{+0.18939}_{-0.00096}$ & $0.16^{+0.74}_{-0.11}$ & $2.0125^{+1.4031}_{-0.0087}$ & $128.2/69$ & NoEx\\[2pt]
060607A\textsuperscript{a,b} & $0.374^{+0.095}_{-0.072}$ & $0.593^{+0.078}_{-0.098}$ & $4.63^{+0.17}_{-0.17}$ & $257.2/156$ & NoEx\\[2pt]
060614\textsuperscript{b} & $0.293^{+0.122}_{-0.085}$ & $0.38^{+0.43}_{-0.22}$ & $2.100^{+0.101}_{-0.064}$ & $54.8/79$ & Ex\\[2pt]
060707\textsuperscript{b} & $0.22^{+0.19}_{-0.16}$ & $0.59^{+0.28}_{-0.38}$ & $2.135^{+0.048}_{-0.069}$ & $41.8/21$ & Ex\\[2pt]
060708\textsuperscript{a} & $0.0597^{+0.0115}_{-0.0080}$ & $0.707^{+0.053}_{-0.056}$ & $2.036^{+0.032}_{-0.019}$ & $88.8/70$ & Ex\\[2pt]
060714\textsuperscript{a} & $0.0557^{+0.0104}_{-0.0072}$ & $0.743^{+0.057}_{-0.050}$ & $2.058^{+0.060}_{-0.035}$ & $73.3/43$ & Ex\\[2pt]
060719 & $0.0512^{+0.0130}_{-0.0051}$ & $0.726^{+0.060}_{-0.067}$ & $2.021^{+0.022}_{-0.012}$ & $74.5/48$ & Ex\\[2pt]
060729 & $0.370^{+0.047}_{-0.039}$ & $0.636^{+0.030}_{-0.036}$ & $2.079^{+0.039}_{-0.033}$ & $584.3/561$ & NoEx\\[2pt]
060801\textsuperscript{a} & $0.0561^{+0.0056}_{-0.0063}$ & $0.050^{+0.053}_{-0.035}$ & $4.84^{+0.11}_{-0.22}$ & $21.1/8$ & NoEx\\[2pt]
060814 & $0.35^{+0.10}_{-0.13}$ & $0.37^{+0.34}_{-0.26}$ & $2.105^{+0.128}_{-0.071}$ & $10.1/12$ & NoEx\\[2pt]
060904B\textsuperscript{a,b} & $0.083^{+0.048}_{-0.015}$ & $0.734^{+0.103}_{-0.078}$ & $2.114^{+0.047}_{-0.056}$ & $50.8/49$ & Ex\\[2pt]
060906 & $0.0500^{+0.0120}_{-0.0042}$ & $0.620^{+0.080}_{-0.076}$ & $2.0115^{+0.0135}_{-0.0067}$ & $166.6/31$ & Ex\\[2pt]
060908 & $0.380^{+0.085}_{-0.113}$ & $0.40^{+0.47}_{-0.28}$ & $2.75^{+0.14}_{-0.13}$ & $54.1/27$ & Ex\\[2pt]
060912A\textsuperscript{b} & $0.227^{+0.152}_{-0.070}$ & $0.53^{+0.25}_{-0.36}$ & $2.134^{+0.100}_{-0.090}$ & $33.6/30$ & NoEx\\[2pt]
060926\textsuperscript{b} & $0.29^{+0.15}_{-0.16}$ & $0.49^{+0.34}_{-0.34}$ & $2.49^{+0.37}_{-0.25}$ & $3.9/3$ & Ex\\[2pt]
060927 & $0.29^{+0.14}_{-0.14}$ & $0.52^{+0.33}_{-0.35}$ & $3.81^{+0.78}_{-0.82}$ & $9.9/10$ & NoEx\\[2pt]
061006\textsuperscript{b} & $0.407^{+0.068}_{-0.173}$ & $0.29^{+0.31}_{-0.20}$ & $2.046^{+0.060}_{-0.035}$ & $7.1/3$ & NoEx\\[2pt]
061007\textsuperscript{b} & $0.31^{+0.13}_{-0.17}$ & $0.53^{+0.30}_{-0.29}$ & $2.50^{+0.27}_{-0.27}$ & $9.4/9$ & Ex\\[2pt]
061021\textsuperscript{b} & $0.133^{+0.060}_{-0.021}$ & $0.56^{+0.17}_{-0.14}$ & $2.035^{+0.032}_{-0.023}$ & $284.8/290$ & Ex\\[2pt]
061121\textsuperscript{b} & $0.094^{+0.118}_{-0.030}$ & $0.858^{+0.079}_{-0.075}$ & $2.474^{+0.087}_{-0.088}$ & $144.3/163$ & Ex\\[2pt]
061126\textsuperscript{a,b} & $0.283^{+0.126}_{-0.077}$ & $0.63^{+0.20}_{-0.24}$ & $2.305^{+0.020}_{-0.064}$ & $211.4/236$ & Ex\\[2pt]
061222A\textsuperscript{a} & $0.0684^{+0.0142}_{-0.0064}$ & $0.596^{+0.068}_{-0.037}$ & $2.459^{+0.057}_{-0.047}$ & $387.6/277$ & Ex\\[2pt]
070103\textsuperscript{b} & $0.29^{+0.15}_{-0.15}$ & $0.52^{+0.33}_{-0.35}$ & $2.60^{+0.28}_{-0.18}$ & $7.8/9$ & Ex\\[2pt]
070110\textsuperscript{b} & $0.33^{+0.12}_{-0.15}$ & $0.43^{+0.33}_{-0.29}$ & $2.130^{+0.154}_{-0.090}$ & $25.1/18$ & Ex\\[2pt]
070125 & $0.26^{+0.12}_{-0.12}$ & $0.40^{+0.24}_{-0.14}$ & $2.34^{+0.28}_{-0.22}$ & $39.2/38$ & NoEx\\[2pt]
070129 & $0.0464^{+0.0056}_{-0.0011}$ & $0.614^{+0.069}_{-0.114}$ & $2.0093^{+0.0124}_{-0.0054}$ & $163.7/77$ & Ex\\[2pt]
070208 & $0.28^{+0.15}_{-0.15}$ & $0.54^{+0.31}_{-0.36}$ & $2.32^{+0.35}_{-0.19}$ & $20.1/13$ & NoEx\\[2pt]
070306\textsuperscript{a} & $0.291^{+0.030}_{-0.033}$ & $0.365^{+0.045}_{-0.044}$ & $2.084^{+0.089}_{-0.046}$ & $137.6/102$ & Ex\\[2pt]
070318\textsuperscript{b} & $0.30^{+0.11}_{-0.12}$ & $0.22^{+0.39}_{-0.15}$ & $2.116^{+0.164}_{-0.090}$ & $96.7/54$ & Ex\\[2pt]
070411 & $0.33^{+0.12}_{-0.16}$ & $0.58^{+0.21}_{-0.38}$ & $2.116^{+0.191}_{-0.075}$ & $21.2/23$ & NoEx\\[2pt]
070419A\textsuperscript{b} & $0.18^{+0.19}_{-0.11}$ & $0.65^{+0.20}_{-0.32}$ & $4.884^{+0.089}_{-0.208}$ & $138.9/103$ & Ex\\[2pt]
070419B & $0.0505^{+0.1462}_{-0.0039}$ & $0.211^{+0.064}_{-0.091}$ & $2.113^{+0.060}_{-0.048}$ & $154.9/174$ & NoEx\\[2pt]
070506\textsuperscript{b} & $0.25^{+0.17}_{-0.16}$ & $0.50^{+0.31}_{-0.34}$ & $2.044^{+0.054}_{-0.028}$ & $10.9/4$ & Ex\\[2pt]
070508\textsuperscript{b} & $0.449^{+0.040}_{-0.081}$ & $0.768^{+0.053}_{-0.616}$ & $2.549^{+0.190}_{-0.052}$ & $466.7/487$ & NoEx\\[2pt]
070521\textsuperscript{b} & $0.154^{+0.161}_{-0.079}$ & $0.41^{+0.29}_{-0.18}$ & $2.31^{+0.32}_{-0.20}$ & $65.1/65$ & Ex\\[2pt]
070529\textsuperscript{b} & $0.25^{+0.16}_{-0.12}$ & $0.70^{+0.21}_{-0.47}$ & $2.26^{+0.13}_{-0.15}$ & $26.1/26$ & NoEx\\[2pt]
070611\textsuperscript{b} & $0.27^{+0.15}_{-0.11}$ & $0.41^{+0.36}_{-0.29}$ & $2.19^{+0.22}_{-0.13}$ & $15.9/4$ & Ex\\[2pt]
070714A & $0.21^{+0.19}_{-0.14}$ & $0.45^{+0.31}_{-0.30}$ & $2.020^{+0.025}_{-0.013}$ & $42.5/6$ & NoEx\\[2pt]
070714B\textsuperscript{b} & $0.33^{+0.11}_{-0.11}$ & $0.835^{+0.059}_{-0.617}$ & $2.670^{+0.212}_{-0.073}$ & $114.7/67$ & NoEx\\[2pt]
070721B\textsuperscript{a} & $0.084^{+0.034}_{-0.029}$ & $0.127^{+0.099}_{-0.086}$ & $2.124^{+0.098}_{-0.065}$ & $45.9/51$ & Ex\\[2pt]
070724A\textsuperscript{b} & $0.27^{+0.16}_{-0.16}$ & $0.54^{+0.33}_{-0.36}$ & $4.55^{+0.30}_{-0.38}$ & $83.1/3$ & NoEx\\[2pt]
070802 & $0.28^{+0.15}_{-0.15}$ & $0.41^{+0.31}_{-0.28}$ & $2.027^{+0.033}_{-0.017}$ & $81.4/8$ & NoEx\\[2pt]
070809\textsuperscript{b} & $0.400^{+0.080}_{-0.333}$ & $0.24^{+0.33}_{-0.17}$ & $2.0121^{+0.0178}_{-0.0087}$ & $54.4/10$ & NoEx\\[2pt]
070810A\textsuperscript{b} & $0.104^{+0.196}_{-0.049}$ & $0.76^{+0.16}_{-0.29}$ & $2.105^{+0.093}_{-0.066}$ & $25.2/27$ & NoEx\\[2pt]
071003\textsuperscript{b} & $0.30^{+0.14}_{-0.18}$ & $0.61^{+0.25}_{-0.33}$ & $2.85^{+0.56}_{-0.37}$ & $66.0/59$ & NoEx\\[2pt]
071010A\textsuperscript{b} & $0.30^{+0.13}_{-0.15}$ & $0.54^{+0.31}_{-0.35}$ & $2.97^{+0.32}_{-0.36}$ & $2.6/3$ & Ex\\[2pt]
071010B\textsuperscript{b} & $0.34^{+0.12}_{-0.22}$ & $0.33^{+0.37}_{-0.23}$ & $2.036^{+0.039}_{-0.023}$ & $7.0/8$ & NoEx\\[2pt]
071020 & $0.34^{+0.13}_{-0.15}$ & $0.62^{+0.27}_{-0.24}$ & $2.135^{+0.038}_{-0.037}$ & $252.4/185$ & Ex\\[2pt]
071021 & $0.26^{+0.16}_{-0.16}$ & $0.50^{+0.32}_{-0.34}$ & $2.061^{+0.060}_{-0.039}$ & $21.3/18$ & Ex\\[2pt]
071025\textsuperscript{b} & $0.30^{+0.14}_{-0.13}$ & $0.48^{+0.31}_{-0.33}$ & $2.734^{+0.113}_{-0.091}$ & $112.0/104$ & NoEx\\[2pt]
071031\textsuperscript{b} & $0.30^{+0.13}_{-0.15}$ & $0.44^{+0.34}_{-0.30}$ & $2.16^{+0.20}_{-0.11}$ & $10.4/3$ & NoEx\\[2pt]
071112C\textsuperscript{b} & $0.398^{+0.071}_{-0.117}$ & $0.74^{+0.19}_{-0.60}$ & $2.77^{+0.21}_{-0.17}$ & $27.4/23$ & Ex\\[2pt]
071117 & $0.29^{+0.14}_{-0.17}$ & $0.48^{+0.33}_{-0.32}$ & $2.19^{+0.18}_{-0.14}$ & $30.0/21$ & Ex\\[2pt]
071122\textsuperscript{b} & $0.25^{+0.16}_{-0.16}$ & $0.53^{+0.34}_{-0.35}$ & $4.30^{+0.52}_{-0.42}$ & $39.5/29$ & Ex\\[2pt]
080129\textsuperscript{b} & $0.19^{+0.20}_{-0.12}$ & $0.73^{+0.20}_{-0.47}$ & $2.37^{+0.14}_{-0.25}$ & $73.4/43$ & Ex\\[2pt]
080207\textsuperscript{b} & $0.31^{+0.13}_{-0.14}$ & $0.49^{+0.35}_{-0.34}$ & $2.92^{+0.15}_{-0.14}$ & $42.6/56$ & NoEx\\[2pt]
080210\textsuperscript{b} & $0.164^{+0.147}_{-0.098}$ & $0.41^{+0.37}_{-0.28}$ & $2.21^{+0.30}_{-0.15}$ & $57.5/25$ & NoEx\\[2pt]
080310\textsuperscript{b} & $0.097^{+0.263}_{-0.052}$ & $0.30^{+0.46}_{-0.16}$ & $2.89^{+0.59}_{-0.88}$ & $57.9/33$ & NoEx\\[2pt]
080319B\textsuperscript{a,b} & $0.098^{+0.046}_{-0.013}$ & $0.620^{+0.082}_{-0.080}$ & $2.780^{+0.018}_{-0.012}$ & $1630.1/1585$ & Ex\\[2pt]
080319C & $0.30^{+0.14}_{-0.15}$ & $0.46^{+0.35}_{-0.32}$ & $2.80^{+0.11}_{-0.11}$ & $44.2/46$ & NoEx\\[2pt]
080411\textsuperscript{b} & $0.097^{+0.362}_{-0.017}$ & $0.771^{+0.034}_{-0.043}$ & $2.097^{+0.093}_{-0.046}$ & $375.7/395$ & Ex\\[2pt]
080413A\textsuperscript{b} & $0.26^{+0.16}_{-0.15}$ & $0.54^{+0.32}_{-0.35}$ & $2.41^{+0.30}_{-0.21}$ & $7.1/5$ & Ex\\[2pt]
080413B\textsuperscript{a} & $0.117^{+0.020}_{-0.021}$ & $0.299^{+0.092}_{-0.174}$ & $2.023^{+0.033}_{-0.011}$ & $317.8/229$ & NoEx\\[2pt]
080430\textsuperscript{a} & $0.0553^{+0.0052}_{-0.0079}$ & $0.587^{+0.060}_{-0.067}$ & $2.0026^{+0.0029}_{-0.0013}$ & $274.1/138$ & Ex\\[2pt]
080603A\textsuperscript{b} & $0.31^{+0.13}_{-0.16}$ & $0.45^{+0.33}_{-0.31}$ & $2.132^{+0.170}_{-0.093}$ & $21.5/14$ & Ex\\[2pt]
080605\textsuperscript{a,b} & $0.360^{+0.087}_{-0.108}$ & $0.49^{+0.18}_{-0.25}$ & $2.542^{+0.068}_{-0.163}$ & $328.0/308$ & Ex\\[2pt]
080607\textsuperscript{b} & $0.26^{+0.16}_{-0.14}$ & $0.54^{+0.30}_{-0.33}$ & $2.65^{+0.39}_{-0.23}$ & $27.9/24$ & Ex\\[2pt]
080707 & $0.30^{+0.14}_{-0.15}$ & $0.44^{+0.35}_{-0.30}$ & $2.21^{+0.26}_{-0.14}$ & $26.7/10$ & Ex\\[2pt]
080710 & $0.244^{+0.092}_{-0.130}$ & $0.20^{+0.15}_{-0.12}$ & $2.089^{+0.107}_{-0.050}$ & $82.8/60$ & Ex\\[2pt]
080721\textsuperscript{a,b} & $0.1117^{+0.0109}_{-0.0083}$ & $0.731^{+0.025}_{-0.021}$ & $2.397^{+0.016}_{-0.014}$ & $1485.3/1382$ & Ex\\[2pt]
080804 & $0.297^{+0.122}_{-0.093}$ & $0.68^{+0.12}_{-0.17}$ & $2.039^{+0.026}_{-0.019}$ & $86.4/95$ & Ex\\[2pt]
080805 & $0.31^{+0.13}_{-0.14}$ & $0.39^{+0.30}_{-0.27}$ & $2.094^{+0.189}_{-0.071}$ & $18.5/12$ & Ex\\[2pt]
080810\textsuperscript{b} & $0.34^{+0.11}_{-0.27}$ & $0.41^{+0.28}_{-0.33}$ & $2.63^{+0.17}_{-0.29}$ & $66.7/70$ & Ex\\[2pt]
080905A\textsuperscript{b} & $0.28^{+0.15}_{-0.16}$ & $0.51^{+0.33}_{-0.35}$ & $4.00^{+0.58}_{-0.46}$ & $18.9/5$ & NoEx\\[2pt]
080905B & $0.161^{+0.179}_{-0.050}$ & $0.58^{+0.22}_{-0.43}$ & $2.46^{+0.13}_{-0.13}$ & $60.8/57$ & NoEx\\[2pt]
080913\textsuperscript{b} & $0.359^{+0.099}_{-0.125}$ & $0.40^{+0.42}_{-0.28}$ & $2.21^{+0.15}_{-0.11}$ & $42.7/4$ & NoEx\\[2pt]
080916A & $0.156^{+0.222}_{-0.086}$ & $0.78^{+0.13}_{-0.28}$ & $2.099^{+0.090}_{-0.062}$ & $47.3/42$ & Ex\\[2pt]
080928\textsuperscript{b} & $0.25^{+0.16}_{-0.17}$ & $0.62^{+0.26}_{-0.33}$ & $3.17^{+0.36}_{-0.56}$ & $83.2/68$ & Ex\\[2pt]
081007\textsuperscript{b} & $0.159^{+0.183}_{-0.066}$ & $0.76^{+0.13}_{-0.19}$ & $2.046^{+0.029}_{-0.025}$ & $65.8/55$ & Ex\\[2pt]
081008 & $0.0610^{+0.0092}_{-0.0089}$ & $0.34^{+0.10}_{-0.14}$ & $2.088^{+0.066}_{-0.056}$ & $40.7/46$ & Ex\\[2pt]
081028\textsuperscript{b} & $0.30^{+0.13}_{-0.15}$ & $0.53^{+0.32}_{-0.36}$ & $2.92^{+0.33}_{-0.34}$ & $22.7/4$ & Ex\\[2pt]
081029\textsuperscript{b} & $0.1615^{+0.0055}_{-0.0061}$ & $0.031^{+0.034}_{-0.022}$ & $2.075^{+0.025}_{-0.021}$ & $140.2/77$ & NoEx\\[2pt]
081109\textsuperscript{b} & $0.26^{+0.16}_{-0.14}$ & $0.57^{+0.30}_{-0.37}$ & $2.241^{+0.345}_{-0.093}$ & $100.5/105$ & Ex\\[2pt]
081121 & $0.281^{+0.143}_{-0.092}$ & $0.760^{+0.096}_{-0.351}$ & $2.463^{+0.102}_{-0.096}$ & $141.0/139$ & Ex\\[2pt]
081203A\textsuperscript{b} & $0.121^{+0.058}_{-0.063}$ & $0.411^{+0.064}_{-0.068}$ & $2.156^{+0.336}_{-0.086}$ & $214.7/216$ & Ex\\[2pt]
081221\textsuperscript{b} & $0.340^{+0.110}_{-0.093}$ & $0.34^{+0.21}_{-0.23}$ & $2.406^{+0.032}_{-0.035}$ & $248.0/255$ & Ex\\[2pt]
081222\textsuperscript{b} & $0.0844^{+0.0081}_{-0.0120}$ & $0.275^{+0.099}_{-0.114}$ & $2.408^{+0.032}_{-0.297}$ & $410.6/398$ & Ex\\[2pt]
090102\textsuperscript{b} & $0.33^{+0.12}_{-0.13}$ & $0.54^{+0.29}_{-0.39}$ & $2.554^{+0.082}_{-0.074}$ & $163.6/134$ & NoEx\\[2pt]
090113\textsuperscript{b} & $0.30^{+0.13}_{-0.14}$ & $0.50^{+0.35}_{-0.35}$ & $2.30^{+0.18}_{-0.13}$ & $12.5/10$ & NoEx\\[2pt]
090205\textsuperscript{a} & $0.0513^{+0.0081}_{-0.0046}$ & $0.22^{+0.12}_{-0.13}$ & $2.045^{+0.054}_{-0.029}$ & $23.5/20$ & NoEx\\[2pt]
090313\textsuperscript{b} & $0.127^{+0.143}_{-0.040}$ & $0.15^{+0.39}_{-0.11}$ & $2.151^{+0.479}_{-0.097}$ & $37.2/40$ & Ex\\[2pt]
090323 & $0.30^{+0.13}_{-0.15}$ & $0.54^{+0.32}_{-0.37}$ & $2.78^{+0.22}_{-0.23}$ & $16.5/10$ & Ex\\[2pt]
090328A\textsuperscript{b} & $0.32^{+0.13}_{-0.17}$ & $0.57^{+0.29}_{-0.36}$ & $2.77^{+0.26}_{-0.29}$ & $12.9/9$ & NoEx\\[2pt]
090407\textsuperscript{a} & $0.300^{+0.042}_{-0.033}$ & $0.245^{+0.065}_{-0.087}$ & $2.044^{+0.032}_{-0.023}$ & $97.3/94$ & Ex\\[2pt]
090417B\textsuperscript{b} & $0.183^{+0.232}_{-0.074}$ & $0.57^{+0.29}_{-0.42}$ & $2.40^{+0.17}_{-0.14}$ & $75.8/107$ & Ex\\[2pt]
090418A & $0.04572^{+0.00225}_{-0.00054}$ & $0.526^{+0.032}_{-0.052}$ & $2.0123^{+0.0220}_{-0.0066}$ & $101.5/108$ & NoEx\\[2pt]
090423 & $0.26^{+0.16}_{-0.14}$ & $0.51^{+0.32}_{-0.35}$ & $2.55^{+0.31}_{-0.16}$ & $16.1/23$ & Ex\\[2pt]
090424\textsuperscript{a} & $0.218^{+0.018}_{-0.014}$ & $0.758^{+0.021}_{-0.020}$ & $2.034^{+0.019}_{-0.015}$ & $568.6/537$ & NoEx\\[2pt]
090426\textsuperscript{b} & $0.31^{+0.13}_{-0.14}$ & $0.48^{+0.32}_{-0.33}$ & $2.220^{+0.091}_{-0.117}$ & $27.6/22$ & Ex\\[2pt]
090510\textsuperscript{b} & $0.387^{+0.080}_{-0.106}$ & $0.57^{+0.22}_{-0.39}$ & $3.34^{+0.38}_{-0.29}$ & $74.8/66$ & Ex\\[2pt]
090516\textsuperscript{a} & $0.0656^{+0.0035}_{-0.0045}$ & $0.317^{+0.039}_{-0.040}$ & $2.043^{+0.039}_{-0.023}$ & $144.8/133$ & NoEx\\[2pt]
090519\textsuperscript{b} & $0.27^{+0.15}_{-0.16}$ & $0.55^{+0.36}_{-0.38}$ & $4.42^{+0.30}_{-0.30}$ & $119.3/15$ & Ex\\[2pt]
090529 & $0.24^{+0.17}_{-0.16}$ & $0.42^{+0.31}_{-0.28}$ & $2.024^{+0.029}_{-0.015}$ & $18.4/5$ & NoEx\\[2pt]
090530 & $0.20^{+0.18}_{-0.11}$ & $0.33^{+0.27}_{-0.22}$ & $2.0080^{+0.0081}_{-0.0043}$ & $88.1/46$ & Ex\\[2pt]
090618\textsuperscript{a,b} & $0.0590^{+0.0025}_{-0.0026}$ & $0.758^{+0.011}_{-0.012}$ & $2.224^{+0.018}_{-0.017}$ & $979.1/843$ & Ex\\[2pt]
090709A\textsuperscript{a} & $0.287^{+0.126}_{-0.091}$ & $0.73^{+0.10}_{-0.29}$ & $2.491^{+0.087}_{-0.073}$ & $167.6/170$ & Ex\\[2pt]
090715B & $0.32^{+0.12}_{-0.15}$ & $0.49^{+0.27}_{-0.25}$ & $2.146^{+0.177}_{-0.099}$ & $38.3/25$ & NoEx\\[2pt]
090726\textsuperscript{b} & $0.183^{+0.195}_{-0.094}$ & $0.45^{+0.38}_{-0.33}$ & $2.49^{+0.40}_{-0.27}$ & $16.7/22$ & NoEx\\[2pt]
090809\textsuperscript{b} & $0.29^{+0.13}_{-0.12}$ & $0.48^{+0.22}_{-0.28}$ & $2.097^{+0.096}_{-0.068}$ & $23.5/16$ & NoEx\\[2pt]
090812\textsuperscript{b} & $0.29^{+0.14}_{-0.15}$ & $0.62^{+0.26}_{-0.43}$ & $2.22^{+0.14}_{-0.12}$ & $39.7/41$ & NoEx\\[2pt]
090902B\textsuperscript{b} & $0.31^{+0.13}_{-0.16}$ & $0.56^{+0.29}_{-0.36}$ & $2.47^{+0.19}_{-0.18}$ & $66.0/70$ & NoEx\\[2pt]
090926A\textsuperscript{b} & $0.31^{+0.13}_{-0.16}$ & $0.58^{+0.28}_{-0.39}$ & $2.52^{+0.22}_{-0.21}$ & $91.2/64$ & NoEx\\[2pt]
090926B & $0.35^{+0.11}_{-0.12}$ & $0.39^{+0.37}_{-0.27}$ & $2.164^{+0.135}_{-0.093}$ & $22.3/18$ & NoEx\\[2pt]
090927 & $0.33^{+0.14}_{-0.25}$ & $0.66^{+0.25}_{-0.26}$ & $2.093^{+0.117}_{-0.059}$ & $32.1/21$ & NoEx\\[2pt]
091003\textsuperscript{b} & $0.31^{+0.13}_{-0.17}$ & $0.58^{+0.28}_{-0.39}$ & $2.39^{+0.21}_{-0.21}$ & $43.8/39$ & Ex\\[2pt]
091018\textsuperscript{b} & $0.30^{+0.14}_{-0.16}$ & $0.56^{+0.30}_{-0.39}$ & $2.72^{+0.23}_{-0.21}$ & $40.2/33$ & NoEx\\[2pt]
091020\textsuperscript{b} & $0.28^{+0.15}_{-0.13}$ & $0.65^{+0.19}_{-0.39}$ & $2.426^{+0.066}_{-0.078}$ & $180.9/185$ & Ex\\[2pt]
091024\textsuperscript{b} & $0.31^{+0.13}_{-0.15}$ & $0.50^{+0.36}_{-0.34}$ & $3.24^{+0.28}_{-0.18}$ & $107.2/19$ & Ex\\[2pt]
091029\textsuperscript{b} & $0.21^{+0.19}_{-0.13}$ & $0.75^{+0.18}_{-0.47}$ & $2.34^{+0.11}_{-0.15}$ & $111.0/105$ & Ex\\[2pt]
091109A & $0.25^{+0.16}_{-0.11}$ & $0.41^{+0.31}_{-0.28}$ & $2.132^{+0.179}_{-0.095}$ & $14.2/17$ & NoEx\\[2pt]
091127\textsuperscript{b} & $0.151^{+0.193}_{-0.071}$ & $0.905^{+0.049}_{-0.078}$ & $2.549^{+0.078}_{-0.098}$ & $354.7/363$ & Ex\\[2pt]
091208B\textsuperscript{a} & $0.0952^{+0.0098}_{-0.0127}$ & $0.19^{+0.25}_{-0.13}$ & $2.082^{+0.056}_{-0.047}$ & $75.4/57$ & Ex\\[2pt]
100117A\textsuperscript{b} & $0.27^{+0.15}_{-0.15}$ & $0.48^{+0.35}_{-0.33}$ & $3.24^{+0.72}_{-0.57}$ & $32.1/9$ & NoEx\\[2pt]
100219A & $0.0481^{+0.0052}_{-0.0024}$ & $0.064^{+0.072}_{-0.045}$ & $2.0118^{+0.0138}_{-0.0071}$ & $66.7/20$ & Ex\\[2pt]
100302A\textsuperscript{b} & $0.19^{+0.20}_{-0.13}$ & $0.37^{+0.31}_{-0.25}$ & $2.0156^{+0.0189}_{-0.0096}$ & $26.3/18$ & Ex\\[2pt]
100316B & $0.151^{+0.226}_{-0.090}$ & $0.79^{+0.14}_{-0.34}$ & $2.079^{+0.082}_{-0.050}$ & $38.7/7$ & Ex\\[2pt]
100418A\textsuperscript{b} & $0.23^{+0.16}_{-0.12}$ & $0.20^{+0.45}_{-0.14}$ & $2.16^{+0.25}_{-0.13}$ & $18.9/11$ & NoEx\\[2pt]
100424A\textsuperscript{b} & $0.17^{+0.22}_{-0.11}$ & $0.60^{+0.28}_{-0.40}$ & $4.07^{+0.29}_{-0.44}$ & $189.6/156$ & NoEx\\[2pt]
100425A\textsuperscript{b} & $0.16^{+0.22}_{-0.10}$ & $0.45^{+0.29}_{-0.31}$ & $2.016^{+0.022}_{-0.010}$ & $38.0/16$ & Ex\\[2pt]
100513A\textsuperscript{b} & $0.28^{+0.15}_{-0.15}$ & $0.45^{+0.32}_{-0.31}$ & $2.170^{+0.067}_{-0.085}$ & $39.5/20$ & Ex\\[2pt]
100615A & $0.28^{+0.15}_{-0.17}$ & $0.48^{+0.34}_{-0.34}$ & $2.16^{+0.21}_{-0.11}$ & $51.3/55$ & NoEx\\[2pt]
100621A\textsuperscript{b} & $0.0477^{+0.0047}_{-0.0020}$ & $0.583^{+0.049}_{-0.047}$ & $2.075^{+0.030}_{-0.029}$ & $212.1/181$ & Ex\\[2pt]
100724A\textsuperscript{b} & $0.28^{+0.15}_{-0.15}$ & $0.51^{+0.31}_{-0.34}$ & $2.25^{+0.12}_{-0.19}$ & $16.7/11$ & Ex\\[2pt]
100728A\textsuperscript{a,b} & $0.112^{+0.052}_{-0.011}$ & $0.722^{+0.119}_{-0.062}$ & $2.438^{+0.073}_{-0.055}$ & $253.2/249$ & Ex\\[2pt]
100728B\textsuperscript{b} & $0.26^{+0.15}_{-0.17}$ & $0.58^{+0.25}_{-0.31}$ & $2.31^{+0.38}_{-0.18}$ & $48.1/28$ & NoEx\\[2pt]
100816A\textsuperscript{b} & $0.29^{+0.14}_{-0.15}$ & $0.45^{+0.33}_{-0.31}$ & $2.22^{+0.21}_{-0.15}$ & $24.8/17$ & Ex\\[2pt]
100901A\textsuperscript{a} & $0.413^{+0.033}_{-0.033}$ & $0.473^{+0.032}_{-0.031}$ & $2.045^{+0.039}_{-0.025}$ & $160.4/99$ & NoEx\\[2pt]
100906A\textsuperscript{a} & $0.0540^{+0.0038}_{-0.0079}$ & $0.307^{+0.031}_{-0.030}$ & $2.0126^{+0.0130}_{-0.0071}$ & $252.6/128$ & NoEx\\[2pt]
101219A & $0.29^{+0.14}_{-0.14}$ & $0.49^{+0.35}_{-0.34}$ & $3.95^{+0.53}_{-0.33}$ & $46.6/5$ & NoEx\\[2pt]
101219B & $0.24^{+0.20}_{-0.17}$ & $0.37^{+0.33}_{-0.25}$ & $2.021^{+0.023}_{-0.012}$ & $19.3/16$ & Ex\\[2pt]
101225A & $0.089^{+0.115}_{-0.032}$ & $0.44^{+0.32}_{-0.23}$ & $3.38^{+1.24}_{-0.39}$ & $157.8/179$ & NoEx\\[2pt]
110106B\textsuperscript{b} & $0.0570^{+0.0167}_{-0.0090}$ & $0.67^{+0.11}_{-0.10}$ & $2.034^{+0.050}_{-0.022}$ & $78.2/48$ & Ex\\[2pt]
110128A\textsuperscript{b} & $0.469^{+0.023}_{-0.047}$ & $0.15^{+0.19}_{-0.11}$ & $2.032^{+0.030}_{-0.020}$ & $34.3/8$ & Ex\\[2pt]
110205A\textsuperscript{b} & $0.388^{+0.077}_{-0.145}$ & $0.69^{+0.25}_{-0.54}$ & $2.759^{+0.088}_{-0.134}$ & $142.1/164$ & Ex\\[2pt]
110213A\textsuperscript{b} & $0.29^{+0.14}_{-0.15}$ & $0.53^{+0.31}_{-0.37}$ & $3.09^{+0.22}_{-0.19}$ & $74.4/79$ & Ex\\[2pt]
110422A\textsuperscript{a,b} & $0.075^{+0.025}_{-0.012}$ & $0.684^{+0.151}_{-0.062}$ & $2.306^{+0.416}_{-0.071}$ & $239.0/254$ & NoEx\\[2pt]
110503A\textsuperscript{a,b} & $0.1057^{+0.0302}_{-0.0079}$ & $0.567^{+0.115}_{-0.051}$ & $2.049^{+0.167}_{-0.020}$ & $403.6/389$ & NoEx\\[2pt]
110715A\textsuperscript{b} & $0.30^{+0.14}_{-0.17}$ & $0.45^{+0.33}_{-0.30}$ & $2.132^{+0.151}_{-0.090}$ & $8.4/14$ & NoEx\\[2pt]
110731A & $0.184^{+0.052}_{-0.035}$ & $0.716^{+0.110}_{-0.065}$ & $2.112^{+0.020}_{-0.019}$ & $275.4/274$ & Ex\\[2pt]
110801A\textsuperscript{a,b} & $0.419^{+0.056}_{-0.075}$ & $0.50^{+0.13}_{-0.16}$ & $2.335^{+0.031}_{-0.041}$ & $168.8/118$ & NoEx\\[2pt]
110808A\textsuperscript{b} & $0.19^{+0.19}_{-0.13}$ & $0.33^{+0.31}_{-0.23}$ & $2.0129^{+0.0170}_{-0.0084}$ & $18.4/6$ & Ex\\[2pt]
110818A\textsuperscript{b} & $0.24^{+0.18}_{-0.16}$ & $0.63^{+0.25}_{-0.35}$ & $2.35^{+0.36}_{-0.20}$ & $28.1/21$ & NoEx\\[2pt]
110918A\textsuperscript{b} & $0.35^{+0.11}_{-0.17}$ & $0.66^{+0.22}_{-0.39}$ & $2.63^{+0.31}_{-0.27}$ & $181.6/159$ & Ex\\[2pt]
111008A & $0.04612^{+0.00259}_{-0.00085}$ & $0.721^{+0.027}_{-0.027}$ & $2.028^{+0.019}_{-0.014}$ & $244.7/130$ & Ex\\[2pt]
111107A\textsuperscript{b} & $0.28^{+0.15}_{-0.15}$ & $0.43^{+0.35}_{-0.29}$ & $2.19^{+0.13}_{-0.13}$ & $13.6/9$ & Ex\\[2pt]
111123A & $0.202^{+0.111}_{-0.072}$ & $0.35^{+0.34}_{-0.19}$ & $2.107^{+0.144}_{-0.063}$ & $45.7/36$ & NoEx\\[2pt]
111209A\textsuperscript{b} & $0.34^{+0.11}_{-0.13}$ & $0.57^{+0.30}_{-0.38}$ & $2.61^{+0.13}_{-0.16}$ & $120.5/75$ & Ex\\[2pt]
111211A\textsuperscript{b} & $0.30^{+0.14}_{-0.15}$ & $0.51^{+0.32}_{-0.34}$ & $2.69^{+0.35}_{-0.43}$ & $35.9/22$ & Ex\\[2pt]
111228A & $0.04591^{+0.00177}_{-0.00070}$ & $0.754^{+0.017}_{-0.021}$ & $2.0057^{+0.0058}_{-0.0033}$ & $304.5/155$ & Ex\\[2pt]
111229A & $0.25^{+0.17}_{-0.15}$ & $0.52^{+0.33}_{-0.36}$ & $4.13^{+0.51}_{-0.56}$ & $10.6/5$ & Ex\\[2pt]
120118B\textsuperscript{b} & $0.29^{+0.14}_{-0.15}$ & $0.50^{+0.32}_{-0.34}$ & $2.22^{+0.32}_{-0.16}$ & $16.4/17$ & NoEx\\[2pt]
120119A\textsuperscript{b} & $0.0510^{+0.0481}_{-0.0046}$ & $0.36^{+0.17}_{-0.24}$ & $2.479^{+0.039}_{-0.377}$ & $140.9/87$ & Ex\\[2pt]
120327A & $0.069^{+0.346}_{-0.017}$ & $0.51^{+0.18}_{-0.25}$ & $2.18^{+0.51}_{-0.14}$ & $88.2/60$ & NoEx\\[2pt]
120404A\textsuperscript{b} & $0.27^{+0.15}_{-0.15}$ & $0.52^{+0.32}_{-0.35}$ & $3.10^{+0.50}_{-0.32}$ & $18.9/20$ & Ex\\[2pt]
120422A & $0.20^{+0.20}_{-0.14}$ & $0.43^{+0.31}_{-0.28}$ & $2.017^{+0.023}_{-0.012}$ & $45.8/4$ & Ex\\[2pt]
120711A & $0.25^{+0.16}_{-0.11}$ & $0.65^{+0.19}_{-0.40}$ & $2.687^{+0.067}_{-0.140}$ & $258.2/239$ & NoEx\\[2pt]
120712A\textsuperscript{b} & $0.301^{+0.125}_{-0.099}$ & $0.878^{+0.039}_{-0.069}$ & $2.211^{+0.032}_{-0.030}$ & $67.6/56$ & Ex\\[2pt]
120802A\textsuperscript{b} & $0.21^{+0.18}_{-0.15}$ & $0.55^{+0.28}_{-0.38}$ & $2.029^{+0.033}_{-0.017}$ & $104.1/18$ & Ex\\[2pt]
120811C & $0.23^{+0.18}_{-0.14}$ & $0.65^{+0.27}_{-0.44}$ & $2.133^{+0.098}_{-0.077}$ & $41.2/42$ & NoEx\\[2pt]
120815A & $0.28^{+0.15}_{-0.15}$ & $0.42^{+0.35}_{-0.30}$ & $2.18^{+0.22}_{-0.13}$ & $41.9/38$ & NoEx\\[2pt]
120907A & $0.28^{+0.15}_{-0.15}$ & $0.41^{+0.31}_{-0.28}$ & $2.218^{+0.040}_{-0.165}$ & $115.8/84$ & Ex\\[2pt]
120909A\textsuperscript{b} & $0.22^{+0.18}_{-0.13}$ & $0.69^{+0.21}_{-0.35}$ & $2.44^{+0.43}_{-0.23}$ & $121.3/115$ & NoEx\\[2pt]
120922A\textsuperscript{b} & $0.30^{+0.14}_{-0.13}$ & $0.41^{+0.32}_{-0.28}$ & $2.130^{+0.116}_{-0.092}$ & $55.8/49$ & NoEx\\[2pt]
121024A\textsuperscript{a} & $0.0565^{+0.0179}_{-0.0080}$ & $0.35^{+0.24}_{-0.22}$ & $2.142^{+0.093}_{-0.081}$ & $43.9/42$ & NoEx\\[2pt]
121027A & $0.058^{+0.076}_{-0.011}$ & $0.61^{+0.21}_{-0.10}$ & $2.116^{+0.054}_{-0.049}$ & $103.2/66$ & Ex\\[2pt]
121128A & $0.27^{+0.15}_{-0.14}$ & $0.51^{+0.31}_{-0.35}$ & $2.98^{+0.15}_{-0.21}$ & $92.8/81$ & Ex\\[2pt]
121201A & $0.364^{+0.098}_{-0.106}$ & $0.39^{+0.48}_{-0.27}$ & $2.269^{+0.053}_{-0.098}$ & $35.2/26$ & NoEx\\[2pt]
121211A & $0.30^{+0.15}_{-0.22}$ & $0.61^{+0.27}_{-0.32}$ & $2.093^{+0.143}_{-0.063}$ & $68.1/61$ & Ex\\[2pt]
121229A & $0.28^{+0.15}_{-0.15}$ & $0.52^{+0.36}_{-0.35}$ & $4.20^{+0.51}_{-0.54}$ & $27.9/16$ & Ex\\[2pt]
\end{longtable}
\end{center}


\section{Linear random utility functions}


\chapter{Model Evaluation}

We have observed the logistic model - foxes, birds, little things and all that. The voting schema looked reasonable. And before that, we have seen the model's predictions. They looked good. But how would we quantify the performance of the model?

\begin{wrapfigure}{o}{1.1\textwidth}
    \includegraphics[scale=0.45]{test-and-score.png}
    \caption{$\;$}
\end{wrapfigure}

Perhaps we should just compute the proportion of the stories for which the model gave the correct answer? This score is called classification accuracy. For example, if we correctly predicted 40 tales out of 44, the classification accuracy would be 40/44 or 91\%.

AUC is another good measure to consider. In essence, the closer these two measures are to 1, the better the model's performance.

The widget that computes the classification accuracy is called \widget{Test and Score}. It needs two inputs: the data to test the model on and the modeling algorithm.\marginnote{\emph{It would make little sense asking whether Rapunzel is about animals after already telling the model that it is not.}\\
\\
Didn't we do this above in Predictions? Indeed, and this is why the predictions there were so excellent. Models should never be tested on the data from which they were constructed.}

\vspace{-0.2cm}
\begin{figure}[h]
  \centering
  \includegraphics[width=\linewidth]{workflow.png}%
  \caption{$\;$}
\end{figure}
\vspace{-0.3cm}

This time, Logistic Regression doesn't need a data input. Instead, it provides a learner, which is a procedure for constructing the model. Test and Score then applies the learner multiple times on different data subsets. Each time, it constructs the model on a selected subset and uses the left-out data for testing. It would make little sense asking the model whether Rapunzel is an Animal Tale, after already telling it that it is not.
\clearpage


    \filetitle{integrate}{Integrate VAR process and data associated with it}{VAR/integrate}

	\paragraph{Syntax}\label{syntax}

\begin{verbatim}
V = integrate(V,...)
\end{verbatim}

\paragraph{Input arguments}\label{input-arguments}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{V} {[} VAR {]} - VAR object whose variables will be integrated
  by one order.
\end{itemize}

\paragraph{Output arguments}\label{output-arguments}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{V} {[} VAR {]} - VAR object with the specified variables
  integrated by one order.
\end{itemize}

\paragraph{Options}\label{options}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{'applyTo='} {[} logical \textbar{} numeric \textbar{}
  \emph{\texttt{Inf}} {]} - Index of variables to integrate; Inf means
  all variables will be integrated.
\end{itemize}

\paragraph{Description}\label{description}

\paragraph{Example}\label{example}


\begin{comment}

 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
 regarding copyright ownership.  The ASF licenses this file
 to you under the Apache License, Version 2.0 (the
 "License"); you may not use this file except in compliance
 with the License.  You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing,
 software distributed under the License is distributed on an
 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations
 under the License.

\end{comment}

\subsection{Linear Regression}
\label{sec:LinReg}

\noindent{\bf Description}
\smallskip

Linear Regression scripts are used to model the relationship between one numerical
response variable and one or more explanatory (feature) variables.
The scripts are given a dataset $(X, Y) = (x_i, y_i)_{i=1}^n$ where $x_i$ is a
numerical vector of feature variables and $y_i$ is a numerical response value for
each training data record.  The feature vectors are provided as a matrix $X$ of size
$n\,{\times}\,m$, where $n$ is the number of records and $m$ is the number of features.
The observed response values are provided as a 1-column matrix~$Y$, with a numerical
value $y_i$ for each~$x_i$ in the corresponding row of matrix~$X$.

In linear regression, we predict the distribution of the response~$y_i$ based on
a fixed linear combination of the features in~$x_i$.  We assume that
there exist constant regression coefficients $\beta_0, \beta_1, \ldots, \beta_m$
and a constant residual variance~$\sigma^2$ such that
\begin{equation}
y_i \sim \Normal(\mu_i, \sigma^2) \,\,\,\,\textrm{where}\,\,\,\,
\mu_i \,=\, \beta_0 + \beta_1 x_{i,1} + \ldots + \beta_m x_{i,m}
\label{eqn:linregdef}
\end{equation}
Distribution $y_i \sim \Normal(\mu_i, \sigma^2)$ models the ``unexplained'' residual
noise and is assumed independent across different records.

The goal is to estimate the regression coefficients and the residual variance.
Once they are accurately estimated, we can make predictions about $y_i$ given~$x_i$
in new records.  We can also use the $\beta_j$'s to analyze the influence of individual
features on the response value, and assess the quality of this model by comparing
residual variance in the response, left after prediction, with its total variance.

There are two scripts in our library, both doing the same estimation, but using different
computational methods.  Depending on the size and the sparsity of the feature matrix~$X$,
one or the other script may be more efficient.  The ``direct solve'' script
{\tt LinearRegDS} is more efficient when the number of features $m$ is relatively small
($m \sim 1000$ or less) and matrix~$X$ is either tall or fairly dense
(has~${\gg}\:m^2$ nonzeros); otherwise, the ``conjugate gradient'' script {\tt LinearRegCG}
is more efficient.  If $m > 50000$, use only {\tt LinearRegCG}.

\smallskip
\noindent{\bf Usage}
\smallskip

{\hangindent=\parindent\noindent\it%
{\tt{}-f }path/\/{\tt{}LinearRegDS.dml}
{\tt{} -nvargs}
{\tt{} X=}path/file
{\tt{} Y=}path/file
{\tt{} B=}path/file
{\tt{} O=}path/file
{\tt{} icpt=}int
{\tt{} reg=}double
{\tt{} fmt=}format

}\smallskip
{\hangindent=\parindent\noindent\it%
{\tt{}-f }path/\/{\tt{}LinearRegCG.dml}
{\tt{} -nvargs}
{\tt{} X=}path/file
{\tt{} Y=}path/file
{\tt{} B=}path/file
{\tt{} O=}path/file
{\tt{} Log=}path/file
{\tt{} icpt=}int
{\tt{} reg=}double
{\tt{} tol=}double
{\tt{} maxi=}int
{\tt{} fmt=}format

}

\smallskip
\noindent{\bf Arguments}
\begin{Description}
\item[{\tt X}:]
Location (on HDFS) to read the matrix of feature vectors, each row constitutes
one feature vector
\item[{\tt Y}:]
Location to read the 1-column matrix of response values
\item[{\tt B}:]
Location to store the estimated regression parameters (the $\beta_j$'s), with the
intercept parameter~$\beta_0$ at position {\tt B[}$m\,{+}\,1$, {\tt 1]} if available
\item[{\tt O}:] (default:\mbox{ }{\tt " "})
Location to store the CSV-file of summary statistics defined in
Table~\ref{table:linreg:stats}, the default is to print it to the standard output
\item[{\tt Log}:] (default:\mbox{ }{\tt " "}, {\tt LinearRegCG} only)
Location to store iteration-specific variables for monitoring and debugging purposes,
see Table~\ref{table:linreg:log} for details.
\item[{\tt icpt}:] (default:\mbox{ }{\tt 0})
Intercept presence and shifting/rescaling the features in~$X$:\\
{\tt 0} = no intercept (hence no~$\beta_0$), no shifting or rescaling of the features;\\
{\tt 1} = add intercept, but do not shift/rescale the features in~$X$;\\
{\tt 2} = add intercept, shift/rescale the features in~$X$ to mean~0, variance~1
\item[{\tt reg}:] (default:\mbox{ }{\tt 0.000001})
L2-regularization parameter~\mbox{$\lambda\geq 0$}; set to nonzero for highly dependent,
sparse, or numerous ($m \gtrsim n/10$) features
\item[{\tt tol}:] (default:\mbox{ }{\tt 0.000001}, {\tt LinearRegCG} only)
Tolerance \mbox{$\eps\geq 0$} used in the convergence criterion: we terminate conjugate
gradient iterations when the $\beta$-residual reduces in L2-norm by this factor
\item[{\tt maxi}:] (default:\mbox{ }{\tt 0}, {\tt LinearRegCG} only)
Maximum number of conjugate gradient iterations, or~0 if no maximum
limit provided
\item[{\tt fmt}:] (default:\mbox{ }{\tt "text"})
Matrix file output format, such as {\tt text}, {\tt mm}, or {\tt csv};
see read/write functions in SystemDS Language Reference for details.
\end{Description}

\begin{table}[t]\small\centerline{%
\begin{tabular}{|ll|}
\hline
Name & Meaning \\
\hline
{\tt AVG\_TOT\_Y}          & Average of the response value $Y$ \\
{\tt STDEV\_TOT\_Y}        & Standard Deviation of the response value $Y$ \\
{\tt AVG\_RES\_Y}          & Average of the residual $Y - \mathop{\mathrm{pred}}(Y|X)$, i.e.\ residual bias \\
{\tt STDEV\_RES\_Y}        & Standard Deviation of the residual $Y - \mathop{\mathrm{pred}}(Y|X)$ \\
{\tt DISPERSION}           & GLM-style dispersion, i.e.\ residual sum of squares / \#deg.\ fr. \\
{\tt PLAIN\_R2}            & Plain $R^2$ of residual with bias included vs.\ total average \\
{\tt ADJUSTED\_R2}         & Adjusted $R^2$ of residual with bias included vs.\ total average \\
{\tt PLAIN\_R2\_NOBIAS}    & Plain $R^2$ of residual with bias subtracted vs.\ total average \\
{\tt ADJUSTED\_R2\_NOBIAS} & Adjusted $R^2$ of residual with bias subtracted vs.\ total average \\
{\tt PLAIN\_R2\_VS\_0}     & ${}^*$Plain $R^2$ of residual with bias included vs.\ zero constant \\
{\tt ADJUSTED\_R2\_VS\_0}  & ${}^*$Adjusted $R^2$ of residual with bias included vs.\ zero constant \\
\hline
\multicolumn{2}{r}{${}^{*\mathstrut}$ The last two statistics are only printed if there is no intercept ({\tt icpt=0})} \\
\end{tabular}}
\caption{Besides~$\beta$, linear regression scripts compute a few summary statistics
listed above.  The statistics are provided in CSV format, one comma-separated name-value
pair per each line.}
\label{table:linreg:stats}
\end{table}

\begin{table}[t]\small\centerline{%
\begin{tabular}{|ll|}
\hline
Name & Meaning \\
\hline
{\tt CG\_RESIDUAL\_NORM}  & L2-norm of conjug.\ grad.\ residual, which is $A \pxp \beta - t(X) \pxp y$ \\
                          & where $A = t(X) \pxp X + \diag (\lambda)$, or a similar quantity \\
{\tt CG\_RESIDUAL\_RATIO} & Ratio of current L2-norm of conjug.\ grad.\ residual over the initial \\
\hline
\end{tabular}}
\caption{
The {\tt Log} file for {\tt{}LinearRegCG} script contains the above \mbox{per-}iteration
variables in CSV format, each line containing triple (Name, Iteration\#, Value) with
Iteration\# being~0 for initial values.}
\label{table:linreg:log}
\end{table}

\noindent{\bf Details}
\smallskip

To solve a linear regression problem over feature matrix~$X$ and response vector~$Y$,
we can find coefficients $\beta_0, \beta_1, \ldots, \beta_m$ and $\sigma^2$ that maximize
the joint likelihood of all $y_i$ for $i=1\ldots n$, defined by the assumed statistical
model~(\ref{eqn:linregdef}).  Since the joint likelihood of the independent
$y_i \sim \Normal(\mu_i, \sigma^2)$ is proportional to the product of
$\exp\big({-}\,(y_i - \mu_i)^2 / (2\sigma^2)\big)$, we can take the logarithm of this
product, then multiply by $-2\sigma^2 < 0$ to obtain a least squares problem:
\begin{equation}
\sum_{i=1}^n \, (y_i - \mu_i)^2 \,\,=\,\,
\sum_{i=1}^n \Big(y_i - \beta_0 - \sum_{j=1}^m \beta_j x_{i,j}\Big)^2
\,\,\to\,\,\min
\label{eqn:linregls}
\end{equation}
This may not be enough, however.  The minimum may sometimes be attained over infinitely many
$\beta$-vectors, for example if $X$ has an all-0 column, or has linearly dependent columns,
or has fewer rows than columns~\mbox{($n < m$)}.  Even if~(\ref{eqn:linregls}) has a unique
solution, other $\beta$-vectors may be just a little suboptimal\footnote{Smaller likelihood
difference between two models suggests less statistical evidence to pick one model over the
other.}, yet give significantly different predictions for new feature vectors.  This results
in \emph{overfitting}: prediction error for the training data ($X$ and~$Y$) is much smaller
than for the test data (new records).

Overfitting and degeneracy in the data is commonly mitigated by adding a regularization penalty
term to the least squares function:
\begin{equation}
\sum_{i=1}^n \Big(y_i - \beta_0 - \sum_{j=1}^m \beta_j x_{i,j}\Big)^2
\,+\,\, \lambda \sum_{j=1}^m \beta_j^2
\,\,\to\,\,\min
\label{eqn:linreglsreg}
\end{equation}
The choice of $\lambda>0$, the regularization constant, typically involves cross-validation
where the dataset is repeatedly split into a training part (to estimate the~$\beta_j$'s) and
a test part (to evaluate prediction accuracy), with the goal of maximizing the test accuracy.
In our scripts, $\lambda$~is provided as input parameter~{\tt reg}.

The solution to least squares problem~(\ref{eqn:linreglsreg}), through taking the derivative
and setting it to~0, has the matrix linear equation form
\begin{equation}
A\left[\textstyle\beta_{1:m}\atop\textstyle\beta_0\right] \,=\, \big[X,\,1\big]^T Y,\,\,\,
\textrm{where}\,\,\,
A \,=\, \big[X,\,1\big]^T \big[X,\,1\big]\,+\,\hspace{0.5pt} \diag(\hspace{0.5pt}
\underbrace{\raisebox{0pt}[0pt][0.5pt]{$\lambda,\ldots, \lambda$}}_{\raisebox{2pt}{$\scriptstyle m$}}
\hspace{0.5pt}, 0)
\label{eqn:linregeq}
\end{equation}
where $[X,\,1]$ is $X$~with an extra column of~1s appended on the right, and the
diagonal matrix of $\lambda$'s has a zero to keep the intercept~$\beta_0$ unregularized.
If the intercept is disabled by setting {\tt icpt=0}, the equation is simply
\mbox{$X^T X \beta = X^T Y$}.

We implemented two scripts for solving equation~(\ref{eqn:linregeq}): one is a ``direct solver''
that computes $A$ and then solves $A\beta = [X,\,1]^T Y$ by calling an external package,
the other performs linear conjugate gradient~(CG) iterations without ever materializing~$A$.
The CG~algorithm closely follows Algorithm~5.2 in Chapter~5 of~\cite{Nocedal2006:Optimization}.
Each step in the CG~algorithm computes a matrix-vector multiplication $q = Ap$ by first computing
$[X,\,1]\, p$ and then $[X,\,1]^T [X,\,1]\, p$.  Usually the number of such multiplications,
one per CG iteration, is much smaller than~$m$.  The user can put a hard bound on it with input
parameter~{\tt maxi}, or use the default maximum of~$m+1$ (or~$m$ if no intercept) by
having {\tt maxi=0}.  The CG~iterations terminate when the L2-norm of vector
$r = A\beta - [X,\,1]^T Y$ decreases from its initial value (for~$\beta=0$) by the tolerance
factor specified in input parameter~{\tt tol}.

The CG algorithm is more efficient if computing
$[X,\,1]^T \big([X,\,1]\, p\big)$ is much faster than materializing $A$,
an $(m\,{+}\,1)\times(m\,{+}\,1)$ matrix.  The Direct Solver~(DS) is more efficient if
$X$ takes up a lot more memory than $A$ (i.e.\ $X$~has a lot more nonzeros than~$m^2$)
and if $m^2$ is small enough for the external solver ($m \lesssim 50000$).  A more precise
determination between CG and~DS is subject to further research.

In addition to the $\beta$-vector, the scripts estimate the residual standard
deviation~$\sigma$ and the~$R^2$, the ratio of ``explained'' variance to the total
variance of the response variable.  These statistics only make sense if the number
of degrees of freedom $n\,{-}\,m\,{-}\,1$ is positive and the regularization constant
$\lambda$ is negligible or zero.  The formulas for $\sigma$ and $R^2$~are:
\begin{equation*}
R^2_{\textrm{plain}} = 1 - \frac{\mathrm{RSS}}{\mathrm{TSS}},\quad
\sigma \,=\, \sqrt{\frac{\mathrm{RSS}}{n - m - 1}},\quad
R^2_{\textrm{adj.}} = 1 - \frac{\sigma^2 (n-1)}{\mathrm{TSS}}
\end{equation*}
where
\begin{equation*}
\mathrm{RSS} \,=\, \sum_{i=1}^n \Big(y_i - \hat{\mu}_i -
\frac{1}{n} \sum_{i'=1}^n \,(y_{i'} - \hat{\mu}_{i'})\Big)^2; \quad
\mathrm{TSS} \,=\, \sum_{i=1}^n \Big(y_i - \frac{1}{n} \sum_{i'=1}^n y_{i'}\Big)^2
\end{equation*}
Here $\hat{\mu}_i$ are the predicted means for $y_i$ based on the estimated
regression coefficients and the feature vectors.  They may be biased when no
intercept is present, hence the RSS formula subtracts the bias.

Lastly, note that by choosing the input option {\tt icpt=2} the user can shift
and rescale the columns of~$X$ to have zero average and the variance of~1.
This is particularly important when using regularization over highly disbalanced
features, because regularization tends to penalize small-variance columns (which
need large~$\beta_j$'s) more than large-variance columns (with small~$\beta_j$'s).
At the end, the estimated regression coefficients are shifted and rescaled to
apply to the original features.

\smallskip
\noindent{\bf Returns}
\smallskip

The estimated regression coefficients (the $\hat{\beta}_j$'s) are populated into
a matrix and written to an HDFS file whose path/name was provided as the ``{\tt B}''
input argument.  What this matrix contains, and its size, depends on the input
argument {\tt icpt}, which specifies the user's intercept and rescaling choice:
\begin{Description}
\item[{\tt icpt=0}:] No intercept, matrix~$B$ has size $m\,{\times}\,1$, with
$B[j, 1] = \hat{\beta}_j$ for each $j$ from 1 to~$m = {}$ncol$(X)$.
\item[{\tt icpt=1}:] There is intercept, but no shifting/rescaling of~$X$; matrix~$B$
has size $(m\,{+}\,1) \times 1$, with $B[j, 1] = \hat{\beta}_j$ for $j$ from 1 to~$m$,
and $B[m\,{+}\,1, 1] = \hat{\beta}_0$, the estimated intercept coefficient.
\item[{\tt icpt=2}:] There is intercept, and the features in~$X$ are shifted to
mean${} = 0$ and rescaled to variance${} = 1$; then there are two versions of
the~$\hat{\beta}_j$'s, one for the original features and another for the
shifted/rescaled features.  Now matrix~$B$ has size $(m\,{+}\,1) \times 2$, with
$B[\cdot, 1]$ for the original features and $B[\cdot, 2]$ for the shifted/rescaled
features, in the above format.  Note that $B[\cdot, 2]$ are iteratively estimated
and $B[\cdot, 1]$ are obtained from $B[\cdot, 2]$ by complementary shifting and
rescaling.
\end{Description}
The estimated summary statistics, including residual standard deviation~$\sigma$ and
the~$R^2$, are printed out or sent into a file (if specified) in CSV format as
defined in Table~\ref{table:linreg:stats}.  For conjugate gradient iterations,
a log file with monitoring variables can also be made available, see
Table~\ref{table:linreg:log}.

\smallskip
\noindent{\bf Examples}
\smallskip

{\hangindent=\parindent\noindent\tt
\hml -f LinearRegCG.dml -nvargs X=/user/biadmin/X.mtx Y=/user/biadmin/Y.mtx
  B=/user/biadmin/B.mtx fmt=csv O=/user/biadmin/stats.csv
  icpt=2 reg=1.0 tol=0.00000001 maxi=100 Log=/user/biadmin/log.csv

}
{\hangindent=\parindent\noindent\tt
\hml -f LinearRegDS.dml -nvargs X=/user/biadmin/X.mtx Y=/user/biadmin/Y.mtx
  B=/user/biadmin/B.mtx fmt=csv O=/user/biadmin/stats.csv
  icpt=2 reg=1.0

}

% \smallskip
% \noindent{\bf See Also}
% \smallskip
%
% In case of binary classification problems, please consider using L2-SVM or
% binary logistic regression; for multiclass classification, use multiclass~SVM
% or multinomial logistic regression.  For more complex distributions of the
% response variable use the Generalized Linear Models script.