% !TEX root = main.tex

%-------------------------------------------------
\section{Simple linear regression}\label{sec:regression}

Let $X$ and $Y$ be continuous random variables. We wish to investigate how $X$ influences the behaviour of $Y$.
\bit
\it $X$ is called the \emph{explanatory variable} or the \emph{independent variable}.
\it $Y$ is called the \emph{response variable} or the \emph{dependent variable}.
\eit

Suppose we observe that $X$ takes the value $x$. Unless $Y$ is completely determined by $X$, we cannot predict its value with certainty so instead we focus on the problem of estimating its conditional expectation $E(Y|X=x)$. This leads us to represent $Y$ as the sum of two random variables:
\[
Y = \mu(X) + \epsilon  \qquad\text{where $\epsilon\sim N(0,\sigma^2)$.}
\]
\bit
\it $\mu(x) = \expe(Y|X=x)$ is called the \emph{regression function};
\it $\epsilon = Y - \expe(Y|X)$ is called the \emph{error variable}.
\eit

%% lemma
%%By the law of total expectation, the expected value of the error variable is zero.
%\begin{lemma}
%The expected value of the error variable is zero.
%%$\expe(\epsilon) = 0$.
%\end{lemma}
%\begin{proof}
%By the law of total expectation,
%\[
%\expe(\epsilon) = \expe\big[Y - \expe(Y|X)\big] = \expe(Y) - \expe\big[\expe(Y|X)\big] = \expe(Y)-\expe(Y) = 0.
%\]
%\end{proof}

%% lemma
%%The law of total variance divides the variance of $Y$ into a component attributed to the explanatory variable $X$, and a component attributed to the error variable $\epsilon$.
%The following lemma shows that the variance of $Y$ can be divided into a component attributed to the explanatory variable $X$, and a component attributed to the error variable $\epsilon$.
%\begin{lemma}
%%$\var(Y) = \var(\mu) + \expe\big[\var(\epsilon|X)\big]$.
%If the error variable $\epsilon$ is independent of the explanatory variable $X$,
%\[\var(Y) = \var(\mu) + \var(\epsilon).
%\]
%\end{lemma}
%\begin{proof}
%$\epsilon = Y - \expe(Y|X)$, so (by the definition of conditional variance),
%\[
%\var(Y|X) = \expe\big(\big[Y-\expe(Y|X)\big]^2|X\big) = \expe(\epsilon^2|X) = \var(\epsilon|X) = \var(\epsilon).
%\]
%By the law of total variance,
%\begin{align*}
%\var(Y)
%	& = \var\big[\expe(Y|X)\big] + \expe\big[\var(Y|X)\big] \\
%	& = \var\big[\mu(X)\big] + \expe\big[\var(\epsilon)\big]
%	& = \var\big[\mu(X)\big] + \var(\epsilon).
%\end{align*}
%\end{proof}
%
%\begin{remark}
%\bit
%\it $\var(\mu) = \var\big[\expe(Y|X)\big]$ is the \emph{explained} variance.
%%\it $\expe\big[\var(\epsilon|X)\big] = \expe\big[\var(Y|X)\big]$ is the \emph{unexplained} variance.
%\it $\var(\epsilon) = \var(Y|X)$ is the \emph{unexplained} variance.
%\eit
%\end{remark}

% lemma
%The law of total variance divides the variance of $Y$ into a component attributed to the explanatory variable $X$, and a component attributed to the error variable $\epsilon$.
%The following lemma shows that the variance of $Y$ can be divided into a component attributed to the explanatory variable $X$, and a component attributed to the error variable $\epsilon$.
\begin{lemma}\label{lem:partition-of-variance}
%$\var(Y) = \var(\mu) + \expe\big[\var(\epsilon|X)\big]$.
$
\var(Y) = \var\big[\mu(X)\big] + \var(\epsilon).
$
\end{lemma}
\begin{proof}
$\epsilon = Y - \expe(Y|X)$, so by the definition of conditional variance,
\[
\var(Y|X) = \expe\big(\big[Y-\expe(Y|X)\big]^2|X\big) = \expe(\epsilon^2|X) = \var(\epsilon|X) = \var(\epsilon).
\]
By the law of total variance,
\begin{align*}
\var(Y)
	& = \var\big[\expe(Y|X)\big] + \expe\big[\var(Y|X)\big] \\
	& = \var\big[\mu(X)\big] + \expe\big[\var(\epsilon)\big] \\
	& = \var\big[\mu(X)\big] + \var(\epsilon).
\end{align*}
\end{proof}
Lemma~\ref{lem:partition-of-variance} expresses the variance of $Y$ as the sum of an \emph{explained variance}, attributed to the variance of the explanatory variable $X$, and an \emph{unexplained variance} attributed to the error variable $\epsilon$.
%\begin{remark}
%\bit
%\it $\var\big[\mu(X)\big] = \var\big[\expe(Y|X)\big]$ is the \emph{explained} variance.
%\it $\var(\epsilon) = \var(Y|X)$ is the \emph{unexplained} variance.
%\eit
%\end{remark}

%-----------------------------
\subsection{Linear models}

\begin{definition}
A \emph{linear model} is a model which is linear in its parameters.
\bit
\it $\mu(x) = \alpha + \beta x + \gamma x^2$ is a linear model.
\it $\mu(x) = \alpha e^{\beta x}$ is not a linear model.
\eit
A \emph{simple linear model} is a linear model of the form $\mu(x) = \alpha + \beta x$.
\end{definition}

The simple linear regression model is
\[
Y = \alpha + \beta X + \epsilon \quad\text{where}\quad \epsilon\sim N(0,\sigma^2).
\]
Given that $X=x$, we have that $Y\sim N(\alpha+\beta x, \sigma^2)$ and in particular,
\[
\expe(Y|X=x) = \alpha + \beta x
\quad\text{and}\quad
\var(Y|X=x) = \sigma^2.
\]

%-----------------------------
\subsection{Test statistics}

Let $(X_1,Y_1),(X_2,Y_2),\ldots,(X_n,Y_n)$ be a random sample from the joint distribution of $X$ and $Y$.
In Week~\ref{chap:likelihood} we saw that the maximum likelihood estimators of $\alpha$, $\beta$ and $\sigma^2$ are respectively
\[
\hat{\alpha} = \bar{Y}-\hat{\beta}\bar{X},\qquad
\hat{\beta} = \frac{\sum_{i=1}^n (X_i-\bar{X})(Y_i-\bar{Y})}{\sum_{i=1}^n (X_i-\bar{X})^2}
\qquad\text{and}\qquad
\hat{\sigma}^2 = \frac{1}{n}\sum_{i=1}^n \hat{\epsilon}_i^2
\]
where $\hat{\epsilon}_i = Y_i-(\hat{\alpha}+\hat{\beta} X_i)$ is the so-called \emph{residual variable} at $X_i$.

%%-----------------------------
%\subsection{Residual variance}
%
%\begin{definition}
%%Let $(x_1,x_2,\ldots,x_n)$ be a realisation of the marginal sample $(X_1,X_2,\ldots,X_n)$.
%%Given $X=x$,
%
%\ben
%\it $\hat{y} = \hat{\alpha} + \hat{\beta}X$ is called the \emph{predicted value of $Y$} at $X$.
%\it $\hat{\epsilon} = Y - \hat{y}$ is called the \emph{residual variable} at $X$.
%\een
%\end{definition}

%% theorem: mle of alpha and beta
%\begin{theorem}
%The maximum likelihood estimator of the error variance $\sigma^2$ is the sample mean of the squared residuals,
%\[
%\hat{\sigma}^2 = \frac{1}{n}\sum_{i=1}^n \hat{\epsilon}_i^2
%\text{\quad where\quad}
%\hat{\epsilon}_i = Y_i-(\hat{\alpha}+\hat{\beta} X_i).
%\]
%\end{theorem}
%
%% proof
%\begin{proof}
%Recall the log-likelihood function:
%\[
%\ell(\alpha,\beta,\sigma^2)
%	= \frac{n}{2}\log(2\pi\sigma^2) + \frac{1}{2\sigma^2}\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big]^2.
%\]
%The first partial derivative of $\ell(\alpha,\beta,\sigma^2)$ with respect to $\sigma^2$ is
%\[
%\frac{\partial\ell}{\partial(\sigma^2)}
%	= \frac{n}{2\sigma^2} - \frac{1}{2(\sigma^2)^2}\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big]^2.
%\]
%Setting this equal to zero,
%\[
%\sigma^2 = \frac{1}{n}\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big]^2.
%\]
%Substituting our estimates for $\alpha$ and $\beta$, we obtain the MLE
%\[
%\hat{\sigma^2} = \frac{1}{n}\sum_{i=1}^n \hat{\epsilon}_i^2 \text{\quad where\quad} \hat{\epsilon}_i = Y_i-(\hat{\alpha}+\hat{\beta} x_i)
%\]
%as required.
%
%\end{proof}
%
%\begin{remark}[Residual Analysis]
%Our model assumes that $\epsilon\sim N(0,\sigma^2)$ and that $\epsilon$ is independent of $X$. To test whether these assumptions hold, we plot the points $(x_i,\hat{\epsilon}_i)$ on a scatter diagram. If the assumptions do indeed hold, the points should be evenly spread about the horizontal axis, and the extent of their spread should not depend on the $x$-coordinate.
%This is an example of \emph{residual analysis}.
%\end{remark}
%
%% exercise
%\begin{exercise}
%Following a class test, 10 students were asked about the number of hours they had revised for the test. The data is shown in the table below.
%\begin{center}
%\begin{tabular}{|l|cccccccccc|} \hline
%Hours studied ($x$)	&  4	 &  9 & 10 & 14 &  4 &  7 & 12 & 22 &  1 & 17 \\
%Test score ($y$)		& 31 & 58 & 65 & 73 & 37 & 44 & 60 & 91 & 21 & 84 \\ \hline
%\end{tabular}
%\end{center}
%Perform a simple linear regression to estimate the relationship between the number of hours studied and the score achieved in the test.
%\begin{answer}
%%It is easy to show that
%%\begin{align*}
%%\sum_{i=1}^n (x_i - \bar{x})(y_i - \bar{y})
%%		& = \sum_{i=1}^n x_iy_i - \frac{1}{n}\left(\sum_{i=1}^n x_i\right)\left(\sum_{i=1}^n y_i\right) \text{ and} \\
%%\sum_{i=1}^n (x_i - \bar{x})^2
%%		& = \sum_{i=1}^n x_i^2 - \frac{1}{n}\left(\sum_{i=1}^n x_i\right)^2.
%%\end{align*}
%%
%%From the table,
%%\bit
%%\it $n=10$,
%%\it $\sum_i x_i = 100$ and $\sum_i y_i = 564$,
%%\it $\sum_i x_i^2 = 1376$ and $\sum_i x_iy_i = 6945$.
%%\eit
%%This yields
%Tedious calculations yield
%\[
%\sum_{i=1}^n(x_i-\bar{x})^2  = 376 \text{\quad and\quad} \sum_{i=1}^n (x_i - \bar{x})(y_i - \bar{y}) = 1305.
%\]
%Thus
%\begin{align*}
%\hat{\beta}
%	& = \displaystyle\frac{\sum_{i=1}^n (x_i - \bar{x})(y_i - \bar{y})}{\sum_{i=1}^n(x_i-\bar{x})^2}
%	= \displaystyle\frac{1305}{376}	= 3.47,\\
%\intertext{and}
%\hat{\alpha}
%	& = \displaystyle\bar{y} - \hat{\beta}\bar{x}
%	= \displaystyle\frac{564}{10} - \left(\frac{1305}{376}\right)\left(\frac{100}{10}\right) = 21.69.
%\end{align*}
%The estimated relationship is therefore $\hat{y} = 21.69 + 3.471 x$.
%\end{answer}
%\end{exercise}

%-----------------------------
%\subsection{}
\bigskip
Let $x_1,x_2,\ldots,x_n$ be a fixed realisation of the marginal sample $X_1,X_2,\ldots,X_n$. Then $\hat{\alpha}$, $\hat{\beta}$ and $\hat{\epsilon}_i$ are all linear combinations of $Y_1,Y_2,\ldots,Y_n$. Because the $Y_i$ are independent normal variables it thus follows that $\hat{\alpha}$, $\hat{\beta}$ and $\hat{\epsilon}_i$ are also normal variables.

\begin{theorem}
The MLEs of $\alpha$, $\beta$ and $\sigma^2$ respectively satisfy
\[
\hat{\alpha} \sim N\left(\alpha,\frac{\sigma^2}{n}\right),
\quad
\hat{\beta} \sim N\left(\beta,\frac{\sigma^2}{\sum_{i=1}^n(x_i-\bar{x})^2}\right)
\quad\text{and}\quad
\frac{n\hat{\sigma}}{\sigma} \sim \chi^2_{n-2}.
\]
%respectively.
\end{theorem}

\begin{proof}
\ben
\it % alpha
The expected value of $\hat{\alpha}$ is
\begin{align*}
\expe(\hat{\alpha})
	= \expe(\bar{Y}-\beta\bar{x})
	& = \expe\left(\frac{1}{n}\sum_{i=1}^n Y_i - \frac{\beta}{n}\sum_{i=1}^n x_i\right) \\
	& = \frac{1}{n}\sum_{i=1}^n \expe(Y_i) - \frac{\beta}{n}\sum_{i=1}^n x_i \\
	& = \frac{1}{n}\sum_{i=1}^n (\alpha+\beta x_i) - \frac{\beta}{n}\sum_{i=1}^n x_i \\
	& = \alpha.
\end{align*}
Because $\var(Y_i)=\sigma^2$, the variance of $\hat{\alpha}$ is
\begin{align*}
\var(\hat{\alpha})
	& = \var\left(\frac{1}{n}\sum_{i=1}^n Y_i - \frac{\beta}{n}\sum_{i=1}^n x_i\right) \\
	& = \frac{1}{n^2}\sum_{i=1}^n \var(Y_i)
	= \frac{\sigma^2}{n}.
\end{align*}
Hence $\hat{\alpha} \sim N(\alpha,\sigma^2/n)$, as required.
\it % beta
Since $\expe(Y_i) = \alpha + \beta x_i$ and $\expe(\bar{Y}) = \alpha + \beta\bar{x}$, the expected value of $\hat{\beta}$ is therefore
\begin{align*}
\expe(\hat{\beta})
	& = \expe\left[\frac{\sum_{i=1}^n (x_i-\bar{x})(Y_i-\bar{Y})}{\sum_{i=1}^n (x_i-\bar{x})^2}\right] \\
	& = \frac{\sum_{i=1}^n (x_i-\bar{x})\expe(Y_i-\bar{Y})}{\sum_{i=1}^n (x_i-\bar{x})^2}
`	= \frac{\sum_{i=1}^n \beta(x_i-\bar{x})^2}{\sum_{i=1}^n (x_i-\bar{x})^2} = \beta.
\end{align*}
Using the fact that $\sum_{i=1}^n x_i = n\bar{x}$, it is easy to see that $\hat{\beta}$ can be rewritten as
\[
\hat{\beta} = \frac{\sum_{i=1}^n (x_i-\bar{x})Y_i}{\sum_{i=1}^n (x_i-\bar{x})^2}.
\]
Because $\var(Y_i)=\sigma^2$, the variance of $\hat{\beta}$ is
\begin{align*}
\var(\hat{\beta})
	= \var\left[\frac{\sum_{i=1}^n (x_i-\bar{x})Y_i}{\sum_{i=1}^n (x_i-\bar{x})^2}\right]
	& = \frac{1}{\left[\sum_{i=1}^n(x_i-\bar{x})^2\right]^2}\sum_{i=1}^n(x_i-\bar{x})^2\var(Y_i) \\
	& = \frac{\sigma^2}{\sum_{i=1}^n(x_i-\bar{x})^2}.
\end{align*}
Hence $\hat{\beta}\sim N\left(\beta,\frac{\sigma^2}{\sum_{i=1}^n(x_i-\bar{x})^2}\right)$, as required.
\it % sigma^2
Recall that
\[
\hat{\sigma}^2 = \frac{1}{n}\sum_{i=1}^n \hat{\epsilon_i}^2
\quad\text{where}\quad
\hat{\epsilon}_i = Y_i-(\hat{\alpha}+\hat{\beta} x_i).
\]
Consider
\begin{align*}
\frac{1}{\sigma^2}\sum_{i=1}^n\big[Y_i-(\alpha+\beta x_i)\big]^2
	& = \frac{1}{\sigma^2}\sum_{i=1}^n\big[(\hat{\alpha}-\alpha) + (\hat{\beta}-\beta)x_i + (Y_i-(\hat{\alpha}+\hat{\beta}x_i))\big]^2 \\
	& = \frac{n(\hat{\alpha}-\alpha)^2}{\sigma^2} + \frac{(\hat{\beta}-\beta)^2}{\sigma^2}\sum_{i=1}^n x_i^2 + \frac{n\hat{\sigma}^2}{\sigma^2}.
\end{align*}
The first three terms in this expression all have chi-squared distribution.
\bit
\it
Because $Y_i\sim N(\alpha+\beta x_i,\sigma^2)$ it follows that $\big[Y_i-(\alpha+\beta x_i)\big]/\sigma\sim N(0,1)$, so
\[
\displaystyle\frac{1}{\sigma^2}\sum_{i=1}^n\big[Y_i-(\alpha+\beta x_i)\big]^2\sim\chi^2_{n}.
\]
\it
Because $\hat{\alpha}\sim N(\alpha,\sigma^2/n)$ it follows that $\sqrt{n}(\hat{\alpha}-\alpha)/\sigma\sim N(0,1)$, so
\[
\displaystyle\frac{n(\hat{\alpha}-\alpha)^2}{\sigma^2} \sim \chi^2_1.
\]
\it
Because $\hat{\beta}\sim N\left(\beta,\sigma^2/\sum_{i=1}^n(x_i-\bar{x})^2\right)$ it follows that $(\hat{\beta}-\beta)\sqrt{\sum_{i=1}^n x_i^2}/\sigma \sim N(0,1)$, so
\[
\frac{(\hat{\beta}-\beta)^2}{\sigma^2}\sum_{i=1}^n x_i^2 \sim \chi^2_1.
\]
\eit
It is easy to see that if $U\sim\chi^2_a$ and $V\sim\chi^2_b$ are independent, then $U+V\sim\chi^2_{a+b}$. It thus follows that $n\hat{\sigma}/\sigma\sim \chi^2_{n-2}$ as required.
\een
\end{proof}

We estimate the error variance using the following unbiased estimator (instead of the MLE),
\[
\hat{\sigma}^2 = \frac{1}{n-2}\sum_{i=1}^n \hat{\epsilon}_i^2
\quad\text{where}\quad
\hat{\epsilon}_i = Y_i-(\hat{\alpha}+\hat{\beta} x_i).
\]
This estimator for $\sigma^2$ yields the following test statistics:
\[
T_1 = \frac{\hat{\alpha}-\alpha}{\sqrt{\hat{\sigma}^2/(n-2)}}
\quad\text{and}\quad
T_2 = \frac{\hat{\beta}-\beta}{\sqrt{n\hat{\sigma}^2/[(n-2)\sum_{i=1}^n(x_i-\bar{x})^2]}}.
\]

\bit
\it Under the null hypothesis $H_0:\alpha=0$,
\[
T_1 = \frac{\hat{\alpha}-\alpha}{\sqrt{\hat{\sigma}^2/(n-2)}} \sim t_{n-2}.
\]
\it Uhder the null hypothesis $H_0:\beta=0$,
\[
T_2 = \frac{\hat{\beta}-\beta}{\sqrt{n\hat{\sigma}^2/[(n-2)\sum_{i=1}^n(x_i-\bar{x})^2]}} \sim t_{n-2}.
\]
\eit

$T_2$ can be used to test whether or not $Y$ depends (linearly) on $X$:
\begin{align*}
H_0: 	&\ Y = \alpha+\epsilon, \\
H_1:	&\ Y = \alpha+\beta X + \epsilon.
\end{align*}

%-----------------------------
\subsection{ANOVA for regression}

%For a fixed realisation $(x_1,x_2,\ldots,x_n)$ of the marginal sample $(X_1,X_2,\ldots,X_n)$,
The \emph{predicted value} of $Y_i$ is the random variable
\[
\hat{Y}_i = \hat{\alpha} + \hat{\beta}X_i.
\]
The total deviation of $Y_i$ from the overall mean $\bar{Y}$ can be divided into two components,
\[
Y_i - \bar{Y} = (Y_i-\hat{Y}_i) + (\hat{Y}_i - \bar{Y}),
\]
from which it follows that
\[
\sum_{i=1}^n(Y_i - \bar{Y})^2 = \sum_{i=1}^n(Y_i-\hat{Y}_i)^2 + \sum_{i=1}^n(\hat{Y}_i - \bar{Y})^2.
\]
As with ANOVA, we define the following sums-of-squares.
\[
\begin{array}{lll}
SST	& = \displaystyle\sum_{i=1}^n (Y_i-\bar{Y})^2
\qquad & \text{The \emph{total} sum-of-squares.} \\
SSR	& = \displaystyle\sum_{i=1}^n (\hat{Y}_i-\bar{Y})^2
\qquad & \text{The \emph{regression} sum-of-squares.} \\
SSE	& = \displaystyle\sum_{i=1}^n (Y_i-\hat{Y}_i)^2
\qquad & \text{The \emph{error} sum-of-squares.}
\end{array}
\]

The total sum-of-squares $SST$ is determined by the marginal sample $(Y_1,Y_2,\ldots,Y_n)$. Substituting for $\hat{Y}_i = \hat{\alpha} + \hat{\beta}X_i$ we see that the regression sum-of-squares satisfies
\[
SSR = \frac{\big[\sum_{i=1}^n (X_i-\bar{X})(Y_i-\bar{Y})\big]^2}{\sum_{i=1}^n (X_i-\bar{X})^2}
\]
The error sum-of-squares is then obtained via $SSE = SST - SSR$.

\bigskip
Under $H_0:\beta=0$,
\[
\frac{1}{\sigma^2}\sum_{i=1}^n (\hat{Y}_i-\bar{Y})^2	\sim \chi^2_1
\text{\quad and\quad}
\frac{1}{\sigma^2}\sum_{i=1}^n (Y_i-\hat{Y}_i)^2		\sim \chi^2_{n-2}.
\]
Thus we have the test statistic
\[
F
	= \frac{SSR}{(n-2)^{-1}SSE}
	= \frac{\sum_{i=1}^n (\hat{Y}_i-\bar{Y})^2}{(n-2)^{-1}\sum_{i=1}^n (Y_i-\hat{Y}_i)^2}
	\sim F_{1,n-2} \quad\text{under $H_0:\beta=0$.}
\]
which provides another means of testing whether or not $Y$ depends linearly on $X$.

%-----------------------------
\subsection{The coefficient of determination}

Recall that the \emph{correlation coefficient} of any pair of random variables $X$ and $Y$ is
\[
\rho(X,Y)
	= \frac{\cov(X,Y)}{\sqrt{\var(X)\var(Y)}}
	= \frac{\expe\big[(X-\expe X)(Y-\expe Y)\big]}
			{\sqrt{\expe\big[(X-\expe X)^2\big]\expe\big[(Y-\expe Y)^2\big]}}
\]

For a bivariate random sample $(X_1,Y_1),(X_2,Y_2),\ldots,(X_n,Y_n)$ the \emph{sample correlation coefficient}, also known as the \emph{Pearson correlation} is defined by
\[
R = \frac{\sum_{i=1}^n(X_i-\bar{X})(Y_i - \bar{Y})}{\sqrt{\sum_{i=1}^n(X_i-\bar{X})^2\sum_{i=1}^n(Y_i-\bar{Y})^2}}
\]

For the simple linear regression model $Y=\alpha+\beta X + \epsilon$, the MLE of $\beta$ can be written as,
\begin{align*}
\hat{\beta}
	& = \frac{\sum_{i=1}^n (X_i-\bar{X})(Y_i-\bar{Y})}{\sum_{i=1}^n (X_i-\bar{X})^2}
	= R\sqrt{\frac{\sum_{i=1}^n(Y_i-\bar{Y})^2}{\sum_{i=1}^n(X_i-\bar{X})^2}}.
\end{align*}

%The square of the empirical correlation coefficient is called the \emph{coefficient of determination}, denoted by $R^2$:
The \emph{coefficient of determination} is the square of the sample correlation, and denoted by $R^2$:
\[
R^2
	= \frac{\big[\sum_{i=1}^n(X_i-\bar{X})(Y_i - \bar{Y})\big]^2}{\sum_{i=1}^n(X_i-\bar{X})^2\sum_{i=1}^n(Y_i-\bar{Y})^2}
	= \frac{SSR}{SST}.
%	= 1 - \frac{SSE}{SST}.
\]

where $SSR$ and $SST$ are the regression sum-of-squares and total sum-of-squares respectively.

\bigskip
Thus $R^2$ is the proportion of the total variation explained by the regression model: it quantifies how well the regression line fits the data points, and as such is an example of a \emph{goodness-of-fit} statistic (the value $R^2=1$ indicates that the regression line perfectly fits the data). The corresponding quantity for one-way ANOVA is the so-called \emph{eta-squared} effect size:
\[
\eta^2 = \frac{SSG}{SST}.
\]

\begin{example}
The table below shows the deaths due to bronchitis ($x$) and corresponding daily temperatures ($y$), averaged over a long period.
\begin{center}
\begin{tabular}{lcccccccccc}\hline
$x$ & 253 & 232 & 210 & 200 & 191 & 187 & 134 & 102 & 81 & 25 \\
$y$ & 35 & 37 & 39 & 41 & 43 & 45 & 47 & 49 & 51 & 53 \\ \hline
\end{tabular}
\end{center}
\ben
\it Use the simple linear model $y = \alpha + \beta x + \epsilon$ to perform a least-squares regression of $y$ against $x$.
\it Test whether the slope of the regression line is significantly different from zero at the 5\% significance level.
\een
\begin{solution}
The various quantities of interest are computed here:
%\bit
%\it $n = 10$.
%\it $\sum x_{i} = 1615$.
%\it $\sum x_{i}^{2}	= 308929$.
%\it $\sum (x_i-\bar{x})^2	= 308929 - (1615)^{2}/10 = 48106.5$.
%\it $\sum y_{i} =  440$.
%\it $\sum y_{i}^{2}  = 19690$.
%\it $\sum (y_i-\bar{y})^2 = 19690 - (440)^{2}/10 = 330$.
%\it $\sum x_{i}y_{i}	= 67209$.
%\it $\sum (x_i-\bar{x})(y_i-\bar{y}) = 67209 - (1615){\times}(440)/10 = -3851$.
%\eit
\bit
\it $n = 10$.
\it $\bar{x} = 161.5$.
\it $\bar{y} = 44$.
\it $\sum (x_i-\bar{x})^2	= 48106.5$.
\it $\sum (y_i-\bar{y})^2 = 330$.
\it $\sum (x_i-\bar{x})(y_i-\bar{y}) = -3851$.
\eit

The MLEs of the regression coefficients are
\begin{align*}
\hat{\beta}		&\quad = \frac{\sum (x_i-\bar{x})(y_i-\bar{y})}{\sum (x_i-\bar{x})^2} = \frac{-3851}{48106.5} = -0.080052 \\
\hat{\alpha}	&\quad = \bar{y}-\hat{\beta}\bar{x} = 44 + 0.080052{\times}161.5 = 56.928326
\end{align*}

The least squares regression line is
\[
y = 56.928326 - 0.080052x.
\]
To test the null hypothesis $H_0:\beta=0$, we compute the regression sum-of-squares:
\[
SSR = \frac{\big[\sum (x_i-\bar{x})(y_i-\bar{y})\big]^{2}}{\sum (x_i-\bar{x})^2}
	=\frac{(-3851)^{2}}{48106.5}
	= 308.2785
\]
and the error sum-of-squares:
\[
SSE	= \sum (y_i-\bar{y})^2 - SSR = 330.0 - 308.2785 =  21.7215.
\]
The test statistic is
\[
F = \frac{SSR}{(n-2)^{-1}SSE} = \frac{308.2785}{21.7215/8} = 113.54.
\]

Critical values of the $F_{1,8}$-distribution are
\bit
\it $5.318$ at sig. level $0.05$
\it $7.570$ at sig. level $0.025$
\it $11.25$ at sig. level $0.001$
\it $14.68$ at sig. level $0.005$
\eit
Thus the null hypothesis $H_0:\beta = 0$ is strongly rejected at the $5\%$ significance level.

\bigskip
Here we have $R^2 = SSR/SST = 308.2785/330 = 0.9342$, which shows that $Y$ depends on $X$ to a very large extent.
\end{solution}
\end{example}


\chapter{Localization}
Before we proceed on to defining an affine scheme,
we will take the time to properly cover one more algebraic construction
that of a \emph{localization}.
This is mandatory because when we define a scheme,
we will find that all the sections and stalks
are actually obtained using this construction.

One silly slogan might be:
\begin{moral}
	Localization is the art of adding denominators.
\end{moral}
You may remember that when we were working with affine varieties,
there were constantly expressions of the form
$\left\{ \frac{f}{g} \mid g(p) \ne 0 \right\}$
and the like.
The point is that we introduced a lot of denominators.
Localization will give us a concise way of doing this in general.

\emph{Notational note}:
moving forward we'll prefer to denote rings by $A$, $B$, \dots,
rather than $R$, $S$, \dots.

\section{Spoilers}
Here is a preview of things to come,
so that you know what you are expecting.
Some things here won't make sense,
but that's okay, it is just foreshadowing.

Let $V \subseteq \Aff^n$, and for brevity let $R = \CC[V]$ be its coordinate ring.
We saw in previous sections how to compute $\OO_V(D(g))$
and $\OO_{V, p}$ for $p \in V$ a point.
For example, if we take $\Aff^1$ and consider a point $p$, then
$\OO_{\Aff^1}(D(x-p)) = \left\{ \frac{f(x)}{(x-p)^n} \right\}$
and $\OO_{\Aff^1, p} = \left\{ \frac{f(x)}{g(x)} \mid g(p) \ne 0 \right\}$.
More generally, we had
\begin{align*}
	\OO_{V}(D(g)) &= \left\{ \frac{f}{g^n} \mid f \in R \right\}
		\quad\text{by \Cref{thm:reg_func_distinguish_open}} \\
	\OO_{V,p} &= \left\{ \frac{f}{g} \mid f,g \in R, g(p) \ne 0 \right\}
		\quad\text{by \Cref{thm:stalks_affine_var}}.
\end{align*}

We will soon define something called a localization,
which will give us a nice way of expressing the above:
if $R = \CC[V]$ is the coordinate ring, then
the above will become abbreviated to just
\begin{align*}
	\OO_{V}(D(g)) &= R_{g} \\
	\OO_{V, p} &= R_{\km} \quad \text{where } \{p\} = \VV(\km).
\end{align*}
The former will be pronounced
``$R$ localized away from $g$''
while the latter will be pronounced
``$R$ localized at $\km$''.

Even more generally,
next chapter we will throw out the coordinate ring $R$
altogether and replace it with a general commutative ring $A$
(which are still viewed as functions).
We will construct a ringed space called $X = \Spec A$,
whose elements are \emph{prime ideals} of $A$
and is equipped with the Zariski topology and a sheaf $\OO_X$.
It will turn out that, in analogy to what we had before,
\begin{align*}
	\OO_X(D(f)) &= A[f\inv] \\
	\OO_{X,\kp} &= A_\kp
\end{align*}
for any element $f \in A$ and prime ideal $\kp \in \Spec A$.
Thus just as with complex affine varieties,
localizations will give us a way to more or less
describe the sheaf $\OO_X$ completely.

\section{The definition}
\begin{definition}
	A subset $S \subseteq A$ is a \vocab{multiplicative set}
	if $1 \in S$ and $S$ is closed under multiplication.
\end{definition}
\begin{definition}
	Let $A$ be a ring and $S \subset A$ a multiplicative set.
	Then the \vocab{localization of $A$ at $S$}, denoted $S\inv A$,
	is defined as the set of fractions
	\[ \left\{ a/s \mid a \in A, s \in S \right\} \]
	where we declare two fractions $a_1 / s_1 = a_2 / s_2$
	to be equal if $s(a_1s_2 - a_2s_1) = 0$ for some $s \in S$.
	Addition and multiplication in this ring
	are defined in the obvious way.
\end{definition}
In particular, if $0 \in S$ then $S\inv A$ is the zero ring.
So we usually only take situations where $0 \notin S$.

We give in brief now two examples which will be
motivating forces for the construction of the affine scheme.
\begin{example}
	[Localizations of {$\CC[x]$}]
	Let $A = \CC[x]$.
	\begin{enumerate}[(a)]
		\ii Suppose we let $S = \left\{ 1, x, x^2, x^3, \dots \right\}$
		be the powers of $x$.
		Then
		\[ S\inv A = \left\{ \frac{f(x)}{x^n}
			\mid f \in \CC[x], n \in \ZZ_{\ge 0} \right\}.  \]
		In other words, we get the Laurent polynomials in $x$.

		You might recognize this as
		\[ \OO_V(U) \text{ where } V = \Aff^1, \; U = V \setminus \{0\}. \]
		i.e.\ the sections of the punctured line.
		In line with the ``hyperbola effect'',
		this is also expressible as $\CC[x,y] / (xy-1)$.

		\ii Let $p \in \CC$.
		Suppose we let $S = \left\{ g(x) \mid g(p) \ne 0 \right\}$,
		i.e.\ we allow any denominators where $g(p) \ne 0$.
		Then
		\[ S\inv A = \left\{ \frac{f(x)}{g(x)}
			\mid f,g \in \CC[x], g(p) \ne 0 \right\}.  \]
		You might recognize this is as the stalk $\OO_{\Aff^1, p}$.
		This will be important later on.
	\end{enumerate}
\end{example}

\begin{remark}
	[Why the extra $s$?]
	We cannot use the simpler $a_1s_2 - a_2s_1 = 0$ since
	otherwise the equivalence relation may fail to be transitive.
	Here is a counterexample: take
	\[ A = \Zc{12} \qquad S = \{ 2, 4, 8 \}. \]
	Then we have for example $\frac12 = \frac24 = \frac64 = \frac32$.
	So we need to have $\frac12=\frac32$ which is only true
	with the first definition.
	Of course, if $A$ is an integral domain (and $0\notin S$)
	then this is a moot point.
\end{remark}

\begin{example}
	[Field of fractions]
	Let $A$ be an integral domain and $S = A \setminus \{0\}$.
	Then $S\inv A = \Frac(A)$.
\end{example}

\section{Localization away from an element}
\prototype{$\ZZ$ localized away from $6$ has fractions $\frac{m}{2^x 3^y}$.}
We now focus on the two special cases of localization we will need the most;
one in this section, the other in the next section.
\begin{definition}
	For $f \in A$, we define the \vocab{localization of $A$ away from $f$},
	denoted $A[1/f]$ or $A[f\inv]$,
	to be $\{1, f, f^2, f^3, \dots\}\inv A$.
	(Note that $\left\{ 1, f, f^2, \dots \right\}$ is multiplicative.)
\end{definition}
\begin{remark}
	In the literature it is more common to
	see the notation $A_f$ instead of $A[1/f]$.
	This is confusing, because in the next section
	we define $A_\kp$ which is almost the opposite.
	So I prefer this more suggestive (but longer) notation.
\end{remark}

\begin{example}
	[Some arithmetic examples of localizations]
	\listhack
	\begin{enumerate}[(a)]
		\ii We localize $\ZZ$ away from $6$:
		\[ \ZZ[1/6] = \left\{ \frac{m}{6^n} \mid m \in \ZZ,
			n \in \ZZ_{\ge 0} \right\}.  \]
		So $A[1/6]$ consist of those rational numbers whose
		denominators have only powers of $2$ and $3$.
		For example, it contains $\frac{5}{12} = \frac{15}{36}$.

		\ii Here is a more confusing example:
		if we localize $\Zc{60}$ away from the element $5$,
		we get $(\Zc{60})[1/5] \cong \Zc{12}$.
		You should try to think about why this is the case.
		We will see a ``geometric'' reason later.
	\end{enumerate}
\end{example}

\begin{example}
	[Localization at an element, algebraic geometry flavored]
	We saw that if $A$ is the coordinate ring of a variety,
	then $A[1/g]$ is interpreted geometrically as $\OO_V(D(g))$.
	Here are some special cases:
	\begin{enumerate}[(a)]
		\ii As we saw, if $A = \CC[x]$,
		then $A[1/x] = \left\{ \frac{f(x)}{x^n} \right\}$
		consists of Laurent polynomials.

		\ii Let $A = \CC[x,y,z]$.
		Then \[ A[1/x] = \left\{ \frac{f(x,y,z)}{x^n} \mid
			f \in \CC[x,y,z], \; n \ge 0 \right\} \]
		is rational functions whose denominators are powers of $x$.

		\ii Let $A = \CC[x,y]$.
		If we localize away from $y-x^2$ we get
		\[ A[(y-x^2)\inv] = \left\{ \frac{f(x,y)}{(y-x^2)^n} \mid
			f \in \CC[x,y], \; n \ge 0 \right\} \]
		By now you should recognize this as $\OO_{\Aff^2}(D(y-x^2))$.
	\end{enumerate}
\end{example}

\begin{example}
	[An example with zero-divisors]
	Let $A = \CC[x,y] / (xy)$
	(which intuitively is the coordinate ring of two axes).
	Suppose we localize at $x$:
	equivalently, allowing denominators of $x$.
	Since $xy = 0$ in $A$, we now have $0 = x\inv (xy) = y$,
	so $y = 0$ in $A$, and thus $y$ just goes away completely.
	From this we get a ring isomorphism
	\[ A[1/x] \cong \CC[x,1/x].\]
\end{example}

\section{Localization at a prime ideal}
\prototype{$\ZZ$ localized at $(5)$ has fractions $\frac{m}{n}$ with $5 \nmid n$.}
\label{sec:localize_prime_ideal}

\begin{definition}
	If $A$ is a ring and $\kp$ is a prime ideal, then we define
	\[ A_\kp \defeq \left( A \setminus \kp \right)^{-1} A. \]
	This is called the \vocab{localization at $\kp$}.
\end{definition}
\begin{ques}
	Why is $S = A \setminus \kp$ multiplicative
	in the above definition?
\end{ques}

%Warning: this notation sort of conflicts with the previous one.
%The ring $A_\kp$ is the localization with multiplicative set $A \setminus \kp$,
%while $A_f$ is the localization with multiplicative set $\{1,f,f^2,\dots\}$;
%these two are quite different beasts!
%In $A_\kp$ the subscript denotes the \emph{forbidden} denominators;
%in $A_f$ the subscript denotes the \emph{allowed} denominators.

This special case is important because we will see that
stalks of schemes will all be of this shape.
In fact, the same was true for affine varieties too.
\begin{example}
	[Relation to affine varieties]
	Let $V \subseteq \Aff^n$, let $A = \CC[V]$
	and let $p = (a_1, \dots, a_n)$ be a point.
	Consider the maximal (hence prime) ideal
	\[ \km = (x_1 - a_1, \dots, x_n - a_n). \]
	Observe that a function $f \in A$ vanishes at $p$
	if and only if $f \pmod{\km} = 0$, equivalently $f \in \km$.
	Thus, by \Cref{thm:stalks_affine_var} we can write
	\begin{align*}
		\OO_{V,p} &= \left\{ \frac{f}{g} \mid f,g \in A, g(p) \ne 0 \right\} \\
		&= \left\{ \frac{f}{g} \mid f \in A, g \in A \setminus \km \right\} \\
		&= \left( A \setminus \km \right)\inv A = A_\km.
	\end{align*}
	So, we can also express $\OO_{V,p}$ concisely as a localization.
\end{example}
Consequently, we give several examples in this vein.

\begin{example}
	[Geometric examples of localizing at a prime]
	\listhack
	\begin{enumerate}[(a)]
		\ii We let $\km$ be the maximal ideal $(x)$ of $A = \CC[x]$.
		Then \[ A_\km = \left\{ \frac{f(x)}{g(x)} \mid g(0) \ne 0 \right\} \]
		consists of the Laurent polynomials.

		\ii We let $\km$ be the maximal ideal $(x,y)$ of $A = \CC[x,y]$.
		Then \[ A_\km = \left\{ \frac{f(x,y)}{g(x,y)} \mid g(0,0) \ne 0 \right\}. \]

		\ii Let $\kp$ be the prime ideal $(y-x^2)$ of $A = \CC[x,y]$.
		Then
		\[ A_\kp = \left\{ \frac{f(x,y)}{g(x,y)} \mid g \notin (y-x^2) \right\}. \]
		This is a bit different from what we've seen before:
		the polynomials in the denominator are allowed to vanish
		at a point like $(1,1)$, as long as they don't vanish on
		\emph{every} point on the parabola.
		This doesn't correspond to any stalk we're familiar with right now,
		but it will later
		(it will be the ``stalk at the generic point of the parabola'').

		\ii Let $A = \CC[x]$ and localize at the prime ideal $(0)$.
		This gives \[ A_{(0)} = \left\{ \frac{f(x)}{g(x)} \mid g(x) \ne 0 \right\}. \]
		This is all rational functions, period.
	\end{enumerate}
\end{example}

\begin{example}
	[Arithmetic examples]
	We localize $\ZZ$ at a few different primes.
	\begin{enumerate}[(a)]
		\ii If we localize $\ZZ$ at $(0)$:
		\[ \ZZ_{(0)} = \left\{ \frac mn \mid n \ne 0 \right\}
			\cong \QQ. \]
		\ii If we localize $\ZZ$ at $(3)$, we get
		\[ \ZZ_{(3)} = \left\{ \frac mn \mid \gcd(n,3) = 1 \right\} \]
		which is the ring of rational numbers
		whose denominators are relatively prime to $3$.
	\end{enumerate}
\end{example}

\begin{example}
	[Field of fractions]
	If $A$ is an integral domain,
	the localization $A_{(0)}$
	is the field of fractions of $A$.
\end{example}

\section{Prime ideals of localizations}
\prototype{The examples with $A = \ZZ$.}
We take the time now to mention how you can
think about prime ideals of localized rings.
\begin{proposition}
	[The prime ideals of $S\inv A$]
	Let $A$ be a ring and $S \subseteq A$ a multiplicative set.
	Then there is a natural inclusion-preserving bijection between:
	\begin{itemize}
		\ii The set of prime ideals of $S\inv A$, and
		\ii The set of prime ideals of $A$ not intersecting $S$.
	\end{itemize}
\end{proposition}
\begin{proof}
	Consider the homomorphism $\iota \colon A \to S\inv A$.
	For any prime ideal $\kq \subseteq S\inv A$,
	its pre-image $\psi\pre(\kq)$ is a prime ideal of $A$
	(by \Cref{prob:prime_preimage}).
	Conversely, for any prime ideal $\kp \subseteq A$
	not meeting $S$,
	$S\inv \kp = \left\{ \frac{a}{s} \mid a \in \kp, s \in S \right\}$
	is a prime ideal of $S\inv A$.
	An annoying check shows that this produces the required bijection.
\end{proof}
In practice, we will almost always use the corollary
where $S$ is one of the two special cases we discussed at length:
\begin{corollary}
	[Spectrums of localizations]
	Let $A$ be a ring.
	\begin{enumerate}[(a)]
		\ii If $\kp$ is a prime ideal of $A$,
		then the prime ideals of $A[1/f]$ are naturally
		in bijection with prime ideals of $A$
		\textbf{do not contain the element} $f$.

		\ii If $\kp$ is a prime ideal of $A$,
		then the prime ideals of $A_\kp$ are naturally
		in bijection with prime ideals of $A$
		which are \textbf{subsets of} $\kp$.
	\end{enumerate}
\end{corollary}
\begin{proof}
	Part (b) is immediate; a prime ideal doesn't meet $A \setminus \kp$
	exactly if it is contained in $\kp$.
	For part (a), we want prime ideals of $A$ not containing
	any \emph{power} of $f$.
	But if the ideal is prime and contains $f^n$,
	then it should contain either $f$ or $f^{n-1}$,
	and so at least for prime ideals these are equivalent.
\end{proof}
Notice again how the notation is a bit of a nuisance.
Anyways, here are some examples, to help cement the picture.
\begin{example}
	[Prime ideals of {$\ZZ[1/6]$}]
	Suppose we localize $\ZZ$ away from the element $6$,
	i.e.\ consider $\ZZ[1/6]$.
	As we saw,
	\[ \ZZ[1/6] = \left\{ \frac{n}{2^x 3^y} \mid n \in \ZZ,
		x,y \in \ZZ_{\ge 0} \right\}.  \]
	consist of those rational numbers whose
	denominators have only powers of $2$ and $3$.
	Note that $(5) \subset \ZZ[1/6]$ is a prime ideal:
	those elements of $\ZZ[1/6]$ with $5$ dividing the numerator.
	Similarly, $(7)$, $(11)$, $(13)$, \dots\
	and even $(0)$ give prime ideals of $\ZZ[1/6]$.

	But $(2)$ and $(3)$ no longer correspond to
	prime ideals; in fact in $A_6$ we have $(2) = (3) = (1)$,
	the whole ring.
\end{example}

\begin{example}
	[Prime ideals of $A_{(5)}$]
	Suppose we localize $\ZZ$ at the prime $(5)$.
	As we saw,
	\[ \ZZ_{(5)} = \left\{ \frac{m}{n} \mid m,n \in \ZZ,
			5 \nmid n \right\}.  \]
	consist of those rational numbers whose
	denominators are not divisible by $5$.
	This is an integral domain, so $(0)$ is still a prime ideal.
	There is one other prime ideal: $(5)$,
	i.e.\ those elements whose numerators are divisible by $5$.

	There are no other prime ideals:
	if $p \ne 5$ is a rational prime,
	then $(p) = (1)$, the whole ring, again.
\end{example}

\section{Prime ideals of quotients}
While we are here, we mention that
the prime ideals of quotients $A/I$
can be interpreted in terms of those of $A$
(as in the previous section for localization).
You may remember this from \Cref{prob:inclusion_preserving}
a long time ago, if you did that problem;
but for our purposes we actually only care about the prime ideals.
\begin{proposition}
	[The prime ideals of $A/I$]
	\label{prop:prime_quotient}
	If $A$ is a ring and $I$ is any ideal (not necessarily prime)
	then the prime (resp.\ maximal) ideals of $A/I$
	are in bijection with prime (resp.\ maximal) ideals of $A$
	which are \textbf{supersets of} $I$.
	This bijection is inclusion-preserving.
\end{proposition}
\begin{proof}
	Consider the quotient homomorphism $\psi \colon A \surjto A/I$.
	For any prime ideal $\kq \subseteq A/I$,
	its pre-image $\iota\pre(\kq)$ is a prime ideal
	(by \Cref{prob:prime_preimage}).
	Conversely, for any prime ideal $\kp$
	with $I \subseteq \kp \subseteq A$,
	we get a prime ideal of $A/I$ by looking at $\kp \pmod I$.
	An annoying check shows that this produces the required bijection.
	It is also inclusion-preserving --- from which
	the same statement holds for maximal ideals.
\end{proof}
\begin{example}
	[Prime ideals of $\Zc{60}$]
	The ring $\Zc{60}$ has three prime ideals:
	\begin{align*}
		(2) &= \left\{ 0, 2, 4, \dots, 58 \right\} \\
		(3) &= \left\{ 0, 3, 6, \dots, 57 \right\} \\
		(5) &= \left\{ 0, 5, 10, \dots, 55 \right\}.
	\end{align*}
	Back in $\ZZ$, these correspond to the three prime ideals
	which are supersets of
	$60\ZZ = \left\{ \dots, -60, 0, 60, 120, \dots \right\}$.
\end{example}

\section{Localization commute with quotients}
\prototype{$(\CC[xy]/(xy))[1/x] \cong \CC[x,x\inv]$.}
While we are here, we mention a useful result from
commutative algebra which lets us compute localizations in quotient rings,
which are surprisingly unintuitive.
You will \emph{not} have a reason to care about this
until we reach \Cref{sec:localize_prime_ideal},
and so this is only placed earlier to emphasize that it's
a purely algebraic fact that we can (and do) state this early,
even though we will not need it anytime soon.

Let's say we have a quotient ring like
\[ A/I = \CC[x,y] / (xy) \]
and want to compute the localization of this ring
away from the element $x$.
(To be pedantic, we are actually localizing away from $x \pmod{xy}$,
the element of the quotient ring, but we will just call it $x$.)
You will quickly find that even the notation becomes clumsy: it is
\begin{equation}
	\left( \CC[x,y] / (xy) \right)[1/x]
	\label{eq:quotient_localization_before}
\end{equation}
which is hard to think about,
because the elements in play are part of the \emph{quotient}:
how are we supposed to think about
\[ \frac{1 \pmod{xy}}{x \pmod{xy}} \]
for example?
The zero-divisors in play may already make you feel uneasy.

However, it turns out that we can actually do the localization
\emph{first}, meaning the answer is just
\begin{equation}
	\CC[x,y,1/x] / (xy)
	\label{eq:quotient_localization_after}
\end{equation}
which then becomes $\CC[x, x\inv, y] / (y) \cong \CC[x,x\inv]$.

This might look like it should be trivial,
but it's not as obvious as you might expect.
There is a sleight of hand present here with the notation:
\begin{itemize}
	\ii In \eqref{eq:quotient_localization_before},
	the notation $(xy)$ stands for an ideal of $\CC[x,y]$
	--- that is, the set $xy \CC[x,y]$.

	\ii In \eqref{eq:quotient_localization_after}
	the notation $(xy)$ now stands for an ideal of $\CC[x,x\inv,y]$
	--- that is, the set $xy \CC[x,x\inv,y]$.
\end{itemize}
So even writing down the \emph{statement} of the theorem
is actually going to look terrible.

In general, what we want to say is that if we have our ring $A$
with ideal $I$ and $S$ is some multiplicative subset of $A$,
then \[ \text{Colloquially: ``}S\inv (A/I) = (S\inv A)/I\text{''}. \]
But there are two things wrong with this:
\begin{itemize}
\ii The main one is that $I$ is not an ideal of $S\inv A$, as we saw above.
This is remedied by instead using $S\inv I$,
which consists of those elements of those elements $\frac xs$
for $x \in I$ and $s \in S$.
As we saw this distinction is usually masked in practice,
because we will usually write $I = (a_1, \dots, a_n) \subseteq A$
in which case the new ideal $S\inv I \subseteq A$ can be denoted
in exactly the same way: $(a_1, \dots, a_n)$,
just regarded as a subset of $S\inv A$ now.

\ii The second is that $S$ is not, strictly speaking,
a subset of $A/I$, either.
But this is easily remedied by instead using the image of $S$
under the quotient map $A \surjto A/I$.

We actually already saw this in the previous example:
when trying to localize $\CC[x,y]/(xy)$,
we were really localizing at the element $x \pmod{xy}$,
but (as always) we just denoted it by $x$ anyways.
\end{itemize}

And so after all those words, words, words, we have the hideous:
\begin{theorem}
	[Localization commutes with quotients]
	\label{thm:localization_commute_quotient}
	Let $S$ be a multiplicative set of a ring $A$,
	and $I$ an ideal of $A$.
	Let $\ol S$ be the image of $S$
	under the projection map $A \surjto A/I$.
	Then
	\[ {\ol S}\inv (A/I) \cong S\inv A / S\inv I \]
	where $S\inv I = \left\{ \frac{x}{s} \mid x \in I, s \in S \right\}$.
\end{theorem}
\begin{proof}
	Do you actually care? No? Cool, I didn't either.
	(Atiyah-Macdonald is the right reference for these
	type of things in the event that you do care.)
\end{proof}
The notation is a hot mess.
But when we do calculations in practice, we instead write
\[ \left( \CC[x,y,z]/(x^2 + y^2 - z^2) \right)[1/x]
	\cong \CC[x,y,z,1/x] / (x^2 + y^2 - z^2) \]
or (for an example where we localize at a prime ideal)
\[ \left( \ZZ[x,y,z]/ (x^2 + yz) \right)_{(x,y)}
	\cong \ZZ[x,y,z]_{(x,y)} / (x^2 + yz) \]
and so on --- the pragmatism of our ``real-life'' notation
which hides some details actually guides our intuition
(rather than misleading us).
So maybe the moral of this section is that whenever
you compute the localization of the quotient ring,
if you just suspend belief for a bit,
then you will probably get the right answer.

We will later see geometric interpretations of these facts
when we work with $\Spec A/I$,
at which point they will become more natural.

\section{\problemhead}
\begin{problem}
	Let $A = \Zc{2016}$, and consider the element $60 \in A$.
	Compute $A[1/60]$, the localization of $A$ away from $60$.
	\begin{sol}
		One should get $A[1/60] = \Zc{7}$.
	\end{sol}
\end{problem}

\begin{problem}
	[Injectivity of localizations]
	Let $A$ be a ring and $S \subseteq A$ a multiplicative set.
	Find necessary and sufficient conditions
	for the map $A \to S\inv A$ to be injective.
	\begin{hint}
		Consider zero divisors.
	\end{hint}
	\begin{sol}
		If and only if $S$ has no zero divisors.
	\end{sol}
\end{problem}

\begin{sproblem}
	[Alluding to local rings]
	Let $A$ be a ring, and $\kp$ a prime ideal.
	How many maximal ideals does $A_\kp$ have?
	\begin{hint}
		Only one!
		A proof will be given a few chapters later.
	\end{hint}
\end{sproblem}

\begin{problem}
	Let $A$ be a ring such that $A_\kp$ is an integral domain
	for every prime ideal $\kp$ of $A$.
	Must $A$ be an integral domain?
	\begin{hint}
		No. Imagine two axes.
	\end{hint}
	\begin{sol}
		Take $A = \CC[x,y] / (xy)$.
	\end{sol}
\end{problem}


{\bfseries\slshape\sffamily\color{ChapterTitleColor} \chapter{Array Support}} \label{chap:arrys}

\section{Introduction}

In most programming languages, the word array usually refers to a data structure that stores identically sized elements in a sequential and continuous block of memory. Lists on the other hand are used to stored non-identically sized elements that might be located anywhere in memory. Arrays are therefore referred to as homogenous collections whereas lists are heterogeneous collections. The advantage of arrays is that they can be used to process information very efficiently. Arrays are often used to store numerical data which require efficient numerical processing. The simplest array is a one dimensional collection of numbers, such as:

\begin{lstlisting}
[1.3, 5.6, 7.8, 4.5, 2.3, 1.2]
\end{lstlisting}

This array has six elements. Since the data in an array is guaranteed to be laid out sequentially in memory, accessing individual elements can be efficiently done by using indexing to the appropriate element. Object Pascal has a variety of ways for declaring arrays. For example an array with a fixed number of elements (a static array) can be declared using:

\begin{lstlisting}
TMyArray = array[1..100] of double;
\end{lstlisting}

It is also possible to declare so-called dynamic arrays which are allocated a size at runtime, and makes them more flexible:

\begin{lstlisting}
var
   myArray : array of double;

setLength (myArray, 100);
\end{lstlisting}

Arrays need not be only one-dimensional but in principle can be any dimension. A common array structure used in mathematics is the two-dimensional matrix:

$$
\setlength{\delimitershortfall}{0pt}
\begin{bmatrix}
3.4 & 6.7 &  8.9\\
1.2 & 4.5 & 3.1 \\
5.6 & 7.7 & 2.5
\end{bmatrix}
$$

Such matrices have very widespread applications in science, engineering, statistics and machine learning. Matrices can easily represented using 2D arrays.

\subsection*{Rhodus Array Syntax}

We've already encountered the basic syntax used with arrays. Rhodus follows to some degree the model used by Python. In particular we repurpose the list syntax to define literal arrays and use a global method, {\tt array} to convert lists into the array data model. For example:

\begin{lstlisting}
>> a = array([[1,2],[3,4]])
>> println (a)
[ 1, 4;
  9, 16]
>> type (a)
array
\end{lstlisting}

The {\tt array} method also can be used to specify the size of a new array, for example:

\begin{lstlisting}
>> a = array(3,4)
>> b = array(5,5,5)
\end{lstlisting}

The first array, {\tt a} is a 3 by 4 array while the second array, {\tt b}, is a three-dimensional array of size 5 in each dimension. By default all elements in a new array are initialized to zero.
For now arrays will only be able to contain floating point values.

Like lists, arrays can be in indexed using the usual indexing syntax, for example:

\begin{lstlisting}
>> a = array(5,5,5)
>> a[1,1,2]
0.0
>> a[1,1,2] = 3.14
>> a[1,1,2]
3.14
\end{lstlisting}

Like lists, indexing starts at zero.

Rhodus has two variants of arrays. The first is an n-dimensional array and the second, derived from arrays, is the 2-dimensional matrix. The two types only differ is what operations can be applied to them. These operations are governed by two built-in libraries, {\tt arrays} and {\tt mat}. Arithmetic operations on arrays are element-wise whereas the operations provided by the matrix library correspond to the classic matrix operations found in linear algebra. For example, multiplying two arrays together is done by multiplying each corresponding element to form a new array of products, for example:

\begin{lstlisting}
>> import arrays
>> a = array([[1,2],[3,4]])
>> println (arrays.mult (a, a))
[ 1, 4;
  9, 16]
\end{lstlisting}

In contrast, multiplication using the {\tt mat} library yields a different result:

\begin{lstlisting}
>> import mat
>> a = array([[1,2],[3,4]])
>> println (mat.mult (a, a))
[   7, 10;
   15  22]
\end{lstlisting}

We will cover more of this topic shortly.

\section{Implementing Arrays}

How do we implement array support? In considering this question, the main point to keep in mind is that access to arrays are meant to be fast and operations on arrays should be as efficient as possible. This requirement will dictate how an array, and in particular a multidimensional arrays is stored. To keep thing simple, let's first consider arrays up to two dimensions. Since we don't know how big arrays will be we need to use dynamic arrays at the Object Pascal level. Secondly, our arrays will be objects that can be garbage collected, this means an array must be derived from the same parent as strings and lists. We begin with a simple array object class:

\begin{lstlisting}
TArrayObject = class (TRhodusObject)
end;
\end{lstlisting}

The parent {\tt TRhodusObject} class includes a number of fields:

\begin{lstlisting}
   blockType : TBlockType;
   objectType : TSymbolElementType;
   methodList : TMethodList;
\end{lstlisting}

The {\tt blockType} is for the garbage collector to know what to do with the object during garbage collection. The {\tt objectType} just tells us what kind of object it is, string, list, user function or array. Finally, the {\tt methodlist} is a new field for version 3 of Rhodus and points to a list of methods that can be applied to the object. This is what lets us do things like:

\begin{lstlisting}
a = "String"
l = a.len()

println ("How long am I".len())
\end{lstlisting}

The first thing to add to the {\tt TArrayObject} is the field that will hold the array data. For now, we will only support arrays that hold floating point numbers.

\begin{lstlisting}
T1DArray = array of double;
TArrayObject = class (TRhodusObject)
   data : T1DArray;
end;
\end{lstlisting}

You may be thinking, ok this will store a 1D array, but what about a 2D array? The best way to handle n-dimensional arrays is to map then into a 1-dimensional array. If we didn't do this we'd have to have special cases for all the possible dimensions, e.g {\tt T1DArray, T2DArray, T3DArray}, etc. I don't think we want to do that.

Figure~\ref{fig:rowMajor} shows how we can map a one-dimensional array to an dimensions. All we do is slice the 1D-array into segments represents the rows, in this case a 2D-array. We can slice at many times as we like to model n-dimensional arrays.

\begin{figure}[htpb]
\centering
\includegraphics[scale=0.55]{rowMajor.pdf}
\caption{Using a 1D-array to represent a 2D-array.}
\label{fig:rowMajor}
\end{figure}

With a bit of simple arithmetic we can convert any 2D-index, such as $(i, j)$, into a single index along the 1D-array. To do this we must know the intended column width of the 2D-array. In the example the column width is 4. By convention, in a 2D index such as $(i, j)$, the $i$ represents the row number and $j$ the column number. Let's say we had a coordinate of $(2,3)$, that represents the second row and third column. One thing we have to be careful of is what is the index of the first element in the array? As with lists, we will always index from zero. this means that the coordinate $(2,3)$ actually means the 3rd row and 4th column. Given a coordinate $(i,j)$, the index along the 1D-array that corresponds to this coordinate will be:
%
$$ \mbox{index} = j + (i \times \mbox{width}) $$
%
If we plug $(2,3)$ into this formula we get: $3 + 2 \cdot 4 = 11$. Hence we would pull the value out of index 11 in the 1D-array. If you look at Figure~\ref{fig:rowMajor} you can confirm this by eye. We can rewrite the above formula is a more general way:
%
$$ \mbox{index} = i \cdot d_j + j $$
%
where $d_j$ is the width or dimension (number of elements) in the $j$th direction. This approach can be extended to any number of dimensions. For example, for a 3D array, with coordinate, $i, j, k$, the formula for computing the index in a linear array can be extended to give:
%
$$ \mbox{index} = i \cdot d_j \cdot d_k + j \cdot d_k + k $$
%
where $d_j$ is the dimension of the $j$th coordinate, and $d_k$ the dimension of the $k$th coordinate. By induction the formula for a 4D array with correlates $i, j, k, l$, would be:
%
$$ \mbox{index} = i \cdot d_j \cdot d_k \cdot d_l + j \cdot d_k \cdot d_l + k \cdot d_l + l $$
%
This can be turned into a function for computing the index of any array with any dimension as follows\footnote{Modified from Heffernan's answer in \url{https://stackoverflow.com/questions/28569850/}}:
%

\begin{lstlisting}
// Given the dimension of the array in the array dimensions
// and the coordinates in the array index, this routine
// will return the index assuming the array is stored
// in a 1D block of memory.
function getIndex (const dimensions, index: array of Integer): integer;
var i: Integer;
begin
  result := idx[0];
  for i := 1 to high(dimensions) do
    result := result*dimensions[i] + index[i];
end;
\end{lstlisting}

For example, if we have a 6 dimensional array of dimensions $5,5,5,5,5,5$, which is 15,625 elements, the index at coordinate $1,2,3,3,0,4$ will be position 4829 in the 1D storage array.

We can create two helper methods for getting and setting a value in an n-dimensional array. The methods will take a dynamic array containing the indices to use.

\begin{lstlisting}
function TArrayObject.getValue (idx : array of integer) : double;
var index : integer;
begin
  for var i := 0 to length (dim) - 1 do
      if idx[i] >= dim[i] then
         raise ERuntimeException.Create(outOfRangeMsg);

  index := getIndex (dim, idx);
  result := data[index];
end;
\end{lstlisting}

For example in the {\tt getValue} method we can use the new open array syntax introduced in XE7.

\begin{lstlisting}
  value := getValue ([1,3,2])
\end{lstlisting}

Likewise we have a {\tt setValue} method that also takes an dynamic array argument for specifying the indices.

\begin{lstlisting}
procedure TArrayObject.setValue (idx : array of integer; value : double);
var index : integer;
    i : integer;
begin
  for i := 0 to length (dim) - 1 do
      if idx[i] >= dim[i] then
         raise ERuntimeException.Create(outOfRangeMsg);

  index := getIndex (dim, idx);
  data[index] := value;
end;
\end{lstlisting}

For example:

\begin{lstlisting}
  setValue ([6,2,7], value)
\end{lstlisting}

In both methods we need to check for index out of bounds errors. The bounds will be stored in a object variable called dim. We can expand the {\tt TArrayObject} to:

\begin{lstlisting}
  TArrayObject = class (TRhodusObject)
    private
    public
     data : T1DArray;
     dim  : TIndexArray;  // Of type array of integer;

     function     getValue (idx : array of integer) : double;
     procedure    setValue (idx : array of integer; value : double);

     function     clone : TArrayObject;

     constructor  Create; overload;
     constructor  Create (dim : TIndexArray); overload;
     destructor   Destroy; override;
\end{lstlisting}

We'll add a bunch of extra methods. One in particular is the ability to clone an array. This is implemented in the clone method:

\begin{lstlisting}
function TArrayObject.clone : TArrayObject;
begin
  result := TArrayObject.Create (dim);
  result.data := copy (self.data);
end;
\end{lstlisting}

There are also a bunch of other methods to make like easier, as well as some arithmetic functions such as {\tt add}, {\tt subtract}, and {\tt multiply}. These are all pair-wise operations.
That is addition of two arrays is archived by summing up pair-wise values in each array.  For this work, both arrays must have the same dimensions. For a 2 by 2 array addition is defined as:

$$
\begin{bmatrix}
  a_1 & a_2 \\
  a_3 & a_4
\end{bmatrix}
+
\begin{bmatrix}
  b_1 & b_2 \\
  b_3 & b_4
\end{bmatrix}
=
\begin{bmatrix}
  a_1+b_1 & a_2+b_2 \\
  a_3 + b_3 & a_4 + b_4
\end{bmatrix}
$$

This idea can be extended to n-dimensional arrays. Subtraction is defined in a similar way except we take the difference between each pair. When we come to multiplication we are confronted with two common multiplication approaches. This includes the simple pair-wise multiplication and, arguably, the more common matrix multiplication. For arrays we will provide pair-wise multiplication. This will be specified using the usual multiply operator {\tt `*'}.

For 2D-arrays, i.e matrices, we need a way to specify matrix multiplication. Python uses {\tt `@'}, so we might as well use that too. We'll have to update the lexical scanner, syntax analysis and AST constructions but that is simple to do. For example, let's compare pair-wise and matrix multiplication:

\begin{lstlisting}
>> a = array([[1,2],[3,4]])
>> println (a*a)
[[   1.0000,     4.0000]],
[    9.0000,    16.0000]]
>> println (a@a)
[[   7.0000,    10.0000]],
[   15.0000,    22.0000]]
\end{lstlisting}

Multiply arrays

[[0]*4]*3
[[0,0,0,0],[0,0,0,0],[0,0,0,0]]

or

\begin{lstlisting}
>>println (a.sqr())
\end{lstlisting}

This squares each element in the array {\tt a}. $n$-dimensional arrays can be created using the {\tt array} method:

There are at least two classes of operations we can apply to arrays, these include manipulating the contents of an array, for example extracting rows and columns, or adding rows and columns, and secondly carrying out arithmetic on arrays. Let's first focus on arithmetic operations.

\section{Matrices}

There isn't much to say about matrices other than they are strictly 2-dimensional arrays. There is nothing intrinsically different between an array and a matrix other than its dimensions. The key difference is that matrix operations are different from the equivalent array operations. This is particularly the case for multiplication which is dot-product based for matrices. This is handled by the {\tt mat} library which also offers other classical matrix operations such as computing the inverse and determinant.

All operations on arrays can also be applied to matrices, but not vice versa, since matrices are strictly 2-dimensional.

As mention in the last section, matrix multiplication comes in two forms, pair-wise and dot product. Pair-wise multiplication can be easily extended to arrays of any dimension. In Rhodus there are two ways to specify pair-wise multiplication, using the normal multiplication operator, {\tt *}:

\begin{lstlisting}
>> a = array([[1,2],[3,4]])
>> b = array([[5,6],[7,8]])
>> x = a*b
>>println (x)
[[   5.0000,    12.0000]],
[   21.0000,    32.0000]]
\end{lstlisting}

Matrix multiplication, as defined in linear algebra, is the generated by taking the dot-product of corresponding rows and columns.  Unlike array multiplication, where the two arrays must be the same size, in matrix multiplication this need not be the case. Moreover, matrix multiplication generally applies to 2-dimensional matrices. When applying matrix multiplication, the number of columns of the first matrix must equal the number of rows in the second matrix. The outer rows and columns can be of any size. In Rhodus there are two ways to specify matrix multiplication, either using the {\tt mult} method from the matrix library or using he special matrix multiplication operator symbol {\tt @}|. For example:

\begin{lstlisting}
>> a = array([[1,2],[3,4]])
>> b = array([[5,6],[7,8]])
>> x = a@b
>> x = mat.mult (a, b)
>>println (x)
[[  19.0000,    22.0000]],
[   43.0000,    50.0000]]
>>
\end{lstlisting}

Application

[m1, m2]

combine (m1, m2)

[m1; m2]

mat.appendRow ([m1, m2])
mat.appendCol ([m1, m2])

m1.appendRow (m2)
m1.appendCol (m2)

m1.hstack (m2)
m1.vstack (m2)

%
%\section{Useful Reading}
%
%\subsection{Introductory Books}
%
%{\bf 1.} Ball, Thorsten. Writing A Compiler In Go. Thorsten Ball, 2018.
%
%{\bf 2.} Kernighan, Brian W.; Pike, Rob (1984). The Unix Programming Environment. Prentice-Hall. ISBN 0-13-937681-X.
%
%{\bf 3.} Nisan, Noam, and Shimon Schocken. The elements of computing systems: building a modern computer from first principles. MIT press, 2005.
%
%{\bf 4.} Parr, Terence. Language implementation patterns: create your own domain-specific and general programming languages. Pragmatic Bookshelf, 2009.
%
%\subsection{More Advanced Books}
%
%{\bf 1.} Jim Smith, Ravi Nair, Virtual Machines: Versatile Platforms for Systems and Processes, Morgan Kauffmann, June 2005
%
%{\bf 2.} Aho, Alfred V., Ravi Sethi, and Jeffrey D. Ullman. Compilers: Principles, Techniques and Tools (also known as The Red Dragon Book), 1986.
%
%\subsection{Source Code}
%
%{\bf 1.} Mak, Ronald. Writing compilers and interpreters: an applied approach/by Ronald Mark. 1991
%
%Note, this is the first edition, 1991. The code is in C, which I found to be understandable. The later editions that use C++ are not as clear. The issue I found is that the object orientated approach that's used tends to obscure the design principles of the interpreter and requires much study to decipher, The C version is much more straightforward.
%
%{\bf 2.} Wren: \url{https://github.com/wren-lang/wren}.\index{wren}
%
%Of the open source interpreters on GitHub, I found this to be the easiest to read. It's written by Bob Nystrom in C, the same person who is writing the web book: Crafting Interpreters \url{https://craftinginterpreters.com/}.
%
%{\bf 3.} Gravity: \index{Gravity} Another open source interpreter worth looking at is Gravity (\url{https://github.com/marcobambini/gravity}). Gravity, like Wren, is also written in C.
%
%{\bf 4.}  If you prefer Go,\index{Go} then the source code to look at is the interpreter written by Thorsten Ball (see book reference above).
%
%There are umpteen BASIC interpreters\index{BASIC} and other languages that can be studied.
%
%\bigskip\medskip

\begin{center}
\pgfornament[width = 8cm, color = cardinal]{83}
\end{center}


\subsubsection*{Matrix and Vector Operations\hfill \hyperlink{linearAlgebraFunctions}{(up)}\hypertarget{matrixOperations}{}} \addcontentsline{toc}{subsubsection}{Matrix and Vector Operations}
\begin{table}[H]
\caption{Matrix and Vector Operations.}
\label{tab:matrixOperations}
\begin{center}
\begin{tabular}{|l|l|}\hline
\hlnkFunc{herm} & Matrix Hermitian\\
\hlnkFunc{jdot} & Complex Vector Conjugate Dot Product\\
\hlnkFunc{gemp} & General Matrix Product\\
\hlnkFunc{gems} & General Matrix Sum \\
\hlnkFunc{kron} & Kronecker Product \\
\hlnkFunc{prod3} & 3 by 3 Matrix Product\\
\hlnkFunc{prod4} & 4 by 4 Matrix Product\\
\hlnkFunc{prod} & Matrix product \\
\hlnkFunc{prodh} & Matrix Hermitian Product\\
\hlnkFunc{prodj} & Matrix Conjugate Product\\
\hlnkFunc{prodt} & Matrix Transpose Product\\
\hlnkFunc{trans} & Matrix Transpose\\
\hlnkFunc{dot} & Vector Dot Product\\
\hlnkFunc{outer} & Vector Outer Product\\
\hline\end{tabular}
\end{center}
%\label{default}
\end{table}
%


% s,fatrix2

\comment{
\newcommand{\A} {\blmath{A}}
\newcommand{\x} {\blmath{x}}
\newcommand{\y} {\blmath{y}}
\newcommand{\z} {\blmath{z}}
}

\subsection{The \fatrixx class}

The pronunciation rhymes with matrix.
Think ``fake matrix''
or ``fancy matrix.''

\subsubsection{Background}

Most iterative algorithms
for image reconstruction
are described conveniently using matrix notation,
but matrices are not necessarily
the most suitable data structure
for actually implementing an iterative algorithm
for large sized problems.
The \fatrixx class
provides a convenient bridge
between matrix notation
and practical system models
used for iterative image reconstruction.

Consider the simple iterative algorithm
for reconstructing \x from data \y
expressed mathematically:
\be
\x^{n+1} = \x^n + \alpha \A' (\y - \A\x^n)
,\ee{e,fatrixx,iter}
where \A is the \emph{system matrix}
associated with
the image reconstruction problem at hand.
%
If \A is small enough
to be stored as a matrix in \matlab
(sparse or full),
then this algorithm translates
very nicely into \matlab as follows.
%\begin{verbatim}
\be
\ty{x = x + alpha * A' * (y - A * x);}
\ee{e,mat2}
%\end{verbatim}
You really cannot get any closer connection
between the math and the program than this!
But often we work
with system models
that are too big to store
as matrices (even sparsely) in \matlab.
Instead,
the models
are implemented by subroutines
that compute the ``forward projection'' operation
$\A \x$
and the ``backprojection operation
$\A' \z$
for input vectors \x and \z respectively.
%
The conventional way
to use one of these system models
in \matlab (or C) would be to rewrite
the above program as follows.
\begin{verbatim}
Ax = forward_project(system_arguments, x)
residual = y - Ax;
correction = back_project(system_arguments, residual)
x = x + alpha * correction
\end{verbatim}
Yuch!
This is displeasing for two reasons.
First,
the code looks a \emph{lot} less like the mathematics.
Second,
usually you end up with a different version of the code
for every different system model
(forward/back-projector subroutine pair)
that you develop.
Having multiple versions
of a simple algorithm
creates a software maintenance headache.

The elegant solution
is to develop \matlab objects
that know how to perform
the following operations:
\blist
\item
\makebox[4em][l]{%
\ty{A * x}
}
(matrix vector multiplication,
operation \ty{mtimes})
\item
\makebox[4em][l]{%
\ty{A'}
}
(\ty{transpose}), and
\item
\makebox[4em][l]{%
\ty{A' * z}
}
(\ty{mtimes} again,
with a transposed object).
\elist
Once such an object is
defined,
one can use \emph{exactly}
the same iterative algorithm
that one would have used
with an ordinary matrix,
\eg,
\eref{e,mat2}.
%
The \fatrixx class
provides a convenient mechanism
for implementing
such linear operators.

\subsubsection{Creating a \fatrixx object}

Suppose \x is of length 1000
and \y is of length 2000.
One can create a corresponding \fatrixx object
using the following call:
\begin{verbatimtab}
A = fatrix2('imask', [1000 1], 'omask', [2000 1], ...
	'arg', system_arguments, ...
	'forw', @forward_project, 'back', @back_project);
\end{verbatimtab}
The resulting \fatrixx object \ty{A}
acts just like a matrix
in most important respects.
In particular,
we can use exactly the same iterative algorithm
\eref{e,fatrixx,iter} as before,
because
\ty{Ax = A * x}
is handled internally
by calling
\[
\ty{
Ax = forward_project(system_arguments, x)
}
\]
and similarly for
\ty{A' * z}.

Basic operations
like \ty{A(:,7)}
are also implemented,
but nonlinear operations
like \ty{exp(A)}
are not
because those cannot be computed readily
using \ty{forward_project}.

For many examples, see
the \ty{systems} subdirectory of \irt.

\subsubsection
{
Operations on a \fatrixx object
that return another \fatrixx object
}

\blist
\item
\ty{B = A'} or \ty{B = ctranspose{A}}
\\
\fatrixx object Hermitian transpose
\item
\ty{C = A * B}
\\
\fatrixx object multiplication
(requires compatible sizes)
\item
\ty{B = 7 * A}
\\
multiplying a scalar times a \fatrixx object
\item
\ty{A = [A1; A2; A3]}
\\
vertical concatenation (\ty{vertcat} is also supported)
(requires compatible sizes)
\item
\ty{A = [A1, A2, A3]}
\\
horizontal concatenation (\ty{horzcat} is also supported) % todo?
(requires compatible sizes)
\item
\ty{A(:,[3 7])} or \ty{A([3 7],:)} or \ty{A(:,2:end)} etc.
\\
these return a smaller sized \fatrixx
%\item
%\ty{A(:,newmask)} % todo
\item
\ty{C = B + A}
\\
\fatrixx object addition
(requires compatible sizes)
\elist

\subsubsection
{
Operations on a \fatrixx object
that return a vector.
}

\blist
\item
\ty{A(:,7)} or \ty{A(7,:)}
\elist

\subsubsection
{
Operations on a \fatrixx object
that return a matrix.
}

These are practical only if \ty{A} has sufficiently small size!

\blist
\item
\ty{A(:,:)} or \ty{full(A)}
\item
\ty{svd(A)}
\item
\ty{eig(A)}
% todo: eigs inv
\elist

\subsubsection
{
Other operations on a \fatrixx object.
}

\blist
\item
\ty{A(7,9)}
\\
This returns a scalar value. % todo
\item
\ty{sparse(A)}
\\
By default, internally this will compute each column of \A
using \ty{A(:,j)}
and then create a sparse matrix
from the nonzero elements of those columns.
For most large cases,
this will be very slow.
However,
a few \fatrixx objects
such as \ty{Gdiag}
have an internal \ty{sparse} method
(provided by the \ty{'sparse'} option
to the \ty{fatrix2} call)
that is very efficient.

\elist

\subsubsection
{
Support for arrays instead of columns
}

For an image reconstruction problem
with the mask illustrated in \fref{fig,reg,mask},
the size of \ty{A} should be
$\nd \times 8$
where \nd is the number of rows of \A.

So if we have a $6 \times 5$ image \ty{x}
and we would like to compute
$\A \x$,
conventionally
one would need to do the following:
\\
\cent{
\ty{y = A * x(mask)}
}

The statement \ty{x(mask)}
extracts the relevant $\np = 8$ values
out of the $6 \times 5$ array \ty{x}.

Objects in the \fatrixx class
often can spare the user the need to use \ty{x(mask)}
if the object is defined properly.

Suppose that in a tomographic image reconstruction problem
corresponding to \fref{fig,reg,mask},
the sinogram size is $9 \times 8$
so $\nd = 72$.

Then if \ty{A} is one of the predefined tomographic system models
in \irt such as \ty{Gtomo2_wtmex},
then the statement
\\
\cent{
\ty{yc = A * x(mask)}
}
\\
will produce \emph{column vector} \ty{yc}
with $\nd = 72$ elements.

On the other hand,
the convenient syntax
\\
\cent{
\ty{ya = A * x}
}
\\
will produce a $9 \times 8$ sinogram \emph{array} output \ty{ya}.
The two different outputs
are related by
\\
\cent{
\ty{yc = ya(:)}
}

In other words,
if the input is a column vector (of length \np),
then the output will also be a column vector,
just like one would expect for an ordinary matrix.
But if the input is an \emph{array},
of the appropriate dimensions,
then the output will also be an array.

When defining a new \fatrixx object,
there are several options
that one can use to specify
the appropriate dimensions.

\blist
\item
\ty{imask}
is the usual input mask.
Default is \ty{true(idim)}
\\
This can also be called just \ty{mask}
for backwards compatibility with \fatrix objects.

\item
\ty{idim}
describes the input array dimensions.
Default is \ty{sum(imask(:))},
\ie, an ordinary column vector.

\item
\ty{omask}
is a rarely used option.
Default is \ty{true(odim)}

\item
\ty{odim}
describes the output array dimensions.
Default is \ty{sum(omask(:))},
\ie, an ordinary column vector.
\elist

For a typical \fatrixx object,
one will use only the \ty{imask} and \ty{odim} options.

For the tomography example above,
we would add the name-value pairs
\ty{'odim', [9 8]}
to the \ty{fatrix2} call.

\subsubsection{Defining \fatrixx methods}

The key methods for a \fatrixx object
are the \ty{forw} and \ty{back} operations.
For the obsolete \fatrix objects,
these methods had to support columns and arrays
and multiples thereof
which made them fairly complicated.
For the new \fatrixx objects,
this complexity is handled by the object itself,
and the user-defined methods are much simpler.

\blist
\item
The \ty{forw} routine
accepts a single input array
of size \ty{idim}
and returns a single output array
of size \ty{odim}.

\item
Caution:
nonzero values in the input array
outside of \ty{imask}
may cause unpredictable results.

\item
Conversely,
the \ty{back} routine
accepts a single input array
of size \ty{odim}
and returns a single output array
of size \ty{idim}.

\item
Caution:
the output array of the \ty{back} routine
must be zero for any pixels
outsize of the \ty{omask}.
%
Sometimes this will happen automatically
because a computationally efficient
\ty{back} routine
will only evaluate the output
within the \ty{omask},
and will set the other values to zero.
%
But for some objects,
like \ty{Gdft},
the \ty{back} routine
first evaluates the entire output
(using \ty{ifftn})
and then performs a \ty{.*} with \ty{omask}
to comply with the requirement.

\elist

\subsubsection{The 1D dilemma}

Unfortunately,
\matlab does not really support 1D arrays.
(\matlab's \ty{size} command always returns
at least a two-element vectors.)
So for any application where
\ty{odim} is 1D,
such as MRI with irregular non-Cartesian k-space samples,
\ty{A' * y}
will produce a 1D array
not a 2D array
even if \ty{imask} is 2D,
because \ty{y} will be ``1D'' in such cases.


\section{Application to color normalization}

Color normalization is the process of imposing the same color palette on a group of images. This color palette is always somehow related to the color palettes of the original images. For instance, if the goal is to cancel the illumination of a scene (avoid color cast), then the imposed histogram should be the histogram of the same scene illuminated with white light. Of course, in many occasions this information is not available. Following Papadakis et al.~\cite{Papadakis_ip11}, we define an in-between histogram, which is chosen here as the regularized OT barycenter.

%The advantage with respect to Papadakis et al.'s method is that the influence of the input histograms on the barycenter can be easily tuned by a change of parameters, in fact it has as a special case any of the original image histograms.

%Some other examples for which color normalization is useful are the color balancing of videos,  or as a preprocessing to register/compare several images taken with different cameras (see \cite{Papadakis_ip11} for more examples).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Algorithm}

Given a set of input images $(X^{0[r]})_{r \in R}$, the goal is to impose on all the images the same histogram $\mu_X$ associated to the barycenter $X$. As for the colorization problem tackled in Section~\ref{sec-appli-color}, the first step is to subsample the original cloud of points $X^{0[r]}$ to make the problem tractable. Thus, for every $X^{0[r]}$ we compute a smaller associated point set $X^{[r]}$ using K-means clustering. Then, we obtain the barycenter $X$ of all the point clouds $(X^{[r]})_{r \in R}$ with the algorithms presented in Section~\ref{algobar}. Figure~\ref{im:baryillu} first row, shows an example on two synthetic cloud of points, $X^{[1]}$ in blue and $X^{[2]}$ in red. The cloud of points in green corresponds to the barycenter $X$, which can change its position depending on the parameter $\rho=(\rho_1,\rho_2)$ in~\eqref{eqbar} from $X^{[1]}$ for $\rho=(1,0)$ to $X^{[2]}$ when $\rho=(0,1)$.  This data set $X$ represents the 3-D histogram we want to impose on all the input images.

Once we have $X$, we compute the regularized and relaxed OT transport maps $T^{[r]}$ between each $X^{[r]}$ and the barycenter $X$, by solving~\eqref{eq-symm-reg-energy}. The line segments in Figure~\ref{im:baryillu} represent the transport between point clouds, i.e. if $\Sig^{[1]}_{i,j}>0$, $X^{[1]}_i$ is linked to $X_j$, and similarly for $\Sig^{[2]}$.

We apply $T^{[r]}$ to $X^{[r]}$, obtaining  $\tilde X^{[r]}$, for all $r \in R$, that is to say, we obtain a set of point clouds $\tilde X^{[r]}$ with a color distribution close to $X$.  Finally, to recover a set of high resolution images, we compute each $\tilde X^{0[r]}$ from $X^{0[r]}$ by up-sampling. A detailed description of the method is given in Algorithm~\ref{alg-norm}.

\begin{algorithm}[ht!]
\caption{Regularized OT Color Normalization}
\label{alg-norm}
% \begin{algorithmic}[1]
\Require Images $\left( X^{0[r]} \right)_{r \in R} \in \RR^{N_0 \times d}$, $\la \in \RR^+ $, $\rho \in [0,1]^{|R|}$ and $k \in \RR^+$.

\Ensure Images $\left( \tilde X^{0[r]} \right)_{r \in R} \in \RR^{N_0 \times d}$.
% \Statex
\begin{enumerate}
	\algostep{Histogram down-sample} Compute $X^{[r]}$ from $X^{0[r]}$ using K-means clustering.
	\algostep{Compute barycenter} Compute with either~\eqref{eq-bar-l2} or~\eqref{eq-bar-l1} a barycenter $\mu_X$ where $X$ is a local minimum of~\eqref{eqbar} using the block coordinate descent described in Section~\ref{algobar}, see Algorithm~\ref{algo-block-barycenters}.
	\algostep{Compute transport mappings} For all $r \in R$ compute $T^{[r]}$ between \\
		$X$ and $X^{[r]}$ by solving~\eqref{eq-symm-reg-energy}, such that $T^{[r]}(X^{[r]}_i) = Z^{[r]}_i$, where $Z^{[r]}= \diag(\Sig^{[r]} \U)^{-1} \Sig^{[r]} X$.
	\algostep{Transport up-sample} For every $T^{[r]}$ compute $\tilde T^{0[r]}$ following~\eqref{eq-upsample}.
	\algostep{Obtain high resolution results} Compute $\foralls r, \tilde X^{0[r]} = \tilde T^{0[r]}(X^{0[r]})$.
\end{enumerate}
% \end{algorithmic}
\end{algorithm}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results}

We now show some example of color normalization using Algorithm~\ref{alg-norm}.

\begin{figure*}[!h]
\centering
\setlength{\arrayrulewidth}{2pt}
%%% OT %%%%
\begin{tabular}{@{}c@{\hspace{1mm}}c@{\hspace{1mm}}c@{\hspace{1mm}}c@{}}
	$\rho=(1,0)$ & $\rho=(0.7,0.3)$ & $\rho=(0.4,0.6)$ & $\rho=(0,1)$ \\
\includegraphics[width=.23\linewidth]{./syntheticbary/Barycenter_DiagsyntheticINVrho-0-ksum1lambda0nnx4QP1png} &
\includegraphics[width=.23\linewidth]{./syntheticbary/Barycenter_DiagsyntheticINVrho-03-ksum1lambda0nnx4QP1png} &
\includegraphics[width=.23\linewidth]{./syntheticbary/Barycenter_DiagsyntheticINVrho-06-ksum1lambda0nnx4QP1png} &
\includegraphics[width=.23\linewidth]{./syntheticbary/Barycenter_DiagsyntheticINVrho-1-ksum1lambda0nnx4QP1png} \\
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag2-syntheticrho-0-ksum1lambda0nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag2-syntheticrho-03-ksum1lambda0nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag2-syntheticrho-06-ksum1lambda0nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag2-syntheticrho-1-ksum1lambda0nnx4QP1}\\
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag1-syntheticrho-0-ksum1lambda0nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag1-syntheticrho-03-ksum1lambda0nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag1-syntheticrho-06-ksum1lambda0nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag1-syntheticrho-1-ksum1lambda0nnx4QP1} \\\hline
%%% Regularized
\includegraphics[width=.23\linewidth]{./syntheticbary/Barycenter_DiagsyntheticINVrho-0-ksum20lambda00005nnx4QP1} &
\includegraphics[width=.23\linewidth]{./syntheticbary/Barycenter_DiagsyntheticINVrho-03-ksum20lambda00005nnx4QP1} &
\includegraphics[width=.23\linewidth]{./syntheticbary/Barycenter_DiagsyntheticINVrho-06-ksum20lambda00005nnx4QP1} &
\includegraphics[width=.23\linewidth]{./syntheticbary/Barycenter_DiagsyntheticINVrho-1-ksum20lambda00005nnx4QP1} \\
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag2-syntheticrho-01-ksum20lambda00005nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag2-syntheticrho-0307-ksum20lambda00005nnx4QP1}&
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag2-syntheticrho-0604-ksum20lambda00005nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag2-syntheticrho-10-ksum20lambda00005nnx4QP1}\\
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag1-syntheticrho-01-ksum20lambda00005nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag1-syntheticrho-0307-ksum20lambda00005nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag1-syntheticrho-0604-ksum20lambda00005nnx4QP1} &
\includegraphics[width=.23\linewidth,height=.2\linewidth]{./syntheticbary/Diag1-syntheticrho-10-ksum20lambda00005nnx4QP1} \\
\end{tabular}
 \caption{Comparison of classical OT (top 3 first rows) and relaxed/regularized OT (bottom 3 last rows). The original input images $X^{0,[1]}$ and $X^{0,[2]}$ are shown in Figure~\ref{im:synth}~(a).
 		Rows \#1 and \#4 shows the 2-D projections of $X^{[1]}$ (blue) and $X^{[2]}$ (red), and in green the barycenter distribution for different values of $\rho$. We display a line between $X^{[r]}_i$ and $X_j$ if $\Sig^{[r]}_{i,j} > 0.1$.
		Rows \#2 and \#5 (resp. \#3 and \#6) show the resulting normalized images $\tilde X^{0[1]}$ (resp. $\tilde X^{0[2]}$),
		for each value of $\rho$.
 	\textbf{Top 3 first rows:} classical OT corresponding to setting $k=1$ and $\la=0$.
	\textbf{Bottom 3 last rows:}  regularized and relaxed OT, with parameters $k=20$ and $\lambda=0.0005$. See main text for comments. \vspace{0.5cm}
}
\label{im:baryillu}
\end{figure*}

%%%%%%%%%%%%%%%%%%
\paragraph{Synthetic example}

Figure~\ref{im:baryillu} shows a comparison of normalization of two synthetic images using classical OT and our proposed relaxed/regularized OT. The results obtained using Algorithm~\ref{alg-norm} (setting $p=q=2$), using the set of two images ($|R|=2$) already used in Figure~\ref{im:synth}~(a), denoting here $X^{0[1]}=X^0$ and $X^{0[2]}=Y^0$. Each column shows the same experiment but with different values of $\rho$, which allows to visualize the interpolation between the color palettes (the colors in the images  evolve from the colors in $X^{[1]}$ towards the colors of $X^{[2]}$).

With classical OT, the structure of the original data sets in not preserved as we change $\rho$, and the consequence on the final images (second and third row), is that the geometry of the original images changes in the barycenters. In contrast to classical OT, for all values of $\rho$ the relaxed/regularized barycenters $X$ have the same number of clusters of the original sets. Note that the consequence of having a transport that maintains the clusters of the original images, is that the geometry is preserved, while the histograms change.

\newcommand{\sidecapY}[1]{ \begin{sideways}\parbox{.19\linewidth}{\centering #1}\end{sideways} }
\newcommand{\myimgY}[1]{\includegraphics[width=.26\linewidth,height=.22\linewidth]{#1}}

\begin{figure*}[!h]
\centering
\begin{tabular}{@{}c@{\hspace{1mm}}c@{\hspace{1mm}}c@{\hspace{1mm}}c@{} }
\sidecapY{ Original $X^{0[1]}$ }  &
\myimgY{star/fleur_1} &
\myimgY{star/wheat_1} &
\myimgY{star/parrot_1} \\
\sidecapY{$\rho=(1,0)$ } &
\myimgY{barycenter/DiagYfleurrho-1-ksum11lambda00009nnx4QP1} &
\myimgY{barycenter/wheat/Diag1-wheatrho-1-ksum13lambda001nnx4QP1} &
\myimgY{barycenter/parrot/Diag1-parrotrho-1-ksum1lambda0001nnx4QP1} \\
\sidecapY{$\rho=(0.7,0.3)$} &
\myimgY{barycenter/DiagYfleurrho-06-ksum11lambda00009nnx4QP1} &
\myimgY{barycenter/wheat/Diag1-wheatrho-06-ksum13lambda001nnx4QP1}  &
\myimgY{barycenter/parrot/Diag1-parrotrho-06-ksum1lambda0001nnx4QP1} \\
\sidecapY{ $\rho=(0.4,0.6)$ } &
\myimgY{barycenter/DiagYfleurrho-03-ksum11lambda00009nnx4QP1} &
\myimgY{barycenter/wheat/Diag1-wheatrho-03-ksum13lambda001nnx4QP1} &
\myimgY{barycenter/parrot/Diag1-parrotrho-03-ksum1lambda0001nnx4QP1} \\
\sidecapY{ $\rho=(0,1)$ } &
\myimgY{barycenter/DiagYfleurrho-0-ksum11lambda00009nnx4QP1} &
\myimgY{barycenter/wheat/Diag1-wheatrho-0-ksum13lambda001nnx4QP1} &
\myimgY{barycenter/parrot/Diag1-parrotrho-0-ksum1lambda0001nnx4QP1} \\
\sidecapY{ Original $X^{0[2]}$ } &
\myimgY{star/fleur_2} &
\myimgY{star/wheat_2} &
\myimgY{star/parrot_2} \\
& (a) &  (b) & (c)
\end{tabular}
\caption{Results for the barycenter algorithm on different images computed with the method proposed in Section~\ref{algobarysobolev}. The parameters were set to \textbf{(a)} $k=1.1,\la=0.0009$, \textbf{(b)} $k=1.3,\la=0.01$, and  \textbf{(b)} $k=1,\la=0.001$. Note how as $\rho$ approaches $(0,1)$, the histogram of the barycenter image becomes similar to the histogram of $X^{0[2]}$.}
\label{im:bar}
\end{figure*}

\paragraph{Example on natural images} Fig.~\ref{im:bar} shows the results of the same experiment as in Fig.~\ref{im:baryillu}, but on the natural images labeled as $X^{0[1]}$ and $X^{0[2]}$ in rows $\#1$ and $\#6$. In this case, we only show the transport from $X$ to $X^{0[1]}$, that is to say, we maintain the geometry of $X^{0[1]}$ (row $\#1$) and match its histogram to the barycenter distribution. As in the previous experiment, note how the colors change smoothly from $(1,0)$ to $(0,1)$ without generating artifacts and match the color and contrast of image $X^{0[2]}$ for $\rho=(0,1)$. The change in contrast is specially visible for the (b) wheat image.

%%%%%%%
\paragraph{Color Normalization}

Computing the barycenter distribution of the histograms of a set of images is useful for color normalization. We show in Figures~\ref{barflower}, and~\ref{barclock} the results obtained with Algorithm~\ref{alg-norm}, and compare them with the standard OT and the method proposed by Papadakis et al.~\cite{Papadakis_ip11}. The improvement of the relaxation and regularization is specially noticeable in Figures~\ref{barflower} where OT creates artifacts such as coloring the leaves on violet for Figure~\ref{barflower}~(a), or introducing new colors on the background in Figure~\ref{barflower}~(c). In Figure~\ref{barclock}, OT and Papadakis et al.'s method introduce artifacts mostly on the sky of Figure~\ref{barclock}~(a) and  Figure~\ref{barclock}~(b), while the relaxed and regularized version displays a smoother result for Figure~\ref{barclock}~(a) and~(c) and a more meaningful color transformation (all the clouds have the same color in the fourth row) for Figure~\ref{barclock}~(b).

\newcommand{\myimgZ}[1]{\includegraphics[width=.28\linewidth,height=.26\linewidth]{#1}}
\newcommand{\myTriTab}[1]{ \begin{tabular}{@{}c@{\hspace{2mm}}c@{\hspace{2mm}}c@{} } #1	\end{tabular} }

\begin{figure*}[ht]
\centering
\myTriTab{
\myimgZ{./barycenter/flowers/flowers-1} &
\myimgZ{./barycenter/flowers/flowers-2} &
\myimgZ{./barycenter/flowers/flowers-3} \\
\myimgZ{./barycenter/flowers/Diag1-OT} &
\myimgZ{./barycenter/flowers/Diag2-OT} &
\myimgZ{./barycenter/flowers/Diag3-OT} \\
\myimgZ{./barycenter/nicoflowers1} &
\myimgZ{./barycenter/nicoflowers2} &
\myimgZ{./barycenter/nicoflowers3} \\
\myimgZ{./barycenter/flowers/Diag1-flowersrho-033333-ksum2lambda0005nnx4QP1} &
\myimgZ{./barycenter/flowers/Diag2-flowersrho-033333-ksum2lambda0005nnx4QP1} &
\myimgZ{./barycenter/flowers/Diag3-flowersrho-033333-ksum2lambda0005nnx4QP1} \\
(a) & (b) & (c)
}
\caption{In the first row, we show the original images. In the following rows, we show the result of computing the barycenter histogram and imposing it on each of the original images, with different algorithms. In the second row, we use OT. In the third row, the results were obtained with the method proposed by Papadakis et al.~\cite{Papadakis_ip11}. On the last row, we show the results obtained with the relaxed and regularized OT barycenter with $k=2,\la=0.005$. Note how the proposed algorithm is the only one that does not produce artifacts on the final images such as (a) color artifacts on the leaves and (c) different colors on the background.}
\label{barflower}
\end{figure*}

\begin{figure*}[ht]
\centering
\myTriTab{
\myimgZ{./barycenter/clockmontague-1} &
\myimgZ{./barycenter/clockmontague-2} &
\myimgZ{./barycenter/clockmontague-3} \\
\myimgZ{./barycenter/Diag1-clockmontaguerho-033333-ksum1lambda0nnx4QP1} & %OT
\myimgZ{./barycenter/Diag2-clockmontaguerho-033333-ksum1lambda0nnx4QP1} &
\myimgZ{./barycenter/Diag3-clockmontaguerho-033333-ksum1lambda0nnx4QP1} \\
\myimgZ{./barycenter/nicoclock1} &
\myimgZ{./barycenter/nicoclock2} &
\myimgZ{./barycenter/nicoclock3} \\
\myimgZ{./barycenter/Diag1-clockmontaguerho-033333-ksum13lambda00005nnx4QP1} &
\myimgZ{./barycenter/Diag2-clockmontaguerho-033333-ksum13lambda00005nnx4QP1} &
\myimgZ{./barycenter/Diag3-clockmontaguerho-033333-ksum13lambda00005nnx4QP1} \\
(a) & (b) & (c)
}
\caption{Experiment as in Figure~\ref{barflower} applied on the images of the first row. Our results, presented in the final row, were obtained with $k=1.3$ and $\la=0.0005$. Contrary to OT (second row) or the method proposed by Papadakis et al.~\cite{Papadakis_ip11} (third row), the proposed method does not create artifacts on the sky and the clock for images (a) and (c).}
\label{barclock}
\end{figure*}

\begin{figure*}[ht]
\centering
\myTriTab{
\myimgZ{./barycenter/clockHD-1} &
\myimgZ{./barycenter/clockHD-2} &
\myimgZ{./barycenter/clockHD-3} \\
\myimgZ{./barycenter/Diag1-clockHDrho-033333-ksum11lambda00005nnx4QP1} &
\myimgZ{./barycenter/Diag2-clockHDrho-033333-ksum11lambda00005nnx4QP1} &
\myimgZ{./barycenter/Diag3-clockHDrho-033333-ksum11lambda00005nnx4QP1} \\
}
\caption{The proposed method can be applied as a preprocessing step in a pipeline for objects detection or image registration, where canceling illumination is important. On the first row, we show a set of pictures of the same object taken at different hours of the day or night, and on the second row, the result of our algorithm setting  $(p,q)=(2,2)$, $k=1$ and $\la=0.0005$. Note how the algorithm is able to normalize the illumination conditions of all the images.}
\label{im:colornorm}
\end{figure*}

As a final example, we would like to show in Figure~\ref{im:colornorm} how this method can be applied as a preprocessing before comparing/registering images of the same object obtained under different illumination conditions.


\section*{Math}

$\vec{A}\cdot\vec{B} = |\vec{A}||\vec{B}|\cos\phi_{\text{AB}} = A_xB_x + A_yB_y +
A_xB_z$
\begin{align*}
  \vec{A}\times\vec{B} &= |\vec{A}||\vec{B}|\sin\phi_{\text{AB}}\hat{n}\\
  &= \quad(A_yB_z-A_zB_y)\hat{x}\\
  &\quad -(A_xB_z-A_zB_x)\hat{y}\\
  &\quad +(A_xB_y-A_yB_x)\hat{z}
\end{align*}
The cross product can be represented as a determinant
\begin{equation*}
  \vec{A}\times\vec{B} = %
  \begin{vmatrix}
    \hat{x} & \hat{y} & \hat{z}\\
    A_x & A_y & A_z \\
    B_x & B_y & B_z \\
  \end{vmatrix}
\end{equation*}

If $ax^2 + bx +c = 0$, $x=\frac{-b \pm \sqrt{b^2 = 4ac}}{2a}$

% Derivatives
$\frac{dx^m}{dx} = mx^{m-1}$\\

% Integrals
$\int x^m\;dx = \frac{x^{m+1}}{m+1} + C$


\newpage
\subsection{Goal}\label{subsec:goal}

The use of first- and second-order methods (Gradient Descent, Conjugate Gradient Descent, Newton's method and Levenberg-Marquardt algorithm) in the tasks of unconstrained nonlinear optimization.

\subsection{Formulation of the problem}\label{subsec:formulation-of-the-problem}

Generate random numbers $\alpha \in (0, 1)$ and $\beta \in (0, 1)$.
Furthermore, generate the noisy data $\{x_k, y_k\}$, where $k = 0, \dots, 100$, according to the following rule:

\begin{equation}
    y_k = \alpha x_k + \beta + \delta_k, x_k = \frac{k}{100},
\end{equation}

where $\delta_k \sim N(0, 1)$ are values of a random variable with standard normal distribution.
Approximate the data by the following linear and rational functions:

\begin{enumerate}
    \item $F(x, a, b) = ax + b$ (linear approximant),
    \item $F(x, a, b) = \frac{a}{1 + bx}$ (rational approximant),
\end{enumerate}

by means of least squares through the numerical minimization (with precision $\varepsilon = 0.001$ of the following function:

\begin{equation}
    D(a, b) = \sum^{100}_{k=0}(F(x_k, a, b) - y_k)^2.
\end{equation}

To solve the minimization problem, use the methods of Gradient Descent, Conjugate Gradient Descent, Newton's method and Levenberg-Marquardt algorithm.
If necessary, set the initial approximations and other parameters of the methods.
Visualize the data and the approximants obtained separately for each type of approximant.
Analyze the results obtained (in terms of number of iterations, precision, number of function evaluations, etc.) and compare them with those from Task 2 for the same dataset.

\subsection{Brief theoretical part}\label{subsec:brief-theoretical-part}

Optimization methods are numerical methods for finding optimal (in some sense) values of objective functions, for example, in the framework of mathematical models of certain processes.

\textit{First-and second-order optimization methods} use to minimize the objective function $f$ on the set $Q$ its value $f(x)$ and the values of its first and second derivatives (gradient, Hessian), respectively.

\paragraph{Gradient descent}

\textit{Gradient descent} is an optimization algorithm that's used when training a machine learning model.
It's based on a convex function and tweaks its parameters iteratively to minimize a given function to its local minimum.

\paragraph{Conjugate gradient method}

\textit{Conjugate gradient methods} represent a kind of steepest descent approach.
With steepest descent, minimization of a function begins $f$ starting at $x_0$ by traveling in the direction of the negative gradient - $f'(x_0)$.
In subsequent steps, the movement continues in the direction of the negative gradient evaluated at each successive point until convergence.

\paragraph{Newton's method}

\textit{Newton's method} is an iterative method for finding the roots of a differentiable function $f$, which are solutions to the equation $f(x) = 0$.
In optimization, Newton's method is applied to the derivative $f'$ of a twice-differentiable function $f$ to find the roots of the derivative (solutions to $f'(x) = 0$), also known as the stationary points of $f$.
These solutions may be minima, maxima, or saddle points.

\paragraph{Levenberg-Marquardt algorithm}

The \textit{Levenberg-Marquardt algorithm} (\textit{LMA}) is a popular trust region algorithm that is used to find a minimum of a function (either linear or nonlinear) over a space of parameters.
Essentially, a trusted region of the objective function is internally modeled with some function such as a quadratic.
When an adequate fit is found, the trust region is expanded.
As with many numerical techniques, the Levenberg-Marquardt method can be sensitive to the initial starting parameters.

\subsection{Results}\label{subsec:results}

It is known that optimization by \textit{linear approximation} has a unique solution, so these methods give similar optimal values for $a$ and $b$, regardless of the choice of initial approximations.
In the case of a \textit{rational approximation}, significant nonlinearities occur, so the result depends on the initial approximations (Figure~\ref{ris:plot}).

\begin{figure}[H]
    \center
    \includegraphics[width=\textwidth]{img/plot.png}
    \caption{Direct methods of optimization.}
    \label{ris:plot}
\end{figure}

From the tables below (Tables~\ref{tbl:direct},~\ref{tbl:order}), we can conclude that first-and second-order optimization methods require orders of magnitude less computational operations, in contrast to direct methods.

\begin{table}[ht]
\caption{Direct methods}
\begin{tabular}{l|l|l|l|}
\cline{2-4}
                                                & \textbf{method}             & \textbf{iterations} & \textbf{function eval} \\ \hline
\multicolumn{1}{|l|}{\multirow{3}{*}{\textbf{linear}}}   & exhaustive\_search & 5448000  & 5448000     \\ \cline{2-4}
\multicolumn{1}{|l|}{}                          & gauss\_method      & 77376     & 77376        \\ \cline{2-4}
\multicolumn{1}{|l|}{}                          & nelder\_mead       & 18         & 3             \\ \hline
\multicolumn{1}{|l|}{\multirow{3}{*}{\textbf{rational}}} & exhaustive\_search & 5448000  & 5448000     \\ \cline{2-4}
\multicolumn{1}{|l|}{}                          & gauss\_method      & 6448      & 6448         \\ \cline{2-4}
\multicolumn{1}{|l|}{}                          & nelder\_mead       & 10         & 2             \\ \hline
\end{tabular}
\label{tbl:direct}
\end{table}

\begin{table}[ht]
\caption{First- and second-order methods}
\begin{tabular}{l|l|l|l|}
\cline{2-4}
                                                         & \textbf{method}              & \textbf{iterations} & \textbf{function eval} \\ \hline
\multicolumn{1}{|l|}{\multirow{4}{*}{\textbf{linear}}}   & gradient\_descent            & 150                 & 150                    \\ \cline{2-4}
\multicolumn{1}{|l|}{}                                   & conjugate\_gradient\_descent & 80                  & 80                     \\ \cline{2-4}
\multicolumn{1}{|l|}{}                                   & newton                       & 250                 & 250                    \\ \cline{2-4}
\multicolumn{1}{|l|}{}                                   & levenberg\_marquardt         & 34                  & 34                     \\ \hline
\multicolumn{1}{|l|}{\multirow{4}{*}{\textbf{rational}}} & gradient\_descent            & 150                 & 150                    \\ \cline{2-4}
\multicolumn{1}{|l|}{}                                   & conjugate\_gradient\_descent & 104                 & 104                    \\ \cline{2-4}
\multicolumn{1}{|l|}{}                                   & newton                       & 208                 & 208                    \\ \cline{2-4}
\multicolumn{1}{|l|}{}                                   & levenberg\_marquardt         & 56                  & 56                     \\ \hline
\end{tabular}
\label{tbl:order}
\end{table}

\paragraph{Newton's method gives a different result}

\begin{theorem}
    \label{theorem:conv_newton}
    For doubly continuously differentiable functions (i.e.\ for $f \in C^2(D)$) with a non-degenerate matrix $\nabla^2 f(y^*)$, there exists a $\varepsilon$-neighborhood of the stationary point $y^*$ of the function $f(y)$ such that for any initial point $y^0$ from this neighborhood, the Newton method will converge superlinearly, and if the Lipschitz condition is met in this neighborhood for Hesse matrices, it will converge quadratically.
\end{theorem}

If we consider the Newton method in more detail, then we can conclude from the theorem above [1]: if the Hesse matrix is not degenerate, but not sign-positive, then the Newton method can converge to a stationary point that is not a minimum point, but a maximum point or a saddle point.
Accordingly, a more careful choice of initial approximations is necessary.

\subsection{Conclusion}\label{subsec:conclusion}

In the course of the laboratory work,  first- and second-order methods were implemented and analyzed within the problem of the unconstrained optimization problem.
This paper also provides a comparative analysis of direct optimization methods and first-and second-order methods.

\subsection{References}\label{subsec:references}
\begin{enumerate}[label={[\arabic*]}]
    \item С.Ю. Городецкий, Лабораторный практикум по методам локальной оптимизации в программной системе LocOpt, 2007.
\end{enumerate}

\subsection{Appendix}\label{subsec:appendix}

The source code is located \href{https://github.com/vanSultan/anal_dev_algo/tree/lab_03}{here}: \url{https://github.com/vanSultan/anal_dev_algo/tree/lab_03}.