\chapter[Quadrature]{Numerical Quadrature}

\section{Introduction}

There are many circumstances in mathematical modelling when we need to
evaluate an integral, like
%
\begin{equation}
 \mint{a}{b}{f(x)}{x} .\label{Qua:eq:1}
\end{equation}
%
However, the integrals that can be evaluated analytically are few and
many integrals of great interest in engineering, like, for example,
the error function,
%
\begin{equation}
 \mbox{Erf}(x) = \mint{-\infty}{x}{e^{-\xi^2}}{\xi} \label{Qua:eq:2}
\end{equation}
%
can only be evaluated numerically.

We discuss two major techniques to evaluate integrals
(\textit{quadrature}): the first is based on \textit{interpolation},
the second on \textit{Gaussian quadrature}.

\section{Polynomial interpolation}

If we want to work with any function $f(x)$ numerically we must expect
to represent it using a finite amount of information. Typically this
is done by assuming that we know only the value of the function at a
finite set of points, call \textit{nodes}, $\{x_j\}$, with
$j=0,\ldots,N$. We denote the function values $f_j \equiv f(x_j)$.

One standard approach is to approximate the function $f$ by another
function $g(x)$ which is easy to manipulate and \textit{interpolates}
$f$ at the nodes, i.e.
%
\begin{equation}
  g(x_j) = f_j = f(x_j), \quad j = 0, \ldots, N.
\end{equation}
%
We then use $g$ in place of $f$ wherever necessary, as we shall see
below.

One standard, simple choice of \textit{interpolating function} $g(x)$
is a polynomial. Given the $N+1$ values $f_j$ at the nodes $x_j$ there
is a \emph{unique} polynomial $g$ of order $N$ that interpolates $f$
(as should be clear, as a polynomial of order $N$ has $N+1$
coefficients to be fixed). There are two standard forms of writing
this polynomial explicitly. Here we just show the Lagrange form to
show the explicit construction is possible. Often the Newton form is
used in numerical calculations instead.

First, given the nodes $x_j$, define the \textit{fundamental
  polynomials} $\ell_j(x)$ by
%
\begin{equation}
  \ell_j(x) = \prod_{\substack{i = 0\\i \ne j}}^N \frac{x - x_i}{x_j -
    x_i}, \quad j = 0, 1, \dots, N.
\end{equation}
%
These have the property that $\ell_j(x_j) = 1$ and $\ell_j(x_k) = 0, j
\ne k$. It follows that
%
\begin{equation}
  g(x) = \sum_{j=0}^N f(x_j) \ell_j(x)
\end{equation}
%
is the polynomial of degree $N$ that interpolates the function $f(x)$
at the nodes $x_j$.

In what follows we shall see methods for solving quadrature and
differential equations based on polynomial interpolation. For these
the defining features are the locations of the nodes and the order of
the interpolating polynomial. In many cases the formulas will simplify
greatly from the general Lagrange form above, so each case is treated
separately.

\section[Using polynomial interpolation]{Numerical integration based on polynomial interpolation}

\subsection{Introduction}

We assume that we can evaluate the function at a finite set of points,
called \textit{nodes}, $\{x_j\}$, with $j=0,\ldots, N$ in the interval
$[a,b]$ and we want to estimate~(\ref{Qua:eq:1}).  The idea behind all
the methods based on polynomial interpolation is that we can replace
the function $f(x)$ with a simpler function $g(x)$, i.e.\ a function
whose integral we can evaluate analytically.  We require that the
function $g(x)$ \textit{interpolates} the function $f(x)$, i.e.\ that
it has the same value as $f(x)$ at the nodes:
%
\begin{equation}
  g(x_j) = f(x_j) , \qquad j = 0,1,\ldots,N. \label{Qua:eq:3}
\end{equation}
%
The simplest choice of class of functions $g(x)$ are polynomials and
the different quadrature interpolation methods are differentiated by
the order of the polynomial interpolation.  Formulae based on
polynomial interpolation with equally spaced nodes go under the
generic name of \textit{Newton-Cotes formulae}.

\subsection{Trapezoidal rule}

\begin{figure}
  \centerline{\includegraphics[width=135mm]{figures/trapez}}
  \caption{\label{fig:trapez} \it The trapezoidal rule is based on the
    approximation of the given function $f(x)$ with an order one
    polynomial (left).  In general one uses a composite trapezoidal
    rule: the integration range is divided in intervals of length $h$
    and the function is approximated by a different polynomial on each
    interval (right).}
\end{figure}

The simplest case results if we choose an interpolating polynomial of
order one, i.e.\ we replace the function with a straight line that
interpolates the function at the end points of the integration range
(see left panel of Figure~\ref{fig:trapez}).   The area under the line
is given by
%
\begin{equation}
  A = \frac{1}{2} (b-a) [f(a) + f(b)]
  \label{Qua:eq:32}
\end{equation}
%
and this is the approximate value of~(\ref{Qua:eq:1}) according to the
\textit{trapezoidal rule}.  It is clear that unless the range $[a,b]$
is very small the error in the estimate obtained using the trapezoidal
rule is very large.  The standard procedure is to make use of the
\textit{composite trapezoidal rule}: the integration range is divided
into $N$ intervals of length $h$ and the function $f(x)$ is
approximated on each interval by a straight line that interpolates the
function at the nodes (see right panel of Figure~\ref{fig:trapez}).
The area under each segment of the piecewise linear curve is
%
\begin{equation}
  A_j = \frac{1}{2} (x_{j}-x_{j-1}) [f(x_{j})+f(x_{j-1})] , \quad
  j = 1,2, \ldots, N.
  \label{Qua:eq:4}
\end{equation}
%
By summing all the areas we obtain the composite trapezoidal rule:
%
\begin{equation}
  \mint{a}{b}{f(x)}{x} = \sum_{j=1}^{N} A_j =
  \frac{1}{2} \sum_{j=1}^{N} (x_{j}-x_{j-1}) [f(x_{j})+f(x_{j-1})]
  \label{Qua:eq:5}
\end{equation}
%
If the nodes are equally spaced, i.e.\ if $(x_{j}-x_{j-1}) = h$ for all
$j=1, \ldots, N$, this formula reduces to:
%
\begin{equation}
 \mint{a}{b}{f(x)}{x} =
  \frac{h}{2} \sum_{j=1}^{N} [f(x_{j})+f(x_{j-1})] =
  \frac{h}{2} ( f_0 + 2 f_1 + 2 f_2 + \ldots + 2 f_{N-1} + f_N ) ,
\label{Qua:eq:12}
\end{equation}
%
where $f_j \equiv f(x_j)$.  The error in the approximation is given by
%
\begin{equation}
  \text{Error} \le \frac{1}{12} h^3 N M_2 =
  \frac{(b-a) h^2}{12} M_2 , \qquad
  M_2 = \max_{x\in[a,b]} |f''(x)| .
  \label{Qua:eq:6}
\end{equation}

\subsection{Simpson's rule}

\begin{figure}
  \centerline{\includegraphics[width=135mm]{figures/simpson}}
  \caption{\label{fig:simpson} \it Simpson's rule is based on the
    approximation of the given function $f(x)$ with an order two
    polynomial (left).  In general one uses a composite Simpson's
    rule: the integration range is divided in intervals of length $2
    h$ and the function is approximated by different parabolas on each
    interval.}
\end{figure}

In this case the function is approximated with a polynomial of order
two (a parabola, see left panel of Figure~\ref{fig:simpson}).  We can
obtain the formula for the quadrature by writing the interpolating
polynomial and integrating it.  It turns out that the area under the
parabola is given by
%
\begin{equation}
  A = \frac{b-a}{6}
  \left [ f(a) + 4 f\left(\frac{a+b}{2}\right) + f(b) \right ],
  \label{Qua:eq:33}
\end{equation}
%
and this is the approximate value of~(\ref{Qua:eq:1}) according to
\textit{Simpson's rule}.  It is clear that unless the range $[a,b]$ is
very small the error in the estimate obtained using the trapezoidal
rule is very large.  The standard procedure is to make use of the
\textit{composite Simpson's rule}: the integration range is divided
into $N/2$ intervals of length $2 h$, with $N$ an even number. The
function $f(x)$ is approximated on each interval by a parabola that
interpolates the function at the nodes (see right panel of
Figure~\ref{fig:simpson}).  For simplicity we assume that the
interpolation points $\{x_j\}$, $j=0,1,2,\ldots,N$ are all equally
spaced so that we can write:
%
\begin{equation*}
  x_j = x_0 + j h .
\end{equation*}
%
The area under the parabola joining the nodes $x_{j-1}$, $x_{j}$ and
$x_{j+1}$ is
%
\begin{equation}
  A_j = \frac{h}{3} \left[f(x_{j-1})+4f(x_{j})+f(x_{j+1})\right]
  \label{Qua:eq:9}
\end{equation}
%
where $j=1,3,5,\ldots, N-1$.  This result is known as
\textit{Simpson's rule}.  If we apply it to all the sub-intervals in
$[a,b]$ we obtain the composite Simpson rule:
%
\begin{equation}
  \mint{a}{b}{f(x)}{x} = \sum_{k=1}^{N/2} A_{2k-1} =
  \frac{h}{3} \left[ f(a) +
    2 \sum_{j=1}^{N/2-1} f(x_{2j}) +
    4 \sum_{j=1}^{N/2} f(x_{2j-1}) + f(b) \right] + R,
  \label{Qua:eq:10}
\end{equation}
%
where the estimate for the error is given by
%
\begin{equation}
  |R|\le \frac{M_4}{180}(b-a)h^4,\label{Qua:eq:11}
\end{equation}
%
where $M_4=\max|f^{(4)}(x)|$ with $a \le x \le b$.  Note that
Simpson's rule is exact for any cubic polynomial as $f^{(4)}(x) \equiv
0$ for all polynomials of order three.

We can obtain an estimate of the error by proceeding as follows.  We
compare the Taylor expansion of Simpson's rule for the integral in the
range $[x_{j-1},x_{j+1}]$, given by equation~(\ref{Qua:eq:9}), to the
Taylor expansion of the exact integral.  The error is the first term
in the difference of the two Taylor expansions.  We start with the
Taylor expansion of the Simpson's rule estimate of
%
\begin{equation}
  \mint{x_{j-1}}{x_{j+1}}{f(x)}{x} \simeq A_j =
  \frac{h}{3} [ f(x_{j-1}) + 4 f(x_{j}) + f(x_{j+1}) ] ,
  \label{Qua:eq:19}
\end{equation}
%
where, to simplify the notation, we use the notation $f_j=f(x_j)$,
$f'_j=f'(x_j)$, and, in general, $f^{(n)}_j$ for the $n$-th derivative
of $f(x)$ evaluated at $x=x_j$.  Using Taylor's theorem to expand
$f(x)$ at $x=x_j$ we can write
%
\begin{equation}
  A_j = 2 h f_j + \frac{h^3}{3} f_j'' + \frac{h^5}{36} f_j^{(4)} + \order{h^6}
  \label{Qua:eq:20}
\end{equation}
%
We now need to compute the Taylor expansion in powers of $h$ of the
exact result.  In order to do this we introduce the function
%
\begin{equation}
  F(t) = \mint{x_j-t}{x_j+t}{f(x)}{x} .
  \label{Qua:eq:21}
\end{equation}
%
Note that with this notation
%
\begin{equation}
  \mint{x_{j-1}}{x_{j+1}}{f(x)}{x} = F(h).
  \label{Qua:eq:14}
\end{equation}
%
Therefore, in order to compute the Taylor expansion in powers of $h$
of the exact result of the integral in equation~(\ref{Qua:eq:14}), we
need to expand $F(t)$ in Taylor series around $t=0$:
%
\begin{equation}
  F(h) = F(0) +
  h \eval{\dv{F}{t}}_{t=0}  +
  \frac{h^2}{2} \, \eval{\dv[2]{F}{t}}_{t=0}  +
  \ldots +
  \frac{h^5}{5!} \, \eval{\dv[5]{F}{t}}_{t=0} +
  \order{h^6}
  \label{Qua:eq:15}
\end{equation}
%
We note that $F(0)=0$.  Moreover, by the fundamental theorem of
calculus (with a little help from the chain rule)
%
\begin{equation*}
  \dv{F}{t} = f(x_j+t) + f(x_j-t) \implies
  \dv[n]{F}{d} = f^{(n-1)}(x_j+t) +
  (-1)^{n+1} f^{(n-1)}(x_j-t) .
\end{equation*}
%
Note that all the even derivatives of $F(t)$ are zero at $t=0$ so that
the Taylor expansion of~(\ref{Qua:eq:15}) becomes
%
\begin{equation}
  2 h f_j + \frac{h^3}{3} f_j'' + \frac{h^5}{60} f_j^{(4)} + \order{h^6}.
  \label{Qua:eq:30}
\end{equation}
%
Combining these two expansions, we have that the local error in the
interval $[x_{j-1},x_{j+1}]$ is, up to order $h^6$,
%
\begin{equation}
  E_j \equiv F(h) - A_j = - \frac{1}{90} h^5 f_j^{(4)} \implies
  \abs{E_j} \le \frac{1}{90} h^5 M_4 ,
  \label{Qua:eq:31}
\end{equation}
%
where $M_4$ is the same as in equation~(\ref{Qua:eq:11}).  Summing the
local errors over all the intervals used in the composite Simpson's
rule gives the total error, $R$,
%
\begin{equation}
  \abs{R} = \abs{\sum_{k=1}^{N/2} E_{2k-1}} \le
  \sum_{k=1}^{N/2} \abs{ E_{2k-1} } \le
  \sum_{k=1}^{N/2} \frac{1}{90} h^5 M_4 = \frac{M_4}{180}(b-a)h^4.
\end{equation}

\subsection{Richardson extrapolation}

This is a very general technique that uses the information on the rate
of decrease of the error to get a better estimate of the numerical
result required.  As such it can be applied to many different
algorithms, not just quadratures.  However, here we illustrate it in
the context of Simpson's rule.

Equation~(\ref{Qua:eq:11}) tells us that the error on the estimate of
the integral decreases with the fourth power of the integration step,
$h$.  However, in order to estimate the error using this formula we
would also need the value of the coefficient $M_4$.  We can get round
this problem by using the following procedure.  Indicate with $I_n$
the estimate of the integral using $n$ intervals and with $I$ the
exact value of the integral.  From~(\ref{Qua:eq:11}) we have that
%
\begin{equation}
  I - I_{2n} \simeq C (2n)^{-4} =
  2^{-4} \left ( C n^{-4} \right ) \simeq 2^{-4} (I - I_n) .
  \label{Qua:eq:16}
\end{equation}
%
In this last step we have assumed that the constant $C$ is the same
whether we use $n$ or $2n$ intervals to estimate the integral.  This
not strictly true, but it is a reasonable approximation if the
integration range is small enough.  From~(\ref{Qua:eq:16}) we have
that the exact value of the integral is approximately
%
\begin{equation*}
  I \simeq \frac{2^4 I_{2 n} - I_n}{2^4 -1}
\end{equation*}
%
and we use this as the new estimate of the integral
(\textit{Richardson extrapolated value}):
%
\begin{equation}
  R_{2 n} = \frac{2^4 I_{2 n} - I_n}{2^4 -1} .
  \label{Qua:eq:22}
\end{equation}
%
The error in the approximation is roughly
%
\begin{equation}
  E_{2n} \equiv \abs{ R_{2 n} - I_{2 n} } = \frac{\abs{I_n - I_{2 n}}}{2^4 -1} .
  \label{Qua:eq:17}
\end{equation}
%
This quantity is called the \textit{computable estimate of the error}.

\subsection{Adaptive quadrature}

Adaptive quadrature methods are intended to compute definite integrals
to a given precision by automatically choosing a set of nodes that
takes into account the behaviour of the integrand by being denser
where the integrand varies more rapidly. Ideally, the user supplies
only the integrand $f$, the interval $[a,b]$, and the accuracy
$\epsilon$ desired for computing the integral
%
\begin{equation}
  \mint{a}{b}{f(x)}{x} . \label{Qua:eq:23}
\end{equation}
%
The program then divides the interval into sub-intervals of varying
length so that numerical integration on these sub-intervals will
produce results of acceptable precision. The main idea is that if
Simpson's rule on a given sub-interval is not sufficiently accurate,
that interval will be divided into two equal parts, and Simpson's rule
will be used on each half. This procedure will be repeated in an
effort to obtain an approximation to the integral with the same
accuracy over all the sub-intervals involved.  A rough algorithm is as
follows:

\begin{enumerate}
  %
\item Compute the integral~(\ref{Qua:eq:23}) using $3$ points and
  estimate the global error.  If the error is smaller than the
  tolerance stop.
  %
\item Divide the interval into two equal parts.  Consider each part in
  turn.
  %
\item Compute the integral on the sub-interval using $3$ points,
  i.e.\ using a finer grid.
%
\item Estimate the error on the evaluation of the integral over the
  sub-interval.  If it is larger than the maximum acceptable local
  error then divide the sub-interval in two and start again from point
  (2).  Otherwise go to point (3) and work on the next sub-interval if
  there are any left.
%
\end{enumerate}

In order to apply this idea we need to firstly to estimate the
\textit{local error}: we can do this using Richardson's extrapolation
and equation~(\ref{Qua:eq:17}).  Secondly, we need to estimate the
\textit{global error} on the integration over the entire interval
$[a,b]$ as a function of the local errors.

\smallskip

\noindent
We indicate with $e_i$ the local error on the integration over the
segment $[x_{i-1},x_i]$. If
%
\begin{equation}
  \abs{e_i} \le \epsilon (x_i-x_{i-1})/(b-a),
  \label{Qua:eq:25}
\end{equation}
%
then the total error will be bounded by
%
\begin{equation}
  \abs{ \sum_{i=1}^n e_i} \le \sum_{i=1}^n \abs{e_i} \le
  \frac{\epsilon}{b-a} \sum_{i=1}^n (x_i-x_{i-1}) = \epsilon.
  \label{Qua:eq:26}
\end{equation}
%
Therefore in the adaptive quadrature algorithm outlined above we have
to ensure that the local error satisfies~(\ref{Qua:eq:25}).

\section{Gaussian Quadrature}

The aim of all quadrature techniques is to create formulae of the type
%
\begin{equation}
  \mint{a}{b}{f(x)}{x} \simeq \sum_{i=1}^n w_i f(x_i) .
 \label{Qua:eq:18}
\end{equation}
%
In the trapezoidal and Simpson's rule the points where the function is
evaluated, $x_i$, are fixed \textit{a priori} and we obtain
that~(\ref{Qua:eq:18}) is exact for all polynomial of degree less than
or equal to $n$.  In the case of the trapezoidal rule we have $n=2$
and the rule is exact for all linear functions.  In the case of
Simpson's rule we have $n=3$ and we would expect the formula to be
exact for all quadratic polynomials.  As a matter of fact, it is exact
for polynomials of order three, but this is due to a happy
cancellation.

However, there is no obligation to fix the nodes.  As a matter of fact
we could fix the coefficients (called \textit{weights}) $w_i$, for
example set them all to one, or we could not fix anything.  The
general criterion to find nodes and weights is to require that
equation~(\ref{Qua:eq:18}) is as accurate as possible, i.e.\ that it is
exact for polynomials of degree as high as possible.  In other words,
we require that the nodes and weights are such that
%
\begin{equation}
  \mint{a}{b}{x^s}{x} = \sum_{i=1}^n w_i x_i^s, \quad s=0,1,...,N,
  \label{Qua:eq:27}
\end{equation}
%
with $N$ as large as possible.

As above, we define the \textit{fundamental polynomials}
$\ell_i(x)$ given by
%
\begin{equation}
  \ell_i(x) = \prod_{\substack{j = 0\\j \ne i}}^N \frac{x - x_j}{x_i -
    x_j}, \quad i = 0, 1, \dots, N,
\end{equation}
%
have the property that $\ell_i(x_i) = 1$ and $\ell_i(x_k) = 0, i \ne
k$. It follows that
%
\begin{equation}
  p(x) = \sum_{i=0}^N f(x_i) \ell_i(x)
\end{equation}
%
is the polynomial of degree $N$ that interpolates the arbitrary
function $f(x)$ at the given nodes $x_i$. If the nodes are known or
fixed we can therefore immediately compute the weights as
%
\begin{equation}
  w_i = \mint{a}{b}{\ell_i(x)}{x}.
\end{equation}
%

If neither the nodes nor the weights are fixed we have $N$ nonlinear
equations for $2n$ unknowns $x_1,...,x_n$ and $w_1,...,w_n$.  One can
show that this problem has a unique solution if $N=2n-1$, i.e.\ that it
is possible to compute exactly the integrals of polynomials of degree
up to $2 n - 1$.  The resulting algorithm is called a {\bf Gaussian
  quadrature}.  There are many formulae for Gaussian quadrature that
depend mainly on the choice of integration interval and on the type of
the integral.  A generic Gauss quadrature formula is
%
\begin{equation}
  \mint{a}{b}{W(x) f(x)}{x} = \sum_{i=1}^n w_i f(x_i)
\end{equation}
%
and the different formulae are summarised in the following table:

\begin{center}
 \begin{tabular}{lll} \hline
  $(a,b)$ & $W(x)$ & Gauss- \\ \hline
  $(-1,1)$ & 1 & Legendre \\
  $(-1,1)$ & $(1-x^2)^{-1/2}$ & Chebychev \\
  $(0,\infty)$ & $x^c e^{-x}$ & Laguerre $c=0,1,\ldots$ \\
  $(-\infty,+\infty)$ & $e^{-x^2}$ & Hermite \\ \hline
 \end{tabular}
\end{center}

\section*{Further reading}

Topics covered here are also covered in
\begin{itemize}
\item Chapter 6 of Linz \& Wang, \textit{Exploring Numerical
    Methods} (QA297 LIN),
\item Chapter 7 of Kincaid \& Cheney, \textit{Numerical Analysis}
  (QA297 KIN),
\item Chapters 7 and 10 of S{\"u}li \& Mayers, \textit{An Introduction
    to Numerical Analysis} (not in library).
\end{itemize}

The issue of representing an arbitrary function by an interpolating
function, which will recur in later chapters, is covered in
considerable detail elsewhere; for example\begin{itemize}
\item Chapters 4 and 5 of Linz \& Wang, \textit{Exploring Numerical
    Methods} (QA297 LIN),
\item Chapter 6 of Kincaid \& Cheney, \textit{Numerical Analysis}
  (QA297 KIN),
\item Chapters 6, 8, 9 and 11 of S{\"u}li \& Mayers, \textit{An Introduction
    to Numerical Analysis} (not in library).
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 7}

Show how to implement a queue using two stacks.
Analyze the running time of the queue operations and argue about their amortized cost.

\subsection*{Solution}

To implement a standard queue with \textsc{Enqueue} and \textsc{Dequeue} operations, using two stacks with pop and push operations, Algorithms \ref{alg61} and \ref{alg62} are respectively proposed, in which number of elements in stack $S$ is noted with $S.length$.

\begin{algorithm}[H]
\caption{\textsc{Enqueue($x$)}}\label{alg61}
\begin{algorithmic}[1]
\State \textsc{Push($S_1$, $x$)}
\State $S_1.length \leftarrow S_1.length + 1$
\end{algorithmic}
\end{algorithm}

\begin{algorithm}[H]
\caption{\textsc{Dequeue()}}\label{alg62}
\begin{algorithmic}[1]
\If {$S_2.length == 0$}
\For {$i \leftarrow S_1.length$ down to $1$}
\State $q \leftarrow$ \textsc{Pop}($S_1$)
\State \textsc{Push}($S_2$, $q$)
\EndFor
\EndIf
\State \Return \textsc{Pop}($S2$)
\end{algorithmic}
\end{algorithm}

It is trivial to show that the runtime for \textsc{Enqueue}$(x)$ using Algorithm \ref{alg61} is $\mathcal{O}(1)$ since each \textsc{Push} operation for an stack takes a constant time $C$.

When stack $S_2$ is not empty, \textsc{Dequeue} operation takes a constant time as well, simply because there is only one \textsc{Pop} operation with cost $C$.

In the worst-case however, when stack $S_2$ is empty and there are $n$ elements in stack $S_1$, the for loop will iterate $n$ times, leading to a run-time of $\mathcal{O}(n)$.
Thus the \textsc{Dequeue} algorithm is linear in number of elements in queue.

An amortize analysis, however, reveal both \textsc{Enqueue} and \textsc{Dequeue} algorithms only take constant time $C$ per item.
This is trivial for \textsc{Enqueue} since each element is \textsc{Push}ed only once.

To prove this statement for \textsc{Dequeue}, suppose a stack $S_1$ of $m$ elements.
It suffices to prove that it takes a multiple of $m$ constant-time cost to dequeue all $m$ elements, thus leading to a $\mathcal{O}(1)$ cost per element.

To dequeue the first element, all $m$ elements are \textsc{Pop}ed from stack $S_1$ and \textsc{Push}ed to stack $S_2$.
Once in $S_2$, they await their turn to be \textsc{Pop}ed again in later \textsc{Dequeue} operations.
This means each individual element of queue is \textsc{Pop}ed, \textsc{Push}ed and \textsc{Pop}ed only once, and since both two operations have constant-time cost, a \textsc{Dequeue} will have an amortized cost of 3, i.e. $\mathcal{O}(1)$.


\documentclass{article}
\title{ALO for LASSO with Intercept through Generalized LASSO}
\author{Yuze Zhou}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{graphicx}
\usepackage{algorithm}
\usepackage{subfigure}
\begin{document}
\maketitle
\section{The Dual of LASSO with intercept}
\paragraph{}For the generalized LASSO problem $\min\limits_{\beta}\frac{1}{2}||y-X\beta||+\lambda||D\beta||_{1}$, the dual problem is derived as:
\begin{center}
$\min\limits_{u}\frac{1}{2}||y-\theta||_{2}^{2}$\\
$\theta \in \{X^{\tau}\theta = D^{\tau}u, ||u||_{\infty} \leq \lambda\}$
\end{center}
\paragraph{}The dual problem could be written in a proximal approach such that:
\begin{center}
$\hat{u} = \textbf{prox}_{R}(y)$\\
$R(u) = \left\{
\begin{aligned}
0 \quad \if \theta \in \{X^{\tau}\theta = D^{\tau}u, ||u||_{\infty} \leq \lambda\}\\
+\infty \quad otherwise
\end{aligned}
\right.
$
\end{center}
\paragraph{}Denote $J$ as the jacobian of the proximal operator at the full data problem $y$, then the ALO estimator could be obtained as:
\begin{center}
$y^{/i} = y_{i} - \frac{\hat{u}_{i}}{J_{ii}}$
\end{center}
\paragraph{}For the case of LASSO with an intercept, we could expand the $X$ with a column of ones in the first column, expand $\beta$ with another dimension and choose $D = [\textbf{0}, I ]$.
\paragraph{}Denote $E := \{j: |X_{i}^{\tau}\theta| = \lambda \}$ to be the active set. Then the Jacobian is locally given as the projection onto the orthogonal complement of the span of $X_{E}$ and the ones vector. Further denote $\tilde{X}_{E} = [\textbf{1}, X_{E}]$, then the Jacobian is given as $I - \tilde{X_{E}}(\tilde{X_{E}}^{\tau}\tilde{X_{E}})\tilde{X_{E}}^{\tau}$.
\end{document}


\chapter{Expression of Angular Eigenstates of Harmonic Oscillators} \label{appA}

In this appendix, we derive some results on harmonic oscillators which are useful to show the abilities of entanglement detection of the SRPT criterion.

\section{Two Dimensional Harmonic Oscillator}

In two dimensions, the Hamiltonian describing a particle of mass $m$ subject to an isotropic harmonic potential is

\[ H =  \frac{p_x^2}{2m}+\frac{p_y^2}{2m} + \frac 1 2 m \omega^2 \left( x^2 + y^2 \right),\]
with $x_i, p_i$ the position and momentum of the particle in the directions $i=x,y$ and $\omega$ a constant. Since $H$ is the sum of two  oscillators Hamiltonians along the directions $x$ and $y$, it is clear that the eigenvalues of the hamiltonian will be written as
\[ \ket{n_x, n_y} \equiv \ket{n_x} \otimes \ket{n_y}, \]
when the $\ket{n_i}$ are the eigenstates of the two one dimensional Hamiltonians. We may define the annihilation operators:
\begin{eqnarray}
    a_x &=& \frac{1}{\sqrt 2} \left( \sqrt{\frac{m\omega}{\hbar}} x + \frac{i}{\sqrt{m \hbar \omega}} p_x \right), \\
    a_y &=& \frac{1}{\sqrt 2} \left( \sqrt{\frac{m\omega}{\hbar}} y +\frac{i}{\sqrt{m \hbar \omega}} p_y \right).
\end{eqnarray}
The creation operators $a^\dagger_i$ are the hermitian conjugates of the above operators. We have
\[ [a_x,a^\dagger_x]= [a_y,a^\dagger_y]=1, \]
while all other commutators between the four operators are zero. Let us finally define the number operators
\begin{eqnarray}
    N_x &=&a_x^\dagger a_x, \\
    N_y &=& a_y^\dagger a_y,
\end{eqnarray}
which allow us to write $H$ as
\[ H = (N_x + N_y +1) \hbar \omega. \]

The eigenstates have the properties
\begin{eqnarray}
    a_i \ket{n_i} &=& \sqrt{n_i} \ket{n_i-1},\\
    a_i^\dagger \ket{n_i} &=& \sqrt{n_i+1} \ket{n_i+1},\\
    N_i \ket{n_i} &=& n_i \ket{n_i},
\end{eqnarray}
and therefore, we have
\[  \ket{n_x,n_y} = \frac{1}{\sqrt{n_x! n_y!}} \left( a_x^\dagger \right)^{n_x}  \left( a_y^\dagger \right)^{n_y} \ket{0,0}, \]
with $\ket{0,0}$ the ground state of the oscillator.

We can also prove that
\[ H \ket{n_x,n_y} = (n_x + n_y +1) \hbar \omega \ket{n_x,n_y}.\]

If we define the total quantum number
\[ n = n_x + n_y, \]
we note that one particular value of $n$ corresponds to the $n+1$ orthogonal eigenstates
\[ \ket{n,0}, \; \ket{n-1,1}, \ldots, \; \ket{0,n}, \]
which shows that the measure of the energy alone does not allow us to pinpoint one proper state. The discrimination between all these states could be achieved by measuring the energy separately in the $x$ and $y$ direction, but there is another method that takes advantage of the angular momentum $L_z$. That operator is defined by
\[ L_z = x p_y - y p_x,\]
or in other terms
\[ L_z= i\hbar(a_x a_y^\dagger-a_x^\dagger a_y), \]
and it is clear that the $\ket{n_x,n_y}$ states are not eigenstates of $L_z$.

However, it is possible to show that
\[ [H,L_z]=0, \]
which indicates there must be a common eigenstate basis for the two operators. That basis is made of the $\ket{\psi_{k,M}}$ eigenvectors which has the following properties:
\begin{eqnarray}
    H \ket{\psi_{k,M}} &=& \hbar \omega(2k+|M|+1) \ket{\psi_{k,M}},\\
    L_z \ket{\psi_{k,M}} &=& \hbar M \ket{\psi_{k,M}}.
\end{eqnarray}

We wish to express any $\ket{\psi_{k,M}}$ state in function of the $\ket{n_x,n_y}$ states. It is quite clear that $n=n_x+n_y=2k+|M|$ and therefore for given $k$ and $M$ we have in general
\[ \ket{\psi_{k,M}} = \sum_{i=0}^n c_i \ket{n_x=n-i, n_y=i}. \label{eq-2Dsum}\]

To find the analytical expression of the $c_i$ coefficients, we need to define an intermediate basis as follow
\begin{eqnarray}
    a_r &=& \frac{1}{\sqrt 2} \left(a_x-ia_y \right), \\
    a_l &=& \frac{1}{\sqrt 2} \left(a_x+ia_y \right),
\end{eqnarray}
where $r$ and $l$ stand for right and left, as those operators can be interpreted as annihilators of right and left  ``circular quanta''. They are very similar to $a_x$ and $a_y$, they follow the commutation rules
\[ [a_r,a^\dagger_r]= [a_l,a^\dagger_l]=1, \]
and all other combinations are zero. It is possible to express the regular ladder operators in function of the circular one and we find that
\begin{eqnarray}
    H&=&(N_r + N_l +1) \hbar \omega,\\
    L_z &=& \hbar (N_r- N_l) ,
\end{eqnarray}
with $N_r=a^\dagger_r a_r$ and $N_l=a^\dagger_l a_l$ the new number operators. We see that there is a $\ket{\varphi_{n_r,n_l}}$ basis in which both $H$ and $L_z$ are diagonal and which behaves exactly like the $\ket{n_x,n_y}$ basis. We also find
\begin{eqnarray}
    n &=& n_r + n_l = 2k+ |M|, \\
    M &=& n_r - n_l,
\end{eqnarray}

This result does allow to associate a definite $\ket{\psi_{k,M}}$ state to a single state $\ket{\varphi_{n_r,n_l}}$. There are two cases depending on the sign of $M$; if $M>0$, it means $n_r>n_l$ and $|M|=n_r-n_l$ and $k=n_l$, if $M<0$, then $|M|=n_l-n_r$ and $k=n_r$. Globally,
\begin{eqnarray}
    \ket{\psi_{k,M}} &=& \ket{\varphi_{n_r=k+|M|,n_l=k}} \:\: \textrm{if } M>0, \\
    \ket{\psi_{k,M}} &=& \ket{\varphi_{n_r=k,n_l=k+|M|}} \:\: \textrm{if } M<0.
\end{eqnarray}

We see that a state with a positive helicity $M$ has more ``right'' quanta than ``left'' quanta and inversely for $M<0$. Now, to express a $\ket{\varphi_{n_r,n_l}}$ into a combination of $\ket{n_x,n_y}$ we have
\begin{eqnarray}
    \ket{\varphi_{n_r,n_l}}&=& \frac{1}{\sqrt{n_r! n_l!}} \left( a_r^\dagger \right)^{n_r}  \left( a_l^\dagger \right)^{n_l} \ket{\varphi_{0,0}}, \\
    &=&   \frac{1}{\sqrt{2^n n_r! n_l!}} \left( a_x^\dagger +ia_y^\dagger \right)^{n_r}  \left( a_x^\dagger -ia_y^\dagger\right)^{n_l} \ket{0,0}, \label{eq-2D2}\\
    &=&   \frac{1}{\sqrt{2^n n_r! n_l!}} \sum_{k=0}^{n_r} \sum_{l=0}^{n_l}  \binom{n_r}{k} \binom{n_l}{l} \left( a_x^\dagger \right)^{n-k-l}  \left(a_y^\dagger \right)^{j+k} (i)^{k-l} \ket{0,0}, \label{eq-2D3}\\
    &=&   \frac{1}{\sqrt{2^n n_r! n_l!}} \sum_{i=0}^{n} \sum_{j}^{\{-i,-i+2,\cdots, i\}}  \binom{n_r}{\frac{i-j}{2}} \binom{n_l}{\frac{i+j}{2}} \left( a_x^\dagger \right)^{n-i}  \left(a_y^\dagger \right)^i (i)^{j} \ket{0,0}, \label{eq-2D4}\\
    &=&   \frac{1}{\sqrt{2^n n_r! n_l!}} \sum_{i=0}^{n} \sum_{j=0}^{i}  \binom{n_r}{j} \binom{n_l}{i-j} \left( a_x^\dagger \right)^{n-i}  \left(a_y^\dagger \right)^i (i)^{2j-i} \ket{0,0},\label{eq-2D5} \\
    &=&   \sum_{i=0}^{n}(-i)^i  \sqrt{ \frac{(n-i)! i!}{2^n n_r! n_l!}}    \sum_{j=0}^{i} (-1)^j \binom{n_r}{j} \binom{n_l}{i-j}  \ket{n-i,i},\label{eq-2D6} \\
    &=&   \sum_{i=0}^{n}(-i)^i  \sqrt{\frac{\binom{n}{n_r} }{2^n \binom{n}{i}}}    \sum_{j=0}^{i}(-1)^j \binom{n_r}{j} \binom{n_l}{i-j}  \ket{n-i,i}, \label{eq-2D7}
\end{eqnarray}
where on line (\ref{eq-2D2}) we simply used the definition of $a_r$ and $a_l$ and noted the ground state is the same in both basis, in (\ref{eq-2D3}) we used the binomial formula twice and in (\ref{eq-2D4}) we applied the change of variables $i=k+l$ and $j=k-l$. In order to span all values of $(k,l)$ only once, we span all ``antidiagonal'' lines with $k+l=i$ $(i=0,1,\ldots,n)$ and along those lines we consider the elements $(k=\frac{i+j}{2},l=\frac{i-j}{2})$ $(j=-i, -i+2,\ldots, i-2,i)$. The values of $j$ should actually only go from $\max\{-i,i-2 n_l\}$ to  $\min\{i,2 n_r -i\}$ in order not to go beyond $k=n_r$ and $l=n_l$ but we can simplify it since the binomial coefficients will yield zero if $k>n_r$ or $l>n_l$. In line (\ref{eq-2D5}) we applied yet another change of variable as $j'=\frac{i+j}{2}$ $(j=0, 1, \ldots, i)$ and renamed $j'$ as $j$, in (\ref{eq-2D6}) we applied the creation operators to the ground state and in (\ref{eq-2D7}) we multiplied the numerator and denominator in the root term by $n!$ and simplified the expression. The final sum on $j$ may be expressed differently as we find that
\[  \sum_{j=0}^{i}(-1)^j \binom{n_r}{j} \binom{n_l}{i-j} = \binom{n_l}{i} {}_2F_1(-i,-n_r; n_l -i+1;-1), \]
with ${}_2F_1$ the hypergeometric function, but we choose to keep the sum as it is for its implementation simplicity.

Now,  we need to express directly the $\ket{\psi_{k,n}}$ state in the $\ket{n_x, n_y}$ basis. Depending on the sign of $M$, we need to consider the state $\ket{\varphi_{k+|M|, k}}$ or $\ket{\varphi_{k,k+|M|}}$. The easiest way to see the effect of a swap of $n_r$ and $n_l$ is in line (\ref{eq-2D3}) where the only difference is $i^{k-l}$  becoming $i^{l-k}=(-i)^{k-l}$. With that consideration, we finally have the $c_i$ coefficients we wanted in (\ref{eq-2Dsum}) up to an overall phase:
\[ c_i =  \left(- \textrm{sign}(M) i \right)^i  \sqrt{\frac{\binom{n}{k} }{2^n \binom{n}{i}}}    \sum_{j=0}^{i}(-1)^j \binom{k+|M|}{j} \binom{k}{i-j}.\]

The last property we want to investigate is the relation
\[ c_i = i^n (-1)^{n_l-i} c_{n-i} . \]
On the left hand side, we have
\begin{eqnarray}
    c_i&=& (-i)^i  \sqrt{\frac{\binom{n}{n_r} }{2^n \binom{n}{i}}}  \sum_{j=0}^{i}(-1)^j \binom{n_r}{j} \binom{n_l}{i-j} ,
\end{eqnarray}
and on the right hand side
\begin{eqnarray}
    c_{n-i}&=&  (-i)^{n-i} \sqrt{\frac{\binom{n}{n_r} }{2^n \binom{n}{n-i}}}    \sum_{j=0}^{n-i}(-1)^j \binom{n_r}{j} \binom{n_l}{n-i-j} , \\
    &=&  (-i)^{n-i}  \sqrt{\frac{\binom{n}{n_r} }{2^n \binom{n}{i}}}    \sum_{j=n_l-i}^{n_r}(-1)^{n_r-j} \binom{n_r}{n_r-j} \binom{n_l}{n-i+j-n_r} , \\
    &=&  (-i)^{n-i} (-1)^{n_r}  \sqrt{\frac{\binom{n}{n_r} }{2^n \binom{n}{i}}}    \sum_{j=n_l-i}^{n_r}(-1)^{j} \binom{n_r}{j} \binom{n_l}{i-j} ,
\end{eqnarray}
where in the second step we used the variable change $j'=n_r-j$ and in the last step simplified the expression. Aside from the phase, the only remaining difference is the borns of the sum but thanks to the binomial coefficients, that difference vanishes. Indeed, for both sums, the condition of having non zero terms is $\max\{ 0, i- n_l\} \le j \le \min\{ i,n_r \} $, so that all terms considered outside those limits are zero. Of course, we have $|c_i|=|c_{n-i}|$.

\section{Three Dimensional Harmonic Oscillator}

In three dimensions, the Hamiltonian describing a particle of mass $m$ subject to an isotropic harmonic potential is

\[ H =  \frac{p_x^2}{2m}+\frac{p_y^2}{2m} + \frac{p_z^2}{2m}+ \frac 1 2 m \omega^2 \left( x^2 + y^2 +z^2\right),\]
with $x_i, p_i$ the position and momentum of the particle in the directions $i=x,y,z$ and $\omega$ a constant. Once again, since $H$ is the sum of three oscillators Hamiltonians, the eigenvalues of the hamiltonian will be written as
\[ \ket{n_x, n_y,n_z} \equiv \ket{n_x} \otimes \ket{n_y} \otimes \ket{n_z}, \]
when the $\ket{n_i}$ are the eigenstates of the three one dimensional Hamiltonians. We may define the third annihilation operator:
\[ a_z = \frac{1}{\sqrt 2} \left( \sqrt{\frac{m\omega}{\hbar}} z + \frac{i}{\sqrt{m \hbar \omega}} p_z \right),\]
which behaves exactly as the others. We also define its number operator
\[ N_z = a_z^\dagger a_z, \]
which allow us to write $H$ as
\[ H = \left(N_x + N_y + N_z +\frac 3 2 \right) \hbar \omega. \]

The eigenstates are of the form
\[  \ket{n_x,n_y,n_z} = \frac{1}{\sqrt{n_x! n_y!n_z!}} \left( a_x^\dagger \right)^{n_x}  \left( a_y^\dagger \right)^{n_y} \left( a_z^\dagger \right)^{n_z} \ket{0,0,0}, \]
with $\ket{0,0,0}$ the ground state of the oscillator. This time for a definite energy $n=n_x+n_y+n_z$ there is a degree of degenerescence $g_n$ of
\[ g_n = \sum_{i=0}^n (i+1) = \frac{n(n+1)}{2} + (n+1) = \frac 1 2 (n+1)(n+2). \]
Just as we introduced $L_z$ in the two dimensional harmonic oscillator, we introduce the additional observable
\[ \mathbf{L}^2 = \frac 1 2 (L_+ L_- + L_- L_+) + L_z^2, \]
with
\begin{eqnarray}
    L_+ &=& \hbar \sqrt 2 (a^\dagger_z a_l - a^\dagger_r a_z), \\
    L_- &=& \hbar \sqrt 2 (a^\dagger_l a_z - a^\dagger_z a_r).
\end{eqnarray}

It can be checked that $[H,\mathbf{L}^2]=0$ and $[L_z,\mathbf{L}^2]=0$ which implies there must be a common set of eigenstates for $H$, $L_z$ and $\mathbf{L}^2$ . Those eigenstates are the $\ket{\psi_{k,l,m}}$ states and have the following properties
\begin{eqnarray}
    H \ket{\psi_{k,l,m}} &=& \hbar \omega(2k+l+\frac 3 2) \ket{\psi_{k,l,m}},\\
    L_z \ket{\psi_{k,l,m}} &=& \hbar m \ket{\psi_{k,l,m}},\\
    \mathbf{L}^2 \ket{\psi_{k,l,m}} &=& \hbar^2 l(l+1) \ket{\psi_{k,l,m}}, \\
    L_\pm \ket{\psi_{k,l,m}} &=& \hbar \sqrt{l(l+1)-m(m\pm1)} \ket{\psi_{k,l,m\pm1}}.\\
\end{eqnarray}

The last relation implies that $m$ can take any integer value from $-l$ to $l$. So by repeatedly applying the $L_-$ operator on a $\ket{\psi_{k,l,l}}$ state, we should be able to generate all states down to $\ket{\psi_{k,l,-l}}$. The first step is to find the expression of the $\ket{\psi_{k,l,l}}$ in the $\ket{\varphi_{n_r,n_l,n_z}}$ basis using the particular property $L_+ \ket{\psi_{k,l,l}} = 0$. By identifying the quantum numbers we find
\begin{eqnarray}
    n&=& n_r + n_l + n_z = 2k+ l, \\
    m &=& n_r - n_l.
\end{eqnarray}

Since $n_r-n_l=m$ we can always write $n_r = K+m$, $n_r=K$ with $K$ a positive integer and therefore we must have $n_z=n-m-2K$, which cannot be negative hence we have $K \le \frac{n-m}{2}$.  We are now able to write
\[ \ket{\psi_{k,l,l}} = \sum_{K=0}^k c_K \ket{\varphi_{n_r=K+l, n_l=K, n_z = 2k-2K}}, \]
since $\frac{n-l}{2} = k$ and with normalized coefficient $c_K$. The effect of $L_+$ on such a decomposition is
\bea
L_+ \ket{\psi_{k,l,l}} &=& \sum_{K=0}^k c_K \sqrt 2 \hbar \left( \sqrt{K} \sqrt{2k-2K+1} \ket{\varphi_{K+l, K-1,2k-2K+1}} \right. \nonumber \\
&& \ - \left. \sqrt{K+l+1}\sqrt{2k-2K} \ket{\varphi_{K+l+1, K,2k-2K-1}} \right),
\eea
which is zero if
\[ c_K  \sqrt{K+l+1}\sqrt{2k-2K} = c_{K+1} \sqrt{K+1} \sqrt{2k-2K-1}. \]

From that relation, we can express all $c_K$ in function of $c_0$ and then normalize all coefficient. We have
\bea
c_K^2 &=& c_0^2 \prod_{j=0}^{K-1} \frac{(j+l+1)(2k-2j)}{(j+1)(2k-2j-1)} ,\\
&=& c_0^2 \frac{(l+1)(l+2)\ldots(l+K)(2k)(2k-2)\ldots(2k-2K+2)}{(1)(2)\ldots K (2k-1)(2k-3)\ldots(2k-2K+1)} ,\\
&=& c_0^2 \frac{(l+K)!}{l!K! } \frac{(2k)!!}{(2k-2K)!!} \frac{(2k-2K-1)!!}{(2k-1)!!}, \\
&=& c_0^2 \binom{l+K}{K} \frac{2^k k! }{2^{k-K}(k-K)! }\frac{(2k-2K)!}{2^{k-K}(k-K)! }\frac{2^{k}(k)! }{(2k)!}, \\
&=& c_0^2 \,2^{2K}\,\frac{ \binom{l+K}{K} \binom{2k-2K}{k-K}  }{ \binom{2k}{k} },
\eea
where $k!!$ is the double factorial of $k$ which has the property
\bea
(2k)!! &=&(2k)(2k-2)\ldots2 = k! 2^{k}, \\
(2k+1)!! &=&(2k+1)(2k-1)\ldots1 = \frac{(2k)!}{k! 2^{k}},
\eea
that we used in the process.

With that expression, we are now able to calculate the value of $c_0$ by normalizing the expression. We have
\bea
\sum_{K=0}^k c_K^2 &=& c_0^2 \sum_{K=0}^k \,2^{2K}\,\frac{ \binom{l+K}{K} \binom{2k-2K}{k-K}  }{ \binom{2k}{k} }, \\
&=& c_0^2 \frac{ \binom{2(k+l+1)}{2k}}{ \binom{k+l+1}{k} },
\eea
Since that expression must be 1, we have he value of $c_0$. Finally, we get up to a global phase
\[ \ket{\psi_{k,l,l}} = \sum_{K=0}^k\,2^{K}\, \sqrt{ \frac{ \binom{l+K}{K} \binom{2k-2K}{k-K} \binom{k+l+1}{k} }{ \binom{2k}{k}  \binom{2(k+l+1)}{2k} }} \ket{\varphi_{K+l, K, 2k-2K}}. \]

%From that state we can get any other  $\ket{\psi_{k,l,m}}$ by applying $L_-$ the right number of time. Indeed, we have
%\[ \ket{\psi_{k,l,m}}=\frac{(L_-/\hbar)^{l-m}}{\prod_{j=m+1}^l \sqrt{l(l+1)-j(j-1)}} \ket{\psi_{k,l,l}}. \]
%
%First, we simplify the product at the denominator
%\bea
%\prod_{j=m+1}^l \sqrt{l(l+1)-j(j-1)} &=& \prod_{j=m+1}^l \sqrt{l+j}\sqrt{l-j+1}, \\
%&=& \sqrt{\frac{(2l)!}{(l+m)!}}\sqrt{(l-m)!}=\sqrt{\binom{2l}{l+m}} (l-m)!.
%\eea
%We also have
%\[ \left(\frac{L_-}{\hbar}\right)^{l-m} = 2^{\frac{l-m}{2}} (a^\dagger_l a_z - a^\dagger_z a_r)^{l-m} = 2^{\frac{l-m}{2}} \sum_{j=0}^{l-m} (-1)^j \binom{l-m}{j} (a^\dagger_l a_z)^{l-m-j} (a^\dagger_z a_r)^j, \]
%and by noting that
%\bea
%a^{k} \ket{n} &=& \sqrt\frac{n!}{(n-k)!} \ket{n-k}, \\
%(a^\dagger)^{k} \ket{n} &=& \sqrt\frac{(n+k)!}{n!} \ket{n+k},
%\eea
%we see that
%\bea
%(a^\dagger_l a_z)^{l-m-j} (a^\dagger_z a_r)^j  \ket{\varphi_{K+l, K, 2k-2K}} &=&   \sqrt\frac{(K+l)!}{(K+l-j)!} \sqrt\frac{(K+l-m-j)!}{K!} \nonumber \\
%&& \ \sqrt\frac{(2k-2K+j)!}{(2k-2K-l+m+2j)!} \sqrt\frac{(2k-2K+j)!}{(2k-2K)!} \nonumber \\
%&& \ \ket{\varphi_{K+l-j, K+l-m-j, 2k-2K-l+m+2j}},
%\eea
%and if we apply the variable change $j'=K+l-j$ ($j'=K+m,\ldots, K+l$)
%\[ (a^\dagger_l a_z)^{l-m-j} (a^\dagger_z a_r)^j  \ket{\varphi_{K+l, K, 2k-2K}} =   \sqrt\frac{(K+l)!(j-m)!(n-K-j)!^2}{j!K!(n+m-2j)!(2k-2K)!} \ket{\varphi_{j, j-m, n+m-2j}}.\]
%
%The expression became
%\[ \ket{\psi_{k,l,m}}=\sum_{K=0}^k   \sum_{j=K+m}^{K+l}  (-1)^j  c_{K,j}\ket{\varphi_{j, j-m, n+m-2j}},\]
%with
%\bea
%c_{K,j}&=&  \sqrt{ \frac{  \binom{l-m}{K+l-j}^2  \binom{l+K}{K} \binom{2k-2K}{k-K} \binom{k+l+1}{k} }{2^{l-m-2K} \binom{2k}{k}  \binom{2(k+l+1)}{2k} \binom{2l}{l+m}}}  \sqrt\frac{(K+l)!(j-m)!(n-K-j)!^2}{(l-m)!^2j!K!(n+m-2j)!(2k-2K)!}
%\eea


\chapter{Reconstructing  CT Signals}

In the previous lecture we focused on sampling of CT signals to produce a DT signal $x[n] = x(nT)$ with sample index $n$ and sample time $T$. In this lecture we consider \emph{reconstruction}, converting from a DT signal $x[n]$ to a CT signal $x(t)$ using a sample time $T$ as the spacing between samples. Ideally a conversion from $x(t)$ to $x[n]$ and back again would result in identical signal.

\section{Reconstruction Theory}
Given a DT signal $x[n]$ and a sample spacing $T$, we can define a corresponding CT signal as
\[
x_p(t) = \sum\limits_{n = -\infty}^{\infty} x[n] \, \delta(t-nT)
\]
the impulse train with each impulse weighted by the DT signal.

CT signal reconstruction can be viewed from two different (but equivalent) perspectives. In the time domain perspective, the CT signal $x(t)$ corresponding to a DT signal $x[n]$ can be viewed as \emph{interpolation}, where the values of the CT signal are equal to the DT signal at intervals of the sample time, i.e. $x(nT) = x[n]$, and in between the value of $x(t)$ is interpolated. If the interpolation is of zero-order, the value at $x(nT)$ is held constant until $x(nT+T)$. This is called a \emph{zero-order hold}, and can mathematically modeled as convolution of the weighted impulse train with a pulse $p(t) = u(t) - u(t-T)$ whose width is the sample time, called the interpolation function.
\[
y(t) = p(t)*x_p(t)
\]
This is illustrated below
\begin{center}
  \includegraphics[scale=1]{graphics/zero-order-hold-interp.pdf}
\end{center}

The zero-order hold is not a very accurate representation of a band-limited signal. So, what interpolation function is optimal?

To answer this question we can turn to the alternative perspective on reconstruction, that of the frequency domain. Recall the sampled signal $x(nT)$ in the frequency domain can be viewed as the summation of the Fourier transform of $x(t)$, $X(j\omega)$, and periodic replicas or images centered at multiples of the sampling frequency. If we assume the original signal was band-limited and sampled appropriately (using the Nyquist criteria), then if we ideal low-pass filter the sampled signal we will preserve the central portion of the Fourier spectrum that corresponds to the original signal, and chop off the images. For this reason the reconstruction filter is also called an anti-imaging filter.

\begin{center}
  \includegraphics[scale=1]{graphics/bandlimitedreconstructed1.pdf}
\end{center}

Recall filtering is multiplication the frequency domain and convolution in the time domain, so the optimal interpolation function corresponds to the impulse response of the ideal low-pass filter with cutoff frequency $\omega_c = 2\pi B$, a sinc function.

\[
h(t) = \mathcal{F}^{-1} \left\{ H(j\omega) \right\} = \frac{1}{2\pi} \int\limits_{\-\omega_c}^{\omega_c} e^{j\omega t} \; d\omega = \frac{1}{\pi t}\sin(\omega_c t)
\]

Thus the ideal ideal interpolation function is the sinc function, and reconstruction is low-pass filtering of the weighted impulse train $x_p(t)$ \footnote{As an aside this also gives an intuitive view of convolution with an impulse train, as interpolation}.

\section{Practical Reconstruction}

As we have seen before we cannot physically represent the impulse train nor the ideal low-pass filter. Thus practical reconstruction uses an approximation of the ideal reconstruction filter by a digital-to-analog converter (DAC), followed by a causal (and thus physically possible) low-pass filter.

\subsection{Zero-order hold using an R-2R ladder}

A zero-order hold DAC can be implemented by a circuit called a resistor ladder. Consider a digital output with $N$ bits and a reference voltage $V_{ref}$ (for example an 8-bit output port on a micro-controller using CMOS 3.3v logic).

If this port is connected to a resistor network consisting of resistor values $R$ and $2R$ as follows

\begin{center}
  \includegraphics[scale=1]{graphics/resistor-ladder.pdf}
\end{center}

then depending on the bit pattern at the output port $V$, the output of the buffer op-amp will be
\[
V_o = V_{ref}\frac{V}{2^N}
\]

If the port value is changed every sample time $T$, then the resister ladder and buffer op-amp combine to implement a zero-order hold circuit.

\subsection{Reconstruction(anti-imaging) filter}

The zero-order hold is followed by the reconstruction (anti-imaging) filter which low-pass filters the output and smooths-out the jumps from value to value.

\begin{center}
  \includegraphics[scale=1]{graphics/sinc-interp.pdf}
\end{center}

In general the reconstruction filter is of a similar, or identical form to the anti-aliasing filter.


%!TEX root=ClassNotes.tex

\section{Applications of Integrals}

\subsection{Improper Integrals}
Improper integrals compute areas of unbounded regions. These integrals are a source of several false paradoxes. As we'll see below, counter-intuitive as it may seem, unbounded regions {\it can} have finite areas.\\

We'll study two types of improper integrals: depending on whether the corresponding region is unbounded along the $x$-axis or along the $y$-axis.
The idea is to {\it view an unbounded region as a limit of bounded regions}.

\begin{definition}
  {\bf Improper integrals} (of the first kind) are integrals where one (or both) of the bounds is $\pm \infty$.\footnote{Note that $\infty$ is not a number here either, but represents a limit.}
  Let $f$ be a continuous function.
  Define,
	\begin{align*}
		\int \limits_a^{\infty} f(t) \: dt
		 & =
		\lim \limits_{x \rightarrow \infty}\int \limits_a^{x} f(t) \: dt \\
		\int \limits_{-\infty}^a f(t) \: dt
		 & =
		\lim \limits_{x \rightarrow -\infty}\int \limits_{x}^{a} f(t) \: dt \\
		\int \limits_{-\infty}^{\infty} f(t) \: dt
		 & =
		\lim \limits_{x \rightarrow \infty}\int \limits_0^{x} f(t) \: dt
    +
    \lim \limits_{x \rightarrow -\infty}\int \limits_{x}^{0} f(t) \: dt
	\end{align*}
  where $a$ is a real number.
\end{definition}

\begin{exercise}
	For the following problems, draw the region whose area the integral is computing (you can use a calculator for graphing) and compute the integral.
	\begin{multicols}{2}
		\begin{enumerate}
			\item $\int \limits_1^{\infty} \dfrac{1}{t} \: dt$
			\item $\int \limits_1^{\infty} \dfrac{1}{t^2} \: dt$
			\item $\int \limits_0^{\infty} \sin t \: dt$
			\item $\int \limits_0^{\infty} 2t e^{-t^2} \: dt$
			\item $\int \limits_{1}^{\infty} \dfrac{9}{(1 - 3t)^4} \: dt$
			\item $\int \limits_{-\infty}^{\infty} \dfrac{6 t^3}{(t^4 + 1)^2} \: dt$
		\end{enumerate}
	\end{multicols}
\end{exercise}

\begin{exercise}
  \begin{enumerate}
    \item The choice of the number 0 in the definition of $\int_{-\infty}^{\infty}$ is not important. We can replace 0 with any real number and the answer would not change.
    Compute
    \begin{align*}
      \lim \limits_{x \rightarrow \infty}\int \limits_1^{x} \dfrac{6 t^3}{(t^4 + 1)^2} \: dt
      +
      \lim \limits_{x \rightarrow -\infty}\int \limits_{x}^1 \dfrac{6 t^3}{(t^4 + 1)^2} \: dt
    \end{align*}
    and compare it with Problem 6 of the previous exercise. Why does the answer not change?

    \item On the other hand, consider the following change.
    Compute
    \begin{enumerate}
      \item $\lim \limits_{x \rightarrow \infty}\int \limits_{-x}^{x} \sin t \: dt$, and
      \item $\lim \limits_{x \rightarrow \infty}\int \limits_0^{x} \sin t \: dt
      +
      \lim \limits_{x \rightarrow -\infty}\int \limits_{x}^{0} \sin t \: dt$
    \end{enumerate}
    Why are the two answers different?
  \end{enumerate}
  Thus, when limits are involved, our intuition does not always provide us with the correct answer.
\end{exercise}

\begin{definition}
  {\bf Improper integrals} (of the second kind) are integrals where the function being integrated has a discontinuity in the interval of integration.
  Let $[a,b]$ be an interval and let $c \in [a, b]$ be a point such that $f(x)$ is discontinuous at $c$ but continuous on the rest of interval. Define,
	\begin{align*}
    \mbox{if } a < c < b, &&
    \int \limits_a^b f(t) \: dt
		 & =
		\lim \limits_{x \rightarrow c^{-}}\int \limits_a^{x} f(t) \: dt
    +
    \lim \limits_{x \rightarrow c^{+}}\int \limits_{x}^{b} f(t) \: dt \\
    \mbox{if } c = a, &&
    \int \limits_c^b f(t) \: dt
		 & =
    \lim \limits_{x \rightarrow c^{+}}\int \limits_{x}^{b} f(t) \: dt \\
    \mbox{if } c = b, &&
    \int \limits_a^c f(t) \: dt
		 & =
		\lim \limits_{x \rightarrow c^{-}}\int \limits_a^{x} f(t) \: dt
	\end{align*}
\end{definition}

\begin{exercise}
	For the following problems, draw the region whose area the integral is computing (you can use a calculator for graphing if you like) and compute the integral.
	\begin{multicols}{2}
		\begin{enumerate}
			\item $\int \limits_0^{3} \dfrac{1}{\sqrt{3 - t}} \: dt$
			\item $\int \limits_{2}^{3} \dfrac{4t}{\sqrt[3]{t^2 - 4}} \: dt$
			\end{enumerate}
	\end{multicols}
\end{exercise}

It is possible to compute integrals of more general unbounded regions using the same ideas.

\subsection{Arc Length}
Recall that the integral $\int_a^b f(t) \: dt$ is defined as the limit of the Riemann sum
\begin{align*}
  \sum_{i = 0}^{n-1} m_i (x_{i+1} - x_{i})
\end{align*}
where $x_i$'s form a partition of the interval $[a,b]$ and $m_i$ is the min (inf) of the $f(x)$ values in the interval $[x_i, x_{i+1}]$. One way to {\it represent} this sum is as the sum of several quantities of the form
\begin{align*}
  y \: \Delta x
\end{align*}
where $\Delta x$ stands for the length of the interval $x_{i+1} - x_i$ which can be thought of as {\bf the change in $x$}, and $y$ stands for {\bf the minimum value of $f(x)$} for $x \in[x_i, x_{i+1}]$. So that
\begin{align}
  \label{eq:sum_limit}
  \lim \limits_{\Delta x \rightarrow 0} \left(\sum y \: \Delta x \right) = \int y \: dx
\end{align}
The reason for doing this, is that this allows us to interpret several {\it naturally occurring} quantities as integrals (which is why the Riemann sum formulation of integral is much more useful than the area formulation). One such example is an {\bf arc length}.\\

{\bf Arc length} is just a fancy name for the length of a curve. For example, the arc length of a circle of radius $r$ is $2 \pi r$. We are interested in computing the arc length of the graph of a function $y=f(x)$.
\begin{figure}[H]
  \centering
  \includegraphics[width=0.5\textwidth]{ArcLength.png}
  \caption*{Approximating the graph $y=f(x)$ by line segments}
\end{figure}
We do this by partitioning the interval $[a,b]$ and approximating the graph $ y = f(x)$ over each partition by a straight line. The length of each small straight line segment equals
\begin{align*}
  \sqrt{\Delta x^2 + \Delta y^2}
\end{align*}
and hence the arc length equals
\begin{align*}
  \lim \limits_{\Delta x \rightarrow 0}  \left(\sum \sqrt{\Delta x^2 + \Delta y^2}\right)
\end{align*}
This looks very close to \eqref{eq:sum_limit}. Next we need to separate a $\Delta x$ term.
\begin{align*}
  \lim \limits_{\Delta x \rightarrow 0} \left(\sum \sqrt{1 + \left(\dfrac{\Delta y}{\Delta x}\right)^2} \Delta x \right)
\end{align*}
This looks exactly like \eqref{eq:sum_limit} with $y$ replaced with $\sqrt{1 + \left(\frac{\Delta y}{\Delta x}\right)^2}$. Finally we notice that
\begin{align*}
  \lim \limits_{\Delta x \rightarrow 0} \dfrac{\Delta y}{\Delta x} = \dfrac{dy}{dx}
\end{align*}
which gives us the formula for the arc length.
\begin{theorem}
  \label{thm:arclength}
  Let $f$ be a differentiable function on the interval $[a,b]$. The arc length of the graph $y = f(x)$ over the interval $[a,b]$ is given by the integral
  \begin{align*}
    \int \limits_a^b \sqrt{1 + \left(\dfrac{dy}{dx} \right)^2}  \: dx
  \end{align*}
\end{theorem}
While the above derivation is not a {\it proof}, it can be converted into a proof using $\sup$'s and $\inf$'s, using the same method we used to prove the Fundamental Theorem of Calculus.

\begin{exercise}
  Using the arc length formula in Theorem \ref{thm:arclength} compute the following:
  \begin{enumerate}
    \item Circumference of a circle of radius r.
    \item Arc length of the parabola $y = x^2$ from 0 to 1.
    \item Arc length of line segment $y = m x$ from $a$ to $b$, where $m$, $a$, $b$ are real numbers. Explain the result geometrically.
  \end{enumerate}
\end{exercise}

\begin{exercise}
  Here we encounter the first example of a {\it naturally occurring} unsolvable integral.
  Find the expression for the arc length of the sine curve $y = \sin x$ from 0 to $a$.
  This integral, called an {\bf elliptic integral}, cannot be algebraically computed.\footnote{Even though elliptic integrals cannot be simplified algebraically, they have deep connections to number theory.
  The impossibility of algebraically evaluating elliptic integrals presents the first obstacle towards a complete understanding of prime numbers!\\
  See: \url{https://en.wikipedia.org/wiki/Elliptic_function}}
\end{exercise}

\begin{exercise}{\bf (Optional)}
  In this problem, you'll need to use the fact that the (curved) surface area of a cylinder is $2 \pi r h$ and the volume is $\pi r^2 h$, where $r$ is the radius and $h$ is the height of the cylinder.
  \begin{figure}[H]
    \centering
    \includegraphics[width=0.5\textwidth]{SurfaceOfRevolution.png}
  \end{figure}
  \end{exercise}
  \begin{enumerate}
    \item Derive the formula for the area of the surface obtained by revolving the curve $y = f(x)$ over the interval $[a,b]$ about the $x$-axis.
   Such a surface is called a {\bf surface of revolution}.
    \item Derive the formula for the volume of the solid obtained by revolving the region under the curve $y = f(x)$ over the interval $[a,b]$ about the $x$-axis.
    Such a solid is called a {\bf solid of revolution}.
  \end{enumerate}
We can do both of these constructions by revolving the curve about the $y$-axis instead of the $x$-axis. In this case, we simply switch $x$ and $y$ and use the function $x=f^{-1}(y)$ instead of $y = f(x)$.\\

\subsection{Differential Equations}
It would be remiss to not mention differential equations while studying applications of calculus.
Calculus was invented by Newton to find solutions to his laws of motion, which are a set of differential equations.

A {\bf differential equation} is any equation involving derivatives $\frac{dy}{dx}$, $\frac{d^2y}{dx^2}$ etc.
Solving a differential equation means finding a function $y = f(x)$ that satisfies the equation.

We've already encountered one very simple differential equation.
The Fundamental Theorem of Calculus is essentially a statement about the solution of a differential equation.
We can rephrase it as saying:\\ {\it The solution to the differential equation
\begin{align*}
  \dfrac{dy}{dx} = g(x)
\end{align*}
is given by
\begin{align*}
  y = \int g(x) \: dx.
\end{align*}
}

In this section, we'll see some examples of differential equations without going into any details.
It is not possible to {\it solve} this differential equation in the same way that we solve algebraic equations. Instead, we {\it guess} a solution and verify it.

\begin{exercise}
  A {\bf separable} differential equation is of the form
  \begin{align*}
    \dfrac{dy}{dx} = f(x) \cdot g(y)
  \end{align*}
  for some functions $f$ and $g$. The solution of this differential equation is given by
  \begin{align*}
    \int \dfrac{1}{g(y)} \: dy  = \int f(x) \: dx
  \end{align*}
  It is called separable because the variables $x$ and $y$ can be ``separated'' i.e. all the terms involving $y$ are on the left hand side and all the terms involving $x$ are on the right hand side. A non-example is $\frac{dy}{dx} = x + y$.

  Find solutions for the following separable differential equations and verify that your solution satisfies the differential equation.
  \begin{multicols}{2}
    \begin{enumerate}
      \item $\dfrac{dy}{dx} = 2y$
      \item $\dfrac{dy}{dx} = xy$
      \item $\dfrac{dy}{dx} = \dfrac{3x^2}{\cos y}$
      \item $\dfrac{dy}{dx} = - \dfrac{y}{x}$
      \item $\dfrac{dy}{dx} = - \dfrac{x}{y}$
      \item $\dfrac{dy}{dx} = e^{-y}(2x - 4) $
    \end{enumerate}
  \end{multicols}
\end{exercise}

\newpage
\begin{exercise}
  The following differential equation is called a {\bf damped harmonic oscillator},
  \begin{align*}
    m \dfrac{d^2y}{dx^2} + c \dfrac{dy}{dx} + k y = 0
  \end{align*}
  where $m$, $k$, and $c$ are constants.
  This differential equation is used to model simple oscillatory systems.
  \begin{enumerate}
    \item Find a number(s) $r$ such that $y=e^{rx}$ solves the differential equation
    \begin{align*}
       \dfrac{d^2y}{dx^2} - 4 y = 0
    \end{align*}
    \item Find a number(s) $r$ such that $y=e^{rx}$ solves the differential equation
    \begin{align*}
       \dfrac{d^2y}{dx^2} + 4 y = 0
    \end{align*}
    \item Find a number(s) $r$ such that $y=e^{rx}$ solves the differential equation
    \begin{align*}
       \dfrac{d^2y}{dx^2} - 2 \dfrac{dy}{dx} + y = 0
    \end{align*}
    For this $r$, verify that $y = x e^{rx}$ is also a solution.
    \item Notice that the three equations have very different kinds of solutions.
    How would you describe the difference between the three equations?
  \end{enumerate}
\end{exercise}

Differential equations in general are very very difficult to solve. Finding solutions to differential equations has been a guiding question for much of modern mathematics. It is still not known whether a solution exists to the differential equation that describes fluid motion.\footnote{See \url{https://en.wikipedia.org/wiki/Navier-Stokes_equations}.} Nevertheless a lot of techniques now exist for solving a variety of differential equations either exactly or numerically.
Differential equations is one of the most active areas of mathematical research today.


\chapter{Linear Regression}
\label{chp:linreg}

%Linear regression is an approach in supervised learning. It is used for predicting a quantitative response and it is widely used for statistical learning methods. It may seem that it is a bit simple compared to some of the more modern statistical learning approaches, but it serves as a good base as newer approaches can be seen as a generalization or extension of the linear regression.
%This chapter presents the key ideas of a linear regression model and it will describe the least squares approach
%which is the most commonly used to fit this model\\

This chapter presents Linear regression which is an approach in supervised learning that can be used for predicting a quantitative response. Furthermore the least squares approach is explained as the most commonly used method for fitting a linear regression model.

\section{Simple Linear Regression}

Simple linear regression is used for predicting a quantitative response, often denoted Y, based on a single predictor, often denoted X.
The relationship between X and Y can be expressed as:

\begin{equation}
Y \approx \beta_0 + \beta_1X
\end{equation}

In equation (2.1) $\beta_0$ represents the intercept and $\beta_1$ which is multiplied with X represents the slope.
%This means that Y is approximately modeled as X or in other words Y is regressing on X.
%$\beta_0$ and $\beta_1$ are two unknown constants that represent the intercept and slope in the linear model. They are also known as model coefficients or parameters.
In linear regression training data containing observations of X and Y is used to estimate $\hat{\beta_0}$ and $\hat{\beta_1}$.
This can be mathematically expressed as:

\begin{equation}
\hat{y} = \hat{\beta_0} + \hat{\beta_1}x
\end{equation}

%$\hat{y}$ is the prediction of Y based on X = x. The \textit{hat} %symbol is used to denote the estimated value for an unknown parameter or coefficient or to predict a value of the response.

%When estimating the coefficients
%$\hat{\beta_0}$ and  $\hat{\beta_1}$ are in practice unknown. So %in order to use (2.1) to make predictions, $\hat{\beta_0}$ and  %$\hat{\beta_1}$ needs to be estimated so the data can be used.

%First Let
%\begin{equation}
%(x_1, y_1), (x_2, y_2),..., (x_n, y_n)
%\end{equation}

%represent n observation pairs where each of them have a measurement of X and a measurement of Y. n can represent lots of thing. can be the number of different markets, or different cities. It depends on the data.

$\hat{\beta_0}$ and  $\hat{\beta_1}$ must be estimated to fit all the observations as good as possible. This can be ensured by using the least squared approach.
%In other words $\hat{\beta_0}$ is the intercept and  $\hat{\beta_1}$ is a slope, so the resulting line is as close as possible to n\{th} data points

%First the prediction for Y will be written as $\hat{y_i} = \hat{\beta_0} + \hat{\beta_1} x_i$ based on the \emph{i}th value of X.
%Then $e_i = y_i - \hat{y_i}$ will represent the \emph{i}th residual, which is the difference between the \emph{i}th observed response value and the \emph{i}th response value that is predicted by the linear model.

Least squared approach determines the $\hat{\beta_0}$ and  $\hat{\beta_1}$ to minimize the residual sum of squares.
The residual is defined as the difference between the predicted and the true response value: $y-\hat{y}$. $\hat{y}$ can be replaced with $\hat{\beta_0} + \hat{\beta_1}x$ and the residual sum of squares can then be defined as:

\begin{equation}
RSS = (y_1 - (\hat{\beta_0} + \hat{\beta_1}X_1))^2 + (y_2 - (\hat{\beta_0} + \hat{\beta_1}X_2))^2 + ... + (y_n - (\hat{\beta_0} + \hat{\beta_1}X_n))^2
\end{equation}

%The least squares approach can then be used to choose $\hat{\beta_0}$ and $\hat{\beta_1}$ to minimize the RSS (2.5).
%The minimization is written as:

The least squares coefficients can then be estimated from the minimizers of RSS defined in equation in (2.4) and (2.5).
\begin{equation}
\hat{\beta_1} = \dfrac{\sum\limits_{i=1}^n(x_i - \bar{x})(y_i - \bar{y})}{\sum\limits_{i=1}^n(x_i - \bar{x})^2}
\end{equation}

\begin{equation}
\hat{\beta_0} = \bar{y} -\hat{\beta_1}\bar{x}
\end{equation}

%n (3.4) $\hat{y} \equiv \dfrac{1}{n}\sum\limits_{i=1}^ny_i$ and %$\hat{x}\equiv \dfrac{1}{n}\sum\limits_{i=1}^nx_i$ are the ample eans and the function is used to define the least squares coefficient estimates for a simple linear regression

\subsection{Lab 3.6.2 - Simple Linear Regression}

In Lab 3.6.2 the data set \emph{Boston} is given. It consists of records from about 506 neighborhoods around Boston. The task is to predict \emph{medv} (median house value) via the predictor \emph{lstat} (percent of households with low socioeconomic status).

The data is initially prepared before the linear regression can be performed. This is shown in listing \ref{lst:lin_reg_prep}. The data is loaded and then the \emph{lstat} data is added to \emph{boston\_X\_train} and the \emph{medv} data is added to \emph{boston\_Y\_train}.

\begin{lstlisting}[caption={Data preparation for linear regression}, label=lst:lin_reg_prep, mathescape=true]
boston = datasets.load_boston()

#Extract 'lstat'-
boston_X_train = boston.data[:, np.newaxis, 12]

#Extract 'medv'
boston_Y_train = boston.target
\end{lstlisting}

To perform the linear regression the library sklearn is used in python. In listing \ref{lst:lin_reg} it is shown how a linear regression model is fitted to the training data \emph{boston\_X\_train} and the \emph{boston\_Y\_train}.

\begin{lstlisting}[caption={Python linear regression function}, label=lst:lin_reg, mathescape=true]
lm = linear_model.LinearRegression()
lm.fit(boston_X_train, boston_Y_train)
\end{lstlisting}

\FloatBarrier

The code in listing \ref{lst:lin_reg_plot} is used to plot \emph{medv} and \emph{lstat} along with the least squares regression line. In line 1 the test data is plotted to see how it corresponds to the least squares regression line. To plot the least squares regression line the command 'plot' is used. This is seen in line 2 where 'boston\_X\_test' is the test data which consist of the \emph{lstat} data. The result is plotted in Figure \ref{fig:lin_reg_plot}. The plots shows that the relation between \emph{lstat} and \emph{medv} is somewhat non-linear.

\begin{lstlisting}[caption={Python plotting of linear regression function}, label=lst:lin_reg_plot, mathescape=true]
plt.scatter(boston_X_test, boston_Y_test,  color='black')
plt.plot(boston_X_test, lm.predict(boston_X_test), color='blue',
linewidth=3)

plt.xticks(())
plt.yticks(())

plt.show()
\end{lstlisting}

\myFigure{lin_reg_plot.PNG}{Linear regression of \emph{medv} and \emph{lstat}}{fig:lin_reg_plot}{0.5}

\section{Multiple Linear Regression}

When using simple linear regression only one predictor is used. But often more than one predictor is available. Instead multiple linear regression uses two or more predictors to predict the response variable Y. The relationship between the predictor variable and response variable with simple linear regression can be described as a one-to-one, while multiple linear regression is a many-to-one relationship, as illustrated in Figure \ref{fig:relation}.

\myFigure{relation.PNG}{Relationship between predictor variables "P.V" and response variables "R.V" in Simple- and Multiple Linear Regression}{fig:relation}{0.8}

\FloatBarrier
As specified in Lab 3.6.2, the predictors are called X, while Y is the response. This means for multiple linear regression, that the relationship between X and Y is expressed as:

\begin{equation}
Y \approx \beta_0 + \beta_1 * X_1 + \beta_2 * X_2 + … + \beta_k * X_k
\end{equation}

Similar to simple linear regression, the regression coefficients are estimated using the least squares approach so that $\beta_0$, $\beta_1$,...$\beta_p$ is chosen to minimize the RSS.

\subsection{Lab 3.6.3 - Multiple Linear Regression}

In this exercise multiple linear regression is used. In Listing \ref{lst:mul_lin_reg} all 13 predictors in the Boston data set is used for performing multiple linear regression.

\begin{lstlisting}[caption={Linear regression of multiple predictors}, label=lst:mul_lin_reg, mathescape=true]
boston_X_train = boston.data
boston_Y_train = boston.target

lm = linear_model.LinearRegression()
lm.fit(boston_X_train, boston_Y_train)
\end{lstlisting}

%In listing \ref{lst:print} the calculated coefficients, intercept and variance for the linear regression model is printed.

%\begin{lstlisting}[caption={Printing regression overview}, label=lst:print, mathescape=true]
%print("Coefficients:")
%print(lm.coef_)
%print("Intercept: ")
%print(lm.intercept_)
%print("Variance score: %.2f" % lm.score(boston_X_test, boston_Y_test))
%\end{lstlisting}

The printed output is shown in figure \ref{fig:multiOutput}. The output first shows the estimated coefficients for the linear regression model. The first value -0.1072 is $\beta_1$ and is multiplied with the predictor \emph{crim}. The next value 0.0464 is $\beta_2$ and is multiplied with the predictors \emph{zn}. In total there are 13 coefficients, one for each predictor.

Furthermore the intercept value $\beta_0$ is printed. This is the value of the interception with the y-axis. Lastly multiple $R^2$ is printed. The $R^2$ of the simple linear regression was only calculated to 0.544, while for the multiple linear regression model including all 13 predictors the $R^2$ coefficient was improved to 0.74. This indicates that the model is improved using multiple linear regression.

\myFigure{MultiLinRegOutput.PNG}{Resulting coefficients, intercept value and multiple $R^2$ for the multiple linear regression model}{fig:multiOutput}{0.8}


\documentclass{article}

\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\title{Solution to the Monty Hall Problem with the Bayesian Theorem}
\date{\today}
\author{Lukas Prokop}

\newcommand{\Pc}[2]{P(#1\,|\,#2)}

\begin{document}
\maketitle

\section{The problem}
%
In a quizshow, the candidate has to select one of three doors.
Behind two doors there is a goat (an undesirable prize).
Behind the one door there is a car.
At the beginning the candidate tries to guess the door with the car.
Afterwards the moderator Monty Hall opens one of the doors with a goat which is not the candidate's door.
So two doors are left where one of them is the one, the candidate has chosen.
Monty asks the candidate whether he would like to change his mind; he is
free to select the other door.

In general, is it better to change the mind, stick to the first selection
or doesn't it matter?

\section{Bayesian approach}
%
Assumption (one of three cases): The candidate selects door \#1.
\begin{center}
  \begin{tabular}{cc}
    $D_1$ & Car behind door \#1 \\
    $D_2$ & Car behind door \#2 \\
    $D_3$ & Car behind door \#3
  \end{tabular}
\end{center}

\noindent
We define the corresponding probabilities:
\[
    P(D_1) = \frac13  \qquad
    P(D_2) = \frac13  \qquad
    P(D_3) = \frac13
\]

We somehow have to model the behavior of the moderator.
Therefore we introduce a hypothesis $H$ that he opens door \#3.
The probability that he selects door \#3 in favor of door \#2
is $\frac12$, because \#1 is already taken by the candidate:
\[
    P(H) = \frac12
\]

\noindent
Furthermore,
\[
    \Pc{H}{D_1} = \frac12  \qquad
    \Pc{H}{D_2} = 1        \qquad
    \Pc{H}{D_3} = 0
\]

So the probability that he opens door \#3 under the condition that door \#1
contains the car is $\frac12$ (moderator will select door \#2 or door \#1
because \#1 is candidate's door). The probability is $1$ if the car is in
door \#2 (door \#1 is candidate's door and \#2 contains the car).
And if the car is in door \#3, the moderator will never show door \#3.

We can spot some asymmetry here. This point is evidently responsible why
an intuition like ``it does not matter'' might be wrong.

So we will use the Bayes' Theorem:
\[
    \underbrace{\Pc{A}{B}}_{\text{posterior probability}} =
      \frac{\overbrace{\Pc{B}{A}}^{\text{Likelihood}} \cdot
        \overbrace{P(A)}^{\text{priori probability}}
      }{
        \underbrace{P(B)}_{\text{expectedness}}
      }
\]

The important question is: Under the assumption that we have selected door \#1
(our first assumption) and Monty opens door \#3 (our hypothesis or
``expectedness''), what is the probability that the car is behind door \#1
compared to door \#2?
\[
    \Pc{D_1}{H} = \frac{\Pc{H}{D_1} \cdot P(D_1)}{P(H)}
        = \frac{\frac12 \cdot \frac13}{\frac12} = \frac13
\] \[
    \Pc{D_2}{H} = \frac{1 \cdot \frac13}{\frac12} = \frac23
\]

So apparently, the probability that door \#2 contains the car is higher than
for door \#1. Or in other words: in this case the candidate should prefer
to change his mind.

Now we have to evaluate this for other hypotheses (moderator opens door \#2)
and other candidate selections (door \#2 and door \#3). The approach is the
same and not carried out here. But the conclusion keeps the same in all cases:
Being open to change your mind will make you probabilistically win the car.

\end{document}


\documentclass[11pt, oneside]{article}

\usepackage{../../shared/preamble}
\addbibresource{../../shared/references.bib}

\usepackage{../sets/sets}
\usepackage{../groups/groups}
\usepackage{../topological-spaces/topological-spaces}
\usepackage{../real-numbers/real-numbers}
\usepackage{vector-spaces}

\title{Vector Spaces}
\author{Arthur Ryman, {\tt arthur.ryman@gmail.com}}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This article contains Z Notation type declarations for vector spaces and some related objects.
It has been type checked by \fuzz.
\end{abstract}

\section{Real Vector Spaces}

Real vector spaces are multidimensional generalizations of real numbers.
They are the objects studied in linear algebra and are foundational to differential geometry.

In the following let $\genT$ denote a set of elements which we'll refer to as {\em vectors}
and let $A$ denote an Abelian group over the vectors in which the binary operation is denoted as addition.
Let $v$ and $w$ denote vectors and
and let $x$ and $y$ denote real numbers.

\subsection{Notation for Vector Addition, Zero, and Negative: \zcmd{addV}, \zcmd{zeroV}, and \zcmd{negV}}

Let $v \addV w$ denote vector addition,
let $\zeroV$ denote the zero vector,
and let $\negV v$ denote the negative vector.

\subsection{Real Scalar Multiplication: \zcmd{mulS}, \zcmd{timesS}, and $RealScalarMultiplication$}

A {\em real scalar multiplication} operation on the vectors is an operation $smul$
that maps the pair $(x, v)$ to another vector, typically denoted $x \mulS v$ or $x \timesS y$,
such that
multiplication by 0 maps all vectors to the group identity element,
multiplication by 1 maps each vector to itself,
multiplication preserves group addition,
and multiplication distributes over both real and group addition.

Let $RealScalarMultiplication$ denote this situation.

\begin{schema}{RealScalarMultiplication}[\genT]
A: \abgroup \genT \\
smul: \R \cross \genT \fun \genT
\where
\LET (\_ \addV \_) == A; \\
\t1	\zeroV == identity\_element~A; \\
\t1	(\_ \mulS \_) == smul @ \\
\t2		\forall x, y: \R; v, w: \genT @ \\
\t3			\zeroR \mulS v = \zeroV \land \\
\t3			\oneR \mulS v = v \land \\
\t3			(x \mulR y) \mulS v = x \mulS (y \mulS v) \land \\
\t3			(x \addR y) \mulS v = x \mulS v \addV y \mulS v \land \\
\t3			x \mulS (v \addV w) = x \mulS v \addV x \mulS w
\end{schema}

\begin{itemize}
	\item Multiplying by $\zeroR$ gives the zero vector.
	\item Multiplying by $\oneR$ gives the same vector.
	\item Scalar multiplication is associative.
	\item Scalar addition distributes over scalar multiplication.
	\item Vector addition distributes over scalar multiplication.
\end{itemize}

\subsection{The Set of All Real Vector Spaces: \zcmd{vecR}}

A {\em real vector space} is a pair $(A, smul)$ where $A$ is an Abelian group and $smul$
is a real scalar multiplication on the elements of $A$.
The elements of $A$ are referred to as \textit{vectors}.

Let $\vecR \genT$ denote the set of all real vector spaces over $\genT$,

\begin{zed}
\vecR \genT == \{~ RealScalarMultiplication[\genT] @ (A, smul) ~\}
\end{zed}

\subsection{Real Linear Transformations: $RealLinearTransformation$}

Let $V_1$ and $V_2$ be real vector spaces and let $f$ be a homomorphism of the underlying
Abelian groups.
The map $f$ is said to be a {\em linear transformation} if $f$ maps scalar multiples of vectors to the
scalar multiple of the mapped vectors.

Let $RealLinearTransformation$ denote this situation.

\begin{schema}{RealLinearTransformation}[\genT, \genU]
f: \genT \fun \genU \\
V_1: \vecR \genT \\
V_2: \vecR \genU
\where
\LET A_1 == first~V_1; (\_ \mulS \_) == second~V_1; \\
\t1	A_2 == first~V_2; (\_ \timesS \_) == second~V_2 @ \\
\t2		f \in \homGroup(A_1, A_2) \land \\
\t2		(\forall x: \R; v: \genT @ \\
\t3			f(x \mulS v) = x \timesS(f~v))
\end{schema}

\begin{itemize}
\item The vector space $V_1$ has Abelian group $A_1$ and scalar multiplication $(\_ \mulS \_)$.
\item The vector space $V_2$ has Abelian group $A_2$ and scalar multiplication $(\_ \timesS \_)$.
\item The map $f$ is a homomorphism of the underlying Abelian groups.
\item The map $f$ maps scalar multiples of vectors in $\genT$ to scalar multiples of the mapped vectors in $\genU$.
\end{itemize}

\subsection{The Set of All Real Linear Transformations: \zcmd{homVecR}}

Let $V_1$ and $V_2$ be real vector spaces.
Let $\homVecR(V_1, V_2)$ denote the set of all linear transformations from $V_1$ to $V_2$.
A linear transformation is also referred to as a \textit{homomorphism} of vector spaces.

\begin{zed}
\homVecR[\genT, \genU] == \\
\t1	(\lambda V_1: \vecR \genT; V_2: \vecR \genU @ \\
\t2		\{~ f: \genT \fun \genU | \\
\t3			RealLinearTransformation[\genT, \genU] ~\})
\end{zed}

\section{Real $n$-tuples}

The preceding section described real vector spaces abstractly.
In this section we define a family of finite-dimensional real vector spaces
whose elements are finite sequences of real numbers, also referred to as \textit{real tuples}.

\subsection{The Set of All Finite Sequences of Real Numbers: \zcmd{Rinf}}

Let $n$ be a natural number.
A finite sequence of $n$ real numbers is called a {\it real $n$-tuple}.
Let $\Rinf$ denote the set of all real $n$-tuples for any $n$.

\begin{zed}
	\Rinf == \seq \R
\end{zed}

\subsection{The Component Projection Function: \zcmd{piRinf}}

The real numbers that comprise an $n$-tuple are called its \textit{components}.
Let $v$ be a real $n$-tuple and let $i$ be an integer where $1 \le i \le n$.
The real number $v(i)$ is the $i$-th component of $v$.
Let $\piRinf(i)$ be the projection function that maps an $n$-tuple $v$ to its $i$-th component $v(i)$.

\begin{axdef}
	\piRinf: \nat_1 \fun \Rinf \pfun \R
\where
	\forall i: \nat_1 @ \\
	\t1	\piRinf(i) = (\lambda v: \Rinf | i \in \dom v @ v(i))
\end{axdef}

\subsection{The Set of All Well-Dimensioned Subsets of $\Rinf$: \zcmd{DeltaRinf}}

A non-empty subset of $\Rinf$ is said to be \textit{well-dimensioned} if each of its elements has the same number
of components.
Let $\DeltaRinf$ denote the family of all well-dimensioned subsets of $\Rinf$ .

\begin{axdef}
	\DeltaRinf: \family~\Rinf
\where
	\DeltaRinf = \{~ S: \power_1 \Rinf | \forall v, w: S @ \# v = \# w ~\}
\end{axdef}

\subsection{The Dimension of a Well-Dimensioned Set of Tuples: \zcmd{dimRinf}}

Let $S \in \DeltaRinf$ be a well-dimensioned set of tuples.
The number of components of each tuple in $S$ is called its \textit{dimension}.
Let $\dimRinf(S)$ denote the dimension of $S$.

\begin{axdef}
\dimRinf: \DeltaRinf \fun \nat
\where
%	\forall n: \nat @ \forall U: \power(\Rtup(n)) @ \\
%	\t1	\dimRinf(U) = n
\forall S: \DeltaRinf @ \\
\t1	\dimRinf S = (\mu v: S @ \# v)
\end{axdef}

\subsection{The Set of All Compatible Pairs of Tuples: \zcmd{RinfDelta}}

The pair of real tuples $(v, w)$ is said to be \textit{compatible} if each member has the same number of components.
Let $\RinfDelta$ denote the set of all compatible pairs of real tuples.
If the pair $(v, w)$ is compatible then $v$ and $w$ are said to be compatible with each other.

\begin{axdef}
	\RinfDelta: \Rinf \rel \Rinf
\where
	\RinfDelta = \{~ v, w: \Rinf | \# v = \# w ~\}
\end{axdef}

\subsection{Addition of Compatible Tuples: \zcmd{addRinf}}

Let $v$ and $w$ be $n$-tuples.
Vector addition of $v$ and $w$ is the $n$-tuple $v \addRinf w$ defined by component-wise addition.

\begin{axdef}
	\_ \addRinf \_: \RinfDelta \fun \Rinf
\where
	\langle \rangle \addRinf \langle \rangle = \langle \rangle
\also
\forall n: \nat_1; v, w: \Rinf | n = \#v = \# w @ \\
\t1	v \addRinf w = (\lambda i: 1 \upto n @ v~i \addR w~i)
\end{axdef}

\subsection{Subtraction of Compatible Tuples: \zcmd{subRinf}}

Vector subtraction is defined similarly.

\begin{axdef}
	\_ \subRinf \_: \RinfDelta \fun \Rinf
\where
	\langle \rangle \subRinf \langle \rangle = \langle \rangle
\also
\forall n: \nat_1; v, w: \Rinf | n = \#v = \# w @ \\
\t1	v \subRinf w = (\lambda i: 1 \upto n @ v~i \subR w~i)
\end{axdef}

\subsection{The Negative of a Tuple: \zcmd{negRinf}}

Let $\negRinf v$ denote the negative of $v$.

\begin{axdef}
\negRinf: \Rinf \fun \Rinf
\where
\negRinf \langle \rangle = \langle \rangle
\also
\forall n: \nat_1; v: \Rinf | n = \# v @ \\
\t1	\negRinf v = (\lambda i: 1 \upto n @ \negR(v~i))
\end{axdef}

\subsection{Scalar Multiplication of a Tuple: \zcmd{smulRinf}}

Let $v$ be an $n$-tuple and let $c$ be a real number.
Scalar multiplication of $v$ by $c$ is the $n$-tuple $c \smulRinf v$ defined by component-wise multiplication.

\begin{axdef}
\_ \smulRinf \_ : \R \cross \Rinf \fun \Rinf
\where
\forall c: \R @ \\
\t1	c \smulRinf \langle \rangle = \langle \rangle
\also
\forall c: \R; n: \nat_1; v: \Rinf | n = \# v @ \\
\t1	c \smulRinf v = (\lambda i: 1 \upto n @ c \mulR (v~i))
\end{axdef}

\begin{remark}
Scalar multiplication is associative in the sense that $(a \mulR b) \smulRinf v = a \smulRinf (b \smulRinf v)$

\begin{zed}
	\forall a, b: \R; v: \Rinf @ \\
	\t1	(a \mulR b) \smulRinf v = a \smulRinf (b \smulRinf v)
\end{zed}

\end{remark}

\subsection{The Set of All Real $n$-tuples: \zcmd{Rtup}}

Let $\Rtup(n)$ denote $\R^n$, the set of all $n$-tuples for some given $n$.
\begin{axdef}
	\Rtup: \nat \fun \power \Rinf
\where
	\forall n: \nat @ \\
	\t1	\Rtup(n) = \{~ v: \Rinf | \# v = n ~\}
\end{axdef}

\begin{remark}

\begin{zed}
	\Rinf = \bigcup \{~ n: \nat @ \Rtup(n) ~\}
\end{zed}

\end{remark}

\begin{remark}
The subset $\Rtup(n)$ is well-dimensioned.

\begin{zed}
	\forall n: \nat @ \\
	\t1	\Rtup(n) \in \DeltaRinf
\end{zed}

\end{remark}

\begin{remark}
The dimension of $\Rtup(n)$ is $n$.

\begin{zed}
	\forall n: \nat @ \\
	\t1	\dimRinf(\Rtup(n)) = n
\end{zed}

\end{remark}

\subsection{Addition of $n$-tuples: $addRtup$}

Let $addRtup(n)$ denote the restriction of addition to $\Rtup(n)$.

\begin{zed}
addRtup == \\
\t1	(\lambda n: \nat @ \\
\t2		(\lambda v, w: \Rtup(n) @ v \addRinf w))
\end{zed}

\begin{example}
The binary operation $addRtup(n)$ defines an Abelian group over $\Rtup(n)$.

\begin{zed}
\forall n: \nat @ \\
\t1	addRtup(n) \in \abgroup(\Rtup(n))
\end{zed}

\end{example}

\subsection{Subtraction of $n$-tuples: $subRtup$}

Let $subRtup(n)$ denote the restriction of subtraction to $\Rtup(n)$.

\begin{zed}
subRtup == \\
\t1	(\lambda n: \nat @ \\
\t2		(\lambda v, w: \Rtup(n) @ v \subRinf w))
\end{zed}

\subsection{The Negative of an $n$-tuple: $negRtup$}

Let $negRtup(n)$ denote the restriction of the negative operation to $\Rtup(n)$.

\begin{zed}
negRtup == \\
\t1	(\lambda n: \nat @ \\
\t2		(\lambda v: \Rtup(n) @ \negRinf v))
\end{zed}

\begin{remark}
The operation $negRtup(n)$ is the inverse operation of the Abelian group $addRtup(n)$.

\begin{zed}
\forall n: \nat @ \\
\t1	negRtup(n) = inverse\_operation(addRtup(n))
\end{zed}

\end{remark}

\subsection{The Zero Real $n$-tuple: \zcmd{zeroRtup}}

Let $\zeroRtup(n)$ denote the $n$-tuple consisting of all zeroes.

\begin{axdef}
	\zeroRtup: \nat \fun \Rinf
\where
	\zeroRtup(0) = \langle \rangle
\also
	\forall n: \nat_1 @ \\
	\t1	\zeroRtup(n) = (\lambda i: 1 \upto n @ \zeroR)
\end{axdef}

\begin{remark}
Every component of $\zeroRtup(n)$ is $\zeroR$.

\begin{zed}
\forall n: \nat @ \\
\t1	\forall i: 1 \upto n @ \\
\t2	(\piRinf i)(\zeroRtup n) = \zeroR
\end{zed}

\end{remark}

\begin{remark}
The tuple $\zeroRtup(n)$ is in $\Rtup(n)$.

\begin{zed}
\forall n: \nat @ \\
\t1	\zeroRtup(n) \in \Rtup(n)
\end{zed}

\end{remark}

\begin{remark}
The tuple $\zeroRtup(n)$ is the identity element of the Abelian group $addRtup(n)$.

\begin{zed}
\forall n: \nat @ \\
\t1	\zeroRtup(n) = identity\_element(addRtup(n))
\end{zed}

\end{remark}

\subsection{Scalar Multiplication of an $n$-tuple: $smulRtup$}

Let $smulRtup(n)$ denote scalar multiplication restricted to $\Rtup(n)$.

\begin{zed}
smulRtup == \\
\t1	(\lambda n: \nat @ \\
\t2		(\lambda c: \R; v: \Rtup(n) @ c \smulRinf v))
\end{zed}

\subsection{The Real Vector Space of $n$-tuples: $vecRtup$}

Let $vecRtup(n)$ denote the real vector space of $n$-tuples.

\begin{zed}
vecRtup == \\
\t1	(\lambda n: \nat @ (addRtup(n), smulRtup(n)))
\end{zed}

\begin{remark}
The pair $vecRtup(n)$ defines a vector space over $\Rtup(n)$.

\begin{zed}
\forall n: \nat @ \\
\t1	vecRtup(n) \in \vecR(\Rtup(n))
\end{zed}

\end{remark}

\subsection{Linear Transformations of $n$-tuples: \zcmd{linRtup}}

Define $\linRtup(n,m)$ to be the set of all linear transformations from $\R^n$ to $\R^m$.
\begin{axdef}
	\linRtup: \nat \cross \nat \fun \power(\Rinf \pfun \Rinf)
\where
	\forall n,m: \nat @ \\
	\t1	\linRtup(n,m) = \homVecR(vecRtup(n), vecRtup(m))
\end{axdef}

\subsection{The Identity Transformation of $n$-tuples: \zcmd{idRtup}}

Let $\idRtup(n)$ denote the identity function on $\Rtup(n)$.

\begin{axdef}
	\idRtup: \nat \fun \Rinf \pfun \Rinf
\where
	\forall n: \nat @ \\
	\t1	\idRtup(n) = \id(\Rtup(n))
\end{axdef}

\begin{remark}
The function $\idRtup(n)$ is a linear transformation.

\begin{zed}
	\forall n: \nat @ \\
	\t1	\idRtup(n) \in \linRtup(n, n)
\end{zed}

\end{remark}

\section{The Metric Topology on Real $n$-tuples}

\subsection{The Dot Product of Tuples: \zcmd{dotRinf}}

The {\it inner} or {\it dot} product of $n$-tuples $v$ and $w$ is the real number $v \dotRinf w$
defined by the sum of the component-wise products.

\begin{axdef}
	\_ \dotRinf \_ : \RinfDelta \fun \R
\where
	\langle \rangle \dotRinf \langle \rangle = \zeroR
\also
	\forall x, y: \R; v, w: \Rinf | \# v = \# w @ \\
	\t1	(\langle x \rangle \cat v) \dotRinf (\langle y \rangle \cat w) = x \mulR y \addR v \dotRinf w
\end{axdef}

Each $\Rtup(n)$ is a real inner product space under the operation of dot product defined above.

\subsection{The Norm of a Tuple: \zcmd{normRinf}}

The norm $\norm{v}$ of the $n$-tuple $v$ is the positive square root of its dot product with itself.
$$
	\norm{v} = \sqrt{v \dotRinf v}
$$

Define $\normRinf(v)$ to be $\norm{v}$.
\begin{axdef}
	\normRinf: \Rinf \fun \R
\where
	\forall v: \Rinf @ \\
	\t1	\normRinf(v) = \sqrtR(v \dotRinf v)
\end{axdef}

The concepts of continuity, limits, and differentiability extend to functions between normed vector spaces such as $\R^n$.

\subsection{The Open Ball at a Tuple: \zcmd{ballRinf}}

Let $\ballRinf(v,r)$ denote the \textit{open ball} in $\Rtup(n)$ of radius $r  \in \Rpos$ centred at $v \in \Rtup(n)$.

\begin{axdef}
\ballRinf: \Rinf \cross \Rpos \fun \power \Rinf
\where
\forall v: \Rinf; r: \Rpos @ \\
\t1	\LET n == \# v @ \\
\t2		\ballRinf(v, r) = \{~ w: \Rtup(n) | \normRinf(v \subRinf w) \ltR r ~\}
\end{axdef}

\subsection{The Set of All Open Balls at an $n$-tuple: \zcmd{ballsRtup}}

Let $\ballsRtup(n)$ denote the family of all open balls in $\Rtup(n)$.

\begin{axdef}
	\ballsRtup: \nat \fun \family~\Rinf
\where
	\forall n: \nat @ \\
	\t1	\ballsRtup(n) =  \{~ v: \Rtup(n); r: \Rpos @ \ballRinf(v,r) ~\}
\end{axdef}

\begin{remark}
The set of all open balls in $\Rtup(n)$ is a family of sets in $\Rtup(n)$.

\begin{zed}
\forall n: \nat @ \\
\t1	\ballsRtup(n) \in \family(\Rtup(n))
\end{zed}

\end{remark}

\subsection{The Usual Topology on $n$-tuples: \zcmd{tauRtup}}

The \textit{usual topology} on $\Rtup(n)$ is the topology generated by the open balls in $\Rtup(n)$.
Let $\tauRtup(n)$ denote the usual topology on $\Rtup(n)$.

\begin{axdef}
	\tauRtup: \nat \fun \family~\Rinf
\where
	\forall n: \nat @ \\
	\t1	\tauRtup(n) = topGen[\Rtup(n)] (\ballsRtup(n))
\end{axdef}

\begin{remark}

If $n \in \nat$ then $\tauRtup(n)$ is a topology on $\Rtup(n)$.

\begin{zed}
	\forall n: \nat @ \tauRtup(n) \in top[\Rtup(n)]
\end{zed}
\end{remark}

\subsection{The Set of All Neighbourhoods of a Tuple: \zcmd{neighRinf}}

Let $v \in \Rtup(n)$.
An open set $U$ in the usual topology $\tauRtup(n)$ that contains $v$
is called a \textit{neighbourhood} of $v$.
Let $\neighRinf(v)$ denote the set of all neighbourhoods of $x$.

\begin{axdef}
\neighRinf: \Rinf \fun \family~\Rinf
\where
\forall n: \nat; v: \Rinf | n = \#v @ \\
\t1	\neighRinf(v) = \{~ U: \tauRtup(n) | v \in U ~\}
\end{axdef}

\begin{remark}

The set of all neighbourhoods of $v \in \Rtup(n)$ is a family of sets in $\Rtup(n)$.

\begin{zed}
\forall n: \nat; v: \Rinf | n = \#v @ \\
\t1	\neighRinf(v) \in \family(\Rtup(n))
\end{zed}

\end{remark}

\subsection{The Topological Space of $n$-tuples: \zcmd{tsRtup}}

Let $\tsRtup(n)$ denote the topological space defined by the usual topology on $\Rtup(n)$.

\begin{axdef}
	\tsRtup: \nat \fun topSpaces[\Rinf]
\where
	\forall n: \nat @ \\
	\t1	\tsRtup(n) = (\Rtup(n), \tauRtup(n))
\end{axdef}

\section{Continuity}

\subsection{Real-Valued Functions That Are Continuous on the Set of All $n$-tuples: \zcmd{CzeroRtup}}

A function $f \in \R^n \fun \R$ is said to be \textit{continuous} if it is continuous with respect to the usual topologies
on $\R^n$ and $\R$.
Let $\CzeroRtup(n)$ denote the set of these continuous functions.

\begin{axdef}
	\CzeroRtup: \nat \fun \power(\Rinf \pfun \R)
\where
	\forall n: \nat @ \\
	\t1	\CzeroRtup(n) = \CzeroTT(\tsRtup(n), \Rtau)
\end{axdef}

\subsection{Real-Valued Functions That Are Continuous on a Subset of $n$-tuples: \zcmd{CzeroSubsetRtup}}

Let $U$ be a subset of $\R^n$.
A function $f \in U \fun \R$ is said to be \textit{continuous on} $U$
if it is continuous with respect to the topology induced on $U$.
Let $\CzeroSubsetRtup(U)$ denote the set of these continuous functions.

\begin{axdef}
	\CzeroSubsetRtup: \DeltaRinf \fun \power(\Rinf \pfun \R)
\where
	\forall U: \DeltaRinf @ \\
	\t1	\LET n == \dimRinf U @ \\
	\t2		\CzeroSubsetRtup(U) = \CzeroTT(\tsRtup(n) \inducedTopSp U, \Rtau)
\end{axdef}

\subsection{Real-Valued Functions That Are Continuous at an $n$-tuple: \zcmd{CzeroPointRtup}}

A partial function $f$ from $\R^n$ to $\R$ is said to be \textit{continuous at} $x \in \R^n$ if its domain contains a neighbourhood $U$ of $x$
such that its restriction to $U$ is continuous on $U$.
Let $\CzeroPointRtup(x)$ denote the set of such functions.

\begin{axdef}
	\CzeroPointRtup: \Rinf \fun \power(\Rinf \pfun \R)
\where
	\forall x: \Rinf @ \\
	\t1	\LET n == \# x @ \\
	\t2		\CzeroPointRtup(x) = \{~ f: \Rtup(n) \pfun \R | \exists U: \neighRinf(x) | U \subseteq \dom f @ U \dres f \in \CzeroSubsetRtup(U) ~\}
\end{axdef}

\subsection{$m$-tuple-Valued Functions That Are Continuous on the Set of All $n$-tuples: \zcmd{CzeroRtupRtup}}

A mapping $f$ from $\Rtup(n)$ to $\Rtup(m)$ is said to be continuous if it is continuous with respect to the usual topologies.
Let $\CzeroRtupRtup(n,m)$ denote the set of these continuous mappings.

\begin{axdef}
	\CzeroRtupRtup: \nat \cross \nat \fun \power(\Rinf \pfun \Rinf)
\where
	\forall n, m: \nat @ \\
	\t1	\CzeroRtupRtup(n,m) = \CzeroTT(\tsRtup(n), \tsRtup(m))
\end{axdef}

\begin{example}
The function $\idRtup(n)$ is continuous.

\begin{zed}
	\forall n: \nat @ \\
	\t1	\idRtup(n) \in \CzeroRtupRtup(n,n)
\end{zed}

\begin{theorem}
Linear functions are continuous.

\begin{zed}
	\forall n, m: \nat@ \\
	\t1	\linRtup(n, m) \subseteq \CzeroRtupRtup(n,m)
\end{zed}

\end{theorem}

\end{example}

\subsection{$m$-tuple-Valued Functions That Are Continuous on a Subset of $n$-tuples: \zcmd{CzeroSubsetRtupRtup}}

Let $U$ be any subset of $\Rtup(n)$.
Let $\CzeroSubsetRtupRtup(U,m)$ denote the set of continuous mappings from the topology induced by $\tsRtup(n)$ on $U$ to $\tsRtup(m)$.

\begin{axdef}
	\CzeroSubsetRtupRtup: \DeltaRinf \cross \nat \fun \power (\Rinf \pfun \Rinf)
\where
	\forall n, m: \nat @ \\
	\t1	\forall U: \DeltaRinf | \dimRinf(U) = n @ \\
	\t2		\CzeroSubsetRtupRtup(U, m) = \CzeroTT(\tsRtup(n) \inducedTopSp U, \tsRtup(m))
\end{axdef}

\begin{remark}

\begin{zed}
	\forall n, m: \nat @ \\
	\t1	\CzeroSubsetRtupRtup(\Rtup(n),m) = \CzeroRtupRtup(n,m)
\end{zed}

\end{remark}

\subsection{$m$-tuple-Valued Functions That Are Continuous at an $n$-tuple:
$VectorConinuous$, \zcmd{CzeroPointRtupRtup}}

Let $x \in \Rtup(n)$ and let $f$ be a partial function from $\Rtup(n)$ to $\Rtup(m)$
whose domain includes some neighbourhood $U$ of $x$ such that $f$ restricted to $U$ is continuous.
In this case $f$ is said to be {\it continuous at $x$}.

\begin{schema}{VectorContinuous}
	n, m: \nat \\
	f: \Rinf \pfun \Rinf \\
	x: \Rinf
\where
	f \in \Rtup(n) \pfun \Rtup(m)
\also
	\exists U: \neighRinf(x) | \\
	\t1	U \subseteq \dom f @ \\
	\t2		U \dres f \in \CzeroSubsetRtupRtup(U,m)
\end{schema}

Let $\CzeroPointRtupRtup(x,m)$ denote the set of all partial functions $f$ from $\Rtup(n)$ to $\Rtup(m)$
that are continuous at $x$.

\begin{axdef}
	\CzeroPointRtupRtup: \Rinf \cross \nat \fun \power (\Rinf \pfun \Rinf)
\where
	\forall n, m: \nat @ \forall x: \Rtup(n)  @ \\
	\t1	\CzeroPointRtupRtup(x, m) = \\
	\t2		\{~ f: \Rtup(n) \pfun \Rtup(m) | VectorContinuous ~\}
\end{axdef}

\begin{example}
The function $\idRtup(n)$ is continuous at every point $x \in \Rtup(n)$.

\begin{zed}
	\forall n: \nat @ \forall x: \Rtup(n) @ \\
	\t1	\idRtup(n) \in \CzeroPointRtupRtup(x, n)
\end{zed}

\end{example}

\begin{theorem}
Linear functions are continuous everywhere.

\begin{zed}
	\forall n, m: \nat @ \\
	\t1	\forall x: \Rtup(n); L: \linRtup(n,m) @ \\
	\t2		L \in \CzeroPointRtupRtup(x, m)
\end{zed}

\end{theorem}

\section{Differentiability}

Let $x \in \R^n$ and let $f: \R^n \pfun \R^m$ be continuous at $x$.
Then $f$ is said to be {\it differentiable at $x$} if there exists a linear transformation $L: \R^n \fun \R^m$
such that $f(x + h) - f(x)$ is approximately linear in $h$ for very small $h$.
$$
f(x + h) - f(x) \approx  L(h) + O(h^2) \quad \text{when} \quad \norm{h} \approx 0
$$

This condition can be written as a limit.
$$
\lim_{h \to 0} \frac{\norm{f(x+h) - f(x) - L(h)}}{\norm{h}} = 0
$$

\subsection{The Difference Quotient: $DifferenceQuotient$ and $diffQuot$}

The limit exists when the following difference quotient function $q: \R^n \pfun \R$ is continuous at $0$.
$$
q(h) =
\begin{cases}
	\frac{\norm{f(x+h) - f(x) - L(h)}}{\norm{h}}	&	\text{if } h \neq 0\\
	0								&	\text{otherwise}
\end{cases}
$$

Given a function $f$ that is continuous at $x$, and a linear transformation $L$,
we can define the difference quotient $q$.
Clearly $q$ is uniquely determined by $f$, $x$, and $L$.
Let $DifferenceQuotient$ denote this situation.

\begin{schema}{DifferenceQuotient}
	VectorContinuous \\
	L: \Rinf \pfun \Rinf \\
	q: \Rinf \pfun \R
\where
	L \in \linRtup(n, m)
\also
	\dom q = \{~ h: \Rtup(n) | x \addRinf h \in \dom f ~\}
\also
	\forall h: \dom q | h \neq \zeroRtup(n) @ \\
	\t1	q(h) = \normRinf(f(x \addRinf h) \subRinf f(x) \subRinf L(h)) \divR \normRinf(h)
\also
	q(\zeroRtup(n)) = \zeroR
\end{schema}

\begin{itemize}
\item $L$ is a linear transformation from $\Rtup(n)$ to $\Rtup(m)$.
\item The difference quotient $q$ is defined on a subset of $\Rtup(n)$ that contains $\zeroRtup(n)$.
\item $q(h)$ is defined as the quotient when $h$ is non-zero.
\item $q(0)$ is defined as zero.
\end{itemize}

Let $diffQuot(f,x,L)$ denote the difference quotient $q$.

\begin{zed}
diffQuot == \{~ DifferenceQuotient @ (f, x, L) \mapsto q ~\}
\end{zed}

\subsection{The Derivative of a Continuous $m$-tuple-Valued Function: $VectorDifferentiable$}

The continuous function $f$ is \textit{differentiable at} $x$ when there exists a linear transformation $L$
such that the difference quotient $q$ is continuous at $0$.
In this case $L$ is unique and is referred to as the \textit{derivative at} $x$.

\begin{schema}{VectorDifferentiable}
VectorContinuous \\
L: \Rinf \pfun \Rinf
\where
\LET q == diffQuot(f, x, L) @ \\
\t1	q \in \CzeroPointRtup(\zeroRtup(n))
\end{schema}

\begin{itemize}
\item The continuous function $f$ is differentiable at $x$ with derivative $L$ if the resulting
difference quotient $q$ is continuous at $\zeroRtup(n)$.
\end{itemize}

\begin{remark}
If $L$ exists then it is unique.
\end{remark}

Let $\smoothRnm(x,m)$ denote the set of all functions $f \in \Rtup(n) \pfun \Rtup(m)$ that are smooth at $x \in \Rtup(n)$.

\printbibliography

\end{document}