\documentclass{article}
\usepackage{amsmath}
\usepackage{amssymb}
\title{Hierarchic Shape Functions}
\author{Brian Granzow}
\date{Apr 14, 2015}

\begin{document}
\maketitle

\section{Introduction}

This document serves to provide an overview of the hierarchic shape functions
provided in apf. Hierarchic shape functions are constructed in a manner so that
the basis $\mathcal{B}^{p+1}$ for the polynomial space $P^{p+1}(\Omega^e)$
over an element $\Omega^e$ is obtained by solely by the addition of new shape
functions to the basis $\mathcal{B}^p$ for the polynomial space
$P^p(\Omega^e)$. The hierarchic shape functions $N$ can be written in terms
of kernel functions $\phi_i$ and barycentric coordinates $A_i$ for elements.

\section{Legendre polynomials}

Legendre polynomials are typically derived over the interval $[-1,1]$. The
Legendre polynomials presented here are derived over the interval $[0,1]$
as it suits the parametric space for our reference tetrahedron and triangle,
as described later. These are commonly referred to as shifted Legendre
polynomials, which can be obtained by using the substitution $x \to 2x - 1$ for
standard Legendre polynomials. The first two shifted Legendre polynomials are
defined as: $L_0 = 1$ and $L_1 = 2x - 1$. The shifted Legendre polynomials
satisfy the recursion relation:
\[
\left\{ \frac{2i-1}{i} (2x-1)\right \} L_{i-1}(x) -
\left\{ \frac{i-1}{i} \right \} L_{i-2}(x) \quad i \geq 2
\]
The first few Legendre polynomials are listed below
\begin{align*}
L_0(x) &= 1 \\
L_1(x) &= 2x - 1 \\
L_2(x) &= 6x^2 - 6x + 1 \\
L_3(x) &= 20x^3 - 30x^2 + 12x - 1 \\
L_4(x) & = 70x^4 - 140x^3 + 90x^2 - 20x + 1
\end{align*}
The Legendre polynomials are orthogonal on $[0,1]$
\[
\int _0 ^1 L_i(x) L_j(x) \; \text{d}x \; =
\begin{cases}
0 &\quad i \neq j \\
1/(2i+1) &\quad i = j
\end{cases}
\]

\section{Lobatto functions}

The Lobatto functions on $[0,1]$ are integrated Legendre polynomials, and
are defined as
\begin{align*}
l_0(x) &= 1-x \\
l_1(x) &= x \\
l_i(x) & = \frac{1}{ \| L_{i-1} \|_2}
\int _0 ^x L_{i-1}(t) \; \text{d}t \quad i \geq 2
\end{align*}
From the orthogonal property of the Legendre polynomials, it is clear that
\[
\frac{1}{ \| L_{i-1} \|_2} = \sqrt{2i - 1}
\]
The first few Lobatto functions are listed below
\begin{align*}
l_0(x) &= 1-x \\
l_1(x) &= x \\
l_2(x) &= \sqrt{3} (x^2 - x) \\
l_3(x) &= \sqrt{5} (2x^3 - 3x^2 + x) \\
l_4(x) &= \sqrt{7} (5x^4 - 10x^3 + 6x^2 - x)
\end{align*}

\section{Kernel functions}

To define shape functions, it will be useful to define kernel functions
$\phi_i$ by decomposing the Lobatto functions as follows:
\[
l_i(x) = l_0(x) l_1(x) \phi_{i-2}(x) \quad i \geq 2
\]
The first few kernel functions are listed below
\begin{align*}
\phi_0(x) &= -\sqrt{3} \\
\phi_1(x) &= -\sqrt{5} (2x - 1) \\
\phi_2(x) &= -\sqrt{7} (5x^2 - 5x + 1)
\end{align*}

\section{Reference Triangle}
We define our reference triangle $\Omega^e_t$ in a parametric space such that
\[
\Omega^e_t = \left\{ \xi \in \mathbb{R}^3 \; : \;
0 < \xi_1, \xi_2 < 1; \; \xi_1 + \xi_2 < 1 \right\}
\]
The barycentric coordinates for the reference triangle can be defined in
terms of the $\xi$ coordinates as
\begin{align*}
A_1 &= 1 - \xi_1 - \xi_2 \\
A_2 &= \xi_1 \\
A_3 &= \xi_2
\end{align*}
There are 3 vertices associated with our parent triangle. These can be
expressed in parametric space as
\begin{align*}
v_1(\xi) &= (0,0) \\
v_2(\xi) &= (1,0) \\
v_3(\xi) &= (0,1)
\end{align*}
They can similarly be expressed in barycentric coordinates as
\begin{align*}
v_1(A) &= (1,0,0) \\
v_2(A) &= (0,1,0) \\
v_3(A) &= (0,0,1)
\end{align*}

\subsection{Vertex shape functions}

There are always only 3 shape functions associated with triangular vertices,
$v_1, v_2$, and $v_3$.
These are
\begin{align*}
N^{v_1}(A) &= A_1 \\
N^{v_2}(A) &= A_2 \\
N^{v_3}(A) &= A_3
\end{align*}
These by themselves would coincide exactly with linear Lagrange shape
functions and are of polynomial order $p=1$.

\subsection{Edge shape functions}
Edge shape functions will be defined in terms of the shape functions
associated with the two vertices $v_i,$ and $v_j$ that bound the edge and the
kernel functions defined previously. Edge shape functions of polyonmial order
$p$ are defined by
\[
N^{e_{ij}}_p = N^{v_i} N^{v_j} \phi_{p-2}(N^{v_j} - N^{v_i})
\quad p \geq 2
\]

\section{Reference Tetrahedron}

We define our reference tetrahedron $\Omega^e_T$ in a parametric space
such that
\[
\Omega^e_T = \left\{ \xi \in \mathbb{R}^3 \; : \;
0 < \xi_1, \xi_2, \xi_3 < 1; \; \xi_1 + \xi_1 + \xi_3 < 1 \right\}
\]
The barycentric coordinates for the reference tetrahedron can be defined in
terms of the $\xi$ coordinates as
\begin{align*}
A_1 &= 1 - \xi_1 - \xi_2 - \xi_3 \\
A_2 &= \xi_1 \\
A_3 &= \xi_2 \\
A_4 &= \xi_3 \\
\end{align*}
There are 4 vertices associated with our parent tetrahedron. These can be
expressed in parametric space as
\begin{align*}
v_1(\xi) &= (0,0,0) \\
v_2(\xi) &= (1,0,0) \\
v_3(\xi) &= (0,1,0) \\
v_4(\xi) &= (0,0,1)
\end{align*}
They can similarly be expressed in barycentric coordinates as
\begin{align*}
v_1(A) &= (1,0,0,0) \\
v_2(A) &= (0,1,0,0) \\
v_3(A) &= (0,0,1,0) \\
v_4(A) &= (0,0,0,1)
\end{align*}

\subsection{Vertex shape functions}

There are always only 4 shape functions associated with tetrahedral vertices,
$v_1, v_2$, $v_3$, and $v_4$.
These are
\begin{align*}
N^{v_1}(A) &= A_1 \\
N^{v_2}(A) &= A_2 \\
N^{v_3}(A) &= A_3 \\
N^{v_4}(A) &= A_4
\end{align*}
These by themselves would coincide exactly with linear Lagrange shape
functions and are of polynomial order $p=1$.

\subsection{Edge shape functions}
Edge shape functions will be defined in terms of the shape functions
associated with the two vertices $v_i,$ and $v_j$ that bound the edge and the
kernel functions defined previously. Edge shape functions of polyonmial order
$p$ are defined by
\[
N^{e_{ij}}_p = N^{v_i} N^{v_j} \phi_{p-2}(N^{v_j} - N^{v_i})
\quad p \geq 2
\]

\section{Future work}
Currently only linear and quadratic hierarchic shape functions are
implemented for tetrahedra. This document will be updated to explain face
and region bubble modes when they are implemented for various element types.

\end{document}


\documentclass{article}
\usepackage{graphicx}

\title{Root finding}
\begin{document}
  \pagenumbering{gobble}
  \maketitle
  \newpage
  \pagenumbering{arabic}

\section*{Relaxation method}

We are trying to figure out the solutions to the equation:

$$x=f(x)$$

where f(x) is a nonlinear function.

We call the solutions $x_0$

Iterations i:

$$x_i = f(x_{i-1}) $$
$$= f(x_0) + (x_{i-1}-x_0)f'(x_0) + ...$$

In each iteration $f(x_0) = x_0$

$$x_i - x_0 = (x_{i-1}-x_0)f'(x_0) + ... $$

$$\Delta x_i \approx \Delta x_{i-1} f'(x_0)$$

Does $\Delta x_i $ shrink?

Yes, only if $|f'(x_0)| < 1$

\section*{Example}

$$x = f(x) = 1 - e^{-2x}$$
$$f'(x) = -2e^{-2x}$$

OK:
$$f'(0.797) = -0.406$$

NOT OK:
$$f'(0) = -2$$

\end{document}


\documentclass{article}
\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\begin{document}
\section*{separable}
\begin{align*}
g(y)\,\mathrm{d}y&=f(t)\,\mathrm{d}t & g(y)y'&=f(t)\\
\int{g(y)\,\mathrm{d}y}&=\int{f(t)\,\mathrm{d}t}
\end{align*}
\section*{first order linear}
\begin{align*}
\frac{\mathrm{d}y}{\mathrm{d}t}+p(t)y&=q(t) & \mu(t)&=e^{\int{p(t)\,\mathrm{d}t}}\\
\frac{\mathrm{d}}{\mathrm{d}t}\left(\mu(t)y\right)&=\mu(t)\frac{\mathrm{d}y}{\mathrm{d}t}+p(t)\mu(t)y & \mu(t)y&=\int{\mu(t)q(t)\,\mathrm{d}t}
\end{align*}
\section*{exact}
\begin{align*}
M(t,y)\,\mathrm{d}t+N(t,y)\,\mathrm{d}y&=0 & \frac{\partial M}{\partial y}&=\frac{\partial N}{\partial y}\\
\int{M(t,y)\,\mathrm{d}t}+\phi(y)&=f(t,y) & \phi'(y)&=N(x,y)-\frac{\mathrm{d}}{\mathrm{d}y}\left(\int{M(t,y)\,\mathrm{d}t}\right)\\
\int{M(t,y)\,\mathrm{d}t}+\int{\phi'(y)\,\mathrm{d}y}&=f(t,y)
\end{align*}
Solution is $f(t,y)=C$
\section*{bernoulli}
\begin{align*}
\frac{\mathrm{d}y}{\mathrm{d}t}+p(t)y&=q(t)y^n & w&=y^{1-n)}\\
\frac{\mathrm{d}w}{\mathrm{d}t}+(1-n)p(t)w&=(1-n)q(t)
\end{align*}
Solve as first order linear, then back substitute
\section*{homogeneous}
\begin{align*}
M(t,y)\,\mathrm{d}t+N(t,y)\,\mathrm{d}y&=0 & M(xt,xy)+N(xt,xy)&=x^n\left(M(t,y)+N(t,y)\right)\\
\end{align*}
substitute with $y=wt$ if $N(t,y)$ is simpler and $t=wy$ if $M(t,y)$ is simpler
\begin{align*}
\mathrm{d}y&=w\,\mathrm{d}t+t\,\mathrm{d}w &\mathrm{d}t&=w\,\mathrm{d}y+y\,\mathrm{d}w
\end{align*}
Solve as a separable equation
\end{document}


% ----------------------------------------------------------------------
\section{Basis and dimension}
\label{sec:basis-and-dimension}

\begin{outcome}
  \begin{enumerate}\setlength\itemsep{0em}
  \item Find a basis for a subspace of $\R^n$.
  \item Use the casting-out algorithm to find a basis for a subspace
    given as a span.
  \item Use basic solutions to find a basis for a subspace given as
    the solution space of a homogeneous system of equations.
  \item Find the coordinates of a vector with respect to a basis.
  \item Find the dimension of a subspace of $\R^n$.
  \item Extend a set of linearly independent vectors to a basis.
  \item Shrink a spanning set to a basis by removing redundant vectors.
  \item Determine whether $k$ vectors form a basis of a $k$-dimensional
    space.
  \end{enumerate}
\end{outcome}

% ----------------------------------------------------------------------
\subsection{Definition of basis}

We saw in Proposition~\ref{prop:span-subspace} that spans are
subspaces of\/ $\R^n$. Interestingly, the converse is also true: every
subspace of\/ $\R^n$ is the span of some finite set of vectors.

\begin{theorem}{Subspaces are spans}{subspaces-are-spans}
  \index{subspace!is a span}%
  Let $V$ be a subspace of\/ $\R^n$. Then there exist linearly
  independent vectors $\set{\vect{u}_1,\ldots,\vect{u}_k}$ in $V$
  such that
  \begin{equation*}
    V= \sspan\set{\vect{u}_1,\ldots,\vect{u}_k}.
  \end{equation*}
  \vspace{-3ex}
\end{theorem}

\begin{proof}
  We proceed as follows.
  \begin{enumerate}
  \item[0.] If $V=\set{\vect{0}}$, then $V$ is the empty span, and we
    are done.
  \item[1.] Otherwise, $V$ contains some non-zero vector.  Pick a
    non-zero vector $\vect{u}_1$ in $V$. If
    $V=\sspan\set{\vect{u}_1}$, we are done.
  \item[2.] Otherwise, pick a vector $\vect{u}_2$ in $V$ that is not
    in $\sspan\set{\vect{u}_1}$. If
    $V=\sspan\set{\vect{u}_1,\vect{u}_2}$, we are done.
  \item[3.] Otherwise, pick a vector $\vect{u}_3$ in $V$ that is not
    in $\sspan\set{\vect{u}_1,\vect{u}_2}$. If
    $V=\sspan\set{\vect{u}_1,\vect{u}_2,\vect{u}_3}$, we are done.
  \item[4.] Otherwise, pick a vector $\vect{u}_4$ in $V$ that is not
    in $\sspan\set{\vect{u}_1,\vect{u}_2,\vect{u}_4}$, and so on.
  \end{enumerate}
  Continue in this way. Note that after the $j\th$ step of this
  process, the vectors $\vect{u}_1,\ldots,\vect{u}_j$ are linearly
  independent. This is because, by construction, no vector is in the
  span of the previous vectors, and therefore no vector is redundant.
  By
  Proposition~\ref{prop:properties-linear-independence}(\ref{properties-linear-independence-c}),
  there can be at most $n$ linearly independent vectors in $\R^n$.
  Therefore the process must stop after $k$ steps for some $k\leq
  n$. But then $V=\sspan\set{\vect{u}_1,\ldots,\vect{u}_k}$, as
  desired.
\end{proof}

In summary, every subspace of\/ $\R^n$ is spanned by a finite,
linearly independent collection of vectors.  Such a collection of
vectors is called a \textbf{basis} of the subspace.

\begin{definition}{Basis of a subspace}{subspace-basis}
  Let $V$ be a subspace of\/ $\R^n$. Then
  $\set{\vect{u}_1,\ldots,\vect{u}_k}$ is a \textbf{basis} for
  $V$ if the following two conditions hold:%
  \index{basis}%
  \index{subspace!basis|see{basis}}%
  \index{vector!basis|see{basis}}%
  \begin{enumerate}
  \item $\sspan\set{\vect{u}_1,\ldots,\vect{u}_k}=V$, and
  \item $\vect{u}_1,\ldots,\vect{u}_k$ are linearly independent.
  \end{enumerate}
\end{definition}

Note that the plural of basis is \textbf{bases}.

% ----------------------------------------------------------------------
\subsection{Examples of bases}

\begin{proposition}{Standard basis of\/ $\R^n$}{standard-basis}
  Let $\vect{e}_i$ be the vector in $\R^n$ whose $i\th$ component is $1$
  and all of whose other components are $0$. In other words, $\vect{e}_i$
  is the $i\th$ column of the identity matrix.
  \begin{equation*}
    \vect{e}_1 = \begin{mymatrix}{c} 1 \\ 0 \\ 0 \\ \vdots \\ 0 \end{mymatrix},\quad
    \vect{e}_2 = \begin{mymatrix}{c} 0 \\ 1 \\ 0 \\ \vdots \\ 0 \end{mymatrix},\quad
    \vect{e}_3 = \begin{mymatrix}{c} 0 \\ 0 \\ 1 \\ \vdots \\ 0 \end{mymatrix},\quad
    \ldots~,\quad
    \vect{e}_n = \begin{mymatrix}{c} 0 \\ 0 \\ 0 \\ \vdots \\ 1\end{mymatrix}.
  \end{equation*}
  Then $\set{\vect{e}_1,\vect{e}_2,\ldots,\vect{e}_n}$ is a basis for
  $\R^n$. It is called the \textbf{standard basis}%
  \index{standard basis}%
  \index{basis!standard}
  of\/ $\R^n$.
\end{proposition}

\begin{proof}
  To see that it is a basis of\/ $\R^n$, first notice that the vectors
  $\vect{e}_1,\vect{e}_2,\ldots,\vect{e}_n$ span $\R^n$. Indeed, every
  vector $\vect{v} = \mat{x_1,\ldots,x_n}^T\in\R^n$ can be written as
  $\vect{v} = x_1\vect{e}_1+\ldots+x_n\vect{e}_n$. Second, the vectors
  $\vect{e}_1,\vect{e}_2,\ldots,\vect{e}_n$ are evidently linearly
  independent, because none of these vectors can be written as a
  linear combination of previous vectors. Since the vectors span
  $\R^n$ and are linearly independent, they form a basis of\/ $\R^n$.
\end{proof}

\begin{example}{A non-standard basis of\/ $\R^3$}{non-standard-basis}
  \index{basis!of Rn@of\/ $\R^n$}%
  Check that the vectors
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 2 \\ 1 \end{mymatrix},\quad
    \vect{u}_2 = \begin{mymatrix}{r} 0 \\ 1 \\ 0 \end{mymatrix},\quad
    \mbox{and}\quad
    \vect{u}_3 = \begin{mymatrix}{r} -1 \\ 0 \\ 1 \end{mymatrix}
  \end{equation*}
  form a basis of\/ $\R^3$.
\end{example}

\begin{solution}
  We must check that the vectors $\vect{u}_1,\vect{u}_2,\vect{u}_3$
  are linearly independent and span $\R^3$. To check linear
  independence, we use the casting-out algorithm.
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 0 & -1 \\
      2 & 1 & 0 \\
      1 & 0 & 1 \\
    \end{mymatrix}
    \quad\roweq\quad
    \begin{mymatrix}{rrr}
      \circled{1} & 0 & -1 \\
      0 & \circled{1} & 2 \\
      0 & 0 & \circled{2} \\
    \end{mymatrix}.
  \end{equation*}
  Since all columns are pivot columns, there are no redundant vectors,
  so $\vect{u}_1,\vect{u}_2,\vect{u}_3$ are linearly independent.
  To check that they span all of\/ $\R^3$, let $\vect{w}=\mat{x,y,z}^T$
  be an arbitrary element of\/ $\R^3$. We must show that $\vect{w}$ is a linear
  combination of $\vect{u}_1,\vect{u}_2,\vect{u}_3$. This amounts to
  solving the system of equations
  \begin{equation*}
    a_1\,\vect{u}_1+a_2\,\vect{u}_2+a_3\,\vect{u}_3 = \vect{w},
  \end{equation*}
  or in augmented matrix form,
  \begin{equation*}
    \begin{mymatrix}{rrr|c}
      1 & 0 & -1 & x \\
      2 & 1 & 0  & y \\
      1 & 0 & 1  & z \\
    \end{mymatrix}
    \quad\roweq\quad
    \begin{mymatrix}{rrr|c}
      1 & 0 & -1 & x    \\
      0 & 1 & 2  & y-2x \\
      0 & 0 & 2  & z-x  \\
    \end{mymatrix}.
  \end{equation*}
  The system is clearly consistent, so it has a solution, and
  therefore $\vect{w}$ is indeed a linear combination of
  $\vect{u}_1,\vect{u}_2,\vect{u}_3$. Since $\vect{w}$ was an
  arbitrary vector of\/ $\R^3$, it follows that
  $\vect{u}_1,\vect{u}_2,\vect{u}_3$ span $\R^3$.
\end{solution}

Generalizing the last example, we find that a set of $n$ vectors forms
a basis of\/ $\R^n$ if and only if the matrix having those vectors as
its columns is invertible. This is the content of the following
proposition.

\begin{proposition}{Invertible matrices and bases of\/ $\R^n$}{invertible-matrices}
  \index{basis!of Rn@of\/ $\R^n$}%
  \index{basis!and invertible matrix}%
  Let $A$ be an $n\times n$-matrix. Then the columns of $A$ form a
  basis of\/ $\R^n$ if and only if $A$ is invertible.
\end{proposition}

We turn to the question of finding bases for subspaces of\/ $\R^n$.

\begin{example}{Basis of a span}{basis-of-span}
  Let
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 2 \\ 0 \\ -2 \end{mymatrix},
    \quad
    \vect{u}_2 = \begin{mymatrix}{r} -1 \\ 0 \\ 1 \end{mymatrix},
    \quad
    \vect{u}_3 = \begin{mymatrix}{r} 1 \\ 3 \\ 5 \end{mymatrix},
    \quad
    \vect{u}_4 = \begin{mymatrix}{r} 3 \\ 5 \\ 7 \end{mymatrix},
    \quad
    \vect{u}_5 = \begin{mymatrix}{r} -1 \\ 1 \\ 3 \end{mymatrix}.
  \end{equation*}
  Find a basis of $\sspan\set{\vect{u}_1,\ldots,\vect{u}_5}$%
  \index{basis!of a span}.
\end{example}

\begin{solution}
  Let $S=\sspan\set{\vect{u}_1,\ldots,\vect{u}_5}$.  By
  Theorem~\ref{thm:linearly-independent-subset}, we know that if we
  remove the redundant vectors from
  $\set{\vect{u}_1,\ldots,\vect{u}_5}$, then the remaining vectors
  will be linearly independent and will still span $S$. In other
  words, the remaining vectors will be a basis for $S$. We use the
  casting-out algorithm to identity the redundant vectors:
  \begin{equation*}
    \begin{mymatrix}{rrrrr}
      2 & -1 & 1 & 3 & -1 \\
      0 & 0 & 3 & 5 & 1 \\
      -2 & 1 & 5 & 7 & 3 \\
    \end{mymatrix}
    ~\roweq~
    \begin{mymatrix}{rrrrr}
      2 & -1 & 1 & 3 & -1 \\
      0 & 0 & 3 & 5 & 1 \\
      0 & 0 & 6 & 10 & 2 \\
    \end{mymatrix}
    ~\roweq~
    \begin{mymatrix}{rrrrr}
      \circled{2} & -1 & 1 & 3 & -1 \\
      0 & 0 & \circled{3} & 5 & 1 \\
      0 & 0 & 0 & 0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  Since columns $2$, $4$, and $5$ are the non-pivot columns, it
  follows that the vectors $\vect{u}_2$, $\vect{u}_4$, and
  $\vect{u}_5$ are redundant. Therefore, the desired basis is
  $\set{\vect{u}_1,\vect{u}_3}$.
\end{solution}

\begin{example}{Basis of the solution space of a homogeneous system of equations}{basis-solution-space}
  Find a basis for the solution space of the system of equations%
  \index{basis!of a solution space}
  \begin{equation*}
    \begin{array}{r@{~}r@{~}r@{~}r@{~}r@{~}r@{~}r@{~}r@{~}r@{~}r@{~}r}
     x &+& y  &-& z &+& 3w &-& 2v &=& 0, \\
     x &+& y &+& z &-& 11w &+& 8v &=& 0, \\
    4x &+& 4y &-& 3z &+& 5w &-& 3v &=& 0. \\
    \end{array}
  \end{equation*}
\end{example}

\begin{solution}
  We solve the system of equations in the usual way:
  \begin{equation*}
    \begin{mymatrix}{rrrrr|r}
      1 & 1 & -1 & 3 & -2 & 0 \\
      1 & 1 &  1 & -11 & 8 & 0 \\
      4 & 4 & -3 & 5 & -3 & 0 \\
    \end{mymatrix}
    ~\roweq~
    \begin{mymatrix}{rrrrr|r}
      1 & 1 & -1 & 3 & -2 & 0 \\
      0 & 0 & 2 & -14 & 10 & 0 \\
      0 & 0 & 1 & -7 & 5 & 0 \\
    \end{mymatrix}
    ~\roweq~
    \begin{mymatrix}{rrrrr|r}
      \circled{1} & 1 & 0 & -4 & 3 & 0 \\
      0 & 0 & \circled{1} & -7 & 5 & 0 \\
      0 & 0 & 0 &  0 & 0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  From the {\rref}, we see that $y$, $w$, and $v$ are free
  variables. The general solution is:
  \begin{equation*}
    \begin{mymatrix}{c} x \\ y \\ z \\ w \\ v \end{mymatrix}
    \quad=\quad
    t \begin{mymatrix}{r} -3 \\ 0 \\ -5 \\ 0 \\ 1 \end{mymatrix}
    ~+~ s \begin{mymatrix}{r} 4 \\ 0 \\ 7 \\ 1 \\ 0 \end{mymatrix}
    ~+~ r \begin{mymatrix}{r} -1 \\ 1 \\ 0 \\ 0 \\ 0 \end{mymatrix}.
  \end{equation*}
  Thus, the solution space is spanned by the vectors
  \begin{equation*}
    \set{\begin{mymatrix}{r} -3 \\ 0 \\ -5 \\ 0 \\ 1 \end{mymatrix},
    \begin{mymatrix}{r} 4 \\ 0 \\ 7 \\ 1 \\ 0 \end{mymatrix},
    \begin{mymatrix}{r} -1 \\ 1 \\ 0 \\ 0 \\ 0 \end{mymatrix}
    }.
  \end{equation*}
  Moreover, these vectors are evidently linearly independent, because
  each vector contains a $1$ in a position where all the previous
  vectors have $0$ (and therefore, none of the vectors can be written
  as a linear combination of previous vectors). It follows that the
  above three vectors form a basis of the solution space.
\end{solution}

Note that the basis vectors of the solution space are exactly what we
called the \textbf{basic solutions}%
\index{basic solution}%
\index{solution!basic}%
\index{solution!basis of}
in Section~\ref{sec:homogeneous-systems}.

% ----------------------------------------------------------------------
\subsection{Bases and coordinate systems}
\label{ssec:bases-and-coordinates}

Let $V$ be a subspace of\/ $\R^n$. A basis of $V$ is essentially the
same thing as a coordinate system for $V$. To see why, let
$B=\set{\vect{u}_1,\ldots,\vect{u}_k}$ be some basis of $V$. This
means that the vectors $\vect{u}_1,\ldots,\vect{u}_k$ are linearly
independent and span $V$. Because the basis vectors are spanning,
every vector $\vect{v}\in V$ can be written as a linear combination of
basis vectors
\begin{equation*}
  \vect{v} = a_1\,\vect{u}_1 + \ldots + a_k\,\vect{u}_k.
\end{equation*}
Moreover, because the basis vectors are linearly independent, it
follows by Theorem~\ref{thm:unique-linear-combination} that the
coefficients $a_1,\ldots,a_k$ are unique. We say that $a_1,\ldots,a_k$
are the \textbf{coordinates of $\vect{v}$ with respect to the basis
$B$}%
\index{coordinate!with respect to basis}%
\index{coordinate system!and basis}, and we write
\begin{equation*}
  \coord{\vect{v}}_B = \begin{mymatrix}{c} a_1 \\ a_2 \\ a_3 \end{mymatrix}.
\end{equation*}
\begin{center}
  \begin{tikzpicture}[scale=0.8]
    \begin{scope}[scale=2.5,x={(1.2cm,-0.2cm)},y={(0.5cm,0.4cm)},z={(0cm,0.8cm)}]
      \draw(-2.5,0,0) -- (2.5,0,0);
      \draw(0,-2.3,0) -- (0,2.3,0);
      \draw(0,0,-1.3) -- (0,0,1.3);
      \draw[->, thick, blue](0,0,0) -- node[below] {$\vect{u}_1$} +(1,0,0);
      \draw[->, thick, blue](0,0,0) -- node[above left=-1ex] {$\vect{u}_2$} +(0,1,0);
      \draw[->, thick, blue](0,0,0) -- node[left] {$\vect{u}_3$} +(0,0,1);
      \draw(-2,0,0) -- +(0,0,-0.1) node[below] {$-2$};
      \draw(-1,0,0) -- +(0,0,-0.1) node[below] {$-1$};
      \draw(0,0,0) -- +(0,0,-0.1) node[below] {$0$};
      \draw(1,0,0) -- +(0,0,-0.1) node[below] {$1$};
      \draw(2,0,0) -- +(0,0,-0.1) node[below] {$2$};
      \draw(0,-2,0) -- +(0,0,-0.1) node[below] {$-2$};
      \draw(0,-1,0) -- +(0,0,-0.1) node[below] {$-1$};
      \draw(0,1,0) -- +(0,0,-0.1) node[below] {$1$};
      \draw(0,2,0) -- +(0,0,-0.1) node[below] {$2$};
      \draw(0,0,-1) -- +(-0.1,0,0) node[left] {$-1$};
      \draw(0,0,1) -- +(-0.1,0,0) node[left] {$1$};
    \end{scope}
    \path(0,-3.2) node {Basis as coordinate system};
  \end{tikzpicture}
\end{center}

\begin{example}{Find a vector from its coordinates in a basis}{vector-from-coordinates}
  Find the vector $\vect{v}$ that has coordinates
  \begin{equation*}
    \coord{\vect{v}}_B = \begin{mymatrix}{r} 1 \\ -1 \\ 2 \end{mymatrix}
  \end{equation*}
  with respect to the basis $B=\set{\vect{u}_1,\vect{u}_2,\vect{u}_3}$
  of\/ $\R^3$, where
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 2 \\ 1 \end{mymatrix},\quad
    \vect{u}_2 = \begin{mymatrix}{r} 0 \\ 1 \\ 0 \end{mymatrix},\quad
    \mbox{and}\quad
    \vect{u}_3 = \begin{mymatrix}{r} -1 \\ 0 \\ 1\end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  This simply means that $\vect{v} = 1\vect{u}_1 - 1\vect{u}_2 +
  2\vect{u}_3$. We calculate
  \begin{equation*}
    \vect{v} =
    1\begin{mymatrix}{r} 1 \\ 2 \\ 1 \end{mymatrix}
    - 1\begin{mymatrix}{r} 0 \\ 1 \\ 0 \end{mymatrix}
    + 2\begin{mymatrix}{r} -1 \\ 0 \\ 1 \end{mymatrix}
    = \begin{mymatrix}{r} -1 \\ 1 \\ 3  \end{mymatrix}.
  \end{equation*}
\end{solution}

In case the basis is the standard basis, the coordinates are just the
usual ones, as the following example illustrates:

\begin{example}{Find a vector from its coordinates in the standard basis}{vector-from-coordinates-standard}
  Find the vector $\vect{v}$ that has coordinates
  \begin{equation*}
    \coord{\vect{v}}_B = \begin{mymatrix}{r} 1 \\ -1 \\ 2 \end{mymatrix},
  \end{equation*}
  where $B$ is the standard basis of\/ $\R^3$.
\end{example}

\begin{solution}
  The standard basis is
  \begin{equation*}
    \vect{e}_1 = \begin{mymatrix}{r} 1 \\ 0 \\ 0 \end{mymatrix},\quad
    \vect{e}_2 = \begin{mymatrix}{r} 0 \\ 1 \\ 0 \end{mymatrix},\quad
    \mbox{and}\quad
    \vect{e}_3 = \begin{mymatrix}{r} 0 \\ 0 \\ 1\end{mymatrix}.
  \end{equation*}
  We have to calculate
  \begin{equation*}
    \vect{v} = 1\vect{e}_1 - 1\vect{e}_2 + 2\vect{e}_3
    = \begin{mymatrix}{r} 1 \\ -1 \\ 2  \end{mymatrix}.
  \end{equation*}
  We see that the coordinates of any vector with respect to the
  standard basis are just the usual components of the vector.
\end{solution}

We can also ask to find the coordinates of a given vector in a given
basis.

\begin{example}{Find the coordinates of a vector with respect to a basis}{coordinates-from-vector}
  Find the coordinates of the vector
  \begin{equation*}
    \vect{v} = \begin{mymatrix}{r} 1 \\ 2 \\ 3 \end{mymatrix}
  \end{equation*}
  with respect to the basis $B=\set{\vect{u}_1,\vect{u}_2,\vect{u}_3}$
  of\/ $\R^3$, where
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 2 \\ 1 \end{mymatrix},\quad
    \vect{u}_2 = \begin{mymatrix}{r} 0 \\ 1 \\ 0 \end{mymatrix},\quad
    \mbox{and}\quad
    \vect{u}_3 = \begin{mymatrix}{r} -1 \\ 0 \\ 1\end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  To find the coordinates, we must solve the system of equations
  $\vect{v} = a_1\,\vect{u}_1+a_2\,\vect{u}_2+a_3\,\vect{u}_3$. We
  solve:
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      1 & 0 & -1 & 1 \\
      2 & 1 & 0 & 2 \\
      1 & 0 & 1 & 3 \\
    \end{mymatrix}
    \quad\roweq\quad
    \begin{mymatrix}{rrr|r}
      1 & 0 & -1 & 1 \\
      0 & 1 & 2 & 0 \\
      0 & 0 & 2 & 2 \\
    \end{mymatrix}
    \quad\roweq\quad
    \begin{mymatrix}{rrr|r}
      1 & 0 & 0 & 2 \\
      0 & 1 & 0 & -2 \\
      0 & 0 & 1 & 1 \\
    \end{mymatrix}.
  \end{equation*}
  Therefore, the unique solution is $(a_1,a_2,a_3) = (2,-2,1)$. The
  coordinates of $\vect{v}$ with respect to the basis $B$ are
  \begin{equation*}
    \coord{\vect{v}}_B
    = \begin{mymatrix}{r} a_1 \\ a_2 \\ a_3 \end{mymatrix}
    = \begin{mymatrix}{r} 2 \\ -2 \\ 1 \end{mymatrix}.
  \end{equation*}
\end{solution}

% ----------------------------------------------------------------------
\subsection{Dimension}

One of the most important properties of bases is that any two bases
for the same space must be of the same size. To show this, we will
need the the following fundamental result, called the Exchange
Lemma. This lemma states that spanning sets have at least as many
vectors as linearly independent sets.

\begin{lemma}{Exchange Lemma}{exchange-lemma}
  \index{exchange lemma!in Rn@in $\R^n$}%
  Suppose $\vect{u}_1,\ldots,\vect{u}_r$ are linearly independent
  elements of $\sspan\set{\vect{v}_1,\ldots,\vect{v}_s}$. Then
  $r\leq s$.
\end{lemma}

\begin{proof}
  Since each $\vect{u}_j$ is an element of
  $\sspan\set{\vect{v}_1,\ldots,\vect{v}_s}$, there exist
  scalars $a_{ij}$ such that
  \begin{equation*}
    \vect{u}_j = a_{1j}\,\vect{v}_1 + \ldots + a_{sj}\,\vect{v}_s.
  \end{equation*}
  Let $A = \mat{a_{ij}}$. Note that this matrix has $s$ rows and $r$
  columns, i.e., it is an $s\times r$-matrix. Now suppose, for the
  sake of obtaining a contradiction, that $r>s$. Then by
  Theorem~\ref{thm:rank-homogeneous-solutions}, the system
  $A\vect{x}=\vect{0}$ has a non-trivial solution $\vect{x}$, i.e.,
  there exists $\vect{x}\neq\vect{0}$ such that $A\vect{x}=\vect{0}$.
  In other words, for all $i=1,\ldots,s$,
  \begin{equation*}
    a_{i1}x_1 + \ldots + a_{ir}x_r = 0.
  \end{equation*}
  Therefore,
  \begin{eqnarray*}
    x_1\,\vect{u}_1 + \ldots + x_r\,\vect{u}_r
    &=&
        x_1(a_{11}\,\vect{v}_1 + \ldots + a_{s1}\,\vect{v}_s)
        + \ldots
        + x_r(a_{1r}\,\vect{v}_1 + \ldots + a_{sr}\,\vect{v}_s)
    \\
    &=&
        (a_{11}x_1 + \ldots + a_{1r}x_r)\vect{v}_1
        + \ldots
        + (a_{s1}x_1 + \ldots + a_{sr}x_r)\vect{v}_s
    \\
    &=& 0\,\vect{v}_1 + \ldots + 0\,\vect{v}_s
    \\
    &=& 0.
  \end{eqnarray*}
  This contradicts the assumption that
  $\vect{u}_1,\ldots,\vect{u}_r$ are linearly independent. Since
  we assumed $r>s$ and obtained a contradiction, it follows that
  $r\leq s$, as desired.
\end{proof}

Armed with the Exchange Lemma, we are now ready to show that any two
bases of a space are of the same size.

\begin{theorem}{Bases are of the same size}{bases-same-size}
  \index{basis!size of}%
  Let $V$ be a subspace of\/ $\R^n$, and let $B_1$ and $B_2$ be
  bases of $V$. Suppose $B_1$ contains $s$ vectors and $B_2$ contains
  $r$ vectors. Then $s=r$.
\end{theorem}

\begin{proof}
  This follows right away from the Exchange Lemma. Indeed, observe
  that $B_1 = \set{\vect{u}_1,\ldots,\vect{u}_s}$ is a spanning
  set for $V$ while $B_2 = \set{\vect{v}_1,\ldots,\vect{v}_r}$ is
  linearly independent, so $s\geq r$. Similarly
  $B_2 = \set{\vect{v}_1,\ldots,\vect{v}_r}$ is a spanning set
  for $V$ while $B_1 = \set{\vect{u}_1,\ldots, \vect{u}_s}$ is
  linearly independent, so $r\geq s$.
\end{proof}

Because every basis of $V$ has the same number of vectors, we give
this number a special name. It is called the \textbf{dimension} of
$V$.

\begin{definition}{Dimension of a subspace}{dimension}
  Let $V$ be a subspace of\/ $\R^n$. Then the \textbf{dimension}%
  \index{dimension!of subspace of $\R^n$}%
  \index{subspace!dimension} of $V$, written $\dim(V)$, is
  defined to be the number of vectors in a basis.
\end{definition}

\begin{example}{Dimension of\/ $\R^n$}{dimension-Rn}
  What is the dimension of $\R^n$?
\end{example}

\begin{solution}
  The standard basis of $\R^n$ is
  $\set{\vect{e}_1,\ldots,\vect{e}_n}$. Since it has $n$ vectors,
  so $\dim(\R^n) = n$.
\end{solution}

\begin{example}{Dimension of a subspace}{dimension-subspace}
  \index{dimension!of solution space}%
  \index{homogeneous system!dimension of solution space}%
  \index{system of linear equations!homogeneous!dimension of solution space}%
  Let
  \begin{equation*}
    V=\set{\left.
      \begin{mymatrix}{c} x \\ y \\ z\end{mymatrix}\in\R^3 ~\right\vert~
      x-y+2z = 0
    }.
  \end{equation*}
  What is the dimension of\/ $V$?
\end{example}

\begin{solution}
  We know that $V$ is a subspace of $\R^3$, because it is the solution
  space of a system of a homogeneous system of equations (in this
  case, one equation in three variables). We can take $y=t$ and $z=s$
  as the free variables and solve for $x=y-2z=t-2s$. Therefore, a
  general element of $V$ is of the form
  \begin{equation*}
    \begin{mymatrix}{c} x \\ y \\ z\end{mymatrix}
    ~=~ \begin{mymatrix}{c} t-2s \\ t \\ s \end{mymatrix}
    ~=~~ t \begin{mymatrix}{c} 1 \\ 1 \\ 0 \end{mymatrix}
    + s \begin{mymatrix}{c} -2 \\ 0 \\ 1 \end{mymatrix}.
  \end{equation*}
  Thus,
  \begin{equation*}
    V = \sspan\set{
      \begin{mymatrix}{c} 1 \\ 1 \\ 0 \end{mymatrix},~
      \begin{mymatrix}{c} -2 \\ 0 \\ 1 \end{mymatrix}
    }.
  \end{equation*}
  Since the two spanning vectors are linearly independent, they form a
  basis of $V$, and thus $\dim(V)=2$.
\end{solution}

Note that the dimension of the solution space of a system of equations
is equal to the number of parameters in the general solution, which is
equal to the number of free variables. For this reason, the dimension
is also sometimes called the number of \textbf{degrees of freedom}%
\index{degree!of freedom}%
\index{freedom!degree of}.

\begin{example}{Dimension of a span}{dimension-basis}
  \index{dimension!of a span}%
  \index{span!dimension of}%
  Let
  \begin{equation*}
    W = \sspan\set{
      \begin{mymatrix}{r} 1 \\ 2 \\ -1 \\ 1 \end{mymatrix},~
      \begin{mymatrix}{r} 1 \\ 3 \\ -1 \\ 1 \end{mymatrix},~
      \begin{mymatrix}{r} 8 \\ 19 \\ -8 \\ 8 \end{mymatrix},~
      \begin{mymatrix}{r} -6 \\ -15 \\ 6 \\ -6 \end{mymatrix},~
      \begin{mymatrix}{r} 1 \\ 3 \\ 0 \\ 1 \end{mymatrix},~
      \begin{mymatrix}{r} 1 \\ 5 \\ 0 \\ 1 \end{mymatrix}
    }.
  \end{equation*}
  What is the dimension of $W$?
\end{example}

\begin{solution}
  Let
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 2 \\ -1 \\ 1 \end{mymatrix},~~
    \vect{u}_2 = \begin{mymatrix}{r} 1 \\ 3 \\ -1 \\ 1 \end{mymatrix},~~
    \vect{u}_3 = \begin{mymatrix}{r} 8 \\ 19 \\ -8 \\ 8 \end{mymatrix},~~
    \vect{u}_4 = \begin{mymatrix}{r} -6 \\ -15 \\ 6 \\ -6 \end{mymatrix},~~
    \vect{u}_5 = \begin{mymatrix}{r} 1 \\ 3 \\ 0 \\ 1 \end{mymatrix},~~
    \vect{u}_6 = \begin{mymatrix}{r} 1 \\ 5 \\ 0 \\ 1 \end{mymatrix},
  \end{equation*}
  so that $W=\sspan\set{\vect{u}_1,\ldots,\vect{u}_6}$.  We use the
  casting-out algorithm to remove any redundant vectors from
  $\vect{u}_1,\ldots,\vect{u}_6$. The remaining vectors will be
  linearly independent, and therefore a basis of the span.
  \begin{equation*}
    \begin{mymatrix}{rrrrrr}
      1 & 1 & 8 & -6 & 1 & 1 \\
      2 & 3 & 19 & -15 & 3 & 5 \\
      -1 & -1 & -8 & 6 & 0 & 0 \\
      1 & 1 & 8 & -6 & 1 & 1
    \end{mymatrix}
    \quad\roweq\quad
    \begin{mymatrix}{rrrrrr}
      \circled{1} & 0 & 5 & -3 & 0 & -2 \\
      0 & \circled{1} & 3 & -3 & 0 & 2 \\
      0 & 0 & 0 & 0 & \circled{1} & 1 \\
      0 & 0 & 0 & 0 & 0 & 0
    \end{mymatrix}.
  \end{equation*}
  Therefore, the vectors $\vect{u}_3$, $\vect{u}_4$, and $\vect{u}_6$
  are redundant, and $\set{\vect{u}_1,\vect{u}_2,\vect{u}_5}$ is a
  basis of $W$. It follows that $\dim(W)=3$.
\end{solution}

% ----------------------------------------------------------------------
\subsection{More properties of bases and dimension}

We begin by noting that every subspace $V$ of $\R^n$ has a basis.

\begin{theorem}{Existence of bases}{subspaces-have-bases}
  \index{basis!existence}%
  Every subspace of $\R^n$ has a basis.
\end{theorem}

\begin{proof}
  This is just a restatement of Theorem~\ref{thm:subspaces-are-spans}.
\end{proof}

Of course, the theorem does not mean that the basis is
unique. Usually, a subspace of $\R^n$ will have many different
bases. The theorem just states that there exists at least one.

Sometimes, when we are looking for a basis of a space, we may already
have a number of linearly independent vectors. We would like to obtain
a basis by adding some {\em additional} linearly independent vectors
to the ones we already have. The following lemma guarantees that this
can always be done.

\begin{lemma}{Linearly independent set can be extended to a basis}{extend-to-basis}
  \index{linear independence!extending to basis}%
  \index{basis!by extending linearly independent set}%
  Let $V$ be a subspace of $\R^n$, and let
  $\vect{u}_1,\ldots,\vect{u}_{\ell}$ be linearly independent elements
  of $V$. Then it is possible to extend
  $\set{\vect{u}_1,\ldots,\vect{u}_{\ell}}$ to a basis of $V$. In other
  words, there exist zero or more vectors
  $\vect{w}_1,\ldots,\vect{w}_s$ such that
  \begin{equation*}
    \set{\vect{u}_1,\ldots,\vect{u}_{\ell},\vect{w}_1,\ldots,\vect{w}_s}
  \end{equation*}
  is a basis of $V$.
\end{lemma}

\begin{proof}
  By Theorem~\ref{thm:subspaces-have-bases}, we know that $V$ has some
  basis, say $\set{\vect{v}_1,\ldots,\vect{v}_k}$. However, this may
  not be the basis we are looking for, because maybe it does not
  contain the vectors $\vect{u}_1,\ldots,\vect{u}_{\ell}$. Consider
  the sequence of $\ell+k$ vectors
  \begin{equation*}
    \vect{u}_1,\ldots,\vect{u}_{\ell},\vect{v}_1,\ldots,\vect{v}_k.
  \end{equation*}
  Since $V$ is spanned by the vectors $\vect{v}_1,\ldots,\vect{v}_k$,
  it is certainly also spanned by the larger set of vectors
  $\vect{u}_1,\ldots,\vect{u}_{\ell},\vect{v}_1,\ldots,\vect{v}_k$.
  From Theorem~\ref{thm:linearly-independent-subset}, we know that we
  can obtain a basis of $V$ by removing the redundant vectors from
  $\vect{u}_1,\ldots,\vect{u}_{\ell},\vect{v}_1,\ldots,\vect{v}_k$. On
  the other hand, $\vect{u}_1,\ldots,\vect{u}_{\ell}$ are linearly
  independent, so none of them can be redundant. It follows that the
  resulting basis of $V$ contains all of the vectors
  $\vect{u}_1,\ldots,\vect{u}_{\ell}$. In other words, we have found a
  basis of $V$ that is an extension of
  $\vect{u}_1,\ldots,\vect{u}_{\ell}$, which is what had to be shown.
\end{proof}

\begin{example}{Extending a linearly independent set to a basis}{extend-to-basis}
  Extend $\set{\vect{u}_1,\vect{u}_2}$ to a basis of $\R^4$, where
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 1 \\ -1 \\ 2 \end{mymatrix},\quad
    \vect{u}_2 = \begin{mymatrix}{r} 1 \\ 2 \\ -2 \\ 4 \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  Let $\set{\vect{e}_1,\ldots,\vect{e}_4}$ be the standard basis of
  $\R^4$. We obtain the desired basis by applying the casting-out
  algorithm to $\vect{u}_1,\vect{u}_2,\vect{e}_1,\vect{e}_2,\vect{e}_3,\vect{e}_4$:
  \begin{equation*}
    \begin{mymatrix}{rrrrrr}
      1  & 1  & 1 & 0 & 0 & 0 \\
      1  & 2  & 0 & 1 & 0 & 0 \\
      -1 & -2 & 0 & 0 & 1 & 0 \\
      2  & 4  & 0 & 0 & 0 & 1 \\
    \end{mymatrix}
    \quad\roweq\quad
    \begin{mymatrix}{rrrrrr}
      \circled{1} &  1 &  1 & 0  & 0 & 0 \\
      0 &  \circled{1} & -1 & 1  & 0 & 0 \\
      0 &  0 &  0 & \circled{1}  & 1 & 0 \\
      0 &  0 &  0 & 0  & \circled{2} & 1 \\
    \end{mymatrix}.
  \end{equation*}
  Therefore, we cast out the vectors $\vect{e}_1$ and $\vect{e}_4$ and
  keep the rest. The resulting basis is
  \begin{equation*}
    \set{\vect{u}_1,\vect{u}_2,\vect{e}_2,\vect{e}_3}
    = \set{
      \begin{mymatrix}{r} 1 \\ 1 \\ -1 \\ 2 \end{mymatrix},~
      \begin{mymatrix}{r} 1 \\ 2 \\ -2 \\ 4 \end{mymatrix},~
      \begin{mymatrix}{r} 0 \\ 1 \\ 0 \\ 0 \end{mymatrix},~
      \begin{mymatrix}{r} 0 \\ 0 \\ 1 \\ 0 \end{mymatrix}
    }.
  \end{equation*}
\end{solution}

\begin{example}{Extending a linearly independent set to a basis}{extend-to-basis2}
  Let
  \begin{equation*}
    V = \set{\left.
        \begin{mymatrix}{c} x \\ y \\ z \\ w \end{mymatrix}
        ~\right\vert~
      x+2y+z-w = 0
    }.
  \end{equation*}
  Note that $\vect{u}_1,\vect{u}_2\in V$, where
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{c} 1 \\ -1 \\ 1 \\ 0 \end{mymatrix},\quad
    \vect{u}_2 = \begin{mymatrix}{c} -2 \\ 1 \\ 1 \\ 1 \end{mymatrix}.
  \end{equation*}
  Extend $\set{\vect{u}_1,\vect{u}_2}$ to a basis of $V$.
\end{example}

\begin{solution}
  We first find a basis of $V$ by solving the linear equation
  $x+2y+z-w = 0$. Taking $y=r$, $z=s$, and $w=t$ as the free
  variables, we get $x=-2y-z+w = -2r-s+t$, and therefore the general
  solution is
  \begin{equation*}
    \begin{mymatrix}{c} x \\ y \\ z \\ w \end{mymatrix}
    ~=~ \begin{mymatrix}{c} -2r-s+t \\ r \\ s \\ t \end{mymatrix}
    ~=~~ r\begin{mymatrix}{c} -2 \\ 1 \\ 0 \\ 0 \end{mymatrix}
    + s\begin{mymatrix}{c} -1 \\ 0 \\ 1 \\ 0 \end{mymatrix}
    + t\begin{mymatrix}{c} 1 \\ 0 \\ 0 \\ 1 \end{mymatrix}.
  \end{equation*}
  Therefore, the vectors $\set{\vect{v}_1,\vect{v}_2,\vect{v}_3}$ form
  a basis of $V$, where
  \begin{equation*}
    \vect{v}_1 = \begin{mymatrix}{c} -2 \\ 1 \\ 0 \\ 0 \end{mymatrix},\quad
    \vect{v}_2 = \begin{mymatrix}{c} -1 \\ 0 \\ 1 \\ 0 \end{mymatrix},\quad
    \vect{v}_3 = \begin{mymatrix}{c} 1 \\ 0 \\ 0 \\ 1 \end{mymatrix}.
  \end{equation*}
  However, this is not the basis we are looking for, because it does
  not extend $\set{\vect{u}_1,\vect{u}_2}$. To get a basis of $V$ that
  extends $\set{\vect{u}_1,\vect{u}_2}$, we perform the casting-out
  algorithm on the vectors
  $\vect{u}_1,\vect{u}_2,\vect{v}_1,\vect{v}_2,\vect{v}_3$:
  \begin{equation*}
    \begin{mymatrix}{rrrrr}
      1  & -2 & -2 & -1 & 1 \\
      -1 & 1  & 1  & 0  & 0 \\
      1  & 1  & 0  & 1  & 0 \\
      0  & 1  & 0  & 0  & 1 \\
    \end{mymatrix}
    \quad\roweq\quad\ldots\quad\roweq\quad
    \begin{mymatrix}{rrrrr}
      \circled{1}  & -2 & -2 & -1 & 1 \\
      0  & \circled{1}  & 1  & 1  & -1 \\
      0  & 0  & \circled{1}  & 1  & -2 \\
      0  & 0  & 0  & 0  & 0 \\
    \end{mymatrix}.
  \end{equation*}
  Casting out $\vect{v}_2$ and $\vect{v}_3$, we find that the desired
  basis is $\set{\vect{u}_1,\vect{u}_2,\vect{v}_1}$.
\end{solution}

We also have a kind of opposite of Lemma~\ref{lem:extend-to-basis}:
every spanning set can be shrunk to a basis.

\begin{lemma}{Spanning set can be shrunk to a basis}{shrink-to-basis}
  \index{spanning set!shrink to basis}%
  \index{basis!by shrinking spanning set}%
  Let $V$ be a subspace of $\R^n$, and let
  $\vect{u}_1,\ldots,\vect{u}_{\ell}$ be a set of vectors spanning
  $V$. Then it is possible to obtain a basis of $V$ by ``shrinking''
  the set, i.e., by removing zero or more vectors from it.
\end{lemma}

\begin{proof}
  This is merely a restatement of
  Theorem~\ref{thm:linearly-independent-subset}. We obtain the
  linearly independent subset by removing the redundant vectors, which
  can be achieved by the casting-out algorithm. See also
  Example~\ref{exa:linearly-independent-subset}.
\end{proof}

The following proposition tells us something about the size of a
linearly independent set of vectors or the size of a spanning set of
vectors.

\begin{proposition}{Size of a linearly independent or spanning set of vectors}{size-linearly-independent-or-spanning}
  Let $V$ be a $k$-dimensional subspace of $\R^n$. Then
  \begin{enumialphparenastyle}
    \begin{enumerate}
    \item Every linearly independent set of vectors in $V$ has at most
      $k$ vectors.
    \item Every spanning set of vectors in $V$ has at least $k$ vectors.
    \end{enumerate}
  \end{enumialphparenastyle}
\end{proposition}

\begin{proof}
  Both properties follow from the Exchange Lemma
  (Lemma~\ref{lem:exchange-lemma}). Since $V$ is $k$-dimensional, it
  has some basis consisting of $k$ vectors
  $\vect{v}_1,\ldots,\vect{v}_k$.
  \begin{enumialphparenastyle}
    \begin{enumerate}
    \item Suppose $\vect{u}_1,\ldots\vect{u}_r$ are linearly
      independent vectors in $V$. Since $\vect{u}_1,\ldots\vect{u}_r$
      are linearly independent and $\vect{v}_1,\ldots,\vect{v}_k$ are
      spanning, the Exchange Lemma implies that $r\leq k$.
    \item Suppose the vectors $\vect{u}_1,\ldots\vect{u}_s$ span
      $V$. Since $\vect{v}_1,\ldots,\vect{v}_k$ are linearly
      independent and $\vect{u}_1,\ldots\vect{u}_s$ are spanning, the
      Exchange Lemma implies that $k\leq s$.
    \end{enumerate}
  \end{enumialphparenastyle}
\end{proof}

The next proposition often comes in handy when we need to check that
some set of vectors is a basis for a subspace $V$, where the dimension
of $V$ is already known. If $\dim(V)=k$, we know that any basis has to
have size $k$. Interestingly, to check that a set of $k$ vectors is a
basis of $V$, it is sufficient to check {\em either} that it is
linearly independent {\em or} that it is spanning. This can save half
the work in checking that some set of vectors is a basis (but it only
works if the number of vectors is exactly $k$, the dimension of $V$).

\begin{proposition}{Basis test for $k$ vectors in $k$-dimensional space}{basis-test-k-vectors}
  \index{basis test}%
  \index{basis!basis test}%
  Let $V$ be a $k$-dimensional subspace of $\R^n$, and consider $k$
  vectors $\vect{u}_1,\ldots,\vect{u}_k$ in $V$.
  \begin{itemize}
  \item If $\vect{u}_1,\ldots,\vect{u}_k$ are linearly
    independent, then they form a basis for $V$.
  \item If $\vect{u}_1,\ldots,\vect{u}_k$ span $V$, then they
    form a basis for $V$.
  \end{itemize}
\end{proposition}

\begin{proof}
  The first claim is an easy consequence of
  Lemma~\ref{lem:extend-to-basis}. Assume that
  $\vect{u}_1,\ldots,\vect{u}_k$ are linearly independent. By
  Lemma~\ref{lem:extend-to-basis}, we can add zero or more vectors to
  $\vect{u}_1,\ldots,\vect{u}_k$ to obtain a basis of $V$. On the
  other hand, since $V$ is $k$-dimensional, every basis must have
  exactly $k$ elements, so that the only possibility is that we have
  added zero vectors. Therefore,
  $\set{\vect{u}_1,\ldots,\vect{u}_k}$ is already a basis of $V$,
  as claimed.

  To prove the second claim, assume that
  $V=\sspan\set{\vect{u}_1,\ldots,\vect{u}_k}$. By
  Theorem~\ref{thm:subspaces-are-spans}, there exists a linearly
  independent subset of $\set{\vect{u}_1,\ldots,\vect{u}_k}$ that
  spans $V$, i.e., that is a basis for $V$. But since $\dim(V)=k$,
  every basis must have exactly $k$ elements, so that the only
  possible such subset is $\set{\vect{u}_1,\ldots,\vect{u}_k}$
  itself. Therefore, $\set{\vect{u}_1,\ldots,\vect{u}_k}$ is a
  basis of $V$, as claimed.
\end{proof}

It is important to note that
Proposition~\ref{prop:basis-test-k-vectors} does {\em not} say that
every linearly independent set of vectors in $V$ is a basis. For
example, a set of $k-1$ or fewer linearly independent vectors will not
be spanning. Also, the proposition does {\em not} say that every
spanning set of vectors in $V$ is a basis. For example, a set of $k+1$
or more spanning vectors will not be linearly independent. Rather,
what the proposition is saying is that if we have exactly $k$ vectors
in a $k$-dimensional space, then linear independence implies spanning
and vice versa.

\begin{example}{Basis test for $3$ vectors in $3$-dimensional space}{check-basis}
  Do the vectors
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ 2 \\ 3 \end{mymatrix},\quad
    \vect{u}_2 = \begin{mymatrix}{r} 1 \\ 1 \\ 1 \end{mymatrix},\quad
    \mbox{and}\quad
    \vect{u}_3 = \begin{mymatrix}{r} 1 \\ 1 \\ 2 \end{mymatrix}
  \end{equation*}
  form a basis of\/ $\R^3$?
\end{example}

\begin{solution}
  This is similar to Example~\ref{exa:non-standard-basis}. But because
  we know that $\R^3$ is a $3$-dimensional space, and because we have
  exactly $3$ vectors, by Proposition~\ref{prop:basis-test-k-vectors},
  we only need to check {\em either} whether
  $\vect{u}_1,\vect{u}_2,\vect{u}_3$ are linearly independent {\em or}
  whether they are spanning. We check whether they are linearly
  independent by using the casting-out algorithm.
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 1 & 1 \\
      2 & 1 & 1 \\
      3 & 1 & 2 \\
    \end{mymatrix}
    \quad\roweq\quad
    \begin{mymatrix}{rrr}
      1 & 1 & 1 \\
      0 & -1 & -1 \\
      0 & 0 & -1 \\
    \end{mymatrix}.
  \end{equation*}
  Since the matrix has rank 3, the vectors
  $\vect{u}_1,\vect{u}_2,\vect{u}_3$ are linearly
  independent. Therefore, by
  Proposition~\ref{prop:basis-test-k-vectors}, they form a basis of
  $\R^3$.
\end{solution}

The following proposition is also a consequence of
Lemma~\ref{lem:extend-to-basis}. It says that smaller subspaces have
smaller dimension.

\begin{proposition}{Subspace of a subspace}{subset-dimension}
  Let $V$ and $W$ be subspaces of\/ $\R^n$, and suppose that
  $V\subseteq W$.  Then $\dim(V)\leq\dim(W)$, with equality only when
  $V=W$.
\end{proposition}

\begin{proof}
  Consider any basis $\vect{u}_1,\ldots,\vect{u}_k$ of $V$. Because
  $V\subseteq W$, the vectors $\vect{u}_1,\ldots,\vect{u}_k$ are
  linearly independent elements of $W$, and therefore can be extended
  to a basis of $W$ by Lemma~\ref{lem:extend-to-basis}. The resulting
  basis of $W$ has at least $k$ elements, i.e., $\dim(V)\leq\dim(W)$.
  To prove the last claim, assume moreover that $\dim(V)=\dim(W)$. In
  that case, $\dim(W)=k$, so that the $k$ linearly independent vectors
  $\vect{u}_1,\ldots,\vect{u}_k$ form a basis of $W$ by
  Proposition~\ref{prop:basis-test-k-vectors}. Since both $V$ and $W$
  are spanned by $\vect{u}_1,\ldots,\vect{u}_k$, we must have $V=W$,
  as claimed.
\end{proof}


\section{Stratified sampling by a stratified probability bound}\label{section:SEBB}

In the previous section we considered different possible EBBs as a way of bounding the error in the context of stratified random sampling, and for this purpose developed a new EBB.
This process of using EBBs involved binding EBBs applied to different strata together using union bounds to create a bound on the stratified sample mean error (via Theorem \ref{triangle_theorem1}).

What is worth noting is that this process of binding EBBs together by probability unions is expected to result in a rather weak bound and that this weakness is expected to increase with larger numbers of strata as there are more probability unions needed to bind it together.
It is noted that the triangle equality $|A+B|\le |A|+|B|$, is only an equality in the event that the elements $A$ and $B$ are of the same sign, and in the context of theorems \ref{triangle_theorem1} and \ref{triangle_theorem2} the bound for the error is developed by effectively assuming all the errors of the estimates of the strata are additive - which is the worst case.
Whereas by assumption, the errors in the strata estimates are independent of each other and hence a overestimation in one stratum estimate is likely to be somewhat countered by an underestimation in another.
By using this knowledge stronger bounds can be created, and in this section we do just that. We have created an empirical (ie. depending on sample variances) concentration inequality specifically for stratified random sampling.

The resulting concentration inequality gives an analytic bound on the error of the stratified mean and explicitly considers the sample variances, data widths, sample numbers, and any additional weights on the strata; and includes factors specifically for strata sampled with and/or without replacement.

We proceed with the derivation of this new bound and method in a series of stages:
\begin{enumerate}
\item	in subsection \ref{subsection:new_bounds} we outlay some lemmas which are the building blocks of further derivations in this section, these are new upper bounds are for the moment generating function of samples and sample squares, as well as the sample means depending on whether sampling is done with or without replacement.
\item	in subsection \ref{sec:constructing} we use these elements to begin the derivation of the new bound, called the Stratified Empirical Bernstein Bound (SEBB)
\item	in subsection \ref{subsection:SECB} we additionally derive a variant of the SEBB which uses Chebyshev's inequality
\item	in subsection \ref{sec:SEBMalgorithm} we describe the algorithm of choosing samples in stratified sampling to minimise the new SEBB bound.
\end{enumerate}

After these subsections, in the next section \ref{section:statistics_results} we consider the numerical performance of minimising these new bounds in the context of stratified sampling.
Let us begin with the elements in these new derivations.
We note in this section, all proofs and demonstrations are our own, and are novel, except where otherwise explicitly noted.

%The derivation involves and utilises much of the same components of the previous section with an additional martingale theorem specifically for any any strata sampled without replacement.

\subsection{Some new bounds on the moment generating function}\label{subsection:new_bounds}

To begin the derivation of our new concentration inequality for stratified random sampling, we build upon some of the results of the previous sections.
Specifically we utilise three upper bounds for various moment generating functions.
The first of which has already been given in the previous section, and is Hoeffding's lemma - Lemma \ref{Hoeffdings_lemma_lemma}.
The other two are given here, and are bounds strongly related to Theorems \ref{hoeffdings1} and \ref{sample_squares} respectively.

The first of these other two upper bounds is very much like Hoeffding's Lemma, except it involves additional information about the variance of the random variable.

\begin{lemma}\label{expectation1}
For a random variable $X$ that is bounded on an interval $a\le X\le b$ with $D=b-a$ and variance $\sigma^2$, and any $s>0$:
\[
\E\left[\exp(s(X-\E[x]))\right]
\le\exp\left(\left(\frac{D^2}{17}+\frac{\sigma^2}{2}\right)s^2\right)
\nonumber
\]
\end{lemma}
\begin{proof}
We assume without loss of generality that $X$ is centred to have a mean of zero.
Then we construct an upper bound for $\E\left[\exp(sX)\right]$ in terms of $D$ by a parabola over $\exp(sX)$ for the permitted values of $X$ in the same way as in the proof of Theorem \ref{hoeffdings1}. By Lemma \ref{thm:parabola} there exists an $\alpha,\beta,\gamma$ such that $\alpha s^2X^2+\beta sX+\gamma\ge \exp(sX)$, and for all $a\le X\le b$, hence:
\[
E\left[\exp(sX)\right] \le \E[\alpha s^2X^2+\beta sX+\gamma] = \alpha s^2\E[X^2]+\gamma = \alpha s^2\sigma^2+\gamma
\nonumber\]
Where it follows that:
\begin{equation}\label{eqn:intermediate_lemma3.5}
E\left[\exp(sX)\right] \le\left(\frac{\sigma^2}{b^2}\exp\left(s\left(b+\frac{\sigma^2}{b}\right)\right) + 1\right)\exp\left(-\frac{s\sigma^2}{b}\right)\left(\frac{\sigma^2}{b^2} + 1\right)^{-1}.
\nonumber
\end{equation}
This relationship is exactly as in Theorem \ref{hoeffdings1}, now we do something slightly different - the expression in \eqref{eqn:intermediate_lemma3.5} is monotonically increasing with $b$, and $D>b$, therefore substituting $D$ for $b$ gives:
\begin{equation}\label{eq:part1}
\log(E\left[\exp(sX)\right]) \le \log\left(\frac{\sigma^2}{D^2}\exp\left(s\left(D+\frac{\sigma^2}{D}\right)\right) + 1\right)-\frac{s\sigma^2}{D} - \log\left(\frac{\sigma^2}{D^2} + 1\right)
\end{equation}
Because it is true that for any $\kappa,x\ge 0$, that:
\begin{equation}\label{eq:part2}
\log(\kappa\exp(x)+1)\le\log(\kappa+1)+\frac{x\kappa}{\kappa+1}+x^2\frac{\frac{1}{17}+\frac{\kappa}{2}}{(\kappa+1)^2}
\end{equation}
Thus letting $\kappa=\frac{\sigma^2}{D^2}$ and $x=s(D+\sigma^2/D)$ if follows that:
\begin{equation}\label{eq:part3}
\log(E\left[\exp(sX)\right]) \le \left(\frac{D^2}{17}+\frac{\sigma^2}{2}\right)s^2\qedhere
\end{equation}
\end{proof}

We note that this process of fitting a parabola over the exponential function is exactly the same process as used to derive Bennett's inequality (Theorem \ref{hoeffdings1}), but that we derive a weakened result from that same approach.

The next lemma that we present, is similar to the former, however this time we consider the random variable $X^2$ instead of $X$, and present a weakened bound on the moment generating function of it via a similar process as was used to derive Theorem \ref{sample_squares}:

\begin{lemma}\label{expectation2}
Let $X$ be a random variable of finite support on an interval $a\le X\le b$, with $D=b-a$ and variance $\sigma^2 = \E[(X-\E[x])^2] = \E[X^2]-\E[X]^2$. Then for any $q>0$:
$$\E[\exp(q(\sigma^2-(X-\E[X])^2))] \le \exp\left(\frac{1}{2}\sigma^2q^2D^2\right)$$
\end{lemma}
\begin{proof}
We assume without loss of generality (and for ease of presentation) that X is centred to have a mean of zero.
We construct an upper bound for $\E\left[\exp(-qX^2)\right]$ in terms of $D$ by a parabola over $\exp(-qX^2)$ for the permitted values of $X$.

For an $\alpha,\gamma$ such that $\alpha X^2+\gamma\ge \exp(-qX^2)$ then:
$$ \E[\exp(-qX^2)] \le \alpha \sigma^2 + \gamma.$$
If $d=\max(b,-a)$ we can choose $\gamma=1$ and $\alpha=(\exp(-q d^2)-1)d^{-2}$ (see figure \ref{fig:graph111}), Thus:
\begin{align*}
\E[\exp(-q X^2)] &\le \frac{\sigma^2}{d^2}\exp(-q d^2)-\frac{\sigma^2}{d^2} + 1\le \frac{\sigma^2}{D^2}\exp(-q D^2)-\frac{\sigma^2}{D^2} + 1 \\
&\le \exp\left(\log\left(\frac{\sigma^2}{D^2}\exp(-q D^2)-\frac{\sigma^2}{D^2} + 1\right)\right)
\end{align*}
Given that for any $0\le \kappa \le 0.5$ and $x\le 0$ that: $$\log\left(\kappa\exp(x)-\kappa + 1\right) \le \kappa x+\frac{1}{2}\kappa(1-\kappa)x^2$$
Letting $\kappa=\frac{\sigma^2}{D^2}$ and $x=-qD^2$, which is valid by Popoviciu's inequality (see \cite{zbMATH05780164}) $\sigma^2\le D^2/4$, then:
$$ \E[\exp(-q X^2)] \le \exp\left(\frac{1}{2}\sigma^2q^2(D^2-\sigma^2)-\sigma^2q\right) \le \exp\left(\frac{1}{2}\sigma^2q^2D^2-\sigma^2q\right)$$
and the result follows by multiplying by $\exp(q\sigma^2)$.
\end{proof}

The inequalities above, Lemmas \ref{expectation1} and \ref{expectation2}, as well as Lemma \ref{Hoeffdings_lemma_lemma}, are used in the derivation of our stratified sampling concentration inequality in Section~\ref{sec:constructing}. One of the primary reasons for utilising these weakened bounds on the moment generating function is that they make the subsequent mathematics far more tractable.
However in order to use these moment generating functions we need to explicitly describe the difference between the moment generating functions of individual random variables (which these are) and the moment generating function of the sample mean of them.

\subsubsection{Some bounds on the moment generating function of sample means}\label{sec:without_replacement}

In order to use the previous bounds on the moment generating function we need a relationship between the moment generating function of a random variable, and the moment generating function of the average of samples of that random variable.
To do this we state two further inequalities, where the first one (Lemma~\ref{martingale1}) is most appropriate for sampling average is taken with replacement, and the second (Lemma~\ref{martingale0}) can optionally be used in the context that the sampling average is without replacement - and may (or may not) give a tighter result.

We first state a lemma that is essentially is a formalisation of the process we are familiar with in the last section (see Lemma \ref{chernoff1}):

\begin{lemma}[Replacement Bound]\label{martingale1}
Let $X$ be a random variable that is bounded $a\le X\le b$ with a mean of zero, with $D=b-a$ and variance $\sigma^2$.
Let $\chi_m = \frac{1}{m}\sum_{i=1}^mX_i$ be the average of $m$ independently drawn (with replacement) samples of this random variable.
If there exists an $\alpha, \beta \ge 0$ such that for any $s>0$ that $\E[\exp(sX)]\le\exp((\alpha D^2 +\beta \sigma^2) s^2)$ then:
$$\textstyle\E[\exp(s\chi_m)]\le\exp(\alpha s^2D^2\frac{1}{m} +\beta s^2\sigma^2 \frac{1}{m}) = \exp((\alpha D^2{\Omega}_m^n +\beta\sigma^2 {\Psi}_m^n)s^2)$$
where ${\Omega}_m^n = {\Psi}_m^n = \frac{1}{m}$
\end{lemma}
\begin{proof}
By the independence of samples, we have:
\[\E[\exp(s\chi_m)]=\E\left[\exp\left(\frac{s}{m}\sum_{i=1}^mX_i\right)\right]=\prod_{i=1}^m\E\left[\exp\left(\frac{s}{m}X\right)\right]\]
Thus:
\[\E[\exp(s\chi_m)] \le \exp\left(\frac{s^2}{m^2}\sum_{i=1}^m\left(\alpha D^2 +\beta \sigma^2\right) \right) \qedhere\]
\end{proof}

%The requirement that the mean of the random variable is zero is useful as further derivations are applied to random variables which are shifted such as to have a mean of zero ie. $X-\E[X]$.
For the case of sampling without replacement, there is an alternative result that can be directly substituted, given in Lemma~\ref{martingale0}, below, which can be tighter in certain cases.
Before this, particular note must be made that the inequality above, Lemma~\ref{martingale1} can be used in the context of either sampling with or without replacement.
In contrast, Lemma~\ref{martingale0} can only be used when sampling without replacement.
This distinction was shown to be true by \cite{hoeffding1}, and is rooted in an already presented Lemma \ref{hoeffdings_reduction}.

We now state Lemma~\ref{martingale0}, an inequality regarding the moment generating function of the average of samples taken specifically \textit{without replacement}.
When the sampling takes place without replacement the inequality of Lemma~\ref{martingale1} can potentially be tightened to take advantage of the finite size of the population.
This inequality extends an important martingale inequality from \cite{bardenet2015}:

\begin{lemma}[Martingale Bound]\label{martingale0}
For finite data $x_1,x_2,\dots x_n$ that is bounded $a\le x_i\le b$, and has a mean of zero and variance $\sigma^2=\frac{1}{n}\sum_{i=1}^nx_i^2$, denote $X_1,X_2,\dots,X_n$ the random variables corresponding to the data sequentially drawn randomly without replacement, and $\chi_m$ the average of the first $m$ of them.
If for any random variable $Z$ with a mean of zero such that $a\le Z\le b$ and $D=b-a$, with variance $\sigma_Z^2$ that there exists an $\alpha, \beta \ge 0$ such that for any $s>0$ that $\E[\exp(sZ)]\le\exp((\alpha D^2 +\beta \sigma_Z^2) s^2)$ then:
\begin{align*}\E[\exp(s\chi_m)]&\le\exp\left(\alpha s^2D^2\sum_{k=m}^{n-1}\frac{1}{k^2} +\beta s^2\sigma^2 \sum_{k=m}^{n-1}\frac{n}{k^2(k+1)}\right)\\ &\le \exp((\alpha D^2\bar{\Omega}_m^n +\beta\sigma^2 \bar{\Psi}_m^n)s^2)\end{align*}
where $\bar{\Omega}_m^n = \sum_{k=m}^{n-1}\frac{1}{k^2}\approx \frac{(m+1)(1-m/n)}{m^2}$ and $\bar{\Psi}_m^n = \sum_{k=m}^{n-1}\frac{n}{k^2(k+1)}\approx \frac{n+1-m}{m^2}$.
\end{lemma}
\begin{proof}
Observe that:
\begin{align*}
\chi_m
& =\frac{1}{m}\sum_{i=1}^{m}X_i = \chi_{m+1}+\frac{1}{m}(\chi_{m+1}-X_{m+1})\\
& =(\chi_m-\chi_{m+1})+(\chi_{m+1}-\chi_{m+2}) + \dots + (\chi_{n-1}-\chi_n)\\
& =\frac{1}{m}(\chi_{m+1}-X_{m+1})+\frac{1}{m+1}(\chi_{m+2}-X_{m+2}) + \dots + \frac{1}{n-1}(\chi_n-X_n).
\end{align*}
Then because:
$$\exp(s\chi_m)=\prod_{k=m}^{n-1}\exp\left(\frac{s}{k}(\chi_{k+1}-X_{k+1})\right),$$
we also have that:
$$\E[\exp(s\chi_m)]=\E\left[\prod_{k=m}^{n-1}\E\left[\exp\left(\frac{s}{k}(\chi_{k+1}-X_{k+1})\right)|\chi_{k+1}\dots \chi_n\right]\right]$$
by repeated application of the Law of total expectation.
Since:
$$\E[X_{k+1}|\chi_{k+1}\dots \chi_n]=\chi_{k+1},$$
then $\chi_{k+1}-X_{k+1}$ is a random variable with a mean of zero bounded within width $D$, and it also has a variance given by:
\begin{equation}\label{approx1}
\sigma_{k+1}^2 = \frac{n\sigma^2-\sum_{j=k+1}^nX_j^2}{n-(n-k-1)} - \chi_k^2 \le \frac{n\sigma^2}{k+1}
\end{equation}
by application of Lemma~\ref{variance1}.
Therefore:
\[\E[\exp(s\chi_m)]\le\exp\left(\sum_{k=m}^{n-1}\left(\alpha D^2 +\beta \frac{n\sigma^2}{k+1}\right) \frac{s^2}{k^2}\right)\qedhere\]
\end{proof}

This martingale result relates the moment generating function bound of the average of finite variables relative to their mean, to the moment generating function bounds of the  differences of the incremental averages relative to their mean.
We note that this result could potentially be made much stronger by working around the use of Equation~\eqref{approx1}, but this comes at a cost of increased mathematical complexity.

Since Lemmas~\ref{martingale0} and~\ref{martingale1} share a common form, and because of Hoeffding's reduction (Lemma~\ref{hoeffdings_reduction}),
all the derivations that follow that invoke Lemma~\ref{martingale1} have direct analogues using Lemma~\ref{martingale0} for the context of sampling without replacement.
Note, however, that the bound without replacement (Lemma~\ref{martingale0}) may or may not be tighter than the bound with replacement (Lemma~\ref{martingale1}). However, the process of substituting one for the other can be done judiciously on a case-by-case basis to create the tightest possible bound.
All the numerical results in Section \ref{section:statistics_results} (that are relevant to sampling without replacement) have been produced with this judicious choice conducted.

\subsection{The Stratified finite Empirical Bernstein Bound (SEBB)}
\label{sec:constructing}

In this section we derive a novel probability bound for the error of the stratified random sampling estimate,
and use it to define a sequential stratified sampling algorithm.
Before this, we begin by precisely defining the context of our derivations, to which our bound applies.

\begin{definition}[Problem context]\label{def:ProblemContext}\hspace{1cm}\\
\begin{itemize}
    \item Let a population consist of $n$ number of strata,
    \item where $n_i$ is the total number of data points in the $i$th stratum.
    \item All values in a stratum are bound within a finite support of width $D_i$.
    \item the mean of the $i$th stratum is $\mu_i$, and its variance $\sigma_i^2$.
    \item the random variables corresponding to the samples drawn from the $i$ stratum are:\\ $X_{i,1},X_{i,2},\dots,X_{i,n_i}$
    \item for each stratum $m_i$ samples are taken
    \item forming the sample mean of each stratum: $\chi_{i,m_i}= \frac{1}{m_i}\sum_{j=1}^{m_i}X_{i,j}$
    \item the biased sample variance of each stratum: $\hat{\sigma}_i^2=\frac{1}{m_i}\sum_j^{m_i}(X_{i,j}-\chi_{i,m_i})^2$
    \item the unbiased sample variance of each stratum: $\doublehat{\sigma}_i^2 = m_i\hat{\sigma}_i^2/(m_i-1)$
    \item we consider the average of the means of the strata as weighted by constant positive factors $\{\tau_i \}_{i\in \{1,\dots,n\}}$
    \item And throughout the derivation we also use temporary arbitrary positive variables $\{\theta_i \}_{i\in \{1,\dots,n\}}$
\end{itemize}
\end{definition}

\noindent The bound is now developed in four theorems, which build on each other in sequence:
\begin{enumerate}
    \item in subsubsection~\ref{subsubsection:variance_assisted_sebb} Theorem~\ref{thm:1} develops a concentration inequality for the error in the stratified population mean estimate $\sum_{i=1}^n\tau_i\chi_{i,m_i}$ in the context of the knowledge of stratum variances.
    \item in subsubsection~\ref{subsubsection:variance_square_error_bound} Theorem~\ref{thm:2} is a concentration inequality of the difference between the stratum variances and sample variances in the context the sum of knowledge of the squared stratum mean errors.
    \item in subsubsection~\ref{subsubsection:a_bound_on_sample_squares} Theorem~\ref{thm:3} is an inequality directly that binds the sum of sample squared stratum mean errors.
    \item in subsubsection~\ref{subsubsection:centerpiece} Theorem~\ref{thm:SEBM_bound} combines the three previous theorems together using two union bounds to create a concentration inequality for the error in the stratified population mean estimate given the sample variances.
\end{enumerate}

\subsubsection{A bound on the sample mean assuming variances}\label{subsubsection:variance_assisted_sebb}

In a similar way to what was done in the previous section, to derive an Empirical Bernstein Bound we begin with a derivation of a probability bound on the error of the sample mean of a random variable in terms of the random variable's variance.
In the previous section the bound in question was the venerable Bennett's inequality (Theorem \ref{hoeffdings1}) however in this section we consider a more relaxed and similar version of it which is more easy for us to manipulate, and in this case the random variable in question is the weighted mean $\sum_{i=1}^n\tau_i\chi_{i,m_i}$.

This is probability bound on the absolute error of the weighted stratified sample means about the weighted strata means, which we call a variance-assisted SEBB (stratified empirical Bernstein bound).

\begin{theorem}[Variance-assisted SEBB]\label{thm:1}
Assuming the context given in Definition~\ref{def:ProblemContext}, and let $\Omega_{m_i}^{n_i}$ and $\Psi_{m_i}^{n_i}$ be given as in Lemma~\ref{martingale1}, then:
\begin{equation}\label{eq1} \pr\left(\left|\sum_{i=1}^n\tau_i(\chi_{i,m_i}-\mu_i)\right|\ge \sqrt{4\log(2/t)\sum_{i=1}^n\left(\frac{1}{17}D_i^2\Omega_{m_i}^{n_i}+\frac{1}{2}\sigma_i^2\Psi_{m_i}^{n_i}\right)\tau_i^2}\right)\le t \end{equation}
\end{theorem}
\begin{proof}
In a similar way as Lemma~\ref{chernoff1} we consider a bound for the weighted mean by the moment generating function of the stratum means:
\begin{align*} \pr\left(\sum_{i=1}^n\tau_i\chi_{i,m_i}-\sum_{i=1}^n\tau_i\mu_i\ge t\right)
&\le\E\left[\exp\left(\sum_{i=1}^n\tau_is\left(\chi_{i,m_i}-\mu_i\right)\right)\right]\exp(-st)\\
&= \prod_{i=1}^n\E\left[\exp\left(\tau_is\left(\chi_{i,m_i}-\mu_i\right)\right)\right]\exp(-st)
\end{align*}
This involves the assumption that the sampling -between- the strata is independent.
This form is sufficient for Lemma~\ref{martingale1} with Lemma~\ref{expectation1} to apply, resulting in a double-sided tail bound:
$$ \pr\left(\left|\sum_{i=1}^n\tau_i(\chi_{i,m_i}-\mu_i)\right|\ge t\right)\le 2\exp\left(\sum_{i=1}^n\left(\frac{1}{17}D_i^2\Omega_{m_i}^{n_i}+\frac{1}{2}\sigma_i^2\Psi_{m_i}^{n_i}\right)\tau_i^2s^2 - st\right) $$
Minimizing with respect to $s$ and rearranging gives result.
\end{proof}

This particular bound assumes knowledge of the variances of the strata, and will be used as a tool for selecting samples primarily for comparison with other sampling methods (such as Neyman sampling) in Section \ref{section:statistics_results}.

It is a concentration inequality for the weighted stratum means, leading to the more general question of what the weights should be. In most cases, the weights $\tau_i$ can be considered as the probability weights $\tau_i=n_i/(\sum_{j=1}^nn_j)$ of standard stratified sampling.
In this context this probability bound can be used as-is for a measure of uncertainty in stratified random sampling if the true variances (or alternatively, upper bounds on the true variances) of the strata are known.
However the weights may be assigned differently - and we will investigate such a case for the Shapley Value in section \ref{section:statistics_results}.

The bound depends on a weighted sum of strata variances $ \sum_{i=1}^n \sigma_i^2 \Psi_{m_i}^{n_i} \tau_i^2 $, and in most situations these values aren't known.
Hence to use this inequality we need to consider the probable error between the weighted sum of strata variances and the weighted sum of strata sample variances.

\subsubsection{A bound on the sample variance in terms of sample error squares}\label{subsubsection:variance_square_error_bound}

To create a bound on the error between the strata variances and the strata sample variances we develop a probability bound for the error estimate of the sum of variances (as weighted by arbitrary $\theta_i$) in terms of the sample square errors (which will also consequently be bounded), as follows:

\begin{theorem}\label{thm:2}
Assuming the context given in Definition~\ref{def:ProblemContext}.
Then with $\Psi_{m_i}^{n_i}$ per Lemma~\ref{martingale1}:
\begin{equation}\label{eq2} \pr\left(\sum_{i=1}^n\theta_i(\sigma_i^2-\hat{\sigma}_i^2 - (\mu_i - \chi_{i,m_i})^2)\ge \sqrt{2\log(1/y)\sum_{i=1}^n\sigma_i^2\theta_i^2D_i^2\Psi_{m_i}^{n_i}}\right) \le y \end{equation}
\end{theorem}

\begin{proof}
To create a probability bound for the sum of variances (weighted by arbitrary positive $\theta_i$), we consider the average square of samples about the strata means.
Applying Lemma~\ref{chernoff1} gives:
\begin{align*}
\pr&\left(\sum_{i=1}^n\theta_i(\sigma_i^2-\frac{1}{m_i}\sum_{j=1}^{m_i}(X_{i,j}-\mu_i)^2)\ge y\right) \\
&\le \E\left[\exp\left(\sum_{i=1}^ns\theta_i\left(\sigma^2-\frac{1}{m_i}\sum_{j=1}^{m_i}(X_{i,j}-\mu_i)^2\right)\right)\right]\exp(-sy)\\
& \le \exp(-sy)\prod_{i=1}^n\E\left[\exp\left(\frac{s\theta_i}{m_i}\sum_{j=1}^{m_i}(\sigma^2-(X_{i,j}-\mu_i)^2)\right)\right]
\end{align*}
And this reason is by the assumption of the independence of the sampling -between- the strata.
This resulting form is sufficient for Lemma~\ref{martingale1} with Lemma~\ref{expectation2} to apply, giving:
$$ \pr\left(\sum_{i=1}^n\theta_i(\sigma_i^2-\frac{1}{m_i}\sum_{j=1}^{m_i}(X_{i,j}-\mu_i)^2)\ge y\right) \le \exp\left(\frac{1}{2}\sum_{i=1}^n\sigma_i^2\theta_i^2s^2D_i^2\Psi_{m_i}^{n_i}-sy\right)$$
Minimizing with respect to $s$, rearranging, and applying Lemma \ref{variance1} gives result.
\end{proof}

This inequality gives the probability bound between the arbitrarily (by $\theta_i$) weighted variances of the strata and also the same weighted (biased estimator) sample variances.
However it also additionally involves the weighted square error of the sample means as a complicating factor.
Although the weighted square error of the sample means may go to zero quickly as additional samples are taken, we nonetheless need to develop another probability bound to incorporate the specific consideration of it.

\subsubsection{A bound on weighted sample error squares}\label{subsubsection:a_bound_on_sample_squares}

In the previous Theorem \ref{thm:2} the weighted sum of sample squares was a complicating factor which we seek to bound and incorporate. The following probability inequality bounds the weighted square error of the sample means directly:

\begin{theorem}\label{thm:3}
Assuming the context given in Definition~\ref{def:ProblemContext}.
Then with $\Omega_{m_i}^{n_i}$ as in Lemma~\ref{martingale1}:
\begin{equation}\label{eq1.5} \pr\left(\sum_{i=1}^n\theta_i(\mu_i - \chi_{i,m_i})^2 \ge \frac{\log(2n/r)}{2}\sum_{i=1}^n\theta_iD_i^2\Omega_{m_i}^{n_i}\right) \le r \end{equation}
\end{theorem}
\begin{proof}
We consider the weighted square error of the sample means, and by probability complimentarity we know:
$$ \pr\left(\sum_{i=1}^n\theta_i(\mu_i - \chi_{i,m_i})^2 \ge r\right) = 1-\pr\left(\sum_{i=1}^n\theta_i(\mu_i - \chi_{i,m_i})^2 < r\right) $$

As the probability that sum of any random variables is less than $r$ is obviously greater than the probability that those random variables individual are all less than specific values that sum to $r$ hence, for $r_i$ such that $\sum r_i=r$:

$$ \pr\left(\sum_{i=1}^n\theta_i(\mu_i - \chi_{i,m_i})^2 \ge r\right) \le 1-\prod_{i=1}^n\pr\left(\theta_i(\mu_i - \chi_{i,m_i})^2 < r_i\right) $$
hence by probability complimentarities again:
\begin{align*}&\pr\left(\sum_{i=1}^n\theta_i(\mu_i - \chi_{i,m_i})^2 \ge r\right) \le \\&\qquad\qquad 1-\prod_{i=1}^n\left(1-\pr\left(\mu_i - \chi_{i,m_i} \ge \sqrt{\frac{r_i}{\theta_i}}\right) - \pr\left( \chi_{i,m_i}-\mu_i \ge \sqrt{\frac{r_i}{\theta_i}}\right)\right)\end{align*}

And this is exactly the form which we want, in terms of the products of the error in each of the strata sample means.
Thus we can apply Lemma~\ref{chernoff1} together with Lemmas~\ref{martingale1} and~\ref{Hoeffdings_lemma_lemma}, which gives:
$$
\pr\left(\sum_{i=1}^n\theta_i(\mu_i - \chi_{i,m_i})^2 \ge r\right) \le 1-\prod_{i=1}^n\left(1-2\exp\left(-\frac{2r_i}{\theta_iD_i^2\Omega_{m_i}^{n_i}}\right)\right)
$$
Next, choosing $r_i$ to minimise the right hand side of this expression gives:
$$
r_i = \frac{r\theta_iD_i^2\Omega_{m_i}^{n_i}}{\sum_j \theta_jD_j^2\Omega_{m_j}^{n_j}}
$$
In this context, we therefore deduce that:
$$
\pr\left(\sum_{i=1}^n\theta_i(\mu_i - \chi_{i,m_i})^2 \ge r\right) \le 1-\prod_{i=1}^n\left(1-2\exp\left( \frac{-2r}{\sum_j\theta_j D_j^2\Omega_{m_j}^{n_j}}\right)\right)
$$
Using the fact that $\log(1-(1-\exp(x))^n)\le x+\log(n)$ for any negative $x$, and rearranging, gives the required result.
\end{proof}

This theorem directly bounds the weighted square errors of the sample means independently of any other specifically unknown factors.

\subsubsection{The centerpiece of the SEBB}\label{subsubsection:centerpiece}

In the previous three theorems we have a bound on the stratified mean estimate in terms of the variances, a bound on the variances in terms of the sample variances by the sample square errors, and a bound on the sample square errors.
In the next step we combine all the inequalities of Equations~\eqref{eq1}, \eqref{eq2} and~\eqref{eq1.5} from Theorems \ref{thm:1}, \ref{thm:2} and \ref{thm:3} together, to complete our derivation of a probability bound for the error in stratified sampling in terms of stratum sample variances - our SEBB.

\begin{theorem}[Stratified Empirical Bernstein Bound (SEBB)]\label{thm:SEBM_bound}
Assuming the context given in Definition~\ref{def:ProblemContext}.
Then with $\Omega_{m_i}^{n_i},\Psi_{m_i}^{n_i}$ per Lemma~\ref{martingale1}:
\begin{equation}\label{big_equation}
%\p\left(\frac{\left|\sum_{i=1}^n\tau_i(\chi_{i,m_i}-\mu_i)\right|}{\sqrt{\log(6/p)}}\ge \sqrt{\begin{matrix*}[l]\sum_{i=1}^n\frac{4}{17}\Omega_{m_i}^{n_i}D_i^2\tau_i^2 \\ +\begin{pmatrix*}[l]\sqrt{\log(3/p)\left(\max_i\tau_i^2{\Psi_{m_i}^{n_i}}^2D_i^2\right)} \\ +\sqrt{\begin{matrix*}[l]2\sum_{i=1}^n\tau_i^2\Psi_{m_i}^{n_i}(m_i-1)\doublehat{\sigma}_i^2/m_i \\ + \log(6n/p)\sum_i\tau_i^2D_i^2\Omega_{m_i}^{n_i}\Psi_{m_i}^{n_i} \\ +\log(3/p)\left(\max_i\tau_i^2{\Psi_{m_i}^{n_i}}^2D_i^2\right)\end{matrix*}} \end{pmatrix*}^2\end{matrix*}}\right)
%\le p
%\end{equation}
%\begin{equation}\label{big_equation_alternate}
\pr\left(\left|\sum_{i=1}^n\tau_i(\chi_{i,m_i}-\mu_i)\right|
\ge \sqrt{\log(6/p)\left( \alpha
+ \left(\sqrt{\beta}
+ \sqrt{\gamma}\right)^2\right) } \right)
\le p
\end{equation}
where:
\begin{align*}
\alpha=&\sum_{i=1}^n\frac{4}{17}\Omega_{m_i}^{n_i}D_i^2\tau_i^2 \\
\beta=&\log(3/p)\left(\max_i\tau_i^2{\Psi_{m_i}^{n_i}}^2D_i^2\right) \\
\gamma=& 2\sum_{i=1}^n\tau_i^2\Psi_{m_i}^{n_i}(m_i-1)\doublehat{\sigma}_i^2/m_i
+ \log(6n/p)\sum_i\tau_i^2D_i^2\Omega_{m_i}^{n_i}\Psi_{m_i}^{n_i} \\
&\quad\quad\quad\quad\quad\quad\quad\quad\quad\quad\quad~+\log(3/p)\left(\max_i\tau_i^2{\Psi_{m_i}^{n_i}}^2D_i^2\right).
\end{align*}

%$$
%\alpha_{m_i}^{n_i}
%=\sum_{i=1}^n\frac{4}{17}\Omega_{m_i}^{n_i}D_i^2\tau_i^2
%$$
%$$
%\beta_{m_i}^{n_i}
%=\log(3/p)\left(\max_i\tau_i^2{\Psi_{m_i}^{n_i}}^2D_i^2\right)
%$$
%and
%\begin{align*}
%\gamma_{m_i}^{n_i}
%= 2\sum_{i=1}^n\tau_i^2\Psi_{m_i}^{n_i}(m_i-1)\doublehat{\sigma}_i^2/m_i
%&+ \log(6n/p)\sum_i\tau_i^2D_i^2\Omega_{m_i}^{n_i}\Psi_{m_i}^{n_i}  \\
%&+\log(3/p)\left(\max_i\tau_i^2{\Psi_{m_i}^{n_i}}^2D_i^2\right)
%\end{align*}
\end{theorem}

\begin{proof}
By widening the bound of Equation~\eqref{eq2} we get:
$$
\pr\left(\begin{matrix*}[l]\sum_{i=1}^n\theta_i\sigma_i^2-\sum_{i=1}^n\theta_i(\hat{\sigma}_i^2+(\mu_i - \chi_{i,m_i})^2)\ge \\ \quad\quad\quad\quad\quad \sqrt{2\log(1/y)(\max_i\theta_iD_i^2\Psi_{m_i}^{n_i})\sum_{i=1}^n\theta_i\sigma_i^2}\end{matrix*}\right) \le y.
$$
Completing the square gives for $\sqrt{\sum_{i=1}^n\theta_i\sigma_i^2}$ gives:
$$
\pr\left(\sqrt{\sum_{i=1}^n\theta_i\sigma_i^2} \ge \begin{matrix*}[l]\quad\sqrt{\begin{matrix*}[l]\sum_{i=1}^n\theta_i(\hat{\sigma}_i^2+(\mu_i - \chi_{i,m_i})^2)\\ ~ + \frac{\log(1/y)}{2}\left(\max_i\theta_iD_i^2\Psi_{m_i}^{n_i}\right)\end{matrix*}} \\  +\sqrt{\frac{\log(1/y)}{2}\left(\max_i\theta_iD_i^2\Psi_{m_i}^{n_i}\right)}\end{matrix*}\right) \le y.
$$
Combining with Equation~\eqref{eq1.5} with a union bound (Lemma~\ref{prob_union}) gives:
\begin{equation}\label{eq:sum_variance_bound_equation}
\pr\left(\sqrt{\sum_{i=1}^n\theta_i\sigma_i^2} \ge \begin{matrix*}[l]\quad\sqrt{\begin{matrix*}[l]\sum_{i=1}^n\theta_i\hat{\sigma}_i^2+ \frac{\log(2n/r)}{2}\sum_i\theta_iD_i^2\Omega_{m_i}^{n_i} \\ ~+ \frac{\log(1/y)}{2}\left(\max_i\theta_iD_i^2\Psi_{m_i}^{n_i}\right)\end{matrix*}} \\ +\sqrt{\frac{\log(1/y)}{2}\left(\max_i\theta_iD_i^2\Psi_{m_i}^{n_i}\right)}\end{matrix*}\right) \le y+r ,
\end{equation}
which is a bound for the weighted sum variances in terms of the sample variances.
Letting $\theta_i = \frac{1}{2}\tau_i^2\Psi_{m_i}^{n_i}$ and combining with \eqref{eq1} with a union bound (Lemma~\ref{prob_union}),
and then assigning $ r=t=y=p/3 $ and rewriting in terms of unbiased sample variance, gives the result.
\end{proof}

This completes the derivation of the SEBB.
In Equation \eqref{big_equation} of Theorem~\ref{thm:SEBM_bound}, we have a concentration inequality for the sum of weighted strata sample mean errors relative to the sample variances.
In this context, the weights $\tau_i$ are flexible but would naturally be probability weights proportional to strata size,  $\tau_i=n_i/(\sum_{j=1}^nn_j)$,
in which case the inequality provides a concentration of measure in stratified random sampling.\\

\subsection{A Stratified Empirical Chebyshev Bound (SECB)}\label{subsection:SECB}

It is also possible to consider another strongly related bound on the error in stratified sampling.
Since the last Theorem \ref{thm:SEBM_bound} ultimately builds upon Theorem \ref{thm:1} which was the embodiment of a simplification of Bennett's inequality per Lemma \ref{expectation1}.
And since we will ultimately compare performance of these bounds against Neyman sampling which is conceptually built upon the minimisation of Chebyshev's inequality (Theorem \ref{thm:chebyshevs}) we will also consider and compare against a empirical Chebyshev's inequality for stratified sampling.

\begin{theorem}[Stratified Empirical Chebyshev Bound (SECB)]\label{thm:SECM_bound}
Assuming the context given in Definition~\ref{def:ProblemContext}.
Then with $\Omega_{m_i}^{n_i},\Psi_{m_i}^{n_i}$ per Lemma~\ref{martingale1}:
\begin{equation}\label{another_big_equation}
\pr\left(\left|\sum_{i=1}^n\tau_i(\chi_{i,m_i}-\mu_i)\right|
\ge \sqrt{\frac{3}{p}}\left(
\sqrt{\alpha+\beta}
+ \sqrt{\beta}\right)  \right)
\le p
\end{equation}
where:
\begin{align*}
\alpha=&\sum_{i=1}^n\tau_i^2\Psi_{m_i}^{n_i}(m_i-1)\doublehat{\sigma}_i^2/m_i+ \frac{\log(6n/p)}{2}\sum_i\tau_i^2\Psi_{m_i}^{n_i}D_i^2\Omega_{m_i}^{n_i} \\
\beta=&\frac{\log(3/p)}{2}\left(\max_i\tau_i^2D_i^2{\Psi_{m_i}^{n_i}}^2\right)
\end{align*}
\end{theorem}
\begin{proof}
We can use Chebyshev's inequality (Theorem \ref{thm:chebyshevs}) for the strata sample estimator giving:
$$ \p\left(\left|\sum_{i=1}^n\tau_i(\chi_{i,m_i}-\mu_i)\right|\ge \frac{1}{\sqrt{k}}\sqrt{\text{Var}\left(\sum_{i=1}^n\tau_i\chi_{i,m_i}\right)}\right) \le k $$
Which is a probability bound for the error of the stratum mean estimator in terms of its variance.
Whereby we can assume the independence of the sampling between the strata and sampling with replacement, giving the decomposition of the variance of the estimator (in a similar process to Equation \ref{eq:variance_decomposition_for_strata_mean}), giving:
$$ \text{Var}\left(\sum_{i=1}^n\tau_i\chi_{i,m_i}\right) = \sum_{i=1}^n\tau_i^2\text{Var}(\chi_{i,m_i}) =  \sum_{i=1}^n\frac{\tau_i^2\sigma_i^2}{m_i} $$
Hence:
$$ \p\left(\left|\sum_{i=1}^n\tau_i(\chi_{i,m_i}-\mu_i)\right|\ge \frac{1}{\sqrt{k}}\sqrt{\sum_{i=1}^n\Psi_{m_i}^{n_i}\tau_i^2\sigma_i^2}\right) \le k $$
Which is a Chebyshev type inequality for the error of the stratum mean estimator in terms of a sum of the stratum variances - and is very analogous to Equation \ref{eq1} of Theorem \ref{thm:1}.
And since equation \ref{eq:sum_variance_bound_equation} (from the previous proof) is a general bound for an arbitrary sum of the stratum variances we can combine with it (in the case of setting $\theta_i = \tau_i^2\Psi_{m_i}^{n_i}$) by a union bound (Lemma~\ref{prob_union}), giving:
\begin{align*}
\pr\left(\left|\sum_{i=1}^n\tau_i(\chi_{i,m_i}-\mu_i)\right| \ge \frac{1}{\sqrt{k}}\left( \begin{matrix*}[l]\quad\sqrt{\begin{matrix*}[l]\sum_{i=1}^n\tau_i^2\Psi_{m_i}^{n_i}\hat{\sigma}_i^2+ \frac{\log(2n/r)}{2}\sum_i\tau_i^2\Psi_{m_i}^{n_i}D_i^2\Omega_{m_i}^{n_i} \\ ~+ \frac{\log(1/y)}{2}\left(\max_i\tau_i^2D_i^2{\Psi_{m_i}^{n_i}}^2\right)\end{matrix*}} \\ +\sqrt{\frac{\log(1/y)}{2}\left(\max_i\tau_i^2D_i^2{\Psi_{m_i}^{n_i}}^2\right)}\end{matrix*}\right)\right)&\\ \le y+&r+k
\end{align*}
And setting $ r=k=y=p/3 $ and rewriting in terms of unbiased sample variance, gives the result.
\end{proof}

We will now turn to how minimising these bounds on the stratified sample mean estimate (theorems \ref{thm:SEBM_bound} and \ref{thm:SECM_bound}) can be used as a method of choosing samples in the context of stratified sampling.
And afterwards, in the next section \ref{section:statistics_results} we will compare how all these methods work in actually sampling stratified data.

\subsection{Sequential sampling using the stratified empirical bernstein method} \label{sec:SEBMalgorithm}

%To evaluate the effectiveness of the process of sequentially minimising the probability bounds on the error of the sample estimated stratified mean, we need a little development of those algorithms before proceeding to give details on the process of testing them in the context of synthetic data.
%It is necessary to consider the way in which the concentration inequality of the previous section (Theorem \ref{thm:SEBM_bound}) should be iteratively minimised; and this process is expounded as Algorithm \ref{alg2}.

In this section, we developed a concentration inequality to bound the error of the sampling mean estimate in stratified random sampling, called the SEBB - as per Theorem \ref{thm:SEBM_bound}.
The process of selecting additional samples to minimise this probability bound on the error is introduced in this section. And we call it the \textit{stratified empirical Bernstein method} (SEBM).

The fundamental principle of the SEBM, is that it is an online method of choosing additional samples which repeatedly scans through possible stratum for optimum sample choice, and the strata which would result in the most reduction of the SEBB bound is then recommended for additional sampling.
The pseudo-code for this process of sampling, is given as Algorithm \ref{alg2}.

Specifically, Algorithm \ref{alg2} is a repetitive process involving a scan through the possible strata and then the selection of one stratum to sample from.
The process of scanning involves calculating the confidence bound width (SEBB) that would result if an additional sample were to be taken from that stratum without changing its sample variance (line numbers 5-17 in Algorithm~\ref{alg2}).
The stratum that yields the smallest confidence bound width in the context of an additional sample is then selected (line 18-21) and sampled (line 24), the sample variance of that stratum is updated (line 26);
this process repeats until the maximum sample budget is reached (per the outer loop, line 1).
In this way the process attempts to iteratively minimise the SEBB in expectation with each additional sample taken; and hence lead to potentially greater accuracy in stratified sampling as a result.

The primary assumption that exists in this method's selection calculus is that it assumes that the sample variance of the strata would likely remain unchanged for the taking of the additional sample from any strata.
While this technically isn't true, the unbiased sample variance is expected to be almost as likely to increase as it is likely to decrease from the taking of an additional sample.
Plus developing an even more complicated probability bound that explicitly takes account of the likely change in error of the stratified mean estimate due to the expected change in the sample variance is beyond the scope of our investigation.

This SEBM method (Algorithm \ref{alg2}) requires the sample variances of all the strata to be calculated. And accordingly, Algorithm \ref{alg2} must be initialised with at least two samples from each stratum so that sample variance can be calculated for it to be able to function.

We also note that Algorithm \ref{alg2} describes a process specific to the sampling without replacement of all strata, and involves the calculation of the SEBB with the tightest possible use cases of Lemmas \ref{martingale0} and~\ref{martingale1}.
In particular, for any stratum $i$ that is sampled without replacement, any specific bound with an associated $\Omega_{m_i}^{n_i}$ and $\Psi_{m_i}^{n_i}$ may be substituted for $\bar{\Omega}_{m_i}^{n_i}$ and $\bar{\Psi}_{m_i}^{n_i}$ to potentially tighten the bound, and this corresponds to choice of Lemma~\ref{martingale0} or Lemma~\ref{martingale1} in the bound's derivation.
Since the SEBB is a composition of such bounds with such choices throughout, there is a structure of valid pairs of substitutions $\Omega,\Psi$ for $\bar{\Omega},\bar{\Psi}$ in the optimal calculation of the SEBB, which is shown in the steps 8-15 of Algorithm \ref{alg2}.
The equivalent algorithm for sampling with replacement simply is the same algorithm altered by replacing all use of $\bar{\Omega},\bar{\Psi}$ with $\Omega,\Psi$ respectively.

Similarly it is elementary to modify the terms of Algorithn \ref{alg2} to be amenable to be minimising SECB bound (Theorem \ref{thm:SECM_bound}).

\begin{algorithm}
\caption{Stratified Empirical Bernstein Method (SEBM) algorithm, with replacement}
\label{alg2}
\begin{algorithmic}[1]
    \REQUIRE probability $p$, strata number $N$, stratum sizes $n_i$, initial sample numbers $m_i$, initial stratum sample variances $\doublehat{\sigma}_i^2$, weights $\tau_i$, widths $D_i$, maximum sample budget $B$
    \WHILE{$\sum_i{m_i}<B$}
        \STATE $beststrata \leftarrow -1$
        \STATE $lowestbound \leftarrow \infty$
    	\FOR{$k=0$ to $N$}
    	    \STATE $m_k \leftarrow m_k + 1$
        	\STATE $a \leftarrow [0,0]$, $b \leftarrow [0,0]$, $c \leftarrow [0,0]$, $d \leftarrow [0,0]$
        	\FOR{$i=0$ to $N$}
        		\STATE $a_0 \leftarrow a_0 + \log(6N/p)D_i^2\bar{\Psi}_{m_i}^{n_i}\min(\bar{\Omega}_{m_i}^{n_i},\Omega_{m_i}^{n_i})\tau^2$
        		\STATE $a_1 \leftarrow a_1 + \log(6N/p)D_i^2\Psi_{m_i}^{n_i}\min(\bar{\Omega}_{m_i}^{n_i},\Omega_{m_i}^{n_i})\tau^2$
        		\STATE $b_0 \leftarrow \max(b_0,\log(3/p)D_i^2\bar{\Psi}_{m_i}^{n_i}\min(\bar{\Psi}_{m_i}^{n_i},\Psi_{m_i}^{n_i})\tau^2)$
        		\STATE $b_1 \leftarrow \max(b_1,\log(3/p)D_i^2\Psi_{m_i}^{n_i}\min(\bar{\Psi}_{m_i}^{n_i},\Psi_{m_i}^{n_i})\tau^2)$
        		\STATE $c_0 \leftarrow c_0 + 2\bar{\Psi}_{m_i}^{n_i}((m_i-1)\doublehat{\sigma}_i^2/m_i)\tau^2$
        		\STATE $c_1 \leftarrow c_1 + 2\Psi_{m_i}^{n_i}((m_i-1)\doublehat{\sigma}_i^2/m_i)\tau^2$
        		\STATE $d_0 \leftarrow d_0 + \frac{4}{17}D_i^2\bar{\Omega}_{m_i}^{n_i}\tau^2$
        		\STATE $d_1 \leftarrow d_1 + \frac{4}{17}D_i^2\Omega_{m_i}^{n_i}\tau^2$
        	\ENDFOR
        	\STATE $boundwidth \leftarrow \sqrt{\log(6/p)\min_j(d_j + (\sqrt{c_j + a_j + b_j} + \sqrt{b_j})^2)}$
    	    \IF{$boundwidth < lowestbound$}
    	        \STATE $beststrata \leftarrow k$
    	        \STATE $lowestbound \leftarrow boundwidth$
    	    \ENDIF
    	    \STATE $m_k \leftarrow m_k - 1$
    	\ENDFOR
    	\STATE take an extra sample from strata: $beststrata$
	    \STATE $m_{beststrata} \leftarrow m_{beststrata} + 1$
    	\STATE recalculate $\doublehat{\sigma}_{beststrata}^2$
    \ENDWHILE
\end{algorithmic}
\end{algorithm}

In the next section \ref{section:statistics_results} we will compare how algorithms \ref{alg3} and \ref{alg2} work in actually sampling stratified data.


%!TEX root = ../notes.tex
\section{February 23, 2022}
\subsection{Chinese Remainder Theorem}
\recall (HW2, Q2) asked us to find $x$ with
\begin{align*}
    x & \equiv 3\mod 7 \\
    x & \equiv 4\mod 9
\end{align*}
We solved this by setting
\begin{align*}
    x = 7y + 3 & \equiv 4\mod 9         \\
    7y         & \equiv 1\mod 9
    \intertext{and taking $7^{-1}\pmod{9}$ which is $4$. So we have}
    y          & \equiv 4\mod 9         \\
    x = 7y+3   & \equiv 31\text{ works}
\end{align*}
\begin{theorem}[Chinese Remainder Theorem]
    Let $\{m_i\}$ be a set of pairwise coprime numbers. That is, $\gcd(m_i, m_j) = 1$ for $i = j$. Then the system
    \begin{align*}
        x & \equiv a_1\mod m_1 \\
        x & \equiv a_2\mod m_2 \\
          & \vdots             \\
        x & \equiv a_n\mod m_n
    \end{align*}
    has a solution.
\end{theorem}
\begin{proof}
    By induction on $n$.
    \begin{description}
        \item[Base case.] $n = 1$, then we take $x = a_1$ which is a solution.
        \item[Mini inductive step.] We first solve for $n=2$. We have
            \begin{align*}
                x & \equiv a_1\mod m_1 \\
                x & \equiv a_2\mod m_2
            \end{align*}
            Let $u_1\cdot m_1 + u_2\cdot m_2 = 1$ (by Bezout's identity). Consider quantity
            \begin{align*}
                u_1m_1a_2 + u_2m_2a_1 & \equiv 0 + 1\cdot a_1 \equiv a_1 \mod m_1 \\
                                      & \equiv 1\cdot a_2 + 0 \equiv a_2\mod m_2
            \end{align*}
            which solves for $n=2$.
        \item[Full inductive step.] Let $n\geq 3$, we solve equation
            \begin{align*}
                x & \equiv a\mod m_1m_2\cdots m_{n-1} \\
                x & \equiv a_{n}\mod m_{n}
            \end{align*}
            where the solution $a$ from the first equation comes from our inductive hypothesis. We can solve this by our mini inductive step (the $n=2$ case).
    \end{description}
    Which concludes this proof.
\end{proof}

\begin{example}
    The Chinese Remainder Theorem allows us to solve congruences with composite moduli. For example, solve
    \[x^2\equiv 18 \mod 21\]
    This is equivalent to solving
    \begin{align*}
        x^2 & \equiv 18 = 0\mod 3 \\
        x^2 & \equiv 18 = 4\mod 7
    \end{align*}
    since $21 = 3\cdot 7$. So this is equivalent to solving
    \begin{align*}
        x & \equiv 0\mod 3     \\
        x & \equiv \pm 2\mod 7
    \end{align*}
    Using CRT, we have
    \[1\cdot 7 + (-2)\cdot 3 = 1\]
    so
    \[x = 0\cdot 1\cdot 7 + (-2)\cdot 2\cdot 2 = -12 \equiv 9\mod 21\]
    we do check that $9^2 = 81\equiv 18\mod 21$.
\end{example}

In general, we claim that square roots mod $p$ are easy to compute.
\begin{proposition}
    Let $p\equiv 3\mod 4$ and $a$ is a square in $\ZZ/p\ZZ$ (that is, $x^2\equiv a$ has a solution).

    Then
    \[x\equiv a^{\frac{p+1}{4}}\]
    is a solution.
\end{proposition}
\begin{proof}
    Say $a\equiv b^2\mod p$. Then
    \[(a^{\frac{p+1}{4}})^2 = a^{\frac{p+1}{2}}\equiv (b^2)^{\frac{p+1}{2}}\equiv b^{p+1} \equiv b^{p-1}\cdot b^2\equiv 1\cdot b^2 = b^2 \equiv a\mod p\]
    which concludes the proof.
\end{proof}
So we can compute square roots modulo a composite number $N$ by taking the prime factor decomposition of $N$, and taking the square root mod each factor, then using CRT.

Conversely, any efficient algorithm to find square roots mod $N$ can be used to factor $N$.

\emph{Why?}
\begin{enumerate}[1.]
    \item We generate an element $x$ mod $N$.
    \item Ask for square root of $x^2$.
    \item Good chance that we get $y\not\equiv \pm x$.
          We now have
          \begin{align*}
              x^2        & \equiv y^2\mod N \\
              (x+y)(x-y) & \equiv 0\mod N
          \end{align*}
    \item We can now calculate $\gcd(x+y, N)$ and find some factors of $N$.
\end{enumerate}

\subsection{Euler's Theorem}
\recall Fermat's Little Theorem which says that
\begin{align*}
    a^{p-1} & \equiv 1\mod p\quad \text{ for }p\nmid a
    \intertext{What happens when we replace $p$ with $N$ where $N$ is composite?}
    a^{N-1} & \overset{?}{\equiv}1\mod N\quad \text{ for }\gcd(N,a)=1
\end{align*}
No! Recall demo showing counterexample.

\begin{proposition}
    If $N = pq$ for primes $p$ and $q$, then
    \[a^{(p-1)(q-1)}\equiv 1\mod N\quad\text{ for }\gcd(N, a) = 1\]
\end{proposition}
\begin{proof}
    \textsc{wlog} taking mod $p$, we have
    \[a^{(p-1)(q-1)}\equiv (a^{p-1})^{q-1}\equiv 1^{q-1}\equiv 1\mod p\]
    Similarly mod $q$ by symmetry. Then it is congruent to $1\mod pq$.
\end{proof}

We can generalize this...
\begin{proposition}[Euler's Theorem]
    For any composite $N$, we have
    \[a^{\varphi(N)}\equiv 1\mod N\quad\text{ for }\gcd(N, a) = 1\]
\end{proposition}

\subsection{Exponentiation}
Given an exponent $x$, we can compute $e^x$ fast. Inverting this gives us the \emph{Discrete Log Problem}. Diffie-Hellman Key Exchange and Elgamal rely on this.

What if we think of this as a function of the base? Given a base $x$, we want to take it to exponent $e$ to get $x^e$. Inverting this is the \emph{Extracting Roots} problem. This is the basis of the RSA cryptosystem (see next time!).

\begin{claim*}
    Let $\gcd(e, p-1) = 1$. We can construct $de\equiv 1 \mod p-1$. Then $(x^e)^d\equiv x$.
\end{claim*}

So \emph{extracting roots} is easy mod prime $p$, but hard mod composites. We'll see this next time.