\documentclass[a4paper]{article}
%\usepackage{html}
\usepackage[dvips]{graphics,color,epsfig}
\usepackage{path}
\usepackage{amssymb}
\usepackage{amsfonts}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{psfrag}
\newcommand{\N}{\ensuremath{\mathbb{N}}}
\newcommand{\F}{\ensuremath{\mathbb{F}}}
\newcommand{\Z}{\ensuremath{\mathbb{Z}}}
\newcommand{\R}{\ensuremath{\mathbb{R}}}
\newcommand{\Q}{\ensuremath{\mathbb{Q}}}
\newcommand{\C}{\ensuremath{\mathbb{C}}}

\newtheorem{lemma}{Lemma}
\newtheorem{assumption}{Assumption}
\newtheorem{definition}{Definition}

\title{Update $Uz$}
\author{Frans Wessendorp}
\begin{document}
\maketitle
\section{The necessity for an additional update}
Suppose in Ratio Test Step~1 variable $x_{i}$ is leaving and variable $x_{j}$ is
entering. If $M_{B \setminus \{i\}}$ is not regular, we no longer have a unique
solution to $(UQP_{j}^{t}(\hat{B} \setminus \{i\}))$ for each value of $t$.
Currently the algorithm then proceeds with Ratio Test Step~2 with
$\hat{B}:=B \cup \{j\}$ and variable $x_{i}$ leaving in the first ratio test in
Ratio Test Step~2. However, this does not always work, for
$B^{\prime}:=B \setminus \{i\} \cup \{j\}$, $M_{B^{\prime}}$ may be regular
whereas $M_{B \cup \{j\}}$ and $M_{B \setminus \{i\}}$ are both singular, such
that both ways to compute the new basis inverse $M_{B^{\prime}}^{-1}$ from
$M_{B}^{-1}$ by `growing' and `shrinking' updates,
that add or remove one column and row per update,
are blocked. The simplest way
to solve this problem is to have a replacement step in phaseII as well.

\section{Update type z}
We use an inversion formula described in~\cite{Zielke} which expresses $(M +
UV^{T})^{-1}$ in terms of $M^{-1}$, $U$ and $V$, where $UV^{T}$ has at
most rank 2 and $M$ as well as $M+UV^{T}$ are both symmetric.
We use the general inversion formula
given in~\cite{Zielke}, Equation~(3), and specialize it to our specific
situation in subsequent sections:
\begin{equation}
\label{eq:inversion_formula}
\left(M + UV^{T}\right)^{-1} =
M^{-1} - M^{-1} U \left(I_{r} +
V^{T}M^{-1}U\right)^{-1}V^{T}M^{-1}
\end{equation}
where
\begin{eqnarray}
\label{def:U}
U &:=& \left(u- u_{i}e_{i}, e_{i} \right) \\
\label{def:V}
V &:=& \left(e_{i}, u \right)
\end{eqnarray}
and
\begin{equation}
\label{def:diff_column}
u:=
\zeta
-
M_{\bullet, i}
\end{equation}
with $\zeta$ being the new column (row).
Thus the correction matrix $C$ has the following form
\begin{equation}
C:=UV^{T}=ue_{i}^{T} + e_{i}u^{T} - u_{i}e_{i}e_{i}^{T}.
\end{equation}
Plugging this into Equation~(\ref{eq:inversion_formula}) yields
\begin{equation}
\label{eq:inversion_formula_spec}
\left(M + UV^{T}\right)^{-1} =
M^{-1} - M^{-1}\left(u - u_{i}e_{i}, e_{i}\right)
Q^{-1}
\left(
\begin{array}{c}
e_{i}^{T} \\
\hline
u^{T}
\end{array}
\right)
M^{-1}
\end{equation}
where
\begin{eqnarray}
Q &:=& I_{2} +
\left(
\begin{array}{c}
e_{i}^{T}  \\
u^{T}
\end{array}
\right)M^{-1}
\left(u - u_{i}e_{i}, e_{i}\right) \\
&=&
\label{def:Q_explicit}
\left(
\begin{array}{cc}
1 + M^{-1}_{i, \bullet}u - u_{i}M_{i,i}^{-1} &
M_{i,i}^{-1} \\
u^{T}M^{-1}u-u_{i}u^{T}M^{-1}_{\bullet, i} &
1 + u^{T}M^{-1}_{\bullet, i}
\end{array}
\right)
\end{eqnarray}
%and $\beta_{i,i}$ is defined as
%\begin{equation}
%\label{def:beta}
%\beta_{i,i}:=M^{-1}_{i,i}
%\end{equation}
The inverse of $Q$,
\begin{equation}
Q=
\left(
\begin{array}{cc}
q_{1} & q_{2} \\
q_{3} & q_{4}
\end{array}
\right)
\end{equation}
is given by
\begin{equation}
Q^{-1}=
\frac{1}{\det(Q)}
\left(
\begin{array}{cc}
q_{4}  & -q_{2} \\
-q_{3} &  q_{1}
\end{array}
\right)
\end{equation}
with
\begin{eqnarray}
\label{def:det_Q}
\det(Q)
&=&
q_{1}q_{4}-q_{2}q_{3}
\nonumber \\
&=&
\left(1+M_{i, \bullet}^{-1}u-u_{i}M_{i,i}^{-1}\right)
\left(1 + u^{T}M^{-1}_{\bullet, i}\right)
\nonumber \\
&&
-M_{i,i}^{-1}\left(u^{T}M^{-1}u-u_{i}u^{T}M_{\bullet,i}^{-1}\right)
\nonumber \\
&=&
\left(1+M_{i, \bullet}^{-1}u\right)^{2}
-u_{i}M_{i,i}^{-1}\left(1+u^{T}M_{\bullet,i}^{-1}\right)
-M_{i,i}^{-1}u^{T}M^{-1}u
\nonumber \\
&&
+ u_{i}M_{i,i}^{-1}u^{T}M_{\bullet,i}^{-1}
\nonumber \\
&=& \left(1+M_{i, \bullet}^{-1}u\right)^{2} -
    M_{i,i}^{-1}\left(u_{i}+ u^{T}M^{-1}u\right)
\end{eqnarray}
Collecting terms in Equation~(\ref{eq:inversion_formula_spec}) then yields
\begin{eqnarray}
\label{eq:inversion_formula_spec_explicit}
\lefteqn{\left(M+ UV^{T}\right)^{-1}=M^{-1} -} \nonumber \\
& & \frac{1}{\det(Q)}
\left[
  M^{-1}_{\bullet, i}
  \left(k_{1}v^{T} + k_{2}M^{-1}_{i, \bullet}\right)
  +v\left(k_{3}v^{T} + k_{1}M^{-1}_{i, \bullet}\right)
\right]
\end{eqnarray}
where
\begin{eqnarray}
\label{def:v}
v     &:=& M^{-1}u \\
\label{def:w}
w     &:=& M^{-1}_{i, \bullet}u \\
\label{def:k1}
k_{1} &:=& 1 + w \\
\label{def:k2}
k_{2} &:=& -\left(u^{T}v + u_{i}\right) \\
\label{def:k3}
k_{3} &:=& -M_{i,i}^{-1}
\end{eqnarray}
With these definitions $\det(Q)$ can then be written in terms of
$k_{1}$, $k_{2}$ and $k_{3}$ as
\begin{equation}
\label{def:det_Q_k}
\det(Q)=k_{1}^{2}-k_{2}k_{3}
\end{equation}
In order to get rid of the difference $u$
in Equation~(\ref{eq:inversion_formula_spec_explicit}) and
Definitions~(\ref{def:v}) to~(\ref{def:k3}) we rewrite the latter ones
\begin{eqnarray}
\label{def:v_zeta}
v
&:=&
M^{-1}u
\nonumber \\
&=&
M^{-1}\left[\zeta - M_{\bullet, i}\right]
\nonumber \\
&=&
M^{-1}\zeta - e_{i}
\end{eqnarray}
\begin{eqnarray}
\label{def:w_zeta}
w
&:=&
M^{-1}_{i, \bullet}u
\nonumber \\
&=&
M^{-1}_{i, \bullet}\left[\zeta - M_{\bullet, i}\right]
\nonumber \\
&=&
M^{-1}_{i, \bullet}\zeta - 1
\end{eqnarray}
\begin{eqnarray}
\label{def:k_1_zeta}
k_{1}
&:=&
1 + w
\nonumber \\
&=&
M^{-1}_{i, \bullet}\zeta
\end{eqnarray}
\begin{eqnarray}
k_{2} &:=& -\left(u^{T}v + u_{i}\right) \nonumber \\
      &=& -\left[\left(\zeta-M_{\bullet,i}\right)^{T}
           M^{-1}\left(\zeta-M_{\bullet,i}\right)
	   + \left(\zeta-M_{\bullet,i}\right)_{i}
	   \right]   \nonumber \\
      &=& -\left[\zeta^{T}M^{-1}\zeta
        +M_{i,\bullet}M^{-1}M_{\bullet,i}
	          -2\zeta^{T}M^{-1}M_{\bullet,i}
         	  +\zeta_{i} - M_{i,i}
	   \right] \nonumber \\
      &=& -\left[\zeta^{T}M^{-1}\zeta + M_{i,i}
                  -2\zeta_{i} + \zeta_{i} - M_{i,i}
            \right] \nonumber \\
      &=& -\zeta^{T}M^{-1}\zeta + \zeta_{i}
\end{eqnarray}

The symmetry of the right-hand side of
Equation~(\ref{eq:inversion_formula_spec_explicit}) becomes obvious when
rewriting it as the sum of three symmetric matrices
\begin{eqnarray*}
\lefteqn{\left(M+ UV^{T}\right)^{-1}=M^{-1}-}  \\
& & \frac{1}{\det(Q)}
\left[
k_{1}\left(
  M^{-1}_{\bullet, i}v^{T}
  +vM^{-1}_{i,\bullet}
\right)
+
k_{2}M^{-1}_{\bullet, i}M^{-1}_{i, \bullet}
+
k_{3}vv^{T}
\right]
\end{eqnarray*}

In case of a regular matrix $M+UV^{T}$ the determinant $\det(Q)$ is
always nonzero as the factorization of
\begin{equation*}
\left(
\begin{array}{cc}
I & -V^{T} \\
U & M
\end{array}
\right)
\end{equation*}
into the products
\begin{equation*}
\left(
\begin{array}{cc}
I & 0 \\
U & M + UV^{T}
\end{array}
\right)
\left(
\begin{array}{cc}
I & -V^{T} \\
0 & I
\end{array}
\right)
=
\left(
\begin{array}{cc}
I & -V^{T}  \\
0 & M
\end{array}
\right)
\left(
\begin{array}{cc}
I + V^{T}M^{-1}U & 0 \\
M^{-1}U          & I
\end{array}
\right)
\end{equation*}
shows.
By taking determinants we obtain
\begin{equation}
\label{eq:det_prod}
\det\left(M + UV^{T}\right) =
\det\left(M\right) \det\left(I + V^{T}M^{-1}U\right)
\end{equation}
by assumption $M$ and $M+UV^{T}$ are regular, therefore by the above
Equation
$\det(I + V^{T}M^{-1}U)=\det(Q) \neq 0$.

A determinant identity that will be useful in the following is subsumed in the
following lemma
\begin{lemma}[Jacobi's determinant identity]
\label{lem:Jacobi_det_id}
Let
\begin{equation}
A:=
\left(
\begin{array}{c|c}
B & C \\
\hline
D & E
\end{array}
\right)
, \quad
A^{-1}:=
\left(
\begin{array}{c|c}
W & X \\
\hline
Y & Z
\end{array}
\right)
\end{equation}
where $B$ and $W$ are $k \times k$ matrices then $\det(B)=\det(A)\det(Z)$.
\end{lemma}
The justification for the new additional update is given by the following
lemma.
\begin{lemma}
\label{lem:main}
If $M_{B}$ is regular with entering variable $x_{j}$, $j \not \in B$, and
leaving variable $x_{i}$, $i \in B$, then if $M_{B \setminus \{i\}}$
is singular $M_{B \setminus \{i\} \cup \{j\}}$ is regular.
\end{lemma}

\begin{proof}
Suppose $M_{B \setminus \{i\} \cup \{j\}}=M_{B}+UV^{T}$ with
$U$, $V$ defined according to Equations~(\ref{def:U}), (\ref{def:V})
and (\ref{def:diff_column}) to be singular, then according to
Equation~(\ref{eq:det_prod}), the regularity of $M_{B}$ and
Equation~(\ref{def:det_Q})
\begin{equation}
\label{eq:det_Q_eq_det_M_B_replaced}
\det \left(I+V^{T}M_{B}^{-1}U\right) =
\det(Q) = 0
\Longleftrightarrow
\det(M_{B \setminus \{i\} \cup \{j\}})=0
\end{equation}
By Equation~(\ref{def:det_Q_k})
\begin{equation}
\label{def:det_Q_k_proof}
\det(Q) = k_{1}^{2} - k_{2}k_{3}
\end{equation}
where
\begin{equation}
\label{def:ks}
k_{1}:=\left(M_{B}^{-1}\right)_{\beta(i), \bullet}\zeta,
\quad
k_{2}:=\zeta_{\beta(i)} - \zeta^{T}M_{B}^{-1}\zeta,
\quad
k_{3}:=-\left(M_{B}^{-1}\right)_{\beta(i), \beta(i)}
\end{equation}
By the assumption of variable $x_{j}$ entering and variable $x_{i}$ leaving
\begin{equation}
\label{precond:pivot_proof}
\left(M_{B}^{-1}\right)_{\beta(i), \bullet}
\zeta
=
q_{x_{i}}
\neq
0
\end{equation}
If the assumption $\det(M_{B\setminus \{i\}})=0$ holds true
we have, according to Lemma~(\ref{lem:Jacobi_det_id}) for $M_{B}$ and
the regularity of $M_{B}$,
\begin{equation}
\label{precond:leaving_proof}
\det(M_{B \setminus \{i\}})=0
\Longleftrightarrow
\left(M_{B}^{-1}\right)_{\beta(i),\beta(i)}=0
\end{equation}
such that by Equations~(\ref{def:ks}), (\ref{precond:pivot_proof})
and~(\ref{precond:leaving_proof}) $\det(Q) \neq 0$ in
Equation~(\ref{def:det_Q_k_proof}).
%If on the other hand the assumptions $\det(M_{B \cup \{j\}})=0$ and
%$D_{j,j}=D_{i,j}$ hold true, we have according to \cite{Sven},
%Equation (6.7),
%\begin{equation*}
%\det(M_{B \cup \{j\}}) = 0
%\Longleftrightarrow
%2D_{j,j} -\zeta^{T}M_{B}^{-1}\zeta = 0
%\end{equation*}
%which together with the assumption $D_{i,j}=D_{j,j}$ and
%the fact $\zeta_{\beta(i)}=2D_{i,j}$ is equivalent to
%\begin{equation}
%\label{precond:entering_proof}
%\det(M_{B \cup \{j\}}) = 0
%\Longleftrightarrow
%\zeta_{\beta(i)} -\zeta^{T}M_{B}^{-1}\zeta = 0
%\end{equation}
%such that by Equations~(\ref{def:ks}), (\ref{precond:pivot_proof})
%and~(\ref{precond:entering_proof}) again $\det(Q) \neq 0$
%in Equation~(\ref{def:det_Q_k_proof}).
%For both cases (\ref{eq:det_Q_eq_det_M_B_replaced}) implies the lemma.
\end{proof}

\section{Update type z in the context of reduced bases}
We distinguish 4 different cases with respect to the entering and leaving
variables and the respective changes of the basis headings they entail:
\begin{description}

\item [$\mathbf{U_{Z_{1}}}$:$\quad j \in O \setminus B$ replaces $i \in B_{O}$]
\begin{equation}
\label{update:o_rep_o}
\begin{array}{ccccccc}
C^{\prime}      &:=&  C     &\quad\quad&
S_{B}^{\prime}  &:=&  S_{B}  \\
B_{O}^{\prime}  &:=&  B_{O} \setminus \{i\} \cup \{j\}  &\quad\quad&
B_{S}^{\prime}  &:=&  B_{S}
\end{array}
\end{equation}

\item [$\mathbf{U_{Z_{2}}}$:$\quad j \in S \setminus B$ replaces $i \in B_{O}$:]
\begin{equation}
\label{update:s_rep_o}
\begin{array}{ccccccc}
C^{\prime}      &:=& C \setminus \{\sigma(j)\}   &\quad\quad&
S_{B}^{\prime}  &:=& S_{B} \cup  \{\sigma(j)\}   \\
B_{O}^{\prime}  &:=& B_{O} \setminus \{i\} &\quad\quad&
B_{S}^{\prime}  &:=&B_{S} \cup \{j\}
\end{array}
\end{equation}

\item [$\mathbf{U_{Z_{3}}}$:$\quad j \in O \setminus B$ replaces $i \in B_{S}$:]
\begin{equation}
\label{update:o_rep_s}
\begin{array}{ccccccc}
C^{\prime}      &:=& C \cup \{\sigma(i)\}   &\quad\quad&
S_{B}^{\prime}  &:=& S_{B} \setminus  \{\sigma(i)\}   \\
B_{O}^{\prime}  &:=& B_{O} \cup \{j\} &\quad\quad&
B_{S}^{\prime}  &:=&B_{S} \setminus \{i\}
\end{array}
\end{equation}

\item [$\mathbf{U_{Z_{4}}}$:$\quad j \in S \setminus B$ replaces $i \in B_{S}$:]
\begin{equation}
\label{update:s_rep_s}
\begin{array}{ccccccc}
C^{\prime}      &:=& C \setminus \{\sigma(j)\} \cup \{\sigma(i)\}  &\quad\quad&
S_{B}^{\prime}  &:=& S_{B} \setminus  \{\sigma(i)\} \cup \{\sigma(j)\}   \\
B_{O}^{\prime}  &:=& B_{O}                                         &\quad\quad&
B_{S}^{\prime}  &:=&B_{S} \setminus \{i\} \cup \{j\}
\end{array}
\end{equation}
\end{description}
Note that with respect to the reduced basis matrix $\check{M}_{B}$
\begin{equation}
\label{def:M_B_red}
\check{M}_{B}:=
\left(
\begin{array}{c|c}
0 & A_{C, B_{O}} \\
\hline
A_{C, B_{O}}^{T} & 2D_{B_{O}, B_{O}}
\end{array}
\right)
\end{equation}
only the Updates~(\ref{update:o_rep_o}) and~(\ref{update:s_rep_s}) are
replacement updates, whereas the Updates~(\ref{update:s_rep_o})
and~(\ref{update:o_rep_s}) are `shrinking' and `growing' updates.
Since the solver uses the reduced basis inverse $\check{M}_{B}^{-1}$ we can
directly apply the update described in the last section only for
Updates~(\ref{update:o_rep_o}) and~(\ref{update:s_rep_s}), although we could
indirectly apply last section's update by expanding the the reduced basis
matrix inverse $\check{M}_{B}^{-1}$ to $M_{B}^{-1}$, apply the update
and shrink back to $\check{M}_{B}^{-1}$ again.
Since we want to work with the reduced basis inverse $\check{M}_{B}^{-1}$
directly we generalize some of the updates described in~\cite{Sven},
Section (6.3.2), in the sense that enlarging the reduced basis matrix by two
rows and columns as well as the removal of two rows and columns of the reduced
basis matrix is possible.

To this end we restate the expression for $M_{B}^{-1}$ in terms of
$\check{M}_{B}^{-1}$ derived in~\cite{Frans_Deg}, Section~(3.7),
given the basis heading
$\left[C, S_{B}, B_{O}, B_{S}\right]$ as
\begin{equation}
\label{eq:M_B_inv_exp}
M_{B}^{-1}=
\left(\begin{array}{c|c|c|c}
        \left(\check{M}_{B}^{-1}\right)_{C,C} &
	0 &
	\left(\check{M}_{B}^{-1}\right)_{C,B_{O}} &
	 \left(\check{M}_{B}^{-1}\right)_{C, B_{O}}\alpha^{T} \\
	\hline
	0 &
	0 &
	0 &
	A_{S_{B},B_{S}} \\
	\hline
	\left(\check{M}_{B}^{-1}\right)_{B_{O}, C} &
	0 &
	\left(\check{M}_{B}^{-1}\right)_{B_{O}, B_{O}} &
          \left(\check{M}_{B}^{-1}\right)_{B_{O},B_{O}}\alpha^{T} \\
	\hline
	\alpha\left(\check{M}_{B}^{-1}\right)_{B_{O},
	C} &
	A_{S_{B}, B_{S}}^{T} &
	\alpha\left(\check{M}_{B}^{-1}\right)_{B_{O},
	B_{O}} &
	\alpha\left(\check{M}_{B}^{-1}\right)_{B_{O}, B_{O}}\alpha^{T}
      \end{array}
\right)
\end{equation}
where
\begin{equation*}
\alpha=-A_{S_{B}, B_{S}}^{T}A_{S_{B}, B_{O}}
\end{equation*}
Furthermore in order to take into account possible permutations within the
different blocks in the above expressions we introduce the following
permutations:
\begin{equation}
\beta: B_{O} \cup B_{S} \rightarrow B_{O} \cup B_{S}
\end{equation}
\begin{equation}
\beta_{O}: B_{O}  \rightarrow  B_{O}, \quad
\beta_{S}: B_{S} \rightarrow  B_{S}
\end{equation}
\begin{equation}
\gamma_{C}: E \cup S_{N} \rightarrow  E \cup S_{N}, \quad
\gamma_{S_{B}}: S_{B} \rightarrow  S_{B}
\end{equation}
where the permutations are defined with respect to the headings in ascending
order.

\subsection{The precondition of the leaving variable}
The precondition of the leaving variable of the update for the basis matrix,
$\det(M_{B\setminus \{i\}})=0$, is
according to Lemma~(\ref{lem:Jacobi_det_id}) equivalent to
\begin{equation}
\label{precond:leaving}
\left(M_{B}^{-1}\right)_{\beta(i),\beta(i)}=0
\end{equation}
For the updates $U_{Z_{1}}$ and $U_{Z_{2}}$ this precondition translates
directly for the reduced basis
matrix $\check{M}_{B}$, for the updates $U_{Z_{3}}$ and $U_{Z_{4}}$ for the
reduced basis matrix $\check{M}_{B}$, however, the translation is a little bit
more involved. Before delving into the details we introduce the following
definitions which will be useful later
\begin{equation}
\label{def:theta_vartheta}
\theta :=
\left(
\begin{array}{c}
A_{C, j} \\
\hline
2D_{B_{O}, j}
\end{array}
\right),
\quad
\vartheta^{T}:=
\left(
\begin{array}{c}
0 \\
\hline
A_{\sigma(i), B_{O}}^{T}
\end{array}
\right)
\end{equation}
\begin{equation}
\label{def:rho_varrho}
\rho:=
\left(\check{M}_{B}^{-1}\right)_{\bullet, \beta_{O}(i)},
\quad
\varrho^{T}:=
\left(\check{M}_{B}^{-1}\right)_{\bullet, \gamma_{C}(\sigma(j))}
\end{equation}
According to Equation~(\ref{eq:M_B_inv_exp})
$\left(M_{B}^{-1}\right)_{\beta(i),\beta(i)}=0$
for $i \in B_{S}$ amounts to
\begin{equation}
\label{eq:precond}
\left(\alpha\left(\check{M}_{B}^{-1}\right)_{B_{O}, B_{O}}
    \alpha^{T}\right)_{\beta_{S}(i), \beta_{S}(i)} = 0
\end{equation}
Simplifying the part involving $\alpha$ in this expression yields
\begin{eqnarray*}
\left(\alpha\right)_{\beta_{S}(i), \bullet} &=&
-\left(A_{S_{B}, B_{S}}^{T}\right)_{\beta_{S}(i), \bullet}A_{S_{B}, B_{O}} \\
&=&
-A_{\sigma(i),i}\left(A_{S_{B},B_{O}}\right)_{\gamma_{S_{B}}(\sigma(i))} \\
&=&
-A_{\sigma(i),i}A_{\sigma(i),B_{O}}
\end{eqnarray*}
such that Equation~(\ref{eq:precond}) becomes
\begin{equation*}
A_{\sigma(i), B_{O}}\left(\check{M}_{B}^{-1}\right)_{B_{O}, B_{O}}
A_{\sigma(i), B_{O}}^{T} =0
\end{equation*}
Using the Definitions~(\ref{def:theta_vartheta}) this can be written as
\begin{equation}
\label{eq:precond_simplified}
\vartheta\check{M}_{B}^{-1}\vartheta^{T}=0
\end{equation}
Subsuming this for later reference we obtain
\begin{equation}
\label{table:leaving_precond}
\begin{array}{c|ccc}
 &
\left(M_{B}^{-1}\right)_{\beta(i), \beta(i)}
&=& 0
 \\
\hline
U_{Z_{1}}
&
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i), \beta_{O}(i)}
&=& 0
 \\
U_{Z_{2}}
&
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i), \beta_{O}(i)}
&=& 0
 \\
U_{Z_{3}}
&
\vartheta\check{M}_{B}^{-1}
\vartheta^{T}
&=& 0
 \\
U_{Z_{4}}
&
\vartheta\check{M}_{B}^{-1}\vartheta^{T}
&=& 0
\end{array}
\end{equation}

\subsection{The pivoting precondition}
Due to the fact variable $x_{i}$ has been selected to be the leaving variable with $x_{j}$ entering by Ratio Test Step~1 we have
\begin{equation}
\label{precond:pivot}
\left(M_{B}^{-1}\right)_{\beta(i), \bullet}\eta \neq 0
\end{equation}
where $\eta$ denotes the entering column.
We distinguish according to Equations~(\ref{update:o_rep_o})
to~(\ref{update:s_rep_s}) the following cases:

\paragraph{$\mathbf{U_{Z_{1}}}$: $j \in O \setminus B$ replaces $i \in B_{O}$.}
According to~(\ref{update:o_rep_o}) we have for the entering column $\eta$
\begin{equation}
\eta:=
\left(
\begin{array}{c}
A_{C, j} \\
\hline
A_{S_{B},j} \\
\hline
2D_{B_{O}, j} \\
\hline
0
\end{array}
\right)
\end{equation}
such that the pivoting precondition~(\ref{precond:pivot})
for the basis matrix inverse in terms of
the reduced basis matrix inverse becomes
\begin{eqnarray}
\left(M_{B}^{-1}\right)_{\beta_{O}(i), \bullet} \eta
&=&
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i),C}A_{C,j}
+\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i), B_{O}}2D_{B_{O},j}
\nonumber \\
&=&
\check{M}_{\beta_{O}(i), \bullet}
\left(
\begin{array}{c}
A_{C,j} \\
\hline
2D_{B_{O},j}
\end{array}
\right)
\neq 0,
\nonumber
\end{eqnarray}
using the Definitions~(\ref{def:theta_vartheta})
and~(\ref{def:rho_varrho}) this can be written as
\begin{equation}
\rho^{T}\theta \neq 0
\end{equation}

\paragraph{$\mathbf{U_{Z_{2}}}$: $j \in S \setminus B$ replaces $i \in B_{O}$.}
According to~(\ref{update:s_rep_o}) and the fact that
\begin{equation}
A_{S_{B},j}=0, \quad 2D_{B_{O}, j}=0 \quad j \in S \setminus B
\end{equation}
we have for the entering column $\eta$
\begin{equation}
\eta:=
\left(
\begin{array}{c}
A_{C, j} \\
\hline
0 \\
\hline
0 \\
\hline
0
\end{array}
\right)
\end{equation}
such that the pivoting precondition~(\ref{precond:pivot})
for the basis matrix inverse in terms of
the reduced basis matrix inverse becomes
\begin{eqnarray}
\left(M_{B}^{-1}\right)_{\beta_{O}(i), \bullet} \eta
&=&
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i),C}A_{C,j}
\nonumber \\
&=&
A_{\sigma(j),j}
 \left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i), \gamma_{C}(\sigma(j))} \neq 0
\nonumber
\end{eqnarray}
Since $A_{\sigma(j),j} \neq 0$ the last inequality amounts to
\begin{equation}
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i), \gamma_{C}(\sigma(j))} \neq 0
\end{equation}

\paragraph{$\mathbf{U_{Z_{3}}}$: $j \in O \setminus B$ replaces $i \in B_{S}$.}
According to~(\ref{update:o_rep_s})
we have for the entering column $\eta$
\begin{equation}
\eta:=
\left(
\begin{array}{c}
A_{C, j} \\
\hline
A_{S_{B},j} \\
\hline
2D_{B_{O},j} \\
\hline
0
\end{array}
\right)
\end{equation}
such that ~(\ref{precond:pivot})
for the basis matrix inverse in terms of
the reduced basis matrix inverse becomes
\begin{eqnarray}
\left(M_{B}^{-1}\right)_{\beta_{S}(i), \bullet} \eta
&=&
\left(\alpha\right)_{\beta_{S}(i), \bullet}
\left(\check{M}_{B}^{-1}\right)_{B_{O},C}A_{C,j}
+\left(A_{S_{B}, B_{S}}^{T}\right)_{\beta_{S}(i), \bullet}A_{S_{B},j}
\nonumber \\
&&
+\left(\alpha\right)_{\beta_{S}(i), \bullet}
 \left(\check{M}_{B}^{-1}\right)_{B_{O}, B_{O}}2D_{B_{O},j}
\nonumber \\
&=&
-A_{\sigma(i),i}A_{\sigma(i),B_{O}}
 \left(\check{M}_{B}^{-1}\right)_{B_{O},C}A_{C,j}
+A_{\sigma(i),i}A_{\sigma(i),j}
\nonumber \\
&&
-A_{\sigma(i),i}A_{\sigma(i),B_{O}}
 \left(\check{M}_{B}^{-1}\right)_{B_{O},B_{O}}2D_{B_{O},j}
\nonumber \\
&=&
A_{\sigma(i),i}
\left[
A_{\sigma(i),j} - A_{\sigma(i), B_{O}}
\left(\check{M}_{B}^{-1}\right)_{B_{O}, \bullet}
\left(
\begin{array}{c}
A_{C,j}  \\
\hline
2D_{B_{O}, j}
\end{array}
\right)
\right]
\neq 0
\nonumber
\end{eqnarray}
Since $A_{\sigma(i), i} \neq 0$ the last inequality above amounts to
\begin{equation}
A_{\sigma(i),j} - A_{\sigma(i), B_{O}}
\left(\check{M}_{B}^{-1}\right)_{B_{O}, \bullet}
\left(
\begin{array}{c}
A_{C,j}  \\
\hline
2D_{B_{O}, j}
\end{array}
\right)
\neq
0,
\nonumber
\end{equation}
using the Definitions~(\ref{def:theta_vartheta}) this can be written as
\begin{equation}
A_{\sigma(i),j} -
\vartheta\check{M}_{B}^{-1}\theta \neq 0
\end{equation}

\paragraph{$\mathbf{U_{Z_{4}}}$: $j \in S \setminus B$ replaces $i \in B_{S}$.}
According to~(\ref{update:o_rep_s})
and the fact that
\begin{equation}
A_{S_{B},j}=0, \quad 2D_{B_{O}, j}=0, \quad j \in S \setminus B
\end{equation}
we have for the entering column $\eta$
\begin{equation}
\eta:=
\left(
\begin{array}{c}
A_{C, j} \\
\hline
0 \\
\hline
0 \\
\hline
0
\end{array}
\right)
\end{equation}
such that the pivoting precondition~(\ref{precond:pivot})
for the basis matrix inverse in terms of
the reduced basis matrix inverse becomes
\begin{eqnarray}
\left(M_{B}^{-1}\right)_{\beta_{S}(i), \bullet} \eta
&=&
\left(\alpha\right)_{\beta_{S}(i), \bullet}
\left(\check{M}_{B}^{-1}\right)_{B_{O},C}A_{C,j}
\nonumber \\
&=&
-A_{\sigma(i),i}A_{\sigma(i),B_{O}}
 \left(\check{M}_{B}^{-1}\right)_{B_{O},C}A_{C,j}
\nonumber \\
&=&
-A_{\sigma(i),i}A_{\sigma(j),j}A_{\sigma(i),B_{O}}
 \left(\check{M}_{B}^{-1}\right)_{B_{O},\gamma_{C}(j)}
 \neq 0
 \nonumber
\end{eqnarray}
Since $A_{\sigma(i), i} \neq 0$ and $A_{\sigma(j),j} \neq 0$
the last inequality above amounts to
\begin{equation}
A_{\sigma(i),B_{O}}
\left(\check{M}_{B}^{-1}\right)_{B_{O},\gamma_{C}(\sigma(j))}
\neq 0,
\nonumber
\end{equation}
using the Definitions~(\ref{def:theta_vartheta})
and~(\ref{def:rho_varrho}) this can be written as
\begin{equation}
\vartheta \varrho^{T} \neq 0
\end{equation}
Subsuming this for later reference we obtain
\begin{equation}
\label{table:pivot_precond}
\begin{array}{c|ccc}
&
\left(M_{B}^{-1}\right)_{\beta(i), \bullet} \eta
&\neq& 0
 \\
\hline
U_{Z_{1}}
&
\rho^{T} \theta &\neq& 0
 \\
U_{Z_{2}}
&
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i), \gamma_{C}(\sigma(j))}
&\neq& 0
 \\
U_{Z_{3}}
&
A_{\sigma(i), j}
-\vartheta\check{M}_{B}^{-1}\theta
&\neq& 0
 \\
U_{Z_{4}}
&
\vartheta \varrho^{T} &\neq& 0
\end{array}
\end{equation}

\section{Updates $U_{Z_{1}} \dots U_{Z_{4}}$}
Since the inverse of a matrix $A$ can be represented as a product of the
cofactor matrix of $A$ and $1/\det(A)$ and we represent $A^{-1}$ in rational
form, we use the following form
\begin{equation}
\label{def:rat_form}
A^{-1}=\frac{\hat{A}^{-1}}{\left|\det\left(A\right)\right|}
\end{equation}
such that the denominator is by convention positive.

\subsection{Updates $\mathbf{U_{Z_{1}}}$, $\mathbf{U_{Z_{4}}}$ and the rational form}
According to~(\ref{update:o_rep_o}) and~(\ref{update:s_rep_s})
and the Definition~(\ref{def:M_B_red}) of the reduced basis matrix
the update described by Equation~(\ref{eq:inversion_formula_spec_explicit})
can directly be applied to the reduced basis $\check{M}_{B}$
in order to implement
the updates $U_{Z_{1}}$ and $U_{Z_{4}}$. In order to obtain the rational form
in~(\ref{def:rat_form}) for Equation~(\ref{eq:inversion_formula_spec_explicit})
\begin{eqnarray}
\label{def:update_Z1_Z4}
\lefteqn{\left(M+ UV^{T}\right)^{-1}=M^{-1} -} \nonumber \\
& & \frac{1}{\det(Q)}
\left[
  M^{-1}_{\bullet, i}
  \left(k_{1}v^{T} + k_{2}M^{-1}_{i, \bullet}\right)
  +v\left(k_{3}v^{T} + k_{1}M^{-1}_{i, \bullet}\right)
\right]
\end{eqnarray}
where
\begin{equation}
\label{def:update_Z1_Z4_entities}
v:= M^{-1}\zeta-e_{i},
\quad
k_{1}:=M_{i, \bullet}^{-1}\zeta,
\quad
k_{2}:=\zeta_{i} - \zeta^{T}M^{-1}\zeta,
\quad
k_{3}:=-M_{i,i}^{-1}
\end{equation}
and
\begin{equation}
\label{eq:det(Q)}
\det(Q)=k_{1}^{2}-k_{2}k_{3}
\end{equation}
we use the following substitutions
\begin{equation}
M^{-1}=:\frac{\hat{M}^{-1}}{d},
\quad
v=:\frac{\hat{v}}{d},
\quad
k_{1}=:\frac{\hat{k}_{1}}{d},
\quad
k_{2}=:\frac{\hat{k}_{2}}{d},
\quad
k_{3}=:\frac{\hat{k}_{3}}{d}
\end{equation}
where $d$ is defined as
\begin{equation}
\label{def:old_denom_Z1_Z4}
d:=\left|\det(M)\right|
\end{equation}
Substituting in the above equation yields,
if we abbreviate $M+UV^{T}$ by $M_{Z}$,
\begin{eqnarray}
M_{Z}^{-1}&=&\frac{\hat{M}^{-1}}{d}
-
\frac{
\left[
  \hat{M}^{-1}_{\bullet, i}
  \left(\hat{k}_{1}\hat{v}^{T} + \hat{k}_{2}\hat{M}^{-1}_{i, \bullet}\right)
  +\hat{v}\left(\hat{k}_{3}\hat{v}^{T}
  +\hat{k}_{1}\hat{M}^{-1}_{i, \bullet}\right)
\right]
}{\det(Q)d^{3}}
\nonumber \\
&=&
\frac{
\det(Q)d^{2}\hat{M}^{-1}
-
\left[
  \hat{M}^{-1}_{\bullet, i}
  \left(\hat{k}_{1}\hat{v}^{T} + \hat{k}_{2}\hat{M}^{-1}_{i, \bullet}\right)
  +\hat{v}\left(\hat{k}_{3}\hat{v}^{T}
  +\hat{k}_{1}\hat{M}^{-1}_{i, \bullet}\right)
\right]
}{\det(Q)d^{3}}
\nonumber
\end{eqnarray}
Using Definitions~(\ref{def:rat_form})
and~(\ref{def:old_denom_Z1_Z4}) and defining the new denominator $\hat{z}$
according to Equation~(\ref{eq:det_prod})
\begin{equation}
\label{def:new_denom_Z1_Z4}
\hat{z}:=\det(Q)d,
\end{equation}
we finally obtain from the last Equation above
\begin{eqnarray}
\label{def:update_Z1_Z4_denom}
M_{Z}^{-1}
&=&
\frac{\hat{z}d\hat{M}^{-1}
-
\left[
  \hat{M}^{-1}_{\bullet, i}
  \left(\hat{k}_{1}\hat{v}^{T} + \hat{k}_{2}\hat{M}^{-1}_{i, \bullet}\right)
  +\hat{v}\left(\hat{k}_{3}\hat{v}^{T}
  +\hat{k}_{1}\hat{M}^{-1}_{i, \bullet}\right)
\right]}
{\hat{z}d^{2}}
\nonumber \\
&=&
\frac{sgn(\hat{z})}{\left|\hat{z}\right|}
%\left[
\frac{\hat{z}d\hat{M}^{-1}
-
\left[
  \hat{M}^{-1}_{\bullet, i}
  \left(\hat{k}_{1}\hat{v}^{T} + \hat{k}_{2}\hat{M}^{-1}_{i, \bullet}\right)
  +\hat{v}\left(\hat{k}_{3}\hat{v}^{T}
  +\hat{k}_{1}\hat{M}^{-1}_{i, \bullet}\right)
\right]}
{d^{2}}
%\right]
\nonumber \\
&=&
\frac{\hat{M}_{Z}^{-1}}{\left|\hat{z}\right|}
\end{eqnarray}

\subsection{Update $\mathbf{U_{Z_{1}}}$}
According to~(\ref{update:o_rep_o}) and the
Definitions~(\ref{def:theta_vartheta})
and~(\ref{def:rho_varrho}) the
Update~(\ref{def:update_Z1_Z4}) applied to $\check{M}_{B}^{-1}$
leads to the following correspondences
in~(\ref{def:update_Z1_Z4_entities})
\begin{eqnarray}
\label{def:zeta_Z1}
\zeta
&=&
\theta + \left(2D_{j,j}-\theta_{\beta_{O}(i)}\right)e_{\beta_{O}(i)}
= \theta+2\left(D_{j,j}-D_{i,j}\right)e_{\beta_{O}(i)} \\
M_{i, \bullet}^{-1}
&=&
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i), \bullet}
= \rho^{T}
\end{eqnarray}
According to table~(\ref{table:leaving_precond})
the leaving variable precondition~(\ref{precond:leaving})
in terms of the reduced basis inverse for $U_{Z_{1}}$ is
\begin{equation}
\label{precond:leaving_Z1}
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i),\beta_{O}(i)}=0
\end{equation}
which, together with the Definitions~(\ref{def:update_Z1_Z4_entities}),
implies
\begin{equation}
\label{eq:k_3_zero}
k_{3} = 0
\end{equation}
Furthermore, according to table~(\ref{table:pivot_precond}) the
pivoting precondition~(\ref{precond:pivot})
in terms of the reduced basis inverse for $U_{Z_{1}}$ is
\begin{equation*}
\rho^{T}\theta
=
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i), \bullet}\theta \neq 0
\end{equation*}
which, together with the leaving variable
precondition~(\ref{precond:leaving_Z1}) and~(\ref{def:zeta_Z1}) is equivalent
to
\begin{equation*}
\rho^{T}\zeta \neq 0,
\end{equation*}
which in turn together with the Definitions~(\ref{def:update_Z1_Z4_entities})
implies
\begin{equation}
\label{ineq:Z_1_k_1}
k_{1} \neq 0
\end{equation}
(\ref{ineq:Z_1_k_1}) and~(\ref{eq:k_3_zero}) together with
Equation~(\ref{eq:det(Q)}) imply, in accordance with Lemma~(\ref{lem:main}),
\begin{equation}
\det(Q)=k_{1}^{2} \neq 0.
\end{equation}
Note that according to the above and
Definition~(\ref{def:new_denom_Z1_Z4}) of the new denominator
$\hat{z}$ we have
\begin{equation}
sgn(\hat{z}) = 1
\end{equation}
Update~(\ref{def:update_Z1_Z4_denom})
applied to $\check{M}_{B}$ then simplifies to
\begin{eqnarray}
\label{def:update_Z1}
\check{M}_{B^{\prime}}^{-1}
&=&
\frac{1}{\left|\hat{z}\right|}
\frac{
\hat{k}_{1}^{2}\hat{\check{M}}_{B}^{-1}
-\left[
  \hat{\rho}
  \left(\hat{k}_{1}\hat{v}^{T} +\hat{k}_{2}\hat{\rho}^{T}\right)
  +\hat{k}_{1}\hat{v}\hat{\rho}^{T}
 \right]
}{d^{2}}
\nonumber \\
&=&
\frac{1}{\left|\hat{z}\right|}
\frac{
\hat{k}_{1}^{2}\hat{\check{M}}_{B}^{-1}
-\left[
  \hat{k}_{2}\hat{\rho}\hat{\rho}^{T}
  +\hat{k}_{1}\left(\hat{\rho}\hat{v}^{T} +\hat{v}\hat{\rho}^{T}\right)
 \right]
}{d^{2}}
\nonumber \\
&=&
\frac{\hat{\check{M}}_{B^{\prime}}}{\left|\hat{z}\right|}
\end{eqnarray}
where $\hat{z}$ is the new denominator and
\begin{equation}
\label{def:update_Z1_entities}
\begin{array}{rclcrcl}
\hat{\rho}
&:=&
\left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, \beta_{O}(i)}
&&
\hat{v}
&:=&
\hat{\check{M}}_{B}^{-1}\zeta - d e_{\beta_{O}(i)}
\\
\hat{k}_{1}
&:=&
\hat{\rho}^{T}\zeta
&&
\hat{k}_{2}
&:=&
d\zeta_{\beta_{O}(i)}- \zeta^{T}\hat{\check{M}}_{B}^{-1}\zeta
\\
\hat{z}
&:=&
\hat{k}_{1}^{2}/d
&&
d
&:=&
\left|\det(\hat{\check{M}}_{B})\right|
\end{array}
\end{equation}

\subsection{Update $\mathbf{U_{Z_{4}}}$}
According to~(\ref{update:s_rep_s}) and the
Definitions~(\ref{def:theta_vartheta})
and~(\ref{def:rho_varrho})
the Update~(\ref{def:update_Z1_Z4})
applied to $\check{M}_{B}^{-1}$ leads to the follwing correspondences
in~(\ref{def:update_Z1_Z4_entities})
\begin{eqnarray}
\label{def:zeta_Z4}
\zeta
&=&
\vartheta^{T}
+
\left(
  \left(\check{M}_{B}\right)_{\gamma_{C}(\sigma(i)),\gamma_{C}(\sigma(i))}
  - \vartheta_{\gamma_{C}(\sigma(j))}
\right)
e_{\gamma_{C}(\sigma(j))}^{T}
=
\vartheta^{T}
\\
M_{i, \bullet}^{-1}
&=&
\left(\check{M}_{B}^{-1}\right)_{\bullet, \gamma_{C}(\sigma(j))}
=
\varrho^{T}
\end{eqnarray}
According to table~(\ref{table:pivot_precond}) the
pivoting precondition~(\ref{precond:pivot})
in terms of the reduced basis inverse for $U_{Z_{4}}$ is together
with~(\ref{def:zeta_Z4})
\begin{equation*}
\vartheta \varrho^{T}= \zeta^{T} \varrho^{T} \neq 0,
\end{equation*}
which together with the Definitions~(\ref{def:update_Z1_Z4_entities}) implies
\begin{equation}
\label{ineq:Z_4_k_1}
k_{1} \neq 0
\end{equation}
Furthermore according to table~(\ref{table:leaving_precond})
the leaving variable precondition~(\ref{precond:leaving})
in terms of the reduced basis inverse for $U_{Z_{4}}$ is
together with~(\ref{def:zeta_Z4})
\begin{equation*}
\vartheta\check{M}_{B}^{-1}\vartheta^{T}
=
\zeta_{\gamma_{C}(\sigma(j))}-\zeta^{T}\check{M}_{B}^{-1}\zeta= 0
\end{equation*}
which, again, together with the Definitions~(\ref{def:update_Z1_Z4_entities})
implies
\begin{equation}
\label{eq:k_2_zero}
k_{2} = 0
\end{equation}
(\ref{ineq:Z_4_k_1}) and~(\ref{eq:k_2_zero}) together with
Equation~(\ref{eq:det(Q)}) imply, in accordance with Lemma~(\ref{lem:main}),
\begin{equation}
\det(Q)=k_{1}^{2} \neq 0
\end{equation}
Note that according to the above and
Definition~(\ref{def:new_denom_Z1_Z4}) of the new denominator
$\hat{z}$ we have
\begin{equation}
sgn(\hat{z}) = 1
\end{equation}
Update~(\ref{def:update_Z1_Z4_denom})
applied to $\check{M}_{B}$ then simplifies to
\begin{eqnarray}
\label{def:update_Z4}
\check{M}_{B^{\prime}}^{-1}
&=&
\frac{1}{\left|\hat{z}\right|}
\frac{
\hat{k}_{1}^{2}\hat{\check{M}}_{B}^{-1}
-\left[
  \hat{k}_{1}\hat{\varrho}^{T}\hat{v}^{T}
  +\hat{v}\left(\hat{k}_{3}\hat{v}^{T}+\hat{k}_{1}\hat{\varrho}\right)
 \right]
}{d^{2}}
\nonumber \\
&=&
\frac{1}{\left|\hat{z}\right|}
\frac{
\hat{k}_{1}^{2}\hat{\check{M}}_{B}^{-1}
-\left[
  \hat{k}_{3}\hat{v}\hat{v}^{T}
  +\hat{k}_{1}\left(\hat{\varrho}^{T}\hat{v}^{T}+\hat{v}\hat{\varrho}\right)
 \right]
}{d^{2}}
\nonumber \\
&=&
\frac{\hat{\check{M}}_{B^{\prime}}^{-1}}{\left|\hat{z}\right|}
\end{eqnarray}
where $\hat{z}$ is the new denominator and
\begin{equation}
\label{def:update_Z4_entities}
\begin{array}{rclcrcl}
\hat{v}
&:=&
\hat{\check{M}}_{B}^{-1}\zeta -de_{\gamma_{C}(\sigma(j))}
&&
\hat{\varrho}^{T}
&:=&
\left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, \gamma_{C}(\sigma(j))}
\\
\hat{k}_{1}
&:=&
\hat{\varrho}\zeta
&&
\hat{k}_{3}
&:=&
-\left(\hat{\check{M}}_{B}^{-1}\right)_{\gamma_{C}(\sigma(j)),
  \gamma_{C}(\sigma(j))}
\\
\hat{z}
&:=&
\hat{k}_{1}^{2}/d
&&
d
&:=&
\left|\det(\hat{\check{M}}_{B})\right|
\end{array}
\end{equation}

\subsection{Updates $\mathbf{U_{Z_{2}}}$ and $\mathbf{U_{Z_{3}}}$ in general}
According to~(\ref{update:s_rep_o}) and~(\ref{update:o_rep_s}) the Updates
$U_{Z_{2}}$ and $U_{Z_{3}}$ require a basis matrix update that is capable of
growing and shrinking the basis matrix by two columns and rows in one step.
Let
\begin{equation}
\label{def:U_W}
M_{>>}=
\left(
\begin{array}{c|c}
M & U \\
\hline
U^{T} & W
\end{array}
\right)
\end{equation}
where $(U^{T}, W)$ are the two rows and $(U^{T}, W^{T})^{T}$ the two columns
to be added, with $W$ being the $2 \times 2$ matrix with the entries
they have in common.
\begin{equation}
\label{product}
M_{>>}=
\left(
\begin{array}{ccc|c}
1	&	 &	&        \\
	& \ddots &	& 0      \\
	&	 & 1	&        \\
\hline
	& X^{T}  &      & I_{2}  \\
\end{array}
\right)
\left(
\begin{array}{ccc|c}
	&	 &	&        \\
	& M      &	& 0      \\
	&	 & 	&        \\
\hline
	& 0      &      & Z      \\
\end{array}
\right)
\left(
\begin{array}{ccc|c}
1	&	 &	&        \\
	& \ddots &	& X      \\
	&	 &  1	&        \\
\hline
	& 0      &      & I_{2}  \\
\end{array}
\right)
\end{equation}
where
\begin{equation}
\label{def:X_Z}
X:=M^{-1}U, \quad Z:=W - U^{T}M^{-1}U
\end{equation}
The inverse $M_{>>}^{-1}$ is then given by
\begin{eqnarray}
M_{>>}^{-1}&=&
\left(
\begin{array}{ccc|c}
1	&	 &	&        \\
	& \ddots &	& -X     \\
	&	 &  1	&        \\
\hline
	& 0      &      & I_{2}  \\
\end{array}
\right)
\left(
\begin{array}{ccc|c}
	&	 &	&        \\
	& M^{-1} &	& 0      \\
	&	 & 	&        \\
\hline
	& 0      &      & Z^{-1} \\
\end{array}
\right)
\left(
\begin{array}{ccc|c}
1	&	 &	&        \\
	& \ddots &	& 0      \\
	&	 & 1	&        \\
\hline
	& -X^{T} &      & I_{2}  \\
\end{array}
\right)
\nonumber \\
&=&
\label{eq:inv_expand_by_2}
\left(
\begin{array}{c|c}
M^{-1}+XZ^{-1}X^{T} & -XZ^{-1}   \\
\hline
-Z^{-1}X^{T}        & Z^{-1}
\end{array}
\right)
\end{eqnarray}
From Jacobi's determinant identity we get
\begin{equation}
\det
\left(
\begin{array}{c|c}
M & 0 \\
\hline
0 & Z
\end{array}
\right)
=
\det\left(M\right)\det\left(Z\right)
\end{equation}
such that from~(\ref{product}) we obtain
\begin{equation}
\label{eq:det_id_Z2_Z3}
\det\left(M_{>>}\right) = \det\left(M\right)\det\left(Z\right)
\end{equation}

\subsubsection{Update $U_{Z_{3}}$ and the rational form}
In order to obtain the rational form of Definition~(\ref{def:rat_form})
for $M_{>>}^{-1}$ in
Equation~(\ref{eq:inv_expand_by_2}) in terms of $\hat{M}^{-1}$
and its common denominator
\begin{equation}
\label{def:old_denom_Z3}
d:=\left|\det\left(M\right)\right|
\end{equation}
we first rewrite the $2 \times 2$-matrix $Z^{-1}$ in terms of $Z$
\begin{equation}
\label{eq:2_by_2_matrix_inverse}
Z^{-1}=\frac{1}{\det\left(Z\right)}PZP^{T}, \quad
P:=
\left(
\begin{array}{cr}
0 & -1 \\
1 & 0
\end{array}
\right),
\end{equation}
and substitute this in Equation~(\ref{eq:inv_expand_by_2}) yielding
\begin{equation}
\label{eq:inv_expand_by_2_Z}
M_{>>}^{-1}=
\frac{1}{\det\left(Z\right)}
\left(
\begin{array}{c|c}
\det\left(Z\right)M^{-1} + XPZP^{T}X^{T} & -XPZP^{T} \\
\hline
-PZP^{T}X^{T}     & PZP^{T}
\end{array}
\right)
\end{equation}
Using the following substitutions, that seem to be coming out of the blue,
\begin{eqnarray}
\label{def:X_hat_Z_hat}
X=:\frac{\hat{X}}{d} & & Z=:\frac{\hat{Z}}{d}
\end{eqnarray}
in Equation~(\ref{eq:inv_expand_by_2_Z}) we obtain
\begin{equation}
\label{eq:inv_expand_2_Z3}
M_{>>}^{-1}=
\frac{1}{\det\left(Z\right)d}
\left(
\begin{array}{c|c}
(\det\left(Z\right)d^{3}M^{-1}+\hat{X}P\hat{Z}P^{T}\hat{X}^{T})/ d^{2} &
-\hat{X}P\hat{Z}P^{T}/d  \\
\hline
-P\hat{Z}P^{T}\hat{X}^{T}/d & P\hat{Z}P^{T}
\end{array}
\right)
\end{equation}
Note that
\begin{equation}
\label{eq:det_hat_Z}
\det(\hat{Z})=d^{2}\det\left(Z\right).
\end{equation}
Using Definitions~(\ref{def:rat_form}) and~(\ref{def:old_denom_Z3})
and defining the new denominator $\hat{z}$ according to
Equation~(\ref{eq:det_id_Z2_Z3})
\begin{equation}
\label{def:new_denom_Z3}
\hat{z}:=\det\left(Z\right) d = \det(\hat{Z}) /d
\end{equation}
we finally obtain from Equation~(\ref{eq:inv_expand_2_Z3})
\begin{eqnarray}
M_{>>}^{-1}&=&
\frac{sgn\left(\hat{z}\right)}{\left|\hat{z}\right|}
\left(
\begin{array}{c|c}
(\det(\hat{Z})\hat{M}^{-1}+\hat{X}P\hat{Z}P^{T}\hat{X}^{T})/d^{2} &
-\hat{X}P\hat{Z}P^{T}/d \\
\hline
-P\hat{Z}P^{T}\hat{X}^{T}/d & P\hat{Z}P^{T}
\end{array}
\right)
\nonumber \\
&=&
\frac{\hat{M}_{>>}^{-1}}{\left|\hat{z}\right|}
\end{eqnarray}
Note that this update works only provided that $\det(Z) \neq 0$.

\subsubsection{Update $U_{Z_{2}}$ and the rational form}
In order to obtain the rational form of Definition~(\ref{def:rat_form})
for $M^{-1}$ in Equation~(\ref{eq:inv_expand_by_2}) in terms of
parts of $\hat{M}_{>>}^{-1}$ and its common denominator
\begin{equation}
\label{def:old_denom_Z2}
d:=\left|\det(M_{>>})\right|
\end{equation}
we multiply Equation~(\ref{eq:inv_expand_by_2}) by the above $d$
\begin{equation}
dM_{>>}^{-1}=
\left(
\begin{array}{c|c}
d\left(M^{-1}+XZ^{-1}X^{T}\right) & \hat{\chi}   \\
\hline
\hat{\chi}^{T}                           & \hat{Z}^{-1}
\end{array}
\right)
\end{equation}
where
\begin{equation}
\label{def:chi}
\hat{\chi}:=-dXZ^{-1},
\quad
\hat{Z}^{-1}:=d Z^{-1}
\end{equation}
and solve for $M^{-1}$ to obtain
\begin{equation}
\label{eq:inv_shrink_by_2_Z}
\left(
\begin{array}{c|c}
M^{-1} & \vdots \\
\hline
\hdots & \bullet
\end{array}
\right)
=
\frac{1}{d}
\left[
\hat{M}_{>>}^{-1}
\left(
\begin{array}{c|c}
dXZ^{-1}X^{T} & \vdots \\
\hline
\hdots        & \bullet
\end{array}
\right)
\right]
\end{equation}
The relevant submatrix $dXZ^{-1}X^{T}$ in the above Equation can be expressed
in terms of the leaving rows and columns $\hat{\chi}$ of the basis inverse
and their intersection $\hat{Z}^{-1}$ using
Equations~(\ref{eq:2_by_2_matrix_inverse}), (\ref{def:chi})
and~(\ref{eq:det_hat_Z})
\begin{eqnarray}
dXZ^{-1}X^{T}
&=&
\left(-dXZ^{-1}\right)\frac{1}{d}Z\left(-dZ^{-1}X^{T}\right) \\
&=&
\frac{1}{\det(Z^{-1})d}
\hat{\chi} PZ^{-1}P^{T}\hat{\chi}^{T} \\
&=&
\frac{1}{\det(\hat{Z}^{-1})}
\hat{\chi} P\hat{Z}^{-1}P^{T}\hat{\chi}^{T}.
\end{eqnarray}
Equation~(\ref{eq:inv_shrink_by_2_Z}) then becomes
\begin{equation}
\label{eq:inv_shrink_by_2_Z2}
\left(
\begin{array}{c|c}
M^{-1}  & \vdots \\
\hline
\hdots  & \bullet
\end{array}
\right)
=
\frac{1}{\det(\hat{Z}^{-1})d}
\left[
\det(\hat{Z}^{-1})\hat{M}_{>>}^{-1}
-
\left(
\begin{array}{c|c}
\hat{\chi} P\hat{Z}^{-1}P^{T} \hat{\chi}^{T} & \vdots \\
\hline
\hdots                           & \bullet
\end{array}
\right)
\right]
\end{equation}
Using Definitions~(\ref{def:rat_form}) and~(\ref{def:old_denom_Z2}) and
defining the new denominator $\hat{z}$ according to
Equation~(\ref{eq:det_id_Z2_Z3})
\begin{equation}
\label{def:new_denom_Z2}
\hat{z}:=\det(Z^{-1})d = \det(\hat{Z}^{-1}) / d
\end{equation}
we finally obtain from Equation~(\ref{eq:inv_shrink_by_2_Z2})
\begin{eqnarray}
\left(
\begin{array}{c|c}
M^{-1}  & \vdots \\
\hline
\hdots  & \bullet
\end{array}
\right)
&=&
\frac{sgn(\hat{z})}{\left|\hat{z}\right|d^{2}}
\left[
\det(\hat{Z}^{-1})\hat{M}_{>>}^{-1}
-
\left(
\begin{array}{c|c}
\hat{\chi} P\hat{Z}^{-1}P^{T} \hat{\chi}^{T} & \vdots \\
\hline
\hdots                           & \bullet
\end{array}
\right)
\right]
\nonumber \\
&=&
\frac{1}{\left|\hat{z}\right|}
\left(
\begin{array}{c|c}
\hat{M}^{-1} & \vdots \\
\hline
\hdots       & \bullet
\end{array}
\right)
\end{eqnarray}
Note that this update works only provided that $\det(Z^{-1}) \neq 0$.

\subsection{Update $\mathbf{U_{Z_{3}}}$}
According to~(\ref{update:o_rep_s}) and Definition~(\ref{def:U_W})
$U$ and $W$ are defined as
\begin{equation}
\label{def:U2}
U:=
\left(
\begin{array}{c|c}
 0                        & A_{C,j}  \\
\hline
A_{\sigma(i), B_{O}}^{T}  & 2D_{B_{O}, j}
\end{array}
\right)
=
\left(
\begin{array}{c|c}
0                     &  \theta_{C} \\
\hline
\vartheta_{B_{O}}^{T} & \theta_{B_{O}}
\end{array}
\right),
\quad
W:=
\left(
\begin{array}{c|c}
0                & A_{\sigma(i), j} \\
\hline
A_{\sigma(i), j} & 2D_{j,j}
\end{array}
\right)
\end{equation}
such that according to the Definitions~(\ref{def:X_Z}) and
Substitutions~(\ref{def:X_hat_Z_hat}) $\hat{X}$ becomes
\begin{eqnarray}
\hat{X}
&:=&
d\check{M}_{B}^{-1}U
\nonumber \\
&=&
\hat{\check{M}}_{B}^{-1}
\left(
\begin{array}{c|c}
\vartheta^{T} & \theta
\end{array}
\right)
\nonumber \\
&=&
\left(
\begin{array}{c|c}
\hat{\check{M}}_{B}^{-1}\vartheta^{T}
&
\hat{\check{M}}_{B}^{-1}\theta  \\
\end{array}
\right)
\end{eqnarray}
using again the Definition~(\ref{def:X_Z}), the
Substitutions~(\ref{def:X_hat_Z_hat}) and the above yields $\hat{Z}$
\begin{eqnarray}
\hat{Z}
&:=&
d\left(W - U^{T}\check{M}_{B}^{-1}U\right)
\nonumber \\
&=&
dW - U^{T}\hat{X}
\nonumber \\
&=&
dW
-
\left(
\begin{array}{c}
\vartheta \\
\hline
\theta^{T}
\end{array}
\right)
\left(
\begin{array}{c|c}
\hat{\check{M}}_{B}^{-1}\vartheta^{T}
&
\hat{\check{M}}_{B}^{-1}\theta  \\
\end{array}
\right)
\nonumber \\
&=&
dW
-
\left(
\begin{array}{c|c}
\vartheta \hat{\check{M}}_{B}^{-1} \vartheta^{T}
&
\vartheta \hat{\check{M}}_{B}^{-1}\theta
\\
\hline
\theta^{T}\hat{\check{M}}_{B}^{-1}\vartheta^{T}
&
\theta^{T}\hat{\check{M}}_{B}^{-1}\theta
\end{array}
\right)
\nonumber \\
&=&
d
\left(
\begin{array}{c|c}
0                & A_{\sigma(i), j} \\
\hline
A_{\sigma(i), j} & 2D_{j,j}
\end{array}
\right)
-
\left(
\begin{array}{c|c}
\vartheta \hat{\check{M}}_{B}^{-1} \vartheta^{T}
&
\vartheta \hat{\check{M}}_{B}^{-1}\theta
\\
\hline
\theta^{T}\hat{\check{M}}_{B}^{-1}\vartheta^{T}
&
\theta^{T}\hat{\check{M}}_{B}^{-1}\theta
\end{array}
\right)
\nonumber
\end{eqnarray}
Since $\nu$ is according to \cite{Sven}, Lemma(2.7), defined as
\begin{equation}
\nu:=
2D_{j,j}
-\left(
  A_{C,j}^{T}\left|\right.A_{S_{B}, j}^{T}\left|\right.
  2D_{B_{O},j}^{T} \left|\right. 2D_{B_{S}, j}^{T}
 \right)
M_{B}^{-1}
\left(
\begin{array}{c}
A_{C, j} \\
\hline
A_{S_{B},j} \\
\hline
2D_{B_{O}, j} \\
\hline
2D_{B_{S}, j}
\end{array}
\right)
\end{equation}
and $D_{B_{S}, j}=0$ we obtain for $\nu$ together with
Definition~(\ref{eq:M_B_inv_exp}) of the basis inverse
and Definitions~(\ref{def:theta_vartheta})
\begin{eqnarray}
\nu
&=&
2D_{j,j}
-\left(
  \theta_{C}^{T}\left|\right.A_{S_{B}, j}^{T}\left|\right.
  \theta_{B_{O}}^{T} \left|\right. 0
 \right)
\left(
\begin{array}{c}
\left(\check{M}_{B}^{-1}\right)_{C,\bullet}\theta
\\
\hline
0
\\
\hline
\left(\check{M}_{B}^{-1}\right)_{B_{O}, \bullet}\theta
\\
\hline
\hdots
\end{array}
\right)
\nonumber \\
&=&
2D_{j,j}- \theta^{T} \check{M}_{B}^{-1} \theta
\end{eqnarray}

With this and the precondition of the leaving variable for $U_{Z_{3}}$
in table~(\ref{table:leaving_precond}) $\hat{Z}$ can then be expressed as
\begin{equation}
\hat{Z}:=
\left(
\begin{array}{c|c}
0
&
dA_{\sigma(i),j}
 -\vartheta\hat{\check{M}}_{B}^{-1}\theta
\\
\hline
dA_{\sigma(i),j}
 -\theta^{T}\hat{\check{M}}_{B}^{-1}\vartheta^{T}
&
d\nu
\end{array}
\right)
\end{equation}
By the pivoting precondition for $U_{Z_{3}}$ in
table~(\ref{table:pivot_precond}) and the symmetry of $\check{M}_{B}^{-1}$
we have
\begin{equation}
\det(Z)=-\left(A_{\sigma(i),j}
 -\vartheta\check{M}_{B}^{-1}\theta
\right)^{2} \neq 0
\end{equation}
in accordance with Lemma~(\ref{lem:main}). Note that according to the above and
the Definition~(\ref{def:new_denom_Z3}) of the new denominator $\hat{z}$
we have
\begin{equation}
\label{eq:sign_of_new_denom_Z3}
sgn(\hat{z})=-1
\end{equation}

If we define
\begin{equation}
\label{def:kappa_hat_nu_hat}
\hat{\kappa}:= dA_{\sigma(i),j}-\vartheta\hat{\check{M}}_{B}^{-1}\theta,
\quad
\hat{\nu}:=d\nu
\end{equation}
\marginpar{check definition of $\kappa$ with last subsection}
we may write
\begin{eqnarray}
-\hat{X}P\hat{Z}P^{T}
&:=&
-
\left(
\begin{array}{c|c}
\hat{\check{M}}_{B}^{-1}\vartheta^{T}
&
\hat{\check{M}}_{B}^{-1}\theta  \\
\end{array}
\right)
\left(
\begin{array}{c|c}
\hat{\nu}     & -\hat{\kappa} \\
\hline
-\hat{\kappa} & 0
\end{array}
\right)
\nonumber \\
&=&
\left(
\begin{array}{c|c}
\hat{\kappa}\hat{\check{M}}_{B}^{-1}\theta
-\hat{\nu}\hat{\check{M}}_{B}^{-1}\vartheta^{T}
&
\hat{\kappa}\hat{\check{M}}_{B}^{-1}\vartheta^{T}
\end{array}
\right)
\end{eqnarray}
as well as
\begin{equation}
\det(\hat{Z})=-\hat{\kappa}^{2},
\end{equation}
if we furthermore define
\begin{equation}
\label{def:phi_hat_varphi_hat}
\hat{\phi}:=\left(\hat{\check{M}}_{B}^{-1}\right)\theta,
\quad
\hat{\varphi}:=\left(\hat{\check{M}}_{B}^{-1}\right)\vartheta^{T}
\end{equation}
we may now express $\hat{X}P\hat{Z}P^{T}\hat{X}^{T}$ as
\begin{eqnarray}
\hat{X}P\hat{Z}P^{T}\hat{X}^{T}
&:=&
-
\left(
\begin{array}{c|c}
\hat{\kappa}\hat{\phi}-\hat{\nu}\hat{\varphi}
&
\hat{\kappa}\hat{\varphi}
\end{array}
\right)
\left(
\begin{array}{c}
\hat{\varphi}^{T}  \\
\hline
\hat{\phi}^{T}
\end{array}
\right)
\nonumber \\
&=&
\left(
  \hat{\nu}\hat{\varphi}-\hat{\kappa}\hat{\phi}
\right)
\hat{\varphi}^{T}
-\hat{\kappa}\hat{\varphi}\hat{\phi}^{T}
\nonumber \\
&=&
\hat{\nu}\hat{\varphi}\hat{\varphi}^{T}
-\hat{\kappa}
 \left[
   \hat{\varphi}\hat{\phi}^{T}+\hat{\phi}\hat{\varphi}^{T}
 \right]
\end{eqnarray}

Subsuming we may finally write
\begin{eqnarray}
\label{def:update_Z3}
\hat{\check{M}}_{B^{\prime}}^{-1}
&=&
-
\left(
\begin{array}{c|c|c}
\left[
  -\hat{\kappa}^{2}\hat{\check{M}}_{B}^{-1}
  +\hat{\nu}\hat{\varphi}\hat{\varphi}^{T}
  -\hat{\kappa}
  \left[
    \hat{\varphi}\hat{\phi}^{T}+\hat{\phi}\hat{\varphi}^{T}
  \right]
\right] / d^{2}
&
\left[
  \hat{\kappa}\hat{\phi} - \hat{\nu}\hat{\varphi}
\right] /d
&
  \hat{\kappa}\hat{\varphi} / d
\\
\hline
\left[
  \hat{\kappa}\hat{\phi}^{T} - \hat{\nu}\hat{\varphi}^{T}
\right] /d
&
\hat{\nu}
&
-\hat{\kappa}
\\
\hline
\hat{\kappa}\hat{\varphi}^{T} /d
&
-\hat{\kappa}
&
0
\end{array}
\right)
\nonumber \\
&=&
\left(
\begin{array}{c|c|c}
\left[
  \hat{\kappa}^{2}\hat{\check{M}}_{B}^{-1}
  -\hat{\nu}\hat{\varphi}\hat{\varphi}^{T}
  +\hat{\kappa}
  \left[
    \hat{\varphi}\hat{\phi}^{T}+\hat{\phi}\hat{\varphi}^{T}
  \right]
\right] / d^{2}
&
\left[
  \hat{\nu}\hat{\varphi} - \hat{\kappa}\hat{\phi}
\right] /d
&
-\hat{\kappa}\hat{\varphi} / d
\\
\hline
\left[
  \hat{\nu}\hat{\varphi}^{T} - \hat{\kappa}\hat{\phi}^{T}
\right] /d
&
-\hat{\nu}
&
\hat{\kappa}
\\
\hline
-\hat{\kappa}\hat{\varphi}^{T} /d
&
\hat{\kappa}
&
0
\end{array}
\right)
\end{eqnarray}
where $\hat{z}$ is the new denominator and
\begin{equation}
\label{def:update_Z3_entities}
\begin{array}{rclcrcl}
\hat{\phi} &:=& \hat{\check{M}}_{B}^{-1}\theta
&&
\hat{\varphi} &:=& \hat{\check{M}}_{B}^{-1}\vartheta^{T}
\\
\hat{\kappa} &:=& dA_{\sigma(i), j}-\vartheta\hat{\check{M}}_{B}^{-1}\theta
&&
\hat{\nu} &:=& 2dD_{j,j}-\theta^{T}\hat{\check{M}}_{B}^{-1}\theta
\\
\hat{z} &:=& \hat{\kappa}^{2} / d
&&
d &:=& \left|\det(\hat{\check{M}}_{B})\right|
\end{array}
\end{equation}

\subsection{Update $\mathbf{U_{Z_{2}}}$}
In analogy with Definitions~(\ref{def:theta_vartheta}) we define
\begin{equation}
\label{def:rho_hat_varrho_hat}
\hat{\rho}:=\left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, \beta_{O}(i)},
\quad
\hat{\varrho}^{T}
:=
\left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, \gamma_{C}(\sigma(j))}
\end{equation}
such that
\begin{equation}
\hat{\chi}:=
\left(
\begin{array}{c|c}
\hat{\varrho}_{C^{\prime}}^{T}     & \hat{\rho}_{C^{\prime}} \\
\hline
\hat{\varrho}_{B_{O}^{\prime}}^{T} & \hat{\rho}_{B_{O}^{\prime}}
\end{array}
\right)
\end{equation}
and
\begin{eqnarray}
\hat{Z}^{-1}
&:=&
\left(
\begin{array}{c|c}
\hat{\varrho}_{\gamma_{C}(\sigma(j))}^{T}
&
\hat{\rho}_{\gamma_{C}(\sigma(j))} \\
\hline
\hat{\varrho}_{\beta_{O}(i)}^{T}
&
\hat{\rho}_{\beta_{O}(i)}
\end{array}
\right)
\nonumber \\
&=&
\left(
\begin{array}{c|c}
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\gamma_{C}(\sigma(j)), \gamma_{C}(\sigma(j))}
&
\left(\hat{\check{M}}_{B}^{-1}\right)_{\gamma_{C}(\sigma(j)), \beta_{O}(i)}
\\
\hline
\left(\hat{\check{M}}_{B}^{-1}\right)_{\beta_{O}(i), \gamma_{C}(\sigma(j))}
&
\left(\hat{\check{M}}_{B}^{-1}\right)_{\beta_{O}(i), \beta_{O}(i)}
\end{array}
\right)
\end{eqnarray}
The above is using the precondition of the leaving variable for $U_{Z_{2}}$ in
table~(\ref{table:leaving_precond}) simplified to
\begin{equation}
\hat{Z}^{-1}=
\left(
\begin{array}{c|c}
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\gamma_{C}(\sigma(j)), \gamma_{C}(\sigma(j))}
&
\left(\hat{\check{M}}_{B}^{-1}\right)_{\gamma_{C}(\sigma(j)), \beta_{O}(i)}
\\
\hline
\left(\hat{\check{M}}_{B}^{-1}\right)_{\beta_{O}(i), \gamma_{C}(\sigma(j))}
&
0
\end{array}
\right)
\end{equation}
By the pivoting precondition for $U_{Z_{2}}$ in
table~(\ref{table:pivot_precond}) we then have
\begin{equation}
\det(Z^{-1}) =
-\left(
\left(\check{M}_{B}^{-1}\right)_{\beta_{O}(i),\gamma_{C}(\sigma(j))}
\right)^{2}
\neq 0
\end{equation}
again, in accordance with Lemma~(\ref{lem:main}). Note that according to the above and
the Definition~(\ref{def:new_denom_Z2}) of the new denominator $\hat{z}$
we have
\begin{equation}
\label{eq:sign_of_new_denom_Z2}
sgn(\hat{z})=-1
\end{equation}

If we define
\begin{equation}
\hat{\kappa}:=
\left(\hat{\check{M}}_{B}^{-1}\right)_{\gamma_{C}(\sigma(j)), \beta_{O}(i)}
\quad
\hat{{\xi}}:=
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\gamma_{C}(\sigma(j)), \gamma_{C}(\sigma(j))}
\end{equation}
we may write
\begin{eqnarray}
-\hat{\chi} P \hat{Z}^{-1}P^{T}\hat{\chi}^{T}
&=&
-\left(
\begin{array}{c|c}
\hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
&
\hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}} \\
\end{array}
\right)
\left(
\begin{array}{c|c}
0              & -\hat{\kappa}  \\
\hline
-\hat{\kappa} & \hat{\xi}
\end{array}
\right)
\left(
\begin{array}{c}
\hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}   \\
\hline
\hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
\end{array}
\right)
\nonumber \\
&=&
\left(
\begin{array}{c|c}
\hat{\kappa}\hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}
&
\hat{\kappa}\hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
 -\hat{\xi}\hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}
\end{array}
\right)
\left(
\begin{array}{c}
\hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}   \\
\hline
\hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
\end{array}
\right)
\nonumber \\
&=&
-\hat{\xi}
 \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}
 \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
+\hat{\kappa}
 \left[
 \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}
 \hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}
 +\hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
 \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
 \right]
\end{eqnarray}
as well as
\begin{equation}
\det(\hat{Z}^{-1})=-\hat{\kappa}^{2},
\end{equation}

Subsuming this we may finally write
\begin{eqnarray}
\label{def:update_Z2}
\hat{\check{M}}_{B^{\prime}}^{-1}
&=&
-
\left(
\begin{array}{c|c}
\left[
  -\hat{\kappa}^{2}\hat{\check{M}}_{B}^{-1}
  -\hat{\xi}
  \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}
  \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
  +\hat{\kappa}
  \left[
    \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}
    \hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}
    +\hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
    \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
  \right]
\right]/d^{2}
&
\vdots
\\
\hline
\hdots
&
\bullet
\end{array}
\right)
\nonumber \\
&=&
\left(
\begin{array}{c|c}
\left[
  \hat{\kappa}^{2}\hat{\check{M}}_{B}^{-1}
  +\hat{\xi}
  \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}
  \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
  -\hat{\kappa}
  \left[
    \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}
    \hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}
    +\hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
    \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
  \right]
\right]/d^{2}
&
\vdots
\\
\hline
\hdots
&
\bullet
\end{array}
\right)
\end{eqnarray}
where $\hat{z}$ is the new denominator and
\begin{equation}
\label{def:update_Z2_entities}
\begin{array}{rclcrcl}
\hat{\rho}
&:=&
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\bullet, \beta_{O}(i)}
&&
\hat{\varrho}^{T}
&:=&
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\bullet, \gamma_{C}(\sigma(j))}
\\
\hat{\kappa}
&:=&
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\gamma_{C}(\sigma(j)), \beta_{O}(i)}
&&
\hat{\xi}
&:=&
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\gamma_{C}(\sigma(j)), \gamma_{C}(\sigma(j))}
\\
\hat{z}
&:=&
\hat{\kappa}^{2}/d
&&
d
&:=&
\left|\det(\hat{\check{M}}_{B})\right|
\end{array}
\end{equation}

\section{Implementation Issues}
Comparing the Updates for $U_{Z_{1}}$ to $U_{Z_{4}}$ we note that the Updates
$U_{Z_{1}}$ and $U_{Z_{4}}$ are en-bloc replacement updates, that is, the to be
replaced row and column need no special treatment, in contrast to the Updates
$U_{Z_{2}}$ and $U_{Z_{3}}$, which change the dimensions of the reduced basis
inverse by $2$. Nevertheless, all the Updates have a similar update
formula in common for parts of the to be updated basis inverses,
for $U_{Z_{1}}$ and $U_{Z_{4}}$ this part comprises the whole
basis inverse $\check{M}_{B}^{-1}$ and for $U_{Z_{2}}$
and $U_{Z_{3}}$ the corresponding part consists of
\begin{equation}
\left(
 \check{M}_{B}^{-1}
\right)_{C \cap C^{\prime} \cup B_{O} \cap B_{O}^{\prime}}
\end{equation}
The general update formula for this part for the Updates
$U_{Z_{1}}$ and $U_{Z_{4}}$ can then be written as
\begin{equation}
\check{M}_{B^{\prime}}^{-1}
=
\frac{
\omega_{0}\check{M}_{B}^{-1}
+\omega_{1}\Psi_{1}\Psi_{1}^{T}
+\omega_{2}\left[\Psi_{1}\Psi_{2}^{T}+\Psi_{2}\Psi_{1}^{T}\right]
}{\omega_{3}},
\end{equation}
The values the above entities assume for the updates $U_{Z_{1}}$ and
$U_{Z_{4}}$ are subsumed in the following table below.
\begin{equation}
\begin{array}{c|cc}
& U_{Z_{1}} & U_{Z_{4}} \\
\hline
C^{\prime}
& C
& C \setminus \{\sigma(j)\} \cup \{\sigma(i)\}
\\
B_{O}^{\prime}
& B_{O} \setminus \{i\} \cup \{j\}
& B_{O}
\\
\Psi_{1}
& \hat{\rho}
& \hat{\check{M}}_{B}^{-1}\zeta-de_{\gamma_{C}(\sigma(j))}
\\
\Psi_{2}
& \hat{\check{M}}_{B}^{-1}\zeta-de_{\beta_{O}(i)}
& \hat{\varrho}^{T}
\\
\omega_{0}
& \left(\hat{\rho}^{T}\zeta\right)^{2}
& \left(\hat{\varrho}\zeta\right)^{2}
\\
\omega_{1}
& -\left(
   d\zeta_{\beta_{O}(i)}-\zeta^{T}\hat{\check{M}}_{B}^{-1}\zeta
  \right)
& \left(
   \hat{\check{M}}_{B}^{-1}
  \right)_{\gamma_{C}(\sigma(j)), \gamma_{C}(\sigma(j))}
\\
\omega_{2}
& -\hat{\rho}^{T}\zeta
& -\hat{\varrho}\zeta
\\
\omega_{3}
& d^{2}
& d^{2}
\end{array}
\end{equation}

Correspondingly for the Updates $U_{Z_{2}}$ and $U_{Z_{3}}$ as
\begin{equation}
\left(
\check{M}_{B^{\prime}}^{-1}
\right)_{C \cap C^{\prime} \cup B_{O} \cap B_{O}^{\prime}}
=
\frac{
\omega_{0}
\left(
\check{M}_{B}^{-1}
\right)_{C \cap C^{\prime} \cup B_{O} \cap B_{O}^{\prime}}
+\omega_{1}\Psi_{1}\Psi_{1}^{T}
+\omega_{2}\left[\Psi_{1}\Psi_{2}^{T}+\Psi_{2}\Psi_{1}^{T}\right]
}{\omega_{3}},
\end{equation}
The values the above entities assume for the updates $U_{Z_{2}}$ and
$U_{Z_{3}}$ are subsumed in the following table below.
\begin{equation}
\begin{array}{c|cccc}
& U_{Z_{2}} & U_{Z_{3}} \\
\hline
C^{\prime}
& C \setminus \{\sigma(j)\}
& C \cup \{\sigma(i)\}
\\
B_{O}^{\prime}
& B_{O} \setminus \{i\}
& B_{O} \cup \{j\}
\\
\Psi_{1}
& \hat{\rho}_{C^{\prime} \cup B_{O}^{\prime}}
& \varphi
\\
\Psi_{2}
& \hat{\varrho}_{C^{\prime} \cup B_{O}^{\prime}}^{T}
& \phi
\\
\omega_{0}
& \left(
   \hat{\check{M}}_{B}^{-1}
  \right)_{\gamma_{C}(\sigma(j)), \beta_{O}(i)}^{2}
& \left(
   dA_{\sigma(i),j}-\vartheta\hat{\check{M}}_{B}^{-1}\theta
  \right)^{2} d
\\
\omega_{1}
& \left(
   \hat{\check{M}}_{B}^{-1}
  \right)_{\gamma_{C}(\sigma(j)), \gamma_{C}(\sigma(j))}
& -\hat{\nu}
\\
\omega_{2}
& -\left(
   \hat{\check{M}}_{B}^{-1}
  \right)_{\gamma_{C}(\sigma(j)), \beta_{O}(i)}
& d A_{\sigma(i), j}-\vartheta\hat{\check{M}}_{B}^{-1}\theta
\\
\omega_{3}
& d^{2}
& d^{2}
\end{array}
\end{equation}
The function performing this part of the Updates $U_{Z_{1}} \dots U_{Z_{4}}$
has the following signature:
\begin{tabbing}
\texttt{z\_update\_inplace($\Psi_{1}, \Psi_{2}, \omega_{0}, \omega_{1},
\omega_{2}, \omega_{3}$)}
\end{tabbing}
\subsection{Interfaces}
%In general entities depending only on the constraint matrix $A$ and on the
%objective function matrix $D$ as well as on the indices $j, i$ of the current
%entering and leaving variables are considered to be global constants.
The QP-solver is represented by the classes \texttt{QP\_solver} and
\texttt{QP\_basis\_inverse} and other classes representing the different
pricing strategies. An instance of the class \texttt{QP\_solver} has
members representing the headings $B_{O},B_{S}, C$ and $S_{B}$,
the constraint matrix $A$, the objective function matrix $D$ as well as a
reference, \texttt{inv\_M\_B}, referring to an instance of the class
\texttt{QP\_basis\_inverse}. An instance of the class
\texttt{QP\_basis\_inverse} however does not have a reference
referring to its ambient solver. This entails that methods of the basis inverse
implementing the Updates $U_{Z_{1}} \dots U_{Z_{4}}$ described in previous
sections can not access these entities kept in the class \texttt{QP\_solver};
we therefore adopt the policy that these entities are passed as parameters of
these methods wherever necessary.

Some of the entities used in the update formulas~(\ref{def:update_Z1}),
(\ref{def:update_Z4}), (\ref{def:update_Z3}) and~(\ref{def:update_Z2})
are already determined by the Ratio Test Step~1 preceeding the update step~1.
Among other entities the Ratio Test Step~1 computes
\begin{equation}
\label{def:q_hat_nu_hat}
\hat{q}:=\hat{\check{M}}_{B}^{-1}\theta
\quad
\hat{\nu}:=2D_{j,j}d-\theta^{T}\hat{\check{M}}_{B}^{-1}\theta
\end{equation}

\subsection{Update $U_{Z_{1}}$}
According to the Definitions~(\ref{def:zeta_Z1})
and~(\ref{def:q_hat_nu_hat}), $\hat{v}$ of the
Definitions~(\ref{def:update_Z1_entities}) can be expressed in terms
of the entering column $\theta$ and $D$ as
\begin{eqnarray*}
\hat{v}
&:=&
\hat{\check{M}}_{B}^{-1}\zeta - de_{\beta_{O}(i)} \\
&=&
\hat{\check{M}}_{B}^{-1}\theta
+2\left(D_{j,j}-D_{i,j}\right)\hat{\check{M}}_{B}^{-1}e_{\beta_{O}(i)}
-de_{\beta_{O}(i)} \\
&=&
\hat{q}+2\left(D_{j,j}-D_{i,j}\right)\hat{\rho}-de_{\beta_{O}(i)}
\end{eqnarray*}
Similarly we obtain for $\hat{k}_{2}$ using Definitions~(\ref{def:zeta_Z1})
and~(\ref{def:q_hat_nu_hat})
\begin{eqnarray*}
\hat{k}_{2}
&:=&
d\zeta_{\beta_{O}(i)}-\zeta^{T}\hat{\check{M}}_{B}^{-1}\zeta \\
&=&
2dD_{j,j}
-\left(\theta+2\left(D_{j,j}-D_{i,j}\right)e_{\beta_{O}(i)}^{T}\right)
\hat{\check{M}}_{B}^{-1}
\left(\theta+2\left(D_{j,j}-D_{i,j}\right)e_{\beta_{O}(i)}\right) \\
&=&
2dD_{j,j}-\theta^{T}\hat{\check{M}}_{B}^{-1}\theta
-4\left(D_{j,j}-D_{i,j}\right)^{2}
e_{\beta_{O}(i)}^{T}\hat{\check{M}}_{B}^{-1}e_{\beta_{O}(i)} \\
&&
-4\left(D_{j,j}-D_{i,j}\right)
e_{\beta_{O}(i)}^{T}\hat{\check{M}}_{B}^{-1}\theta \\
&=&
\hat{\nu}
-4\left(D_{j,j}-D_{i,j}\right)^{2}
\left(\hat{\check{M}}_{B}^{-1}\right)_{\beta_{O}(i), \beta_{O}(i)}
-4\left(D_{j,j}-D_{i,j}\right)\hat{\rho}^{T}\theta \\
&=&
\hat{\nu}
-4\left(D_{j,j}-D_{i,j}\right)\hat{\rho}^{T}\theta \\
&=&
\hat{\nu}
-4\left(D_{j,j}-D_{i,j}\right)\hat{q}_{\beta_{O}(i)}
\end{eqnarray*}
Using the above, the entities used
in~(\ref{def:update_Z1_entities}) can be expressed as
\begin{equation}
\label{def:update_Z1_entities_prep}
\begin{array}{rclcrcl}
\hat{\rho}
&:=&
\left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, \beta_{O}(i)}
&&
\hat{v}
&:=&
\hat{q}+2\left(D_{j,j}-D_{i,j}\right)\hat{\rho}-de_{\beta_{O}(i)}
\\
\hat{k}_{1}
&:=&
\hat{q}_{\beta_{O}(i)}
&&
\hat{k}_{2}
&:=&
\hat{\nu}
-4\left(D_{j,j}-D_{i,j}\right)\hat{q}_{\beta_{O}(i)}
\\
\hat{z}
&:=&
\hat{k}_{1}^{2}/d
&&
d
&:=&
\left|\det(\hat{\check{M}}_{B})\right|
\end{array}
\end{equation}
such that the part in the class \texttt{QP\_basis\_inverse} of
Update $U_{Z_{1}}$ can be outlined as
\begin{tabbing}
\texttt{z\_replace\_original\_by\_original($y, s_{\Delta}, s_{\nu}, k_{i}$) \{}
\\
\texttt{  } \= // compute $\hat{v}, \hat{\rho}, \hat{k}_{1}, \hat{k}_{2}$ \\
\> \texttt{$\hat{k}_{1}:=y_{k_{i}}$} \\
\> \texttt{$\hat{\rho}:=
  \left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, k_{i}}$} \\
\> \texttt{$\hat{v}:=y + s_{\Delta} \hat{\rho} - de_{k_{i}}$} \\
\> \texttt{$\hat{k}_{2}:=s_{\nu} - 2s_{\Delta}\hat{k}_{1}$} \\
\> \texttt{z\_update\_in\_place($\hat{\rho}, \hat{v}, \hat{k}_{1}^{2},
-\hat{k}_{2}, -\hat{k}_{1}, d^{2}$)} \\
\> // store new denominator $\hat{z}$ \\
\> \texttt{$d:=\hat{k}_{1}^{2}/d$} \\
\texttt{\}}
\end{tabbing}
The calling counterpart in the class \texttt{QP\_solver} can be outlined as
\begin{tabbing}
\texttt{z\_replace\_variable\_original\_by\_original() \{} \\
\texttt{  } \= // compute $k_{i}, \hat{k}_{2}$ \\
\> \texttt{$s_{\Delta}:= 2\left(D_{j,j} - D_{i,j}\right)$} \\
\> \texttt{$s_{\nu}:= \nu$} \\
\> \texttt{$k_{i}:=\beta_{O}(i)$} \\
\> // update headings \\
\> // ... \\
\> // update basis inverse \\
\> \texttt{inv\_M\_B.z\_replace\_original\_by\_original($\hat{q},
s_{\Delta}, s_{\nu}, k_{i}$)} \\
\texttt{\}}
\end{tabbing}

\subsection{Update $U_{Z_{4}}$}
According to the Definitions~(\ref{def:zeta_Z4}),
%and~(\ref{def:q_hat_nu_hat}),
$\zeta=\vartheta$, such that the entities $\hat{v}$ and $\hat{k}_{1}$ used
in Definitions~(\ref{def:update_Z4_entities}) can be expressed in terms
of the entering column $\vartheta$ as
\begin{equation}
\label{def:update_Z4_entities_prep}
\begin{array}{rclcrcl}
\hat{v}
&:=&
\hat{\check{M}}_{B}^{-1}\vartheta^{T} -de_{\gamma_{C}(\sigma(j))}
&&
\hat{\varrho}^{T}
&:=&
\left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, \gamma_{C}(\sigma(j))}
\\
\hat{k}_{1}
&:=&
\hat{\varrho}\vartheta^{T}
&&
\hat{k}_{3}
&:=&
-\left(\hat{\check{M}}_{B}^{-1}\right)_{\gamma_{C}(\sigma(j)),
  \gamma_{C}(\sigma(j))}
\\
\hat{z}
&:=&
\hat{k}_{1}^{2}/d
&&
d
&:=&
\left|\det(\hat{\check{M}}_{B})\right|
\end{array}
\end{equation}
such that, by taking into account that according to
Definition~(\ref{def:theta_vartheta}), $\vartheta_{C}=0$,
the part in the class \texttt{QP\_basis\_inverse} of
Update $U_{Z_{4}}$ can be outlined as
\begin{tabbing}
\texttt{z\_replace\_slack\_by\_slack($u, k_{j}$) \{} \\
\texttt{  } \= // compute $\hat{v}, \hat{\varrho}, \hat{k}_{1}, \hat{k}_{3}$ \\
\> \texttt{$\hat{v}:=
    \left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, B_{O}}u-de_{k_{j}}$} \\
\> \texttt{$\hat{\varrho}^{T}:=
  \left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet,k_{j}}$} \\
\> \texttt{$\hat{k}_{1}:=\hat{\varrho}_{B_{O}}u$} \\
\> \texttt{$\hat{k}_{3}:=
  -\left(\hat{\check{M}}_{B}^{-1}\right)_{k_{j}, k_{j}}$} \\
\> \texttt{z\_update\_in\_place($\hat{v}, \hat{\varrho}, \hat{k}_{1}^{2},
-\hat{k}_{3}, -\hat{k}_{1}, d^{2}$)} \\
\> // store new denominator $\hat{z}$ \\
\> \texttt{$d:=\hat{k}_{1}^{2}/d$} \\
\texttt{\}}
\end{tabbing}
The calling counterpart in the class \texttt{QP\_solver} can be outlined as
\begin{tabbing}
\texttt{z\_replace\_variable\_slack\_by\_slack() \{} \\
\texttt{  } \= // compute $u, k_{j}$ \\
\> \texttt{$u:=\vartheta_{B_{O}}^{T}$} \\
\> \texttt{$k_{j}:=\gamma_{C}(\sigma(j))$} \\
\> // update headings \\
\> // ... \\
\> // update basis inverse \\
\> \texttt{inv\_M\_B.z\_replace\_slack\_by\_slack($u, k_{j}$)}
\\
\texttt{\}}
\end{tabbing}

\subsection{Update $U_{Z_{2}}$}
Using Definitions~(\ref{def:q_hat_nu_hat}) the entities used
in~(\ref{def:update_Z2_entities}) can be expressed as
\begin{equation}
\label{def:update_Z2_entities_prep}
\begin{array}{rclcrcl}
\hat{\rho}
&:=&
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\bullet, \beta_{O}(i)}
&&
\hat{\varrho}^{T}
&:=&
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\bullet, \gamma_{C}(\sigma(j))}
\\
\hat{\kappa}
&:=&
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\gamma_{C}(\sigma(j)), \beta_{O}(i)}
&&
\hat{\xi}
&:=&
\left(
  \hat{\check{M}}_{B}^{-1}
\right)_{\gamma_{C}(\sigma(j)), \gamma_{C}(\sigma(j))}
\\
\hat{z}
&:=&
\hat{\kappa}^{2}/d
&&
d
&:=&
\left|\det(\hat{\check{M}}_{B})\right|
\end{array}
\end{equation}
such that the part in the class \texttt{QP\_basis\_inverse} of
Update $U_{Z_{2}}$ can be outlined as
\begin{tabbing}
\texttt{z\_replace\_original\_by\_slack() \{} \\
\texttt{  } \= // compute $\hat{\rho}, \hat{\varrho}, \hat{\kappa},
\hat{\xi}$ \\
\> \texttt{$\hat{\rho}:=
  \left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, k_{i}}$} \\
\> \texttt{$\hat{\varrho}:=
  \left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, k_{j}}$} \\
\> \texttt{$\hat{\kappa}:=
  \left(\hat{\check{M}}_{B}^{-1}\right)_{k_{j}, k_{i}}$} \\
\> \texttt{$\hat{\xi}:=
  \left(\hat{\check{M}}_{B}^{-1}\right)_{k_{j}, k_{j}}$} \\
\> \texttt{z\_update\_in\_place($\hat{\rho}, \hat{\varrho}^{T},
\hat{\kappa}^{2}, \hat{\xi}, -\hat{\kappa}, d^{2}$)} \\
\> // store new denominator $\hat{z}$ \\
\> \texttt{$d:=\hat{\kappa}^{2}/d$} \\
\texttt{\}}
\end{tabbing}
The calling counterpart in the class \texttt{QP\_solver} can be outlined as
\begin{tabbing}
\texttt{z\_replace\_variable\_original\_by\_slack() \{} \\
\texttt{  } \= // compute $u, k_{j}$ \\
\> \texttt{$k_{j}:=\gamma_{C}(\sigma(j))$} \\
\> \texttt{$k_{i}:=\beta_{O}(i)$} \\
\> // update headings \\
\> // ... \\
\> // prepare basis inverse \\
\> \texttt{inv\_M\_B.swap\_variable($k_{i}$)} \\
\> \texttt{inv\_M\_B.swap\_constraint($k_{j}$)}\\
\> // update basis inverse \\
\> \texttt{inv\_M\_B.z\_replace\_original\_by\_slack()}
\\
\texttt{\}}
\end{tabbing}

\subsection{Update $U_{Z_{3}}$}
Using Definitions~(\ref{def:q_hat_nu_hat})
as well as Definitions~(\ref{def:theta_vartheta}) the entities used
in~(\ref{def:update_Z3_entities}) can be expressed as
\begin{equation}
\label{def:update_Z3_entities_prep}
\begin{array}{rclcrcl}
\hat{\phi} &:=& \hat{q}
&&
\hat{\varphi}
&:=&
\hat{\check{M}}_{B}^{-1}\vartheta^{T}
\\
\hat{\kappa} &:=& dA_{\sigma(i), j}-\vartheta\hat{q}
&&
\hat{\nu} &:=& 2dD_{j,j}-\theta^{T}\hat{\check{M}}_{B}^{-1}\theta
\\
\hat{z} &:=& \hat{\kappa}^{2}/d
&&
d &:=& \left|\det(\hat{\check{M}}_{B})\right|
\end{array}
\end{equation}
such that, by taking into account that according to
Definition~(\ref{def:theta_vartheta}), $\vartheta_{C}=0$,
the part in the class \texttt{QP\_basis\_inverse} of
Update $U_{Z_{3}}$ can be outlined as
\begin{tabbing}
\texttt{z\_replace\_slack\_by\_original($y,u,\kappa,\nu$) \{}
\\
\texttt{  } \= // compute $\hat{\phi}, \hat{\varphi}, \hat{\kappa},
\hat{\nu}$ \\
\> \texttt{$\hat{\phi}:=y$} \\
\> \texttt{$\hat{\varphi}:=
 \left(\hat{\check{M}}_{B}^{-1}\right)_{\bullet, B_{O}}u$} \\
\> \texttt{$\hat{\kappa}:=\kappa$} \\
\> \texttt{$\hat{\nu}:=\nu$} \\
\> \texttt{z\_update\_in\_place($\hat{\varphi}, \hat{\phi},
\hat{\kappa}^{2}, -\hat{\nu}, \hat{\kappa}, d^{2}$)} \\
\> // append new rows and columns \\
\> $\left(\hat{\check{M}}_{B^{\prime}}\right)_{s, \bullet}:=
\left[
  (\hat{\nu}\hat{\varphi}-\hat{\kappa}\hat{\phi})_{C}/d
  \left|\right. -\hat{\nu}
  \left|\right.(\hat{\nu}\hat{\varphi}-\hat{\kappa}\hat{\phi})_{B_{O}}/d
  \left|\right. \hat{\kappa}
\right]$ \\
\> $\left(\hat{\check{M}}_{B^{\prime}}\right)_{l+b, \bullet}:=
\left[
  -\hat{\kappa}\hat{\varphi}^{T}_{C}/d
  \left|\right. \hat{\kappa}
  \left|\right. -\hat{\kappa}\hat{\varphi}^{T}_{B_{O}}/d
  \left|\right. 0
\right]$ \\
\> // store new denominator $\hat{z}$ \\
\> \texttt{$d:=\hat{\kappa}^{2}/d$} \\
\texttt{\}}
\end{tabbing}
The calling counterpart in the class \texttt{QP\_solver} can be outlined as
\begin{tabbing}
\texttt{z\_replace\_variable\_slack\_by\_original() \{} \\
\texttt{  } \= // compute $u, k_{j}$ \\
\> \texttt{$y:=\hat{q}$} \\
\> \texttt{$u:=\vartheta_{B_{O}}$} \\
\> \texttt{$\kappa:=dA_{\sigma(i), j}-\vartheta_{B_{O}}\hat{q}_{B_{O}}$} \\
\> \texttt{$\nu:=\hat{\nu}$} \\
\> // update headings \\
\> // ... \\
\> // update basis inverse \\
\> \texttt{inv\_M\_B.z\_replace\_slack\_by\_original($y, u, \kappa, \nu$)}
\\
\texttt{\}}
\end{tabbing}

\section{Verification}
\subsection{Update $U_{Z_{1}}$}
According to Equations  we have
\begin{eqnarray}
\hat{\rho}^{T}\check{M}_{B}=de_{\beta_{O}(i)}
\end{eqnarray}
as well as
\begin{equation}
\hat{v}^{T}\check{M}_{B}=
d\left(\theta^{T}
   -\left(\check{M}_{B}\right)_{\beta_{O}(i), \bullet}
 \right)
=
du^{T}
\end{equation}
such that $\hat{\check{M}}_{B^{\prime}}^{-1}\check{M}_{B}$ becomes
\begin{eqnarray}
\hat{\check{M}}_{B^{\prime}}^{-1}
\check{M}_{B}
&=&
\frac{
\hat{k}_{1}^{2}\hat{\check{M}}_{B}^{-1}
-\left[
  \hat{k}_{2}\hat{\rho}\hat{\rho}^{T}
  +\hat{k}_{1}\left(\hat{\rho}\hat{v}^{T} +\hat{v}\hat{\rho}^{T}\right)
 \right]
}{d^{2}}
\check{M}_{B}
\nonumber \\
&=&
\frac{
  d\hat{k}_{1}^{2}I
  -d
  \left[
    \hat{k}_{2}\hat{\rho}e_{\beta_{O}(i)}^{T}
    +\hat{k}_{1}
    \left(
      \hat{\rho}u^{T}
      +\hat{v}e_{\beta_{O}(i)}^{T}
    \right)
  \right]
}{d^{2}}
\nonumber \\
&=&
\hat{z}
-\frac{
  \hat{k}_{2}\hat{\rho}e_{\beta_{O}(i)}^{T}
  +\hat{k}_{1}\hat{\rho}u^{T}
  +\hat{k}_{1}\hat{v}e_{\beta_{O}(i)}^{T}
}{d}
\end{eqnarray}

According to Equations we have
\begin{eqnarray}
\hat{\check{M}}_{B}^{-1}UV^{T}
&=&
\hat{\check{M}}_{B}^{-1}ue_{\beta_{O}(i)}^{T}
+\hat{\check{M}}_{B}^{-1}e_{\beta_{O}(i)}u^{T}
-u_{i}\hat{\check{M}}_{B}^{-1}e_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
\hat{\check{M}}_{B}^{T}
\left(\theta-\left(\check{M}_{B}\right)_{\bullet, \beta_{O}(i)}\right)
e_{\beta_{O}(i)}^{T}
+\hat{\rho}u^{T}
-u_{\beta_{O}(i)}\hat{\rho}e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
\left(
  \hat{\check{M}}_{B}^{-1}\theta -de_{\beta_{O}(i)}
\right)
e_{\beta_{O}(i)}^{T}
+\hat{\rho}u^{T}
-u_{\beta_{O}(i)}\hat{\rho}e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
\hat{v}e_{\beta_{O}(i)}^{T}
+\hat{\rho}u^{T}
-u_{\beta_{O}(i)}\hat{\rho}e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
\hat{v}e_{\beta_{O}(i)}^{T}
+\hat{\rho}\left(u^{T}-u_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}\right)
\end{eqnarray}
and
\begin{eqnarray}
\hat{\rho}^{T}UV^{T}
&=&
\hat{\rho}^{T}ue_{\beta_{O}(i)}^{T}
+\hat{\rho}^{T}e_{\beta_{O}(i)}u^{T}
-u_{\beta_{O}(i)}\hat{\rho}^{T}e_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
\hat{\rho}^{T}
\left(\theta-\left(\check{M}_{B}\right)_{\bullet, \beta_{O}(i)}\right)
e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
\left(\hat{k}_{1}-d\right)e_{\beta_{O}(i)}^{T}
\end{eqnarray}
as well as
\begin{eqnarray}
\hat{v}^{T}UV^{T}
&=&
\hat{v}^{T}ue_{\beta_{O}(i)}^{T}
+\hat{v}^{T}e_{\beta_{O}(i)}u^{T}
-u_{\beta_{O}(i)}\hat{v}^{T}e_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
\hat{v}^{T}
\left(\theta-\left(\check{M}_{B}\right)_{\bullet, \beta_{O}(i)}\right)
e_{\beta_{O}(i)}^{T}
+\hat{v}_{\beta_{O}(i)}u^{T}
-u_{\beta_{O}(i)}\hat{v}_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
-\hat{k}_{2}e_{\beta_{O}(i)}^{T}
-\hat{v}^{T}
\left(\check{M}_{B}\right)_{\bullet, \beta_{O}(i)}e_{\beta_{O}(i)}^{T}
+\hat{v}_{\beta_{O}(i)}u^{T}
-u_{\beta_{O}(i)}\hat{v}_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
-\hat{k}_{2}e_{\beta_{O}(i)}^{T}
-du_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}
+\hat{v}_{\beta_{O}(i)}u^{T}
-u_{\beta_{O}(i)}\hat{v}_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}
\nonumber \\
&=&
-\hat{k}_{2}e_{\beta_{O}(i)}^{T}
-u_{\beta_{O}(i)}\left(d+\hat{v}_{\beta_{O}(i)}\right)e_{\beta_{O}(i)}^{T}
+\hat{v}_{\beta_{O}(i)}u^{T}
\nonumber \\
&=&
-\hat{k}_{2}e_{\beta_{O}(i)}^{T}
-u_{\beta_{O}(i)}\hat{k}_{1}e_{\beta_{O}(i)}^{T}
+\left(\hat{k}_{1}-d\right)u^{T}
\nonumber \\
&=&
-\hat{k}_{2}e_{\beta_{O}(i)}^{T}
+\hat{k}_{1}
\left(u^{T}-u_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}\right)
-du^{T}
\end{eqnarray}
such that $\hat{\check{M}}_{B^{\prime}}^{-1}UV^{T}$ becomes
\begin{eqnarray}
\hat{\check{M}}_{B^{\prime}}^{-1}UV^{T}
&=&
\frac{
\hat{k}_{1}^{2}\hat{\check{M}}_{B}^{-1}
-\left[
  \hat{k}_{2}\hat{\rho}\hat{\rho}^{T}
  +\hat{k}_{1}\left(\hat{\rho}\hat{v}^{T} +\hat{v}\hat{\rho}^{T}\right)
 \right]
}{d^{2}}
UV^{T}
\nonumber \\
&=&
\frac{
\hat{k}_{1}^{2}\hat{v}e_{\beta_{O}(i)}^{T}
+\hat{k}_{1}^{2}
\hat{\rho}\left(u^{T}-u_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}\right)
-\hat{k}_{2}\left(\hat{k}_{1}-d\right)\hat{\rho}e_{\beta_{O}(i)}^{T}
}{d^{2}}
\nonumber \\
&&
\frac{
-\hat{k}_{1}\hat{\rho}
\left(
  -\hat{k}_{2}e_{\beta_{O}(i)}^{T}
  +\hat{k}_{1}
  \left(u^{T}-u_{\beta_{O}(i)}e_{\beta_{O}(i)}^{T}\right)
  -du^{T}
\right)
}{d^{2}}
\nonumber \\
&&
\frac{
-\hat{k}_{1}\left(\hat{k}_{1}-d\right)\hat{v}e_{\beta_{O}(i)}^{T}
}{d^{2}}
\nonumber \\
&=&
\hat{k}_{2}d\hat{\rho}e_{\beta_{O}(i)}^{T}
+\hat{k}_{1}d\hat{\rho} u^{T}
+\hat{k}_{1}d\hat{v}e_{\beta_{O}(i)}^{T}
\end{eqnarray}
Finally we have
\begin{equation}
\hat{\check{M}}_{B}^{-1}\left(M_{B}+UV^{T}\right)=\hat{z}
\end{equation}
as expected..
\begin{thebibliography}{99}
\bibitem{Sven} Sven Sch\"{o}nherr. Quadratic Programming in Geometric Optimization:
Theory, Implementation, and Applications, Dissertation, Diss. ETH No 14738, ETH
Z\"{u}rich, Institute of Theoretical Computer Science, 2002.
\bibitem{Chvatal} Va\v{s}ek Chv\'{a}tal. \textit{Linear Programming}. W. H. Freeman and Company,
New York, Chapter 8, 1983
\bibitem{Zielke} Gerhard Zielke. Inversion of Modified Symmetric Matrices.
\textit{Journal of the Association for Computing Machinery}, Vol. 15, No. 3,
July 1968, pp. 402-408
\bibitem{Frans_Deg} Degeneracy
\end{thebibliography}
\end{document}


\documentclass[a4paper,10pt]{article}
\usepackage[utf8]{inputenc}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage[square,numbers]{natbib}
\bibliographystyle{unsrtnat}

\title{Deep Learning for High Schoolers}
\author{Sharad Vikram}

\begin{document}

\maketitle

\section{Introduction}
Artificial intelligence dominated
the headlines when Google Deepmind's
AlphaGo defeated Go professional
Lee Sedol. The event was a testament
to the progress of artificial intelligence
research over the past decade.
One of the main components that
enabled AlphaGo to win,
however, has been around for
more than 30 years, namely
the neural network.
Over the past few years,
neural network research
has had a resurgence,
resulting in a new
area of research for artificial
intelligence, called \emph{deep learning}.

In this writeup, I explain
from the ground up the core mathematics
and algorithms
used in deep learning.
This writeup will act as lecture notes
for a presentation to Torrey Pines High School
math students, hence the name.
The prerequisites are
a basic understanding of multivariable calculus
and linear algebra.

\section{Background}
Artificial intelligence is a broad
term that encompasses several areas of
research, but in general,
the goal of artificial intelligence
is to write computer programs
that exhibit intelligent behavior.
There are several approaches
to writing such a computer program.
<describe approaches>

In this writeup, we focus on
\emph{machine learning},
where computers
learn patterns from
data to perform a task.

\subsection*{What is machine learning?}
Machine learning is the subfield of
artificial intelligence
dealing with learning from data.
Machine learning itself can be split up
into
supervised and unsupervised learning.

In supervised learning,
the computer is provided
data in the form of \emph{labeled examples},
or input-output pairs,
and is tasked with creating
some function that maps inputs
to outputs, while also generalizing
to never before seen inputs.
In unsupervised learning,
the problem is much less specified.
The computer is provided with data
in the form of just inputs,
and is tasked with discovering
underlying patterns in the inputs.

Unsupervised learning is considered
much harder than supervised learning.
Consider the classic supervised learning problem
of determining if an email is spam or not.
Our program is provided with
a set of emails, annotated with whether they are spam or not.
It's not unreasonable that a computer program performs
some sort of word association, to figure out
which words are more associated with spam emails
and which are not.
However, now consider the same task,
but we do not know which emails are spam beforehand.
An unsupervised learning algorithm's goal
would be to discover the properties of
spammy emails, without knowledge of which ones
are spam in the first place.

In this writeup, we concentrate on a particular
flavor of supervised learning algorithms,
neural networks. Machine learning with
``deep neural networks'', or deep learning
as it is called today, has skyrocketed
in popularity.

\section{Supervised Learning}
The supervised learning problem can be expressed very concisely.
Given a set of input-output pairs $\{(x_1, y_1)\, (x_2, y_2), \ldots, (x_N, y_N)\}$,
return a function $h(x)$ that when given an
input $x$, predicts its corresponding output $y$.
In general, all $x$'s are vectors of identical dimension,
but the form of $y$ can vary from task to task.

Supervised learning problems can be broken down into
two closely related tasks, \emph{classification} and \emph{regression},
both of which relate to the type of prediction at hand.
In classification, we want a function $h(x)$ that, from an input,
predicts a category, for example
deciding whether an email is spam or not. Each input
can only belong to a finite set of categories. Other examples
are facial recognition, where an picture of a face is mapped to a person,
and song recognition, mapping snippets of noisy audio to the song they
originated from.
In regression, we want $h(x)$ to output a real number. Examples of
regression tasks include extrapolating stock market prices
and predicting atmospheric gas levels from noisy sensor readings.
Although these two tasks are inherently different,
the algorithms for each are very closely related to each other.

Now, consider the following scenario. You're provided
with a of input-output pairs $\{(x_1, y_1)\, (x_2, y_2), \ldots, (x_N, y_N)\}$,
which we'll call a \emph{dataset}.
How do we even begin coming up with an $h(x)$? A simple approach
would be to ``memorize'' the dataset. When we pass an $x$
into $h(x)$, it simply looks up the known examples
in the dataset for a matching $x_i$, and returns the corresponding
$y_i$. Although this approach is simple, it fails at \emph{generalizing}.
Given an $x$ not in the dataset, $h(x)$ will not know what to do.
We can't possibly expect this $h(x)$ to work in a real-life
scenario with unexpected events and noise.

An approach to better generalize
is to pick a functional form, or \emph{model},
of $h(x)$.
There are so many possible options for $h(x)$,
so we typically start by constraining the
possible types of functions that $h(x)$ could be.
For example, we could assert that $h(x)$ must be
a \emph{linear function}, that is,
if $x$ is a vector,
\begin{align}
  h(x) &= w^Tx + b
\end{align}
Now, the problem of finding
best $h(x)$ is much easier.
Instead of deciding on the best $h(x)$
to fit the data from the space of all possible
functions, we just need to
find the best $w$ and $b$,
a task called \emph{linear regression}.
We'll discuss exactly how find such a $w$ and $b$ later.

Of course, asserting that $h(x)$
is linear has downsides.
For example, a linear function is ill-suited
for any classification tasks,
since linear functions map real-valued inputs
to a real-valued output.
Moreover, if the relationship
between the inputs and outputs
is far more complex than linear,
our linear $h(x)$ would not be a great predictor.

%Thus, we want to open the door to
%more complex functional forms.
%A logical step up is
%the $\emph{quadratic function}$.
%For input vector $x$, our function
%is constrained to the form
%\begin{align}
%h(x) = x^TAx + b^Tx + c
%\end{align}
%The ``unknowns''
%are matrix $A$, vector $b$ and scalar $c$.
%Amazingly, any function
%that could have been produced
%by linear regression,
%could be produced by quadratic regression (plug in
%a matrix of zeros for $A$).

We can imagine using more complex functions.
Maybe a quadratic or a cubic
or a 10th degree polynomial
will fit the data better?
Why wouldn't we then always use
a more complex model?
Why wouldn't we always use quadratic regression
over linear regression? There are a couple
issues that come up when we make our functions more complicated.
In linear regression, where we had to
only decide values $w$ and $b$, a vector
and a scalar, we now have to decide
values for a matrix, a vector,
and a scalar. We now see a tradeoff
with choosing a complicated functional form.
As our functions grow more complex,
we need to learn more unknowns from the data.

The linear regression model isn't
naturally built for classification.
We can make it work though,
by thresholding.
First, imagine we're doing
binary classification,
where we're deciding between two
classes, $0$ and $1$.
We could take the output of our linear $h(x)$,
and predict class $1$ if it's positive
and class $0$ otherwise.
What if we can make this linear classifier
even better? Let's make it so instead of
$h(x)$ predicting an arbitrary real value,
it outputs
the \emph{probability} of an input $x$
belonging to class $1$.
If the output is greater than $0.5$,
then $P(\text{class}\; 1) > P(\text{class}\; 0)$
and we'll output class $1$,
and otherwise, we'll output class $0$.
In order to do this, we need to force
$h(x)$ to output a number between 0 and 1,
requiring the use of the \emph{logistic function}.

The logistic function is
\begin{align}
  \theta(z) = \frac{1}{1 + e^{-z}}
\end{align}
It has the effect of \emph{squashing} its
input if it gets too high or low.
As $z \rightarrow \infty$, $\theta(z) \rightarrow 1$,
and as $z \rightarrow -\infty, \theta(z) \rightarrow 0$.
Furthermore, $\theta(0) = 0.5$.
Let's take the output of a linear function of $x$
and pass it into the logistic function.
We now get the model
\begin{align}
  h(x) = \frac{1}{1 + e^{-(w^Tx + b)}}
\end{align}
This function has the nice benefit of being a
linear classifier (if $w^Tx + b > 0$, then
$P(\text{class} \; 1) > 0.5$),
while also outputting a probability.
It is a natural fit for binary classification,
and the model is called \emph{logistic regression}.
We'll soon see that logistic regression is a
core building block of the neural network.

\section{Optimization}

\section{Neural Networks}
Let's dive right into neural networks.
Neural networks are a framework
for building functions
for both classification and regression tasks,
\emph{inspired} by the biology of the brain.
Let's quickly discuss how the brain works.
The brain consists of cells called
\emph{neurons}, which can be roughly divided
into two parts, the dendrite and the axon.
A dendrite receives signals from other neighboring neurons,
combines them, and
fires a signal through its axon if the combined
input signal is strong enough.
A neuron will saturate if
the input signal is strong enough.
This means that the neuron's output
caps as the input gets stronger.

This idea of receiving inputs,
combining them, and sending an output
that saturates
is the basis for what we call
the \emph{artificial neuron}.
The artificial neuron is an extremely
rough model for a real neuron,
so by no means is a neural
network ``simulating'' the brain.
The artificial neuron is a building block
for a neural network just as the neuron
is a building block for the brain.

Every neuron outputs a scalar number,
and receives input from several other neurons
in the form of a vector $x$.
The output of the neuron $f(x)$ combines
the inputs in the following way:

\begin{align*}
  f(x) = \theta(w \cdot x + b)
\end{align*}

In this equation, $\theta$
is some function we choose beforehand
and $w$ and $b$ are unknowns as before.
$f(x)$ looks oddly similar to our
logistic regression function,
if we just choose $\theta$
to be the logistic function.
In fact, a single artificial neuron
is a generalization of logistic regression.
Each element of the vector $x_i$
represents the output of some
other artificial neuron.
We take a linear combination of the inputs
as we do in linear regression, but then we apply
$\theta$, a \emph{non-linear} function.
This non-linear function often saturates,
just like the logistic function,
where the output converges as
the input gets extremely positive or negative.
Again, just like linear regression and
logistic regression, the ``unknowns''
in our function are $w$ and $b$.

%Why is it crucial that $\theta$ be non-linear?
%Imagine we were to string a bunch of neurons
%together without applying $\theta$.
%We would repeatedly be taking
%linear combinations of an input $x$.
%The problem is that multiple
%linear combinations of $x$
%can be represented by a single one,
%so any number of linear neurons
%could be equivalently represented by
%just one.

The next big difference between
a neural network and the brain
is how we connect neurons together.
In the real brain,
there are cycles in neuron connectivity
(neuron A can connect to neuron B which
can connect back to A). In
neural networks, we only allow
\emph{feed-forward} connections.
Neurons are only allowed to connect
forwards, towards the desired output
and there are no cycles allowed.
It may seem like we're removing
a critical component that makes the brain powerful,
but cycles can make it almost impossible
to do anything practical. We'll see why later.

How do feed-forward connections work?
It's easiest to see them layer-wise.
Imagine the input to our network, $x$ (the data
we wish to predict an output for),
as a series of stacked nodes.
Each node feeds into a layer of stacked artificial neurons.
Each of these artificial neurons performs its calculation (a linear combination
and a non-linear function), and has its own output.
This output itself can be fed into another layer of
stacked neurons. Eventually, the final layer of our network
will represent the network's prediction $y$.
We can pick and choose how many layers
there are in the network, and how many neurons
are in each layer. This is where
the power of neural networks comes in. Remember
that there could be scenarios where
a simple linear or logistic function isn't
powerful enough. To make a neural network
more complex, we simply make the layers
bigger or add more layers.

Let's look at a neural network
in more mathematical detail.
So each neuron
performs a dot product, adds a scalar bias
term, and then applies a non-linearity.
Imagine that each neuron in a layer
uses the same non-linear function.
We can now vectorize
the entire layer's computation!
Each neuron receives the previous layer's output $x$
as input.
Let's say we have $d$ neurons in the current layer.
The output of the $i$-th neuron
in the layer can be written as:
\begin{align*}
  o_i = \theta(w_i \cdot x + b_i)
\end{align*}
Since each neuron outputs a scalar,
we can take each of the $d$ outputs
and make a $d$-dimensional vector
$o = [o_1, \ldots, o_d]$.
It turns out that $o$ can be written as
\begin{align*}
  o = \theta(Wx + b)
\end{align*}
where $W$ is a matrix
whose rows are each $w_i$
and $b$ is a vector
whose entries are each $b_i$.
$W$ is $d_o \times d_i$,
where $d_o$ is the size of our layer
and $d_i$ is dimension
of the input vector.

Each layer will perform a similar
calculation. Let's assume
for simplicity, every layer in the network
uses the same non-linear function $\theta$.
For a 2-layer network,
our final $h(x)$ can be written compactly as
\begin{align*}
  h(x) = \theta(W_2\theta(W_1x + b_1) + b_2)
\end{align*}

Adding a layer to the network nests the output
of one network in yet another
linear combination and non-linearity.

What non-linear functions do we typically use?
Well, the logistic is perhaps the simplest one.
Another popular one is the hyperbolic tangent function,
or \emph{tanh}. It looks like
\begin{align*}
  \theta(z) = \frac{e^z - e^{-z}}{e^z + e^{-z}}
\end{align*}

The tanh behaves exactly like the logistic function,
but saturates at $-1$ and 1, instead of 0 and 1.

\bibliography{dlhs}

\end{document}


\subsection{Degrees of Freedom}
So there was something I neglected to explicitly tell you in the previous section\footnote{Hahahaha I'm so evil.}. Not all gases are ideal. In fact, most gases aren't ideal\footnote{Although most are almost ideal.}. Now's when I explain why this is. And to do that, I need to explain degrees of freedom.
\subsubsection{What are Degrees of Freedom?}
What a degree of freedom is can depend on the subject. For the purposes of thermodynamics, it's a way in which a particle can store energy. For example, the point-like particles that I talked about in ideal gas law can store energy in their motion along each directional axis. This gives them three degrees of freedom. That being said, there's a bunch of ways particles can store energy. They can vibrate, rotate, and have potential energy. The more of these a particle can do, the more degrees of freedom it has. However, not all degrees of freedom survive, because of reasons arising from quantum mechanics,\footnote{As you will learn next term, in quantum mechanics, energy is discretized (i.e. not continuous). If the amount of energy that could be stored in a potential degree of freedom is too small to get to the next discrete energy level, it'll be "frozen out". When that occurs, no energy can be stored in that way at all.} but essentially, each degree of freedom must store a minimum amount of energy to exist at all.
\subsubsection{The Equipartition Theorem}
The equipartition theorem states that:
\begin{center}
    At thermal equilibrium, the thermal energy of a system is equally divided amongst all available degrees of freedom.
\end{center}
The equipartition theorem tells us that a a single quadratic degree of freedom \footnote{A quadratic degree of freedom is simply one where the energy goes by the square of some property; for example, the kinetic energy in the x direction is $E_{x} = \frac{1}{2}mv_x^2$, which is quadratic in the $v_x$ term.}, and therefore that one dimension of kinetic energy, has a magnitude of:
\begin{equation*}
    \epsilon_{k}=\frac{1}{2}{k_{b}}T.
\end{equation*}
 We also know that the total internal energy of the system is simply the sum of the energy of each particle. So
\begin{gather}
    E_{th}=\frac{\chi}{2}N{k_{b}}T \\
    E_{av}=\frac{\chi}{2}{k_{b}}T
\end{gather}
Here, $N$ represents the number of particles in the system, and $k_b$ is the Boltzmann constant, a fundamental constant of nature with a precisely defined value. $\chi$ is the number of degrees of freedom in a molecule, which happens to be $3$ for monotomic gases (corresponding to three translational degrees of freedom). A more comprehensive list is given in the section below. $T$ is temperature, in \textbf{Kelvin}. In questions 2 and 3 below, you will show that:
\begin{equation}
E_{th} = \frac{\chi}{2}nRT
\end{equation}
Where $n$ is the number of mols of gas, and $R$ is the gas constant ($R = 8.3145 \textrm{ J} \textrm{ mol}^{-1} \textrm{ K}^{-1}$). Now, we may define:
\[ c_v = \frac{\chi}{2}R \]
to be the specific heat capacity at a fixed volume. We now obtain the useful formula:
\begin{equation}
    E_{th} = nc_v T
\end{equation}
This formula is extremely useful for relating energy and temperature. Assuming that the amount and kind of gas you have doesn't change, this also allows us to look at the change in temperature and energy:
\begin{equation}
    \label{eqn:(13)}
    \Delta E_{th} = nc_v \Delta T
\end{equation}
The above formula will hold for any process where the amount of gas you have doesn't change, making it quite useful. It also has some immediately clear consequences. For one, it tells us that for a given amount of energy we give to a system, how much its temperature changes depends on two things; it depends on:
\begin{enumerate}
    \item The amount of stuff we have in the system, represented by the number of mols, $n$. This is fairly intuitive; the more stuff I have, the more energy I need to increase its temperature!
    \item The degrees of freedom of the molecules we are looking at. Imagine we were comparing two molecules, where one was simple molecule with few degrees of freedom (say, monotomic Helium), and a complex molecule with many degrees of freedom (say, a triatomic molecule), the former would have a lower heat capacity $c_v$ (check this from the definition!). Therefore, for a given amount of energy $\Delta E$ we give to a system, the system composed of molecules with fewer degrees of freedom will have a greater change in temperature (check this as well!). Physically, we might intuit this from the fact that molecules with a greater number of degrees of freedom have more degrees to which the energy can be distributed, and hence a smaller proportion of the energy goes into increasing the temperature.
\end{enumerate}

Now, you may wonder why I have chosen to label $c_v$ the way I have (What do I mean by specific heat capacity at fixed volume?). This notation will soon become clear when we discuss the different thermodynamic processes. \\

Brief note before we move on; If you take the equipartition theorem to be a fundamental result\footnote{unfortunately, we will not be proving it here}, we obtain a derivation for our temperature definition in equation \ref{eqn:(8)} that works for any molecule (not just an ideal one). Any molecule in three dimensions would have three translational degrees of freedom, or three directions of kinetic energy. Therefore, the theorem would tell us that:
\[\epsilon_{kx}+\epsilon_{ky}+\epsilon_{kz} = \frac{1}{2}k_bT + \frac{1}{2}k_bT + \frac{1}{2}k_bT \]
So if we group the leftmost three terms into a general average kinetic energy term:
\[\epsilon_{kavg} = \frac{3}{2}k_bT \]
And rearranging for temperature:
\[T = \frac{2}{3k_b}\epsilon_{kavg} \]
So we have recovered our definition! In fact, you could combine this temperature definition with equation \ref{eqn:(2)} which we derived from first principles:
\[ \frac{3}{2} P V=N \epsilon_{kavg} \]
And actually derive the ideal gas law from scratch (without having to rely on it as an experimental result)!

\subsubsection{Degrees of Freedom of Different Molecule Types}
I've said that monoatomic, neutral gases have only three degrees of freedom. But as any good chemist knows\footnote{Don't confirm this with Chris or John} most elements don't like to be alone. So most gases aren't monoatomic. For example nitrogen gas, the most abundant gas in the atmosphere, is annoyingly diatomic. The ideal gas law doesn't apply to it (though, for reasonably low pressures\footnote{i.e. The range of pressures we would consider in Science One problems}, its behavior does not deviate far from an ideal gas, and we can apply the ideal gas law to it and be approximately correct. But the equation for energy does, if we could just figure out how many degrees of freedom it had, and the same is true for any gas. Anyways I'll stop prattling on now and tell you how many degrees of freedom these gases have (they're all kinetic degrees of freedom). \\
\newline
\begin{center}
\begin{tabular}{|c|c|c|}
    \hline
    Gas Type & \# of Deg of Freedom & Types of Deg of Freedom \\
    \hline
    Monoatomic & 3 & 3 translational \\
    \hline
    Diatomic & 5 & 3 translational, 2 rotational \\
    \hline
    Triatomic & 6 & 3 translational, 3 rotational \\
    \hline
\end{tabular}
\end{center}
\begin{figure}[h!]
    \centering
    \begin{tikzpicture}
    \filldraw[fill=black, draw=black] (0,0) circle (5pt);
    \draw[->, blue] (0.13,0) -- (1.125,0);
    \draw[->, blue] (0,0.13) -- (0,1.125);
    \draw[->, blue] (-0.1,-0.1) -- (-0.5,-0.9);
    %\node[below] at (1,0) {$y$};
    %\node[left] at (0,1) {$z$};
    %\node[left] at (-0.5,-0.85) {$x$};

    \filldraw[fill=black, draw=black] (3,0) circle (5pt);
    \filldraw[fill=black, draw=black] (4,0) circle (5pt);
    \draw[black] (3,0) -- (4,0);
    \draw[->, red] plot [smooth, tension=1] coordinates {(3,0.5) (3.25,0.7) (3.5,0.75) (3.75,0.7) (4,0.5)};
    \draw[->, red] plot [smooth, tension=1] coordinates {(3.75,-0.3) (3.25,-0.4) (3.75,-0.5)};

    \filldraw[fill=black, draw=black] (6,-0.5) circle (5pt);
    \filldraw[fill=black, draw=black] (7,-0.5) circle (5pt);
    \filldraw[fill=black, draw=black] (6.5,0.5) circle (5pt);
    \draw[black] (6,-0.5) -- (7,-0.5);
    \draw[black] (6,-0.5) -- (6.5,0.5);
    \draw[black] (7,-0.5) -- (6.5,0.5);
    \draw[->, red] plot [smooth, tension=1] coordinates {(6,0.5) (6.25,0.7) (6.5,0.75) (6.75,0.7) (7,0.5)};
    \draw[->, red] plot [smooth, tension=1] coordinates {(6.75,-0.6) (6.25,-0.7) (6.75,-0.8)};
    \draw[->, red] plot [smooth, tension=1] coordinates {(5.8,-0.25) (5.9,0.25) (6,-0.25)};

    \end{tikzpicture}
    \caption{Visualization of the degrees of freedom of monoatomic, diatomic, and triatomic molecules. Monoatomic molecules only have three translational degrees of freedom, diatomic molecules have three translational plus two rotational degrees (it cannot rotate along the rotation axis parallel to its length), and the triatomic molecules have three translational and rotational degrees of freedom each.}
\end{figure}

\noindent
Liquids and solids have their own characteristic number of degrees of freedom too, I just didn't write them here. James/Chris might cover this in lecture however, so just because I didn't write it doesn't mean you don't need to know it\footnote{Really attach this disclaimer to the whole document}!
\newpage
\subsubsection{(Optional) Why Can't Monoatomic Molecules Rotate?}
After looking at that chart of degrees of freedom, you might be wondering: "Wait; why don't monoatomic gases have rotational degrees of freedom? What's stopping it from rotating?" The answer lies in the fact that the radius of atoms is extremely small; these leads to the atoms having a very small moment of inertia (you may recall that the moment of inertia of a sphere is $I = \frac{2}{5}MR^2$ where $M$ is the mass and $R$ the radius; if $R$ is small, then $R^2$ and therefore $I$ are extremely small). This has the consequence of the quantum energy levels of the rotational kinetic energy being extremely far apart. Hence these rotations are negligeble (i.e. not thermodynamically accessible) because they require a lot of energy to become excited; therefore, we do not consider the rotations as a degree of freedom\footnote{Since it's hard to get an intuition for why this is true, we can also consider a simpler (although oversimplified) explanation; if we approximate atoms as point particles of zero radius, then it becomes clear that they cannot rotate, and therefore the rotations cannot constitute a degree of freedom!}. You might be able to see why this argument generalizes to the diatomic molecule, as well (where you might have been wondering why it can't rotate along its length); the moment of inertia about the rotation axis along the length of the molecule is very small compared to the other two rotation axes; hence, rotations about that axis are not excited.