% ------------------------------------------------------------------
\chapter{Implementation details}\label{s:impl}
% ------------------------------------------------------------------

This chapter contains calculations and details.

% ------------------------------------------------------------------
\section{Convolution}\label{s:impl-convolution}
% ------------------------------------------------------------------

It is often convenient to express the convolution operation in matrix form. To this end, let $\phi(\bx)$ the \verb!im2row! operator, extracting all $W' \times H'$ patches from the map $\bx$ and storing them as rows of a $(H''W'') \times (H'W'D)$ matrix. Formally, this operator is given by:
\[
   [\phi(\bx)]_{pq} \underset{(i,j,d)=t(p,q)}{=} x_{ijd}
\]
where the index mapping $(i,j,d) = t(p,q)$ is
\[
 i = i''+i'-1, \quad
 j = j''+j'-1, \quad
 p = i'' + H'' (j''-1), \quad
 q = i' + H'(j'-1) + H'W' (d-1).
\]
It is also useful to define the ``transposed'' operator \verb!row2im!:
\[
   [\phi^*(M)]_{ijd}
   =
   \sum_{(p,q) \in t^{-1}(i,j,d)}
   M_{pq}.
\]
Note that $\phi$ and $\phi^*$ are linear operators. Both can be expressed by a matrix $H\in\real^{(H''W''H'W'D) \times(HWD)}$ such that
\[
  \vv(\phi(\bx)) = H \vv(\bx), \qquad
  \vv(\phi^*(M)) = H^\top \vv(M).
\]
Hence we obtain the following expression for the vectorized output (see~\cite{kinghorn96integrals}):
\[
 \vv\by =
 \vv\left(\phi(\bx) F\right)
 =
 \begin{cases}
 (I \otimes \phi(\bx)) \vv F, & \text{or, equivalently,} \\
 (F^\top \otimes I) \vv \phi(\bx),
 \end{cases}
\]
where $F\in\mathbb{R}^{(H'W'D)\times K}$ is the matrix obtained by reshaping the array $\bff$ and $I$ is an identity matrix of suitable dimensions. This allows obtaining the following formulas for the derivatives:
\[
\frac{dz}{d(\vv F)^\top}
=
\frac{dz}{d(\vv\by)^\top}
(I \otimes \phi(\bx))
= \vv\left[
\phi(\bx)^\top
\frac{dz}{dY}
\right]^\top
\]
where $Y\in\real^{(H''W'')\times K}$ is the matrix obtained by reshaping the array $\by$. Likewise:
\[
\frac{dz}{d(\vv \bx)^\top}
=
\frac{dz}{d(\vv\by)^\top}
(F^\top \otimes I)
\frac{d\vv \phi(\bx)}{d(\vv \bx)^\top}
=
\vv\left[
\frac{dz}{dY}
F^\top
\right]^\top
H
\]
In summary, after reshaping these terms we obtain the formulas:
\[
\boxed{
\vv\by =
 \vv\left(\phi(\bx) F\right),
\qquad
\frac{dz}{dF}
=
\phi(\bx)^\top\frac{d z}{d Y},
\qquad
\frac{d z}{d X}
=
\phi^*\left(
\frac{d z}{d Y}F^\top
\right)
}
\]
where $X\in\real^{(H'W')\times D}$ is the matrix obtained by reshaping $\bx$. Notably, these expressions are used to implement the convolutional operator; while this may seem inefficient, it is instead a fast approach when the number of filters is large and it allows leveraging fast BLAS and GPU BLAS implementations.

% ------------------------------------------------------------------
\section{Convolution transpose}\label{s:impl-convolution-transpose}
% ------------------------------------------------------------------

In order to understand the definition of convolution transpose, let $\by$ obtained from $\bx$ by the convolution operator as defined in Section~\ref{s:convolution} (including padding and downsampling).  Since this is a linear operation, it can be rewritten as $\vv \by = M \vv\bx$ for a suitable matrix $M$; convolution transpose computes instead $\vv \bx = M^\top \vv \by$.  While this is simple to describe in term of matrices, what happens in term of indexes is tricky. In order to derive a formula for the convolution transpose, start from standard convolution (for a 1D signal):
\[
   y_{i''} = \sum_{i'=1}^{H'} f_{i'} x_{S (i''-1) + i' - P_h^-},
   \quad
    1 \leq i'' \leq 1 + \left\lfloor \frac{H - H' + P_h^- + P_h^+}{S} \right\rfloor,
\]
where $S$ is the downsampling factor, $P_h^-$ and $P_h^+$ the padding, $H$ the length of the input signal, $\bx$ and $H'$ the length of the filter $\bff$. Due to padding, the index of the input data $\bx$ may exceed the range $[1,H]$; we implicitly assume that the signal is zero padded outside this range.

In order to derive an expression of the convolution transpose,  we make use of the identity $\vv \by^\top (M \vv \bx) = (\vv \by^\top M) \vv\bx = \vv\bx^\top (M^\top \vv\by)$. Expanding this in formulas:
\begin{align*}
\sum_{i''=1}^b y_{i''}
\sum_{i'=1}^{W'} f_{i'} x_{S (i''-1) + i'  -P_h^-}
&=
\sum_{i''=-\infty}^{+\infty}
\sum_{i'=-\infty}^{+\infty}
y_{i''}\ f_{i'}\ x_{S (i''-1) + i'  -P_h^-}
\\
&=
\sum_{i''=-\infty}^{+\infty}
\sum_{k=-\infty}^{+\infty}
y_{i''}\ f_{k-S(i'' -1) + P_h^-}\ x_{k}
\\
&=
\sum_{i''=-\infty}^{+\infty}
\sum_{k=-\infty}^{+\infty}
y_{i''}%
\ %
f_{%
(k-1+ P_h^-) \bmod S +
S \left(1 -i''  + \left\lfloor \frac{k-1+ P_h^-}{S} \right\rfloor\right)+1
}\ x_{k}
\\
&=
\sum_{k=-\infty}^{+\infty}
x_{k}
\sum_{q=-\infty}^{+\infty}
y_{\left\lfloor \frac{k-1+ P_h^-}{S} \right\rfloor + 2 - q}
\ %
f_{(k-1+ P_h^-)\bmod S +S(q - 1)+1}.
\end{align*}
Summation ranges have been extended to infinity by assuming that all signals are zero padded as needed. In order to recover such ranges, note that $k \in [1,H]$ (since this is the range of elements of $\bx$ involved in the original convolution). Furthermore, $q\geq 1$ is the minimum value of $q$ for which the filter $\bff$ is non zero; likewise, $q\leq \lfloor (H'-1)/2\rfloor +1$ is a fairly tight upper bound on the maximum value (although, depending on $k$, there could be an element less). Hence
\begin{equation}\label{e:convt-step}
 x_k =
 \sum_{q=1}^{1 + \lfloor \frac{H'-1}{S} \rfloor}
y_{\left\lfloor \frac{k-1+ P_h^-}{S} \right\rfloor + 2 - q}\ %
f_{(k-1+ P_h^-)\bmod S +S(q - 1)+1},
\qquad k=1,\dots, H.
\end{equation}
Here $H$, $H'$ and $H''$ are related by:
\[
H'' = 1+ \left\lfloor \frac{H - H' + P_h^- + P_h^+}{S} \right\rfloor.
\]
If $H''$ is now given as input, it is not possible to recover $H$ uniquely; instead, all the following values are possible
\[
   S_h (H''-1) +H' -  P_h^- - P_h^+ \leq H < S_h H'' +H' -  P_h^- - P_h^+.
\]
We use the tighter definition and set $H =  S_h (H''-1) +H' -  P_h^- - P_h^+$.
Note that the summation extrema in \eqref{e:convt-step} can be refined slightly to account for the finite size of $\by$ and $\bw$:
\begin{multline*}
\max\left\{
1,
\left\lfloor \frac{k-1 + P_h^-}{S} \right\rfloor + 2 - H''
\right\}
\leq q \\
\leq
1 +\min\left\{
\left\lfloor \frac{H'-1-(k-1+ P_h^-)\bmod S}{S} \right\rfloor,
\left\lfloor \frac{k-1 + P_h^-}{S} \right\rfloor
\right\}.
\end{multline*}

% ------------------------------------------------------------------
\section{Spatial pooling}\label{s:impl-pooling}
% ------------------------------------------------------------------

Since max pooling simply select for each output element an input element, the relation can be expressed in matrix form as
$
    \vv\by = S(\bx) \vv \bx
$
for a suitable selector matrix $S(\bx)\in\{0,1\}^{(H''W''D) \times (HWD)}$. The derivatives can the be written as:
$
\frac{d z}{d (\vv \bx)^\top}
=
\frac{d z}{d (\vv \by)^\top}
S(\bx),
$
for all but a null set of points, where the operator is not differentiable (this usually does not pose problems in optimization by stochastic gradient). For max-pooling, similar relations exists with two differences: $S$ does not depend on the input $\bx$ and it is not binary, in order to account for the normalization factors. In summary, we have the expressions:
\begin{equation}\label{e:max-mat}
\boxed{
\vv\by = S(\bx) \vv \bx,
\qquad
\frac{d z}{d \vv \bx}
=
S(\bx)^\top
\frac{d z}{d \vv \by}.
}
\end{equation}

% ------------------------------------------------------------------
\section{Activation functions}\label{s:impl-activation}
% ------------------------------------------------------------------

% ------------------------------------------------------------------
\subsection{ReLU}\label{s:impl-relu}
% ------------------------------------------------------------------

The ReLU operator can be expressed in matrix notation as
\[
\vv\by = \diag\bfs \vv \bx,
\qquad
\frac{d z}{d \vv \bx}
=
\diag\bfs
\frac{d z}{d \vv \by}
\]
where $\bfs = [\vv \bx > 0] \in\{0,1\}^{HWD}$ is an indicator vector.

% ------------------------------------------------------------------
\subsection{Sigmoid}\label{s:impl-sigmoid}
% ------------------------------------------------------------------

The derivative of the sigmoid function is given by
\begin{align*}
\frac{dz}{dx_{ijk}}
&=
\frac{dz}{d y_{ijd}}
\frac{d y_{ijd}}{d x_{ijd}}
=
\frac{dz}{d y_{ijd}}
\frac{-1}{(1+e^{-x_{ijd}})^2} ( - e^{-x_{ijd}})
\\
&=
\frac{dz}{d y_{ijd}}
y_{ijd} (1 - y_{ijd}).
\end{align*}
In matrix notation:
\[
\frac{dz}{d\bx} = \frac{dz}{d\by} \odot
\by \odot
(\mathbf{1}\mathbf{1}^\top - \by).
\]

% ------------------------------------------------------------------
\section{Normalization}\label{s:normalization}
% ------------------------------------------------------------------

% ------------------------------------------------------------------
\subsection{Cross-channel normalization}\label{s:impl-ccnormalization}
% ------------------------------------------------------------------

The derivative is easily computed as:
\[
\frac{dz}{d x_{ijd}}
=
\frac{dz}{d y_{ijd}}
L(i,j,d|\bx)^{-\beta}
-2\alpha\beta x_{ijd}
\sum_{k:d\in G(k)}
\frac{dz}{d y_{ijk}}
L(i,j,k|\bx)^{-\beta-1} x_{ijk}
\]
where
\[
 L(i,j,k|\bx) = \kappa + \alpha \sum_{t\in G(k)} x_{ijt}^2.
\]

% ------------------------------------------------------------------
\subsection{Batch normalization}\label{s:impl-bnorm}
% ------------------------------------------------------------------

The derivative of the input with respect to the network output is computed as follows:
\[
\frac{dz}{dx_{ijkt}} = \sum_{i''j''k''t''}
\frac{dz}{d y_{i''j''k''t''}}
\frac{d y_{i''j''k''t''}}{d x_{ijkt}}.
\]
Since feature channels are processed independently, all terms with $k''\not=k$ are null. Hence
\[
\frac{dz}{dx_{ijkt}} = \sum_{i''j''t''}
\frac{dz}{d y_{i''j''kt''}}
\frac{d y_{i''j''kt''}}{d x_{ijkt}},
\]
where
\[
\frac{d y_{i''j''kt''}}{d x_{ijkt}}
=
w_k
\left(\delta_{i=i'',j=j'',t=t''} - \frac{d \mu_k}{d x_{ijkt}}\right)
\frac{1}{\sqrt{\sigma^2_k + \epsilon}}
-
\frac{w_k}{2}
\left(x_{i''j''kt''} - \mu_k\right)
\left(\sigma_k^2 + \epsilon \right)^{-\frac{3}{2}}
\frac{d \sigma_k^2}{d x_{ijkt}},
\]
the derivatives with respect to the mean and variance are computed as follows:
\begin{align*}
\frac{d \mu_k}{d x_{ijkt}} &= \frac{1}{HWT},
\\
\frac{d \sigma_k^2}{d x_{i'j'kt'}}
&=
\frac{2}{HWT}
\sum_{ijt}
\left(x_{ijkt} - \mu_k \right)
\left(\delta_{i=i',j=j',t=t'} - \frac{1}{HWT} \right)
=
\frac{2}{HWT} \left(x_{i'j'kt'} - \mu_k \right),
\end{align*}
and $\delta_E$ is the indicator function of the event $E$. Hence
\begin{align*}
\frac{dz}{dx_{ijkt}}
&=
\frac{w_k}{\sqrt{\sigma^2_k + \epsilon}}
\left(
\frac{dz}{d y_{ijkt}}
-
\frac{1}{HWT}\sum_{i''j''kt''}
\frac{dz}{d y_{i''j''kt''}}
\right)
\\
&-
\frac{w_k}{2(\sigma^2_k + \epsilon)^{\frac{3}{2}}}
\sum_{i''j''kt''}
\frac{dz}{d y_{i''j''kt''}}
\left(x_{i''j''kt''} - \mu_k\right)
\frac{2}{HWT} \left(x_{ijkt} - \mu_k \right)
\end{align*}
i.e.
\begin{align*}
\frac{dz}{dx_{ijkt}}
&=
\frac{w_k}{\sqrt{\sigma^2_k + \epsilon}}
\left(
\frac{dz}{d y_{ijkt}}
-
\frac{1}{HWT}\sum_{i''j''kt''}
\frac{dz}{d y_{i''j''kt''}}
\right)
\\
&-
\frac{w_k}{\sigma^2_k + \epsilon}
\,
\frac{x_{ijkt} - \mu_k}{\sqrt{\sigma^2_k + \epsilon}}
\,
\frac{1}{HWT}
\sum_{i''j''kt''}
\frac{dz}{d y_{i''j''kt''}}
\left(x_{i''j''kt''} - \mu_k\right).
\end{align*}

% ------------------------------------------------------------------
\subsection{Spatial normalization}\label{s:impl-spnorm}
% ------------------------------------------------------------------

The neighborhood norm $n^2_{i''j''d}$ can be computed by applying average pooling to $x_{ijd}^2$ using \verb!vl_nnpool! with a $W'\times H'$ pooling region, top padding $\lfloor \frac{H'-1}{2}\rfloor$, bottom padding $H'-\lfloor \frac{H-1}{2}\rfloor-1$, and similarly for the horizontal padding.

The derivative of spatial normalization can be obtained as follows:
\begin{align*}
\frac{dz}{dx_{ijd}}
&= \sum_{i''j''d}
\frac{dz}{d y_{i''j''d}}
\frac{d y_{i''j''d}}{d x_{ijd}}
\\
&=
\sum_{i''j''d}
\frac{dz}{d y_{i''j''d}}
(1 + \alpha n_{i''j''d}^2)^{-\beta}
\frac{dx_{i''j''d}}{d x_{ijd}}
-\alpha\beta
\frac{dz}{d y_{i''j''d}}
(1 + \alpha n_{i''j''d}^2)^{-\beta-1}
x_{i''j''d}
\frac{dn_{i''j''d}^2}{d (x^2_{ijd})}
\frac{dx^2_{ijd}}{d x_{ijd}}
\\
&=
\frac{dz}{d y_{ijd}}
(1 + \alpha n_{ijd}^2)^{-\beta}
-2\alpha\beta x_{ijd}
\left[
\sum_{i''j''d}
\frac{dz}{d y_{i''j''d}}
(1 + \alpha n_{i''j''d}^2)^{-\beta-1}
x_{i''j''d}
\frac{dn_{i''j''d}^2}{d (x_{ijd}^2)}
\right]
\\
&=
\frac{dz}{d y_{ijd}}
(1 + \alpha n_{ijd}^2)^{-\beta}
-2\alpha\beta x_{ijd}
\left[
\sum_{i''j''d}
\eta_{i''j''d}
\frac{dn_{i''j''d}^2}{d (x_{ijd}^2)}
\right],
\quad
\eta_{i''j''d}=
\frac{dz}{d y_{i''j''d}}
(1 + \alpha n_{i''j''d}^2)^{-\beta-1}
x_{i''j''d}
\end{align*}
Note that the summation can be computed as the derivative of the
\verb!vl_nnpool! block.

% ------------------------------------------------------------------
\subsection{Softmax}\label{s:impl-softmax}
% ------------------------------------------------------------------

Care must be taken in evaluating the exponential in order to avoid underflow or overflow. The simplest way to do so is to divide from numerator and denominator by the maximum value:
\[
 y_{ijk} = \frac{e^{x_{ijk} - \max_d x_{ijd}}}{\sum_{t=1}^D e^{x_{ijt}- \max_d x_{ijd}}}.
\]
The derivative is given by:
\[
\frac{dz}{d x_{ijd}}
=
\sum_{k}
\frac{dz}{d y_{ijk}}
\left(
e^{x_{ijd}} L(\bx)^{-1} \delta_{\{k=d\}}
-
e^{x_{ijd}}
e^{x_{ijk}} L(\bx)^{-2}
\right),
\quad
L(\bx) = \sum_{t=1}^D e^{x_{ijt}}.
\]
Simplifying:
\[
\frac{dz}{d x_{ijd}}
=
y_{ijd}
\left(
\frac{dz}{d y_{ijd}}
-
\sum_{k=1}^K
\frac{dz}{d y_{ijk}} y_{ijk}.
\right).
\]
In matrix for:
\[
  \frac{dz}{dX} = Y \odot \left(\frac{dz}{dY}
  - \left(\frac{dz}{dY} \odot Y\right) \bone\bone^\top\right)
\]
where $X,Y\in\real^{HW\times D}$ are the matrices obtained by reshaping the arrays
$\bx$ and $\by$. Note that the numerical implementation of this expression is straightforward once the output $Y$ has been computed with the caveats above.

% ------------------------------------------------------------------
\section{Losses and comparisons}\label{s:impl-losses}
% ------------------------------------------------------------------

% ------------------------------------------------------------------
\subsection{Log-loss}\label{s:impl-loss}
% ------------------------------------------------------------------

The derivative is
\[
\frac{dz}{dx_{ijd}} = - \frac{dz}{dy} \frac{1}{x_{ijc}} \delta_{\{d = c\}}.
\]

% ------------------------------------------------------------------
\subsection{Softmax log-loss}\label{s:impl-sfloss}
% ------------------------------------------------------------------

The derivative is given by
\[
\frac{dz}{dx_{ijd}}
= - \frac{dz}{dy} \left(\delta_{d=c} - y_{ijc}\right)
\]
where $y_{ijc}$ is the output of the softmax layer. In matrix form:
\[
\frac{dz}{dX}
= - \frac{dz}{dy} \left(\bone^\top \bfe_c - Y\right)
\]
where $X,Y\in\real^{HW\times D}$ are the matrices obtained by reshaping the arrays
$\bx$ and $\by$ and $\bfe_c$ is the indicator vector of class $c$.

% ------------------------------------------------------------------
\subsection{$p$-distance}\label{s:impl-pdistance}
% ------------------------------------------------------------------

The derivative of the operator without root is given by:
\begin{align*}
\frac{dz}{dx_{ijd}}
&=
\frac{dz}{dy_{ij}}
p |x_{ijd} - \bar x_{ijd}|^{p-1} \operatorname{sign} (x_{ijd} - \bar x_{ijd}).
\end{align*}
The derivative of the operator with root is given by:
\begin{align*}
\frac{dz}{dx_{ijd}}
&=
\frac{dz}{dy_{ij}}
\frac{1}{p}
\left(\sum_{d'} |x_{ijd'} - \bar x_{ijd'}|^p \right)^{\frac{1}{p}-1}
p |x_{ijd} - \bar x_{ijd}|^{p-1} \sign(x_{ijd} - \bar x_{ijd})
\\
&=
\frac{dz}{dy_{ij}}
\frac{|x_{ijd} - \bar x_{ijd}|^{p-1} \sign(x_{ijd} - \bar x_{ijd})}{y_{ij}^{p-1}}.
\end{align*}
The formulas simplify a little for $p=1,2$ which are therefore implemented as special cases.


\section{M/G/1 Queues}
\label{sec:M-G-1-queues}

A $M/G/1$ is a queue where
(i) the arrival process is Poissonian with rate $\lambda$,
(ii) the service process is generically distributed,
(iii) there is one servers,
(iv) the buffer has infinite capacity,
(v) the scheduling policy is FCFS.

%\begin{figure}[tp]
%\label{fig:M-G-1-queue}
%	\centering
%	\includegraphics{fig/M-G-1-Queue}
%	\caption{M/G/1 queue}
%\end{figure}

In general, we have that

\begin{equation}
\label{sec:M-G-1-Mean-Queue-Excess-Time}
\expected{T_{Q}}=\frac{\varrho}{1-\varrho}\expected{S_{e}}
\end{equation}

where $\expected{T_{e}}$ is the \textit{excess of service time}, that is the remaining service time of the job in service, given that there is some job in service.

\subsection{P-K Formula}
\label{sec:PK-Formula}
The Pollaczek-Khinchin (P-K Formula) \cite{pollaczek1930aufgabe,khinchin1967mathematical} is written in several equivalent forms:

\begin{equation}
\expected{T_{Q}}=\frac{\varrho}{1-\varrho}\cdot\frac{\expected{S^{2}}}{2\expected{S}}
\end{equation}

\begin{equation}
\expected{T_{Q}}=\frac{\varrho}{1-\varrho}\cdot\frac{\expected{S}}{2}\cdot(C_{S}^{2}+1)
\end{equation}

\begin{equation}
\expected{T_{Q}}=\frac{\lambda\expected{S^{2}}}{2(1-\varrho)}
\end{equation}


\section{Homework}

\subsection{Non-Linear Case}

Given the conditions $x(0) = 0.5$ and $F(t) = F$, we
evaluate the Laplace transform of \ref{eq: system equation}
and obtain

\begin{equation}
  s \mathcal{X} - 0.5 + \frac{k}{b} \mathcal{X} =
\frac{1}{b} \frac{F}{s}
\end{equation}

Solving for $\mathcal{X}$.

\begin{equation}
 \mathcal{X} = \frac{F}{b}
\frac{1}{s(s+\frac{k}{b})} + \frac{0.5}{s+\frac{k}{b}}
\label{eq: non-linear system equation laplace}
\end{equation}

Using partial fractions, find an equivalent expression for
\ref{eq: non-linear system equation laplace} .

\begin{equation*}
 \begin{split}
  \frac{1}{s(s+\frac{k}{b})}  & = \frac{A}{s} +
\frac{B}{s+\frac{k}{b}}\\
A & = \frac{b}{k}\\
B &= -\frac{b}{k}\\
 \end{split}
\end{equation*}

\begin{equation}
 \frac{1}{s(s+\frac{k}{b})}   = \frac{1}{s} \frac{b}{k} -
\frac{b}{k}
\frac{1}{s+\frac{k}{b}}
\label{eq: partial fractions}
\end{equation}

Substitute \ref{eq: partial fractions} in \ref{eq:
non-linear system equation laplace} and then obtain the
inverse Laplace transform of the equation.

\begin{equation*}
 \begin{split}
   \mathcal{X} & = \frac{F}{b}
\Bigl ( \frac{1}{s} \frac{b}{k} -
\frac{b}{k}
\frac{1}{s+\frac{k}{b}} \Bigr) + \frac{0.5}{s+\frac{k}{b}}
\\
& = \frac{F}{k} \frac{1}{s}  -
\frac{F}{k}
\frac{1}{s+\frac{k}{b}} + \frac{0.5}{s+\frac{k}{b}}
\\
 \end{split}
\end{equation*}

\begin{equation}
 \mathfrak{L}^{-1} \Bigl \{ \mathcal{X} \Bigr \}
\xrightarrow{} x(t) = \frac{F}{k} - \frac{F}{k}
\mathrm{e}^{-\frac{k}{b}t} + 0.5 \mathrm{e}^{-\frac{k}{b}t}
\label{eq: non-linear system equation}
\end{equation}

Test the principles of proportionality and superposition on
\ref{eq: non-linear system equation}.

\begin{equation*}
 \begin{split}
 x(\alpha t) & = \frac{F}{k} - \frac{F}{k}
\mathrm{e}^{-\alpha \frac{k}{b}t} + 0.5
\mathrm{e}^{-\alpha \frac{k}{b}t}\\
 \alpha x(t) &= \alpha \frac{F}{k} - \alpha \frac{F}{k}
\mathrm{e}^{-\frac{k}{b}t} + 0.5 \alpha
\mathrm{e}^{-\frac{k}{b}t}\\
x(\alpha t) & \neq \alpha x(t)\\
  x( t_1 + t_2) & = \frac{F}{k} - \frac{F}{k}
\mathrm{e}^{-\frac{k}{b}(t_1+t_2)} + 0.5
\mathrm{e}^{-\frac{k}{b}(t_1+t_2)}\\
x(t_1) +  x(t_2) & = 2 \frac{F}{k} -
\frac{F}{k} \Bigl (
\mathrm{e}^{- \frac{k}{b}t_1} -\mathrm{e}^{- \frac{k}{b}t_2}
\Bigr ) + 0.5
\Bigl (\mathrm{e}^{- \frac{k}{b}t_1}
 +
\mathrm{e}^{- \frac{k}{b}t_2} \Bigr) \\
x(t_1 + t_2) & \neq x(t_1) + x(t_2)\\
 \end{split}
\end{equation*}

We conclude that the system in Non-linear for the given
initial conditions.


\section{Gaussian elimination}
\label{sec:gaussian-elimination}

\begin{outcome}
  \begin{enumerate}
  \item Find the {\ef} of a matrix.
  \item Determine whether a system of linear equations has no
    solution, a unique solution or an infinite number of solutions
    from its {\ef}.
  \item Solve a system of linear equations using Gaussian elimination
    and back substitution.
  \item Find the rank of a matrix.
  \item Determine whether a consistent system of linear equations has
    a unique solution or an infinite number of solutions from its
    rank.
  \end{enumerate}
\end{outcome}

In the previous section, we saw examples of how to solve a system of
equations using elementary row operations (and sometimes back
substitution). But it is not clear whether every system of equations
can be solved this way. How do we know which elementary row operation
to apply next? In this section, you will learn a procedure called {\em
  Gaussian elimination} by which every system of linear equations can
be solved systematically.

Before we start, let's figure out what it means to be ``done''. At
what point should we stop performing row operations? The answer is
that we will stop performing row operations when the system of
equations is in a special form called {\em {\ef}}, which we now define.

\begin{definition}{\Ef}{echelon-form}
  An entry of an augmented matrix is called a \textbf{leading entry}%
  \index{leading entry} or \textbf{pivot entry}%
  \index{pivot entry} if it is the leftmost non-zero entry of a row.
  An augmented matrix is in \textbf{\ef}%
  \eindex{\ef} (also called \textbf{row echelon form}%
  \index{row echelon form|see {\ef}}) if
  \begin{enumerate}
  \item All rows of zeros are below all non-zero rows.

  \item Each leading entry of a row is in a column to the right of the
    leading entry of any row above it.
  \end{enumerate}
  A column containing a pivot entry is also called a \textbf{pivot
    column}%
  \index{pivot column}.
\end{definition}

The word {\em echelon} comes from French {\em \'echelle}, which means
ladder. This is because an {\ef} looks a bit like a ladder or
staircase. Here are some examples of {\ef}s.

\begin{example}{Matrices in {\ef}}{matrices-not-rref}
  The following augmented matrices are in {\ef}. We have circled the
  pivot entries for clarity.
  \begin{equation*}
    \begin{mymatrix}{rrrrr|r}
      0 & \circled{5} & 2 & 1 & 3 & 5 \\
      0 & 0 & 0 & 0 & \circled{1} & 6 \\
      0 & 0 & 0 & 0 & 0 & 0 \\
      0 & 0 & 0 & 0 & 0 & 0
    \end{mymatrix}, \quad \begin{mymatrix}{rrrrr|r}
      \circled{1} & 4 & 0 & 0 & 5 & 0 \\
      0 & 0 & \circled{1} & 0 & 2 & 0 \\
      0 & 0 & 0 & \circled{1} & 4 & 0 \\
      0 & 0 & 0 & 0 & 0 & \circled{1}
    \end{mymatrix}, \quad \begin{mymatrix}{rrr|r}
      \circled{3} & 0 & 6 & 2 \\
      0 & \circled{1} & 4 & 0 \\
      0 & 0 & \circled{2} & 1 \\
      0 &  0 & 0 & 0
    \end{mymatrix}
  \end{equation*}
\end{example}

\begin{example}{Not in {\ef}}{matrices-not-echelon}
  The following augmented matrices are not in {\ef}.

  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      0 & 0 & 0 & 0 \\
      \circled{1} & 2 & 3 & 3 \\
      0 & \circled{1} & 0 & 2 \\
      0 & 0 & 0 & \circled{1} \\
      0 & 0 & 0 & 0
    \end{mymatrix}, \quad \begin{mymatrix}{rr|r}
      \circled{1} & 2 & 3 \\
      \circled{2} & 4 & -6 \\
      \circled{4} & 0 & 7
    \end{mymatrix}, \quad \begin{mymatrix}{rrr|r}
      0 & \circled{2} & 3 & 3 \\
      \circled{1} & 5 & 0 & 2 \\
      0 & 0 & \circled{1} & 0 \\
      0 & 0 & 0 & 0
    \end{mymatrix}
  \end{equation*}
  In the first matrix, a row of zeros is above a non-zero row. In the
  second and third matrix, the leading entries of some rows are not to
  the right of the leading entries of previous rows.
\end{example}

An augmented matrix can always be converted to {\ef} by using
elementary row operations. The following algorithm shows how to do
this.

\begin{algorithm}{Gaussian elimination}{gaussian-elimination}
  This algorithm provides a method for using row operations to take a
  matrix to its {\ef}%
  \eindex{\ef!algorithm}%
  \index{Gaussian elimination}.  We begin with the matrix in its
  original form.

  \begin{enumerate}
  \item Starting from the left, find the first non-zero column. This is
    the first pivot column, and the position at the top of this column
    will be the position of the first pivot entry. Switch rows if
    necessary to place a non-zero number in the first pivot position.

  \item Use row operations to make the entries below the first pivot
    entry (in the first pivot column) equal to zero.

  \item Ignoring the row containing the first pivot entry, repeat
    steps 1 and 2 with the remaining rows.  Repeat the process until
    there are no more non-zero rows left.

  \end{enumerate}
\end{algorithm}

Most often we will apply this algorithm in order to solve a system of
linear equations. This works by first converting the system to {\ef},
then using back substitution to find the solutions. The next few
examples show how to do this.

\begin{example}{Solving a system of equations: one solution}{system-with-one-solution}
  Solve the following system of equations:
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      x + 4y + 3z &=& 11 \\
      2x + 10y + 7z &=& 27 \\
      x + y + 2z &=& 5.
    \end{array}
  \end{equation*}
\end{example}

\begin{solution} The augmented matrix for this system is
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      1 & 4 & 3 & 11 \\
      2 & 10 & 7 & 27 \\
      1 & 1 & 2 & 5
    \end{mymatrix}.
  \end{equation*}
  In order to find the solution(s) to this system, we first use
  Algorithm~\ref{algo:gaussian-elimination} to carry the augmented
  matrix to {\ef}. Notice that the first column is non-zero, so this is
  our first pivot column. The first entry in the first row, $1$, is
  the first pivot entry. We will use row operations to create zeros in
  the entries below the $1$.
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 4 & 3 & 11 \\
      2 & 10 & 7 & 27 \\
      1 & 1 & 2 & 5
    \end{mymatrix}
    \stackrel{R_2\rowop R_2-2R_1}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 4 & 3 & 11 \\
      0 & 2 & 1 & 5 \\
      1 & 1 & 2 & 5
    \end{mymatrix}
    \stackrel{R_3\rowop R_3-R_1}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 4 & 3 & 11 \\
      0 & 2 & 1 & 5 \\
      0 & -3 & -1 & -6
    \end{mymatrix}
  \end{equation*}
  Now the entries in the first column below the pivot position are
  zeros. We now look for the second pivot column, which in this case
  is column two.  Here, the $2$ in the second row and second column is
  in the pivot entry. We could create a zero below the $2$ with a
  single row operation by adding $\frac{3}{2}$ times the second row
  from the third row. But it is sometimes more convenient not to work
  with fractions, and therefore we start instead by multiplying the
  third row by $2$.
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 4 & 3 & 11 \\
      0 & \circled{2} & 1 & 5 \\
      0 & -3 & -1 & -6
    \end{mymatrix}
    \stackrel{R_3\rowop 2R_3}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 4 & 3 & 11 \\
      0 & \circled{2} & 1 & 5 \\
      0 & -6 & -2 & -12
    \end{mymatrix}
    \stackrel{R_3\rowop R_3+3R_2}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 4 & 3 & 11 \\
      0 & \circled{2} & 1 & 5 \\
      0 & 0 & 1 & 3
    \end{mymatrix}
  \end{equation*}
  The final matrix is our desired {\ef}.
  \begin{equation}\label{eqn:system-with-one-solution-1}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 4 & 3 & 11 \\
      0 & \circled{2} & 1 & 5 \\
      0 & 0 & \circled{1} & 3
    \end{mymatrix}
  \end{equation}
  Now we do back substitution to solve for $z$, $y$, and then $x$. The
  last equation of the {\ef} gives us $z=3$. Substituting this into
  the second equation, we get $2y+3=5$, which we can solve for $y$ to
  get $y=1$. Finally, substituting $y=1$ and $z=3$ into the first
  equation, we have $x+4(1)+3(3)=11$, which we can solve to get
  $x=-2$. Therefore we have the solution $(x,y,z)=(-2,1,3)$.

  At this point, it is a good idea to double-check this solution using
  the original equations
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      x+4y+3z &=& 11 \\
      2x+10y+7z &=& 27 \\
      x+y+2z &=& 5.
    \end{array}
  \end{equation*}
  For example, we can double-check that $(-2)+4(1)+3(3)$ is indeed
  equal to $11$, and similarly for the other two equations.
  Double-checking the solution against the original equations is an
  excellent way to guard against any errors that might have happened
  during the row operations or back substitution.

  Finally, we note that $(x,y,z)=(-2,1,3)$ is the {\em only} solution
  to this system of equations. There cannot be any other solutions,
  because by Theorem~\ref{thm:elementary-operations-and-solutions}, any
  solution of the original system of equations would also have to be a
  solution of {\eqref{eqn:system-with-one-solution-1}}, and the back
  substitution leaves us no choice except $z=3$, $y=1$, and $x=-2$.
\end{solution}

\begin{example}{Solving a system of equations: no solution}{system-with-no-solution}
  Solve the following system of equations:
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      y + 2z &=& 2 \\
      2x + y - 2z &=& 3 \\
      4x - y - 10z &=& 4.
    \end{array}
  \end{equation*}
\end{example}

\begin{solution} The augmented matrix for this system is
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      0 &  1 &   2 & 2 \\
      2 &  1 &  -2 & 3 \\
      4 & -1 & -10 & 4
    \end{mymatrix}.
  \end{equation*}
  We use Algorithm~\ref{algo:gaussian-elimination} to carry the
  augmented matrix to {\ef}. The first column is non-zero and will be the
  first pivot column. We switch the first two rows to move a non-zero
  number into the pivot position:
  \begin{equation*}
    \stackrel{R_2\rowswap R_1}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{2} &  1 &  -2 & 3 \\
      0 &  1 &   2 & 2 \\
      4 & -1 & -10 & 4
    \end{mymatrix}.
  \end{equation*}
  To create zeros below the pivot entry, we subtract $2$ times the
  first row from the third row:
  \begin{equation*}
    \stackrel{R_3\rowop R_3-2R_1}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{2} &  1 &  -2 & 3 \\
      0 &  1 &   2 & 2 \\
      0 & -3 & -6 & -2
    \end{mymatrix}.
  \end{equation*}
  This finishes the first column. The second pivot column will be
  column two, with the $1$ in the second row and column as the pivot
  entry. We add $3$ times the second row to the third row to create a
  zero below the pivot:
  \begin{equation*}
    \stackrel{R_3\rowop R_3+3R_2}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{2} &  1 &  -2 & 3 \\
      0 &  \circled{1} &   2 & 2 \\
      0 & 0 & 0 & \circled{4}
    \end{mymatrix}.
  \end{equation*}
  This matrix is in {\ef}. Note that the final pivot entry is on the
  right-hand side. The last row corresponds to the equation
  \begin{equation*}
    0x + 0y + 0z = 4.
  \end{equation*}
  This equation has no solution, because for all $x,y,z$, the
  left-hand size will equal $0$ and not $4$. Therefore, there is no
  solution to the given system of equations. In other words, the
  system is inconsistent.
\end{solution}

\begin{example}{Solving a system of equations: an infinite set of solutions}{infinite-set-of-solution}
  Solve the following system of equations:
  \begin{equation}
    \begin{array}{r@{~}c@{~}l}
      3x  -y + 5z  &=& 8 \\
      y - 10z &=& 1 \\
      6x  -y       &=& 17.
    \end{array}
  \end{equation}
\end{example}

\begin{solution} The augmented matrix of this system is
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      3 &  -1  &  5 & 8 \\
      0 &   1 & -10 & 1 \\
      6 &  -1  &  0 & 17
    \end{mymatrix}.
  \end{equation*}
  We use Gaussian elimination to carry the augmented matrix to
  {\ef}. The first column is the first pivot column, and $3$ is the
  pivot entry. We use row operating to create zeros beneath the pivot
  entry. We subtract $2$ times the first row from the third row and get:
  \begin{equation*}
    \stackrel{R_3\rowop R_3-2R_1}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{3} &  -1  &  5 & 8 \\
      0 &   1 & -10 & 1 \\
      0 &   1 & -10 & 1
    \end{mymatrix}
  \end{equation*}
  Now, we have created zeros beneath the pivot entry in the first
  column, so we move on to the second pivot column (which is the
  second column) and repeat the procedure. Subtracting the second row
  from the third row, we get:
  \begin{equation*}
    \stackrel{R_3\rowop R_3-R_2}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{3} &  -1  &  5 & 8 \\
      0 &   \circled{1} & -10 & 1 \\
      0 &   0 & 0 & 0
    \end{mymatrix}
  \end{equation*}
  This matrix is now in {\ef}. Observe that the first two columns are
  pivot columns, and the third column is not. We call the
  corresponding variables $x$ and $y$ \textbf{pivot variables}%
  \index{pivot variable}%
  \index{variable!pivot}, and the variable $z$ is a \textbf{free
    variable}%
  \index{free variable}%
  \index{variable!free}.  The equations corresponding to this {\ef}
  are
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      3x - y + 5z &=& 8 \\
      y - 10z &=& 1.
    \end{array}
  \end{equation*}
  Observe that the free variable $z$ is not constrained by any
  equation. In fact, $z$ can equal any number. We choose $t$ to be any
  number and let $z = t$.  In this context $t$ is called a
  \textbf{parameter}%
  \index{parameter}. We then use back substitution to solve for the
  pivot variables $y$ and $x$. From the second equation, we have
  $y = 1+10z = 1+10t$. From the first equation, we have
  $3x = 8+y-5z = 8+(1+10t)-5t = 9+5t$, and therefore
  $x=3+\frac{5}{3}t$. Therefore, the general solution of this system
  is
  \begin{equation*}
    \begin{array}{l}
      x=3+\frac{5}{3}t \\
      y=1+10t \\
      z=t,
    \end{array}
  \end{equation*}
  where $t$ is arbitrary. The system has an infinite set of solutions
  which are given by these equations. For any value of the parameter
  $t$ we select, $x$, $y$, and $z$ will be given by the above
  equations. For example, if we choose $t=4$ then the corresponding
  solution would be
  \begin{equation*}
    \begin{array}{l}
      x = 3 + \frac{5}{3}(4) = \frac{29}{3} \\
      y = 1+10(4) = 41 \\
      z = 4.
    \end{array}
  \end{equation*}
\end{solution}

In Example~\ref{exa:infinite-set-of-solution} the solution involved a
parameter. It may happen that the solution to a system involves more
than one parameter, as shown in the following example.

\begin{example}{Solving a system of equations: a two parameter set of solutions}{two-parameter-set-of-solution}
  Solve the following system of equations:
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      x + 2y - 2z + 2w &=& 3 \\
      x + 2y - z  + 3w &=& 5 \\
      x + 2y - 3z + w &=& 1.
    \end{array}
  \end{equation*}
\end{example}

\begin{solution} The augmented matrix is
  \begin{equation*}
    \begin{mymatrix}{rrrr|r}
      1 & 2 & -2 & 2 & 3 \\
      1 & 2 & -1 & 3 & 5 \\
      1 & 2 & -3 & 1 & 1
    \end{mymatrix}.
  \end{equation*}
  We carry this matrix to {\ef} using row operations.
  \begin{equation*}
    \begin{mymatrix}{rrrr|r}
      1 & 2 & -2 & 2 & 3 \\
      1 & 2 & -1 & 3 & 5 \\
      1 & 2 & -3 & 1 & 1
    \end{mymatrix}
    \stackrel{R_2\rowop R_2-R_1}{\stackrel{R_3\rowop R_3-R_1}{\roweq}}
    \begin{mymatrix}{rrrr|r}
      \circled{1} & 2 & -2 & 2 & 3 \\
      0 & 0 & 1 & 1 & 2 \\
      0 & 0 & -1 & -1 & -2
    \end{mymatrix}
    \stackrel{R_3\rowop R_3+R_2}{\roweq}
    \begin{mymatrix}{rrrr|r}
      \circled{1} & 2 & -2 & 2 & 3 \\
      0 & 0 & \circled{1} & 1 & 2 \\
      0 & 0 & 0 & 0 & 0
    \end{mymatrix}.
  \end{equation*}
  This matrix is in {\ef} and we can see that the first and third
  columns are pivot columns, whereas the second and fourth columns are
  not. Therefore, $x$ and $z$ are pivot variables and $y$ and $w$ are
  free variables. We assign parameters $y=s$ and $w=t$ to the free
  variables. Then we do back substitution to solve for $z$ and $x$.
  From the second equation, we have $z=2-w=2-t$. From the first
  equation, we have $x=3-2y+2z-2w = 3-2s+2(2-t)-2t =
  7-2s-4t$. Therefore, the general solution is given by
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      x &=& 7-2s-4t \\
      y &=& s \\
      z &=& 2-t \\
      w &=& t.
    \end{array}
  \end{equation*}
  It is customary to write this solution in the form
  \begin{equation}\label{two-parameters-2}
    \begin{mymatrix}{c}
      x \\
      y \\
      z \\
      w
    \end{mymatrix} =\begin{mymatrix}{c}
      7-2s-4t \\
      s \\
      2-t \\
      t
    \end{mymatrix}.
  \end{equation}
\end{solution}

In
Examples~\ref{exa:system-with-one-solution}--\ref{exa:two-parameter-set-of-solution},
we have seem systems of equations with one solution, no solution, and
infinitely many solutions with one parameter as well as two
parameters. Moreover, in each case, we have been able to determine the
number of solutions by looking at the {\ef} of the augmented matrix.
To summarize, we have the following possibilities for a system of equations:

\begin{enumerate}
\item {\em No solution:} If the {\ef} has a row of the form
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      0 & 0 & 0 & b
    \end{mymatrix},
  \end{equation*}
  where $b\neq 0$, then the system is inconsistent and has no
  solution.

\item {\em One solution:} For a consistent system of equations: If
  every column of the coefficient matrix of the {\ef} is a pivot
  column, the system has exactly one solution. The following is an
  example of an augmented matrix in {\ef} for a system of equations
  with one solution.
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 1 & -2 & 5 \\
      0 & \circled{2} & 3 & 0 \\
      0 & 0 & \circled{1} & 2 \\
      0 & 0 & 0 & 0
    \end{mymatrix}.
  \end{equation*}

\item {\em Infinitely many solutions:} For a consistent system of
  equations: If not all columns of the coefficient matrix of the {\ef}
  are pivot columns, then the system has infinitely many solutions.
  In this case, each variable corresponding to a non-pivot column is a
  {\em free variable}%
  \index{free variable}%
  \index{variable!free} and can be assigned a {\em parameter}%
  \index{parameter}. The remaining variables are {\em pivot
    variables}%
  \index{pivot variable}%
  \index{variable!pivot} and can be expressed in terms of the
  parameters. Therefore, the number of parameters in the general
  solution is equal to the number of non-pivot columns.  The following
  are examples of {\ef}s for systems of equations with infinitely many
  solutions.
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 0 & 1 & 5 \\
      0 & \circled{1} & 2 & -3 \\
      0 & 0 & 0 & 0 \\
      0 & 0 & 0 & 0
    \end{mymatrix}
  \end{equation*}
  or
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 2 & 3 & 5 \\
      0 & 0 & \circled{4} & 6
    \end{mymatrix}.
  \end{equation*}
\end{enumerate}

There is a special name for the number of pivot variables in a system
of equations. It is called the {\em rank} of the system.

\begin{definition}{Rank of a matrix}{rank}
  Let $A$ be a matrix and consider any {\ef} of $A$.  Then, the number
  $r$ of pivot entries of $A$ does not depend on the {\ef} we choose,
  and is called the \textbf{rank}%
  \index{matrix!rank}%
  \index{rank} of $A$. We denote it by $\rank(A)$.

  The rank of a system of linear equations is the rank of its
  coefficient matrix (i.e., the matrix on the left-hand side).
  It is equal to the number of pivot variables.
\end{definition}

\begin{example}{Finding the rank of a matrix}{rank-of-a-matrix}
  Consider the matrix
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 2 & 3 \\
      1 & 5 & 9 \\
      2 & 4 & 6
    \end{mymatrix}
  \end{equation*}
  What is its rank?
\end{example}

\begin{solution}
  First, we need to find an {\ef} of $A$. Through the usual algorithm,
  we find that this is
  \begin{equation*}
    \begin{mymatrix}{rrr}
      \circled{1} & 0 & -1 \\
      0 & \circled{3} & 6 \\
      0 & 0 & 0
    \end{mymatrix}.
  \end{equation*}
  Here we have two pivot entries, and therefore the rank of $A$ is
  $r=2$.
\end{solution}

Suppose we have a system of $m$ equations in $n$ variables, and
suppose that $n>m$. Further assume that the system is consistent. From
our above discussion, we know that this system will have infinitely
many solutions. This is because there can be at most one pivot entry
per row, and therefore at most $m$ variables can be pivot
variables. It follows that there are at least $n-m$ free variables.
Therefore, the general solution of this system has at least $n-m$
parameters.

Notice that if $n=m$ or $n<m$, it is possible for the system to have a
unique solution or infinitely many solutions. In all cases ($n>m$,
$n=m$, or $n<m$), it is also possible for the system to be
inconsistent (have no solutions).

By refining the above argument, we get the following theorem:

\begin{theorem}{Rank and solutions of consistent system of equations}{rank-consistent-solutions}
  Consider a system of $m$ equations in $n$ variables, and assume that
  the coefficient matrix has rank $r$. Assume further that the system
  is consistent.
  \begin{enumerate}
  \item If $r=n$, then the system has a unique solution.
  \item If $r<n$, then the system has infinitely many solutions, with $n-r$ parameters.
  \end{enumerate}
\end{theorem}

Here is a final summary of how the rank affects the number of
solutions:

\begin{enumerate}
\item {\em No solution.} If the system of equations is inconsistent,
  then it has no solution, regardless of the rank.

\item {\em Unique solution.} For a consistent system, suppose
  $r=n$. Then there is a pivot position in every column of the
  coefficient matrix of $A$. Hence, there is a unique solution.

\item {\em Infinitely many solutions.} For a consistent system, suppose
  $r<n$. Then there are less pivot positions than columns in the
  coefficient matrix, meaning that not every column is a pivot
  column. The columns which are {\em not} pivot columns correspond to
  parameters. In fact, in this case we have $n-r$ parameters. The
  system has infinitely many solutions.
\end{enumerate}


\section{Calculus}

\formdesc{Tangent and Secant Lines}

\begin{itemize}
  \item Tangent line is the line on $f(x)$ at point $x = a$ that just touches the
	graph of the function
	    \begin{itemize}
	        \item The slope of a non-linear function at one point
	    \end{itemize}

	    \begin{center}
	        \includegraphics{tangent_line}
	    \end{center}

  \item Secant line is the line between $A$ and $B$ on a curve
      \begin{center}
          \includegraphics{secant_line}
      \end{center}
\end{itemize}

\textit{Note}: As $A$ gets	closer to $B$ the secant slope approaches the tangent line
\hformbar

\formdesc{Average Rate of Change}

\begin{equation}
    ARC = f(a, a + \Delta) = \frac{\Delta y}{\Delta x} = \frac{f(a + \Delta x) - f(a)}{\Delta x}
\end{equation}

over interval $[a, a + \Delta x]$
\hformbar

\formdesc{Derivative}

The instantaneous rate of change of a function $f(x)$ at point $x = a$

\begin{equation}
    f'(a) = \lim_{\Delta x \to 0} \frac{\Delta y}{\Delta x} = \lim_{\Delta x \to 0} \frac{f(a + \Delta x) - f(a)}{\Delta x}
\end{equation}

``the derivative of $y$ w.r.t. $x$'' $\equiv \frac{dy}{dx} \equiv \lim_{\delta \rightarrow 0} \frac{\delta y}{\delta x}$

\hformbar

\formdesc{Simple Derivatives}

\begin{center}
\[
\def\arraystretch{2.5}
 \begin{array}{cc}
  \textit{y}   &      \textit{y'}             \\
  \midrule
  k            & 0                             \\
  x            & 1                             \\
  x^n          & nx^{n-1}                      \\
  \sqrt{x}     & \dfrac{1}{2\sqrt{x}}          \\
  \sqrt[n]{x}  & \dfrac{1}{n\sqrt[n]{x^{n-1}}} \\
  \dfrac{1}{x} & \dfrac{-1}{x^2}               \\
  e^x          & e^x                           \\
  a^x          & a^x\ln(a)                     \\
  x^x          & xx^{x-1}+x^x\ln(x)            \\
  \ln(x)       & \dfrac{1}{x}                  \\
  \log_a(x)    & \dfrac{1}{x}\log_a(e)
 \end{array}
\]
\end{center}
\hformbar

\formdesc{Composite Derivatives}

\begin{center}
\[
\def\arraystretch{2.5}
 \begin{array}{cc}
  \textit{y}   &      \textit{y'}               \\
  \midrule
  u^n          & nu^{n-1}u'                     \\
  \sqrt{u}     & \dfrac{u'}{2\sqrt{u}}          \\
  \sqrt[n]{u}  & \dfrac{u'}{n\sqrt[n]{u^{n-1}}} \\
  \dfrac{1}{u} & \dfrac{-u'}{u^2}               \\
  e^u          & e^u u'                         \\
  a^u          & a^u\ln(a)u'                    \\
  u^v          & vu^{v-1}u'+u^v\ln(u) v'        \\
  \ln(u)        & \dfrac{u'}{u}                 \\
  \log_a(u)    & \dfrac{u'}{u}\log_a(e)
 \end{array}
\]
\end{center}
\hformbar

\formdesc{Derivative Operations}

\begin{center}
  \begin{tabular}{ll}
  Sum         & $(f(x)+g(x))' = f'(x) + g'(x)$    \\
  Difference  & $(f-g)'(x) = f'(x) - g'(x)$        \\
  Product     & $(fg)'(x) = f'(x)g(x) + f(x)g'(x)$ \\
  Quotient    & $ \left(\frac{f(x)}{g(x)}\right)' = \frac{f'(x)g(x) - f(x)g'(x)}{g(x)^2}$ \\
  Chain rule  & $(f(g))'(x) = f'(g(x))g'(x)$        \\
  Inverse     & $(f^{-1})'(x) = \frac{1}{f'(x)}$
  \end{tabular}
\end{center}
\hformbar

\formdesc{Limits}

\begin{equation}
    \lim_{x \rightarrow C} f(x) = L
\end{equation}

or, ``the limit of $f(x)$, as $x$ approaches $c$, is $L$.'' $\lim_{x \rightarrow C} f(x)$ is a \textit{single number} that describes the behavior of the function $f(x)$ \textit{near} but not \textit{at} the point $x = c$.

Introduced to make calculating rate of change at 0 feasible, by making the $\Delta$ so infinitesimal the difference is between it and 0 is negligible---``allows'' division by 0

\subsection*{Example}

\begin{center}
    \includegraphics{linear_limits2}
\end{center}

where hollow points are undefined and solid points are defined

\begin{enumerate}
    \item $\lim_{x \rightarrow 6} f(x) = 1$
    \item $\lim_{x \rightarrow 4} f(x) = 3$
        \begin{itemize}
            \item \textit{Note}: even though $x=4$ is undefined, we're only concerned with the area \textit{around} 4, so we can still find the limit
        \end{itemize}
    \item $\lim_{x \rightarrow 2} f(x) = 5$
\end{enumerate}

\subsection*{Example: Determining Limits of Non-Linear Functions}

\begin{center}
    \includegraphics{limits_ex2}
\end{center}

where hollow points are undefined and solid points are defined

\begin{enumerate}
    \item $\lim_{x \rightarrow 1} f(x) = 2$---values where $x$ is close to but not equal to 1 are near 2
    \item $\lim_{x \rightarrow 2} f(x) = 3$---even though $f(2)$ is undefined, only values \textit{near} $f(2)$ are important
    \item $\lim_{x \rightarrow 3} f(x) = 1$---even though $f(3)$ is actually 2
    \item $\lim_{x \rightarrow 4} f(x) =$ \textit{does not exist}: Can't determine a single number because $f(4)$ from the right is about 2, and from the left about 3
\end{enumerate}

\subsection*{Example: Determining Limits Using Algebra}

Factor equation to simplest form and plug in $c$ (assuming function at $c$ is defined):

\begin{equation}
    \lim_{x \rightarrow 5} \frac{x^2 - 6x + 8}{x - 4} = \frac{25 - 30 + 8}{1} = 3
\end{equation}
\hformbar

\formdesc{Limits of Broken Functions}

Some functions are continuous but in an unusual way---they appear ``broken'' when graphed---and so there is a \textit{left} and \textit{right} limit

\begin{description}
    \item[Left] ``The limit coming from the left''; values of $f(x)$ as $f(x)$ nears $x$ and left of $c$, $x < c$
        \begin{equation}
            \lim_{x \rightarrow c^-} f(x) = L
        \end{equation}
    \item[Right] ``This limit coming from the right''; values of $f(x)$ as $f(x)$ nears $x$ and right of $c$, $x > c$
        \begin{equation}
            \lim_{x \rightarrow c^+} f(x) = L
        \end{equation}

\end{description}

\textit{Note}: If left and rights limits are not the same, limit \textit{doesn't exist}.
\hformbar

\formdesc{Continuity}

A function $f$ is continuous at $x = a$ iff $\lim_{x \rightarrow a} f(x) = f(a)$,
i.e., if the limit of $x$ at $a$ is equals $f(a)$, i.e., no breaks or jumps

\subsection*{Example}

\begin{center}
    \includegraphics{continuity}
\end{center}

where $H$ indicates a hole---where the graph is defined but could be made continuous by changing the point---and $J$ a jump---where the left and right limits are not the same.

\begin{itemize}
    \item Continuous at 1 since $\lim_{x \rightarrow 1} f(x) = f(1) = 2$
    \item Not continuous at 2, 3, or 4
        \begin{itemize}
            \item $\lim_{x \rightarrow 2} f(x) =2 \neq f(2) = 1$
            \item $\lim_{x \rightarrow 3} f(x) =$ doesn't exist $\neq f(3) = 2$
            \item $\lim_{x \rightarrow 4} f(x) = 2 \neq f(4) =$ undefined
        \end{itemize}

\end{itemize}
\hformbar

\formdesc{Calculating Derivatives}

\subsection*{Example: Using Formal Definition}

Find derivative of $f(x) = 2x^2 - 16x + 35$.

\begin{enumerate}
    \item Assemble using the formal definition
        \begin{equation}
            \frac{\left[ 2(x - h)^2 - 16(x + h) + 35 \right] -
                \left[ 2x^2 - 16x - 35 \right] }{h}
        \end{equation}

    \item Factor---\textit{cannot plug $h=0$ because no division by zero!}
        \begin{eqnarray}
            &=& \frac{2x^2 + 4xh + 2h^2 - 16x - 16h + 35 - 2x^2 + 16x - 35}{h} \\
            &=& \lim_{h \rightarrow 0} \frac{4xh + 2h^2 - 16h}{h}
        \end{eqnarray}

    \item Factor out $h$ in numerator to cancel $h$ in denominator
        \begin{eqnarray}
            f'(x) &=& \lim_{h \rightarrow 0} \frac{h(4x + 2h - 16)}{h} \\
            &=& \lim_{h \rightarrow 0} 4x + 2h - 16 \\
            &=& 4x - 16
        \end{eqnarray}

\end{enumerate}
\hformbar

\formdesc{Implicit Differentiation}

The process to find $y' = f'(x)$ when $f(x)$ is difficult or impossible to use with explicit differentiation, by assuming $y$ is a function of $x$:

\subsection*{Example}

Implicitly differentiate $x^2 + y^2 = 25$

\begin{enumerate}
    \item Differentiate each side, treating $y$ as a function
    \begin{eqnarray}
        \frac{d}{dx} \left( x^2 + y^2 \right) = \frac{d}{dx} 25 \\
        &\Rightarrow& \frac{d}{dx} x^2 + \frac{d}{dx} y^2 = 0 \\
        &\Rightarrow& 2x + 2y \frac{dy}{dx} = 0 \\
        &\Rightarrow& 2x + 2yy' = 0
    \end{eqnarray}

    \item Algebraically solve for $y'$
    \begin{eqnarray}
    	2yy' = -2x \\
	    &\Rightarrow& y' = \frac{-2x}{2y} \\
	    &\Rightarrow& y' = -\frac{x}{y}
    \end{eqnarray}
\end{enumerate}
\hformbar

\formdesc{Definite Integral}

The definite integral of a positive function $f(x)$ over an interval $[a, b]$ is the area between $f$, the $x$-axis, $x = a$, and $x = b$.

\begin{equation}
    \int_a^b f(x)\, dx
\end{equation}

where $a$ and $b$ are the ``limits of integration'' and $f(x)$ is the integrand

    \begin{eqnarray}
        \int_a^a f(x)\, dx &=& 0                   \\
        \int_a^b f(x)\, dx &=& -\int_b^a f(x)\, dx  \\
        \int_a^b f(x)\, dx &=& \int_a^c f(x)\, dx + \int_c^b f(x)\, dx
    \end{eqnarray}

where $a<c<b$
\hformbar

\formdesc{Integration Operations}

\begin{center}
  \begin{tabular}{ll}
  Sum           & $\int u+v\,dx = \int u\,dx + \int v\,dx$  \\
  Difference    & $\int u-v\,dx = \int u\,dx - \int v\,dx$  \\
  Product       & $\int af(x)\,dx = a\int f(x)\,dx$         \\
  Parts         & $\int u\,dv = uv - \int v\,du$            \\
  Substitution  & $\int f(u)u'\,dx = \int f(u)\,du$
  \end{tabular}
\end{center}
\hformbar

\formdesc{Growth, Concavity, and Extrema}

\flushleft
			\begin{description}
				\item[Growth]
				      \begin{itemize}
					      \item[]
					      \item $\forall x\in I\ f'(x)\geq 0$ $\Rightarrow$ $f$ is increasing in $I$.
					      \item $\forall x\in I\ f'(x)\leq 0$ $\Rightarrow$ $f$ is decreasing in $I$.
				      \end{itemize}
				\item[Concavity]
				      \begin{itemize}
					      \item[]
					      \item $\forall x\in I\ f''(x)\geq 0$ $\Rightarrow$ $f$ is concave up in $I$.
					      \item $\forall x\in I\ f''(x)\leq 0$ $\Rightarrow$ $f$ is concave down in $I$.
				      \end{itemize}
				\item[Extrema] If $f'(a)=0$ (critical point)
				      \begin{itemize}
					      \item $f''(a)<0$ $\Rightarrow$ $f$ has a local maximum at $x=a$.
					      \item $f''(a)>0$ $\Rightarrow$ $f$ has a local minimum at $x=a$.
				      \end{itemize}
			\end{description}

\begin{figure}[h]
\centering
\begin{minipage}{.5\linewidth}
  \centering
  \includegraphics[trim={0 45pt 0 0},clip,width=.8\linewidth]{concave_up}
\end{minipage}%
\begin{minipage}{.5\linewidth}
  \centering
  \includegraphics[trim={0 45pt 0 0},clip,width=.8\linewidth]{concave_down}
\end{minipage}
\caption{Concave up (left) and concave down (right)}
\end{figure}
\hformbar

\formdesc{Increasing and Decreasing Functions}

\begin{itemize}
    \item $f(x)$ is \textit{increasing} iff $\forall~ x_1, x_2$ in interval $I$ is such that $x_1 < x_2$ and $f(x_1) < f(x_2)$
    \item $f(x)$ is \textit{decreasing} iff $\forall~ x_1, x_2$ in interval $I$ is such that $x_1 > x_2$ and $f(x_1) > f(x_2)$
    \item Determine all intervals where $f(x)$ is in/decreasing:
    \begin{enumerate}
        \item Find all critical points (via first deriative)
        \item For each critical point, select a number $a$ in that range and see if $f'(a)$ is positive or negative
    \end{enumerate}
\end{itemize}

\hformbar

\formdesc{Inflection Points}

\begin{itemize}
    \item Where the second derivative changes signs
    \item The point(s) on a graph where the concavity of a function changes from up to down
    \item functions can be increasing (positive derivative) or decreasing (negative derivative) regardless of concavity,
\end{itemize}
\hformbar

\formdesc{Maxima And Minima}

A \textit{critical point} is a point where either $f'(a) = 0$ or $f'(a)$ is undefined, and is a \textit{candidate} of being a local or global extreme

\subsection*{Local}

\begin{description}
    \item[maximum] at $a$ if $f(a) \geq f(x)~ \forall x$ near $a$
    \item[minimum] at $a$ if $f(a) \leq f(x)~ \forall x$ near $a$
    \item[extreme] at $a$ if $f(a)$ is a local maximum or minimum
\end{description}

\subsection*{Global}

\begin{description}
    \item[maximum] at $a$ if $f(a) \geq f(x)~ \forall x$ in domain of $f$
    \item[minimum] at $a$ if $f(a) \leq f(x)~ \forall x$ in domain of $f$
    \item[extreme] at $a$ if $f(a)$ is a global maximum or minimum
\end{description}

\subsection*{Example}

Find the critical point of $f(x) = x^3 - 6x^2 + 9x + 2$

\begin{enumerate}
    \item Find $f'(x)$
        \begin{eqnarray}
            f'(x)  =  3x^2 - 12x + 9  \\
                  &=& 3(x^2 - 4x + 3) \\
                  &=& 3(x - 1)(x - 3) \\
         \end{eqnarray}
     \item Find where $f'(x) = 0$, which is 1 and 3
     \item Put $x=1$ and $x=3$ into $f(x)$ to find the critical points
         \begin{eqnarray}
             (1, f(1)) &=& (1, 6) \\
             (3, f(2)) &=& (3, 2)
         \end{eqnarray}
\end{enumerate}
\hformbar

\formdesc{Antiderivatives}

\begin{itemize}
    \item \textit{An} antiderivative of a function $f(x)$ is any function $F(x)$ such that $F'(x) = f(x)$
    \item \textit{The} antiderivative is an entire family of functions, written $F(x) + c$
    \item Also known as the \textit{indefinite integral} (with no limit markers):

    \begin{equation}
        \int f(x)~ dx
    \end{equation}
\end{itemize}

\subsection*{Example}

\textit{An} antiderivative of $\int 2x~ dx$ is $x^2 - 5.2$; \textit{the} antiderative is $x^2 + C$
\hformbar

\formdesc{Definite v. Indefinite Integrals}

Indefinite integrals do not have limits to integration where definite integrals do
\hformbar

\formdesc{Integration by Substitution}

A method to algebraically manipulate an integrand so it is amenable to antiderivative rules; especially useful when there is a product in the integral.

Substitute $u$ for $g(x)$ where necessary, making $\frac{du}{dx} = g'(x)$, so $du = g'(x)~ dx$. Since

\begin{equation}
    \frac{du}{dx} = g'(x) \equiv du = g'(x)~ dx
\end{equation}

we can substitute so that

\begin{equation}
    \int f'(g(x)) g'(x)~ dx \equiv \int f'(u)~ du
\end{equation}

Now integrate $f'(u)~ du$. (Note that $g(x) \equiv u$ and $g'(x)~ dx \equiv du$.)

\begin{enumerate}
    \item Set one part of the integrand to $u$, one ``level'' into the integral
    \item Compute $du = \frac{du}{dx} ~dx$ (the derivative of $u$)
    \item Convert $x$'s to $u$'s in original integral, even including in $dx$
    \item Integrate new $u$ integral
    \item Substitute $u$'s back to $x$'s in integral
\end{enumerate}

\subsection*{Example}

Integrate $\int (x + 1)^3~ dx$

\begin{enumerate}
	\item Rearrange so that $u = x + 1$ and $du = 1~ dx$
		\begin{equation}
			= \int (x + 1)^3 \cdot 1 ~ dx
		\end{equation}
	\item Substitute in $u$ and $du$
		\begin{equation}
			\int u^3 ~ du
		\end{equation}
	\item Integrate
		\begin{equation}
			= \frac{u^4}{4} + C
		\end{equation}
	\item Add $u$ back in
		\begin{equation}
			= \frac{(x + 1)^4}{4} + C
		\end{equation}
\end{enumerate}

\hformbar

\formdesc{Integration by Parts}

Integrate a complex function by rewriting it as a product of two simpler functions $u$ and $du$, using two possible forms:

\begin{eqnarray}
  \int u ~dv     &=& uv - \int v ~du \\
  \int_a^b u ~dv &=& uv |_a^b - \int_a^b v ~du
\end{eqnarray}

\subsection{Example: First Form}

Integrate $\int xe^x ~dx$:

\begin{enumerate}
  \item Break into two parts: $u = x$ and $dv = e^x ~dx$
  \item Calculate the derivative of $u$, $~du$, and $v$, the integral of $dv$
	\begin{eqnarray}
	  du &=& \left( \frac{d}{dx} x \right) ~dx = 1 ~dx \\
	  v &=& \int dv = \int e^x ~dx = e^x
	\end{eqnarray}
  \item Using the first formula, noting the prior forms from 1 and 2
	\begin{eqnarray}
      \int u ~dv &=& \int x e^x ~dx \\
	             &=& xe^x - \int e^x ~dv \\
				 &=& xe^x - e^x + C
     \end{eqnarray}
\end{enumerate}

\subsection{Example: Second Form}

Integrate $\int_1^4 6x^2 ln x ~dx$:

\begin{enumerate}
	\item Break into two parts: $u = ln x$ and $dv = 6x^2$
	\item Calculate the derivative of $u$, $du$, and the integral of $dv$, $v$:
	  \begin{eqnarray}
	    du &=& \frac{d}{dx} ln x = \frac{1}{x} ~dx \\
		v  &=& \int 6x^2 ~dx = 6 \int x^2 ~dx = 6 \cdot \frac{x^3}{3} = 2x^3
	  \end{eqnarray}
	\item Use the second formula
	  \begin{eqnarray}
	    \int_1^4 6x^2 ln x ~dx &=& 2x^3 ln x |_1^4 - \int_1^4 2x^3 \frac{1}{x} ~dx \\
		&=& 2x^3 ln x |_1^4 - 3x^2 |_1^4
	  \end{eqnarray}
	\item Find the integral the usual way:
	  \begin{eqnarray}
	    \left[ (2 \cdot 4^3 ln(4)) - (2 \cdot 1^3 ln(1)\right] - \left[ (3 \cdot 4^2) - (3 \cdot 1^2)  \right] \\
		  = 128 \cdot ln(4) - 45 \\
		  \approx 132.446
	  \end{eqnarray}
\end{enumerate}

\hformbar

\formdesc{Antiderivative Rules}

\begin{center}
\[
\def\arraystretch{2.5}
 \begin{array}{cc}
  \int a\,dx              & ax+C                   \\
  \int x^n\,dx            & \dfrac{u'}{2\sqrt{u}}  \\
  \int e^x\,dx            & e^x+C                  \\
  \int a^x\,dx           & \dfrac{a^x}{\ln(a)}+C  \\
  \int \dfrac{1}{x}\, dx  &  \ln|x|+C
 \end{array}
\]
\end{center}
\hformbar

\newpage


\documentclass{subfile}

\begin{document}
	\section{Introduction}\label{sec:intro}% power mean, cauchy schwarz
	Let us start with the most fundamental inequality.
		\begin{align}
			x^2
				& \geq0\label{ineq:mother}
		\end{align}
	The first author calls it the \index{mother of all inequality}mother of all inequality. Equality occurs if and only if $x=0$. We can extend it for $n$ variables.
		\begin{align}
			x_1^2+\ldots+x_n^2
				& \geq0\label{ineq:extendedmother}
		\end{align}
	Equality occurs if $x_i=0$ for all $1\leq i\leq n$. We immediately get some useful results substituting $x$ with appropriate expressions. Substituting $x$ with $a-b$, we get
		\begin{align*}
			(a-b)^2
				& \geq0\\
			a^2+b^2
				& \geq2ab
		\end{align*}
	This is true for any real numbers $a,b$ and equality occurs if $a=b$. If $a,b$ are positive, then replacing $a$ and $b$ by $\sqrt{a}$ and $\sqrt{b}$ respectively, we get
		\begin{align*}
			a+b
				& \geq2\sqrt{ab}\\
			\iff\sqrt{\dfrac{a}{b}}+\sqrt{\dfrac{b}{a}}
				& \geq2\\
			\iff x+\dfrac{1}{x}
				& \geq2
		\end{align*}
	where $x=\dfrac{a}{b}$. This can be generalized to the following result.
		\begin{theorem}[\itshape \index{Arithmetic-Geometric Inequality}Arithmetic-Geometric Inequality]\label{thm:amgm}
			Let $a_1,\ldots,a_n$ be positive real numbers. Then
				\begin{align*}
					\dfrac{a_1+\ldots+a_n}{n}
						& \geq\sqrt[n]{a_1\cdots a_n}\\
					\iff a_1+\ldots+a_n
						& \geq n\sqrt[n]{a_1\cdots a_n}\\
					\iff \left(\dfrac{a_1+\ldots+a_n}{n}\right)^n
						& \geq a_1\cdots a_n
				\end{align*}
		\end{theorem}
	We will show a classical proof of this result here. The proof is due to \textcite{cauchy_1821}. Later in \autoref{sec:powermean}, \autoref{sec:mazorization} we will show more proofs.
		\begin{proof}

		\end{proof}
	Note the following.
		\begin{align*}
			\dfrac{a+b}{2}
				& \geq\sqrt{ab}
		\end{align*}
	Now, $\frac{a+b}{2}$ is the \index{arithmetic mean}\textit{arithmetic mean} of $a$ and $b$. On the right side, $\sqrt{ab}$ is the \index{geometric mean}\textit{geometric mean} of $a$ and $b$. So the inequality states that the arithmetic mean of two positive real numbers is greater than or equal to their geometric mean. We can also rewrite it as the following.
		\begin{align*}
			\sqrt{ab}\left(\dfrac{a+b}{2}\right)
				& \geq ab\\
			\sqrt{ab}
				& \geq\dfrac{2ab}{a+b}\\
			\sqrt{ab}
				& \geq\dfrac{2}{\frac{1}{a}+\frac{1}{b}}
		\end{align*}
	$\dfrac{2ab}{a+b}$ is the \index{harmonic mean}\textit{harmonic mean} of $a$ and $b$. So, this form of the inequality states that the geometric mean is larger than the harmonic mean. This can be extended for three variables.
		\begin{align*}
			\dfrac{a+b+c}{3}
				& \geq\dfrac{3}{\dfrac{1}{a}+\dfrac{1}{b}+\dfrac{1}{c}}
		\end{align*}
	In fact, this result can be extensively generalized. First, we can consider $n$ variables $a_1,\ldots,a_n$ instead of just $a$ and $b$. Second, we can generalize the fact that $AM\geq GM\geq HM$. We will talk about this generalization in \autoref{sec:powermean}.
		\begin{problem}
			If $a$ is a real number greater than one, prove that $\log{a}+\log_{a}{e}\geq2$.
		\end{problem}
	Throughout the book, if the base of logarithm is unspecified, then $\log{a}$ shall mean $\log_{e}{a}$. Also, try the next problem in a similar manner.
		\begin{problem}
			Prove the inequality
				\begin{align*}
					\dfrac{x^2}{1+x^4}
						& \leq\dfrac{1}{2}
				\end{align*}
		\end{problem}
	Recall from the definition of $e$ that
		\begin{align*}
			e^x
				& = 1+x+\dfrac{x^2}{2!}+\dfrac{x^3}{3!}+\ldots\\
				& = \sum\limits_{i\geq0}\dfrac{x^i}{i!}
		\end{align*}
	You may also know from elementary differentiation that
		\begin{align*}
			\lim\limits_{n\to\infty}\left(1+\dfrac{1}{n}\right)^n
				& = e
		\end{align*}
	In other words, if $x_n=\left(1+\frac{1}{n}\right)^n$, then $x_n$ converges to $e$. However, we also have the relation that $x_n\leq x_{n+1}$. While we can prove this using induction, we will show a better proof with arithmetic-geometric mean inequality here.
		\begin{problem}
			Let $x_n=\left(1+\frac{1}{n}\right)^n$. Prove that $x_n\leq x_{n+1}$.
				\begin{solution}
					Apply the arithmetic-geometric mean inequality for $a_1=1,a_2=1+\frac{1}{n},\ldots,a_{n+1}=1+\frac{1}{n}$,
						\begin{align*}
							1+\left(1+\dfrac{1}{n}\right)+\ldots+\left(1+\dfrac{1}{n}\right)
								& \geq(n+1)\sqrt[n+1]{1\cdot\left(1+\dfrac{1}{n}\right)\cdots\left(1+\dfrac{1}{n}\right)}\\
							n+2
								& \geq(n+1)\sqrt[n+1]{\left(1+\dfrac{1}{n}\right)^n}\\
							1+\dfrac{1}{n+1}
								& \geq\sqrt[n+1]{\left(1+\dfrac{1}{n}\right)^n}\\
							\left(1+\dfrac{1}{n+1}\right)^{n+1}
								& \geq\left(1+\dfrac{1}{n}\right)^n
						\end{align*}
				\end{solution}
		\end{problem}
	Note that we can generalize the idea used in this problem.
		\begin{problem}
			For positive real numbers $x,y$ show that
				\begin{align*}
					\left(\dfrac{x+ny}{n+1}\right)^{n+1}
						& \geq xy^n
				\end{align*}
			Equality occurs only for $x=y$.
		\end{problem}
	Similarly, we can show the following.
		\begin{problem}
			Let $y_n=\left(1+\frac{1}{n}\right)^{n+1}$. Show that $y_n\geq y_{n+1}$.
		\end{problem}
	So $y_n$ is decreasing. Using the two problems above, show that $2<e<4$. In fact, we can prove the following using elementary means
		\begin{align*}
			\lim\limits_{n\to\infty}y_{n}
				& = \lim\limits_{n\to\infty}x_{n}=e
		\end{align*}

		\begin{problem}
			For a positive integer $n$, show that
				\begin{align*}
					\left(\dfrac{n+1}{2}\right)^{n}
						& \geq n!
				\end{align*}
			Equality occurs only for $n=1$.
				\begin{solution}
					We use arithmetic-geometric inequality for $1,2,\ldots,n$.
						\begin{align*}
							1+\ldots+n
								& \geq n\sqrt[n]{1\cdots n}\\
							\dfrac{n(n+1)}{2}
								& \geq n\sqrt[n]{n!}\\
							\dfrac{n+1}{2}
								& \geq\sqrt[n]{n!}
						\end{align*}
					Equality occurs if and only if $1=\ldots=n$ which is possible only when $n=1$.
				\end{solution}
		\end{problem}
	Let us again go back to the mother of inequality. Setting $x_1=a-b,x_2=b-c,x_3=c-a$ in \ref{ineq:extendedmother},
		\begin{align*}
			(a-b)^2+(b-c)^2+(c-a)^2
				& \geq0\\
			\iff a^2+b^2+c^2-ab-bc-ca
				& \geq0\\
			\iff a^2+b^2+c^2
				& \geq ab+bc+ca
		\end{align*}
	We could prove this using $a^2+b^2\geq2ab$ repeatedly.
		\begin{align*}
			a^2+b^2
				& \geq2ab\\
			b^2+c^2
				& \geq2bc\\
			c^2+a^2
				& \geq2ca\\
			2(a^2+b^2+c^2)
				& \geq2(ab+bc+ca)\\
			a^2+b^2+c^2-ab-bc-ca
				& \geq0
		\end{align*}
	Multiplying both sides by $(a+b+c)$ and using the fact $a^3+b^3+c^3-3abc=(a+b+c)(a^2+b^2+c^2-ab-bc-ca)$,
		\begin{align*}
			a^3+b^3+c^3-3abc
				& \geq0\\
			a^3+b^3+c^3
				& \geq3abc
		\end{align*}
	Replacing $a^3,b^3,c^3$ by $u,v,w$,
		\begin{align}
			u+v+w
				& \geq3\sqrt[3]{uvw}\label{ineq:amgm3}
		\end{align}
	Before we go into any theory or method of solving problems, here are some basic tactics which are used very often for solving problems. Let $x,y,z>0$ be real numbers.
		\begin{enumerate}[\itshape (i)]
			\item If $\dfrac{1}{x}\geq\dfrac{1}{y}$, then $\dfrac{1}{x-a}\geq\dfrac{1}{y}$ for $a\geq0$. Similarly, $\dfrac{1}{x}\geq\dfrac{1}{y-b}$ for $b\geq0$. They are true because $x-a\leq x$ implies $\dfrac{1}{x-a}\geq\dfrac{1}{x}$ and $y+b\geq y$ implies $\dfrac{1}{y+b}\leq\dfrac{1}{y}$.
			\item Check if you can assume an ordering on the variables. For example, if the inequality is symmetric or cyclic on $x,y,z$, you may possibly assume without loss of generality that $x\geq y\geq z$ or $x\leq y\leq z$. We will discuss more on this in \autoref{sec:buffalo}.
			\item If you cannot assume an ordering on the variables e.g. $a\geq b\geq c$, can you assume that it has a \textit{maximal element} e.g. $a=\max(a,b,c)$? Sometimes this helps in unexpected ways. See the example below.
			\item See if you can get an \textit{if and only if} way of proving an inequality. We will do this very often. If we can use if and only if (or \textit{iff}), then we are free to prove either the if part or the only if part.
			\item Check if you can get some familiar expressions with some basic manipulations such as making the numerator or denominator equal. For example, see the transformation in \ref{eqn:nesbittf}. This often helps us get a clue on what to do with the inequality.
			\item Does some substitutions such as $x=a+b,y=b+c,z=c+a$ or $x=a-b,y=b-c,z=c-a$ help? One could say we used substitution in our first proof of Nesbitt's inequality (see \ref{eqn:nesbittsub}). Also, see the transformation $a-c=a-b+b-c$ used in the example below. We will check more on substitutions in \autoref{sec:subs}.
			\item Check if the inequality holds even if you put some restrictions on it. For example, you may be allowed to assume that one of the variables is $1$ due to some scaling. We will talk about this in \autoref{sec:homonorm}.
			\item Can you reduce the number of variables without any assumption? See that we have already used it to prove $a^2+b^2+c^2\geq ab+bc+ca$ in a proof above. Also, see a proof of Nesbitt's inequality below where it is enough to prove the inequality for reduced number of variables.
			\item Induction works great in many cases. We will use induction on many occasions.
		\end{enumerate}
	For demonstration purposes, let us prove $a^2+b^2+c^2\geq ab+bc+ca$ again exploiting symmetry. Note that if we let
		\begin{align*}
			f(a,b,c)
				& = a^2+b^2+c^2-ab-bc-ca
		\end{align*}
	then $f(a,b,c)$ is \textit{symmetric} on $a,b,c$. We can verify this by the fact that $f(a,b,c)=f(b,a,c)=f(c,a,b)$ and so on. So, without loss of generality, we assume that $a\geq b\geq c$. Then see the following.
		\begin{align*}
			a^2+b^2+c^2
				& \geq ab+bc+ca\\
			a(a-b)+b(b-c)-c(a-c)
				& \geq0\\
			a(a-b)+b(b-c)-c(a-b+b-c)
				& \geq0\\
			a(a-b)+b(b-c)-c(a-b)-c(b-c)
				& \geq0\\
			(a-c)(a-b)+(b-c)^2
				& \geq0
		\end{align*}
	The last inequality immediately follows from the assumption that $a-c\geq0,a-b\geq0,(b-c)^2\geq0$. Also, note that we did not actually require the condition $a\geq b\geq c$. Just assuming $a=\max(a,b,c)$ was enough in this case to claim that the inequality holds.
		\begin{theorem}[\itshape \index{Nesbitt's inequality}Nesbitt's inequality]\label{thm:nesbitt}
			Let $a,b,c$ be real positive numbers. Then
				\begin{align*}
					\dfrac{a}{b+c}+\dfrac{b}{c+a}+\dfrac{c}{a+b}
						& \geq\dfrac{3}{2}
				\end{align*}
			and equality occurs if and only $a=b=c$.
		\end{theorem}
	The thing about inequalities is that they can be solved in more than one ways most of the times. We will  prove this inequality along with some other classical results such as \textit{Cauchy-Schwarz inequality} in more than one ways. Some proofs will be discussed later when we develop some certain techniques. For now, we present some proofs using what we have already developed.
	First, we will try to \textit{familiarize} the expression on the left side.
		\begin{align}
			S
				& = \dfrac{a}{b+c}+\dfrac{b}{c+a}+\dfrac{c}{a+b}\nonumber\\
				& = \dfrac{a+b+c}{b+c}-1+\dfrac{a+b+c}{c+a}-1+\dfrac{a+b+c}{a+b}-1\nonumber\\
				& = (a+b+c)\left(\dfrac{1}{b+c}+\dfrac{1}{c+a}+\dfrac{1}{a+b}\right)-3\nonumber\\
				& = \dfrac{1}{2}(a+b+b+c+c+a)\left(\dfrac{1}{b+c}+\dfrac{1}{c+a}+\dfrac{1}{a+b}\right)-3\label{eqn:nesbittf}
		\end{align}

		\begin{proof}[\itshape Classical proof]
			Setting
				\begin{align}
					x=a+b,
						& y=b+c,z=c+a\label{eqn:nesbittsub}
				\end{align}
				\begin{align*}
					S
						& = \dfrac{1}{2}(x+y+z)\left(\dfrac{1}{x}+\dfrac{1}{y}+\dfrac{1}{z}\right)-3\\
						& = \dfrac{1}{2}\left(1+\dfrac{x}{y}+\dfrac{x}{z}+\dfrac{y}{x}+1+\dfrac{y}{z}+\dfrac{z}{x}+\dfrac{z}{y}+1\right)-3\\
						& = \dfrac{3}{2}+\dfrac{1}{2}\left(\dfrac{x}{y}+\dfrac{y}{x}+\dfrac{y}{z}+\dfrac{z}{y}+\dfrac{z}{x}+\dfrac{x}{z}\right)-3\\
						& = \dfrac{3}{2}+\dfrac{1}{2}\left(u+\dfrac{1}{u}+v+\dfrac{1}{v}+w+\dfrac{1}{w}\right)-3
				\end{align*}
			where $u=\dfrac{x}{y},v=\dfrac{y}{z},w=\dfrac{z}{x}$. Evidently, $u+\frac{1}{u}\geq2,v+\frac{1}{v}\geq2,w+\frac{1}{w}\geq2$ and we have
				\begin{align*}
					S
						& \geq\dfrac{3}{2}+\dfrac{1}{2}(2+2+2)-3
				\end{align*}
			Thus, $S\geq\frac{3}{2}$. Equality occurs if $u=1,v=1,z=1$ or $x=y=z$ or $a=b=c$.
		\end{proof}

		\begin{proof}[\itshape Proof by arithmetic-harmonic mean inequality]
			We write the arithmetic-harmonic mean inequality for $u,v,w$ as below.
				\begin{align*}
					\dfrac{u+v+w}{3}
						& \geq\dfrac{3}{\dfrac{1}{u}+\dfrac{1}{v}+\dfrac{1}{v}}\\
					\iff(u+v+w)\left(\dfrac{1}{u}+\dfrac{1}{v}+\dfrac{1}{w}\right)
						& \geq9
				\end{align*}
			Using this on \ref{eqn:nesbittf},
				\begin{align*}
					S
						& \geq\dfrac{1}{2}\cdot9-2
				\end{align*}
		\end{proof}

		\begin{proof}[\itshape \index{Variable reduction}Variable reduction proof]
			Let us clear the denominators in the original inequality.
				\begin{align*}
					\dfrac{a}{b+c}+\dfrac{b}{c+a}+\dfrac{c}{a+b}
						& \geq\dfrac{3}{2}\\
					\iff2(a^3+b^3+c^3)
						& \geq a^2b+ab^2+b^2c+bc^2+c^2a+ca^2
				\end{align*}
			Notice the cyclic nature in the expression on the right side. There are $6$ terms on the right side and if we count each of $a^3,b^3,c^3$ twice, there are $6$ terms on the left side as well. So rearranging the inequality above as below
				\begin{align*}
					(a^3+b^3)+(b^3+c^3)+(c^3+a^3)
						& \geq (a^2b+ab^2)+(b^2c+bc^2)+(c^2a+ca^2)
				\end{align*}
			tells us that if we can prove $x^3+y^3\geq x^2y+xy^2$, we will be done if we simply sum them up for $(x,y)=(a,b),(b,c),(c,a)$. Here, we reduced the inequality from $3$ variables to $2$. And fortunately, this inequality is a lot easier to prove.
				\begin{align*}
					x^3+y^3
						& \geq x^2y+xy^2\\
					\iff x(x^2-y^2)-y(x^2-y^2)
						& \geq0\\
					\iff (x-y)(x^2-y^2)
						& \geq0\\
					\iff(x-y)^2(x+y)
						& \geq0
				\end{align*}
			The last inequality is evidently true.
		\end{proof}

		\begin{proof}[\itshape Proof using arithmetic-geometric mean inequality]
			Using \ref{ineq:amgm3},
				\begin{align*}
					(a+b)+(b+c)+(c+a)
						& \geq3\sqrt[3]{(a+b)(b+c)(c+a)}\\
					\dfrac{1}{a+b}+\dfrac{1}{b+c}+\dfrac{1}{c+a}
						& \geq3\sqrt[3]{\dfrac{1}{(a+b)(b+c)(c+a)}}
				\end{align*}
			We can use this observation on \ref{eqn:nesbittf} and get the following.
				\begin{align*}
					S
						& \geq\dfrac{1}{2}\cdot3\sqrt[3]{(a+b)(b+c)(c+a)}\cdot3\sqrt[3]{\dfrac{1}{(a+b)(b+c)(c+a)}}-2\\
						& \geq\dfrac{9}{2}-2=\dfrac{3}{2}
				\end{align*}
			This again proves the inequality.
		\end{proof}
	\textcite{anovic_pecaric_2011} proves the following generalization of \nameref{thm:nesbitt}.
		\begin{theorem}[Generalization of Nesbitt's inequality]
			Let $x,y,z$ be positive real numbers and
				\begin{align*}
					M
						& = \max\left\{\dfrac{x}{y+z}+\dfrac{2(y+z)}{2x+y+z},\dfrac{y}{z+x}+\dfrac{2(z+x)}{2y+z+x},\dfrac{z}{x+y}+\dfrac{2(x+y)}{2z+x+y}\right\}\\
					m
						& =  \min\left\{\dfrac{x}{y+z}+\dfrac{2(y+z)}{2x+y+z},\dfrac{y}{z+x}+\dfrac{2(z+x)}{2y+z+x},\dfrac{z}{x+y}+\dfrac{2(x+y)}{2z+x+y}\right\}
				\end{align*}
			Then
				\begin{align*}
					\dfrac{x}{y+z}+\dfrac{y}{z+x}+\dfrac{z}{x+y}
						& \geq M\geq m\geq \dfrac{3}{2}
				\end{align*}
		\end{theorem}
	We have already showed that $a+b\geq2\sqrt{ab}$ which is a special case of the arithmetic-geometric mean inequality. Let us consider the following question. We are given four positive real numbers $a,b,c,d$ such that $S=a+b=c+d$. Which of the products between $ab$ and $cd$ is the smaller one?
		\begin{align*}
			4ab
				& = (a+b)^2-(a-b)^2\\
				& = S^2-(a-b)^2\\
			4cd
				& = (c+d)^2-(c-d)^2\\
				& = S^2-(c-d)^2
		\end{align*}
	As we can see here, the sign in $4ab?4cd$ ($?$ to be replaced by one of $>,<,\geq,\leq$) will be dictated by which of the differences $a-b,c-d$ is smaller. Since $x^2\geq0$, if $(a-b)^2<(c-d)^2$, we have $4ab>4cd$. Now, consider the product $a_1\cdots a_n$. We want to see how the product changes as $a_i$ varies with respect to $\bar{a}=\frac{a_1+\ldots+a_n}{n}$.

	If all the $a_i$ are equal to each other, then we have nothing to check. Otherwise, there are at least two positive integers $i$ and $j$ such that $a_i$ and $a_j$ are not equal to $\bar{a}$. Moreover, one of them is greater than $\bar{a}$ and the other is smaller than $\bar{a}$ because all of them cannot be greater (or smaller) than $\bar{a}$. Without loss of generality, assume that $a_1,a_2$ are those two numbers and $a_1=\bar{a}-h,a_2=\bar{a}+k$. Now, consider two other positive numbers $c$ and $d$ which keeps the sum fixed, for example $c=\bar{a},d=\bar{a}+k-h$. We have
		\begin{align*}
			cd
				& = \bar{a}(\bar{a}+k-h)\\
				& = \bar{a}^2+\bar{a}k-\bar{a}h\\
			a_1a_2
				& = (\bar{a}-h)(\bar{a}+k)\\
				& = \bar{a}^2+\bar{a}k-\bar{a}h-hk\\
				& = cd-hk < cd\\
			a_1a_2\cdots a_n
				& < cd\cdots a_n
		\end{align*}
	This basically tells us that we can increase the product further if there are other $a_i$ which are not equal to $\bar{a}$ and the product is maximum when all $a_i$ is equal to $\bar{a}$.
		\begin{problem}
			Show that for a positive integer $n$,
				\begin{align*}
					n!
						& > \left(\dfrac{n}{4}\right)^n
				\end{align*}
		\end{problem}
	We can easily prove this with induction. This can be improved to $n!>\left(\dfrac{n}{3}\right)^n$. In fact, we can prove the following.
		\begin{align*}
			n!
				& > \left(\dfrac{n}{e}\right)^n
		\end{align*}
	This is another nice result. For example, setting $n=2019$,
		\begin{align*}
			2019!
				& > 673^{2019}
		\end{align*}
	We can even bound $n!$ from both sides with the next result.
		\begin{problem}
			For a positive integer $n$, prove the inequality
				\begin{align*}
					e\left(\dfrac{n+1}{e}\right)^{n+1}
						& > n!>\left(\dfrac{n}{e}\right)^n
				\end{align*}
		\end{problem}
	Equality is not possible because on both sides we have non-integers.
		\begin{problem}
			Let $x,y,z$ be positive real numbers. Prove that
				\begin{align*}
					\dfrac{2}{x+y}+\dfrac{2}{y+z}+\dfrac{2}{z+x}
						& \geq \dfrac{9}{x+y+z}
				\end{align*}

				\begin{solution}

				\end{solution}
		\end{problem}
	A function $f$ is \textit{non-decreasing} (resp. \textit{increasing}) on the interval $I$ if for any $a,b\in I$, $(a-b)(f(a)-f(b))\geq0$ (resp. $(a-b)(f(a)-f(b))>0$). Similarly, $f$ is \textit{non-increasing} (resp. \textit{decreasing}) on the interval $I$ if for any $a,b\in I$, $(a-b)(f(a)-f(b))\leq0$ (resp. $(a-b)(f(a)-f(b))<0$). If either of these two conditions apply for $f$, then $f$ is a \index{monotone function}\textit{monotone function}. We say that $f$ is \textit{monotonic}.
	Note that the slope between any two points $(a,f(a))$ and $(b,f(b))$ is $m=\frac{f(a)-f(b)}{a-b}$ and the quantity we have used for the definition is
		\begin{align*}
			(a-b)(f(a)-f(b))
				& = (a-b)^2\dfrac{f(a)-f(b)}{f(a)-f(b)}\\
				& = (a-b)^2m
		\end{align*}
	So, the sign of this quantity is the same as the sign of the slope $m$.

	A sequence $(a_n)$ is \textit{non-decreasing} if $a_i\leq a_{i+1}$ for all $i\in\mathbb{N}$. $(a_n)$ is \textit{strictly increasing} if $a_i<a_{i+1}$. Similarly, $(a_n)$ is \textit{non-increasing} if $a_i\geq a_{i+1}$ for all $i$. $(a_n)$ is \textit{strictly decreasing} if $a_i>a_{i+1}$. If $(a_n)$ is either increasing or decreasing, then $(a_n)$ is a \textit{monotone sequence}. We say that $(a_n)$ is \textit{monotonic}.
		\begin{theorem}[\itshape Abel formula]\label{thm:abelform}
			Let $(a_{n})$ and $(b_{n})$ be two sequences of real numbers. If $c_{k}=b_{1}+\ldots+b_{k}$ for $1\leq k\leq n$, then
				\begin{align*}
					a_{1}b_{1}+\ldots+a_{n}b_{n}
						& = (a_{1}-a_{2})c_{1}+(a_{2}-a_{3})c_{2}+\ldots+(a_{n-1}-a_{n})c_{n}+a_{n}c_{n}
				\end{align*}

				\begin{proof}

				\end{proof}
		\end{theorem}

		\begin{theorem}[\itshape Abel's Inequality]\label{thm:abelineq}
			Let $(a_n)$ and $(b_n)$ be two sequences of real numbers such that $b_1\geq \ldots\geq b_n\geq0$. For $1\leq k\leq n$, define
				\begin{align*}
					s_k
						& = a_1+\ldots+a_k\\
					m
						& = \min_{1\leq i\leq n}(s_i)\\
					M
						& = \max_{1\leq i\leq n}(s_i)
				\end{align*}
			Then we have
				\begin{align*}
					mb_1
						& \leq a_1b_1+\ldots+a_nb_n\leq  Mb_1
				\end{align*}
		\end{theorem}

		\begin{proof}
			Write the sum $a_1b_1+\ldots+a_nb_n$ as the following.
				\begin{flalign*}
					a_1b_1+\ldots+a_nb_n
						& = s_1b_1+(s_2-s_1)b_2+\ldots+(s_n-s_{n-1})b_n\\
						& = s_1(b_1-b_2)+s_2(b_2-b_3)+\ldots+s_{n-1}(b_{n-1}-b_n)+s_nb_n
				\end{flalign*}
			Using $m\leq s_i\leq M$,
				\begin{flalign*}
					m(b_1-b_2)
						& \leq s_1(b_1-b_2) \leq M(b_1-b_2)\\
						& \vdots\\
					m(b_{n-1}-b_{n})
						& \leq s_{n-1}(b_{n-1}-b_n)\leq M(b_{n-1}-b_n)
				\end{flalign*}
			We additionally have $mb_n\leq s_nb_n\leq Mb_n$. Summing these inequalities together,
				\begin{flalign*}
					m(b_1-b_2+b_2-b_3+\ldots+b_{n-1}-b_n)+mb_n
						& \leq s_1(b_1-b_2)\\
						& +\ldots+s_{n-1}(b_{n-1}-b_n)+s_nb_n\\
						& \leq M(b_1-b_2+b_2-b_3+\ldots\\
						& +b_{n-1}-b_n)+Mb_n\\
					mb_1
						& \leq a_1b_1+\ldots+a_nb_n\leq Mb_1
				\end{flalign*}
			This proves the inequality.
		\end{proof}
	% What about Aczel's inequality?
	\section{Warm Up Problems}
	Try the following problems as warm up exercises.
		\begin{problem}
			For real numbers $x,y$, prove that
				\begin{align*}
					|x+y|
						& \leq|x|+|y|\\
					||x|-|y||
						& \leq|x+y|\\
					(|x|-|y|)^2
						& \leq|x^2-y^2|
				\end{align*}
			When does equality occur in the last inequality?
		\end{problem}
	The first inequality in this problem is also known as the \index{triangle inequality}\textit{triangle inequality}.
		\begin{problem}
			Prove \index{Jordan's inequality}\textit{Jordan's inequality}.
				\begin{align*}
					\dfrac{2}{\pi}
						& \leq\dfrac{\sin{\theta}}{\theta}<1
				\end{align*}
			for $0<|\theta|<\frac{\pi}{2}$.
		\end{problem}

		\begin{problem}
			Prove \index{Redheffer's inequality}\textit{Redheffer's inequality}.
				\begin{align*}
					\dfrac{\sin{\theta}}{\theta}
						& \geq\dfrac{\pi^2-\theta^2}{\pi^2+\theta^2}
				\end{align*}
		\end{problem}
	The two inequalities above are not derived from each other. Also, you may have to put a bit of extra effort to prove them. Use calculus if that gives a faster solution.
		\begin{problem}[\itshape IMO $1960$]
			For which real value of $x$ does the inequality
				\begin{align*}
					\dfrac{4x^2}{(1-\sqrt{1+2x})^2}
						& <2x+9
				\end{align*}
			hold?
		\end{problem}
\end{document}


\chapter{The detal numerics}
\section{4th order central differnce}
{
\footnotesize
The 4th order central difference is given by
\begin{eqnarray}
&& \frac{\partial \phi}{\partial x}
=\frac{-\phi_{i+2}+8\phi_{i+1}-8\phi_{i-1}+\phi_{i+2}}{12\Delta x} = 0
\end{eqnarray}
where
\begin{eqnarray}
&& \phi_{i+2} = \phi_{i}
+ 2 \Delta x \left(\frac{\partial \phi}{\partial x}\right)_{i}
+ 2 \Delta x^2 \left(\frac{\partial^2 \phi}{\partial x^2}\right)_{i}
+ \frac{4 \Delta x^3}{3} \left(\frac{\partial^3 \phi}{\partial x^3}\right)_{i}
+ \frac{2 \Delta x^4}{3} \left(\frac{\partial^3 \phi}{\partial x^4}\right)_{i}
+ O(\Delta x^5)\\
&& \phi_{i+1} = \phi_{i}
+ \Delta x \left(\frac{\partial \phi}{\partial x}\right)_{i}
+ \frac{\Delta x^2}{2} \left(\frac{\partial^2 \phi}{\partial x^2}\right)_{i}
+ \frac{\Delta x^3}{6} \left(\frac{\partial^3 \phi}{\partial x^3}\right)_{i}
+ \frac{\Delta x^4}{24} \left(\frac{\partial^3 \phi}{\partial x^4}\right)_{i}
+ O(\Delta x^5)\\
&& \phi_i = \phi_i\\
&& \phi_{i-1} = \phi_{i}
- \Delta x \left(\frac{\partial \phi}{\partial x}\right)_{i}
+ \frac{\Delta x^2}{2} \left(\frac{\partial^2 \phi}{\partial x^2}\right)_{i}
- \frac{\Delta x^3}{6} \left(\frac{\partial^3 \phi}{\partial x^3}\right)_{i}
+ \frac{\Delta x^4}{24} \left(\frac{\partial^3 \phi}{\partial x^4}\right)_{i}
+ O(\Delta x^5)\\
&& \phi_{i+2} = \phi_{i}
- 2 \Delta x \left(\frac{\partial \phi}{\partial x}\right)_{i}
+ 2 \Delta x^2 \left(\frac{\partial^2 \phi}{\partial x^2}\right)_{i}
- \frac{4 \Delta x^3}{3} \left(\frac{\partial^3 \phi}{\partial x^3}\right)_{i}
+ \frac{2 \Delta x^4}{3} \left(\frac{\partial^3 \phi}{\partial x^4}\right)_{i}
+ O(\Delta x^5)
\end{eqnarray}
Therefore,
\begin{eqnarray}
&&
\frac{-\phi_{i+2}+8\phi_{i+1}-8\phi_{i-1}+\phi_{i+2}}{12\Delta x}
= \left(\frac{\partial \phi}{\partial x}\right)_{i}+ O(\Delta x^4)\\
&&
\frac{(-\phi_{i+2}+7\phi_{i+1}+7\phi_{i}-\phi_{i-1})-
(-\phi_{i+1}+7\phi_{i}+7\phi_{i-1}-\phi_{i-2})}{12\Delta x}
= \left(\frac{\partial \phi}{\partial x}\right)_{i}+ O(\Delta x^4)
\end{eqnarray}

\section{Flux Corrected Transport scheme}
Equation (\ref{eq:tracer_int}) can be written as
\begin{eqnarray}
%%
\left(\rho q\right)^{n+1}_{i,j,k}
&=& \left(\rho q\right)^{n}_{i,j,k}
- \frac{1}{\Delta x \Delta y \Delta z}
\big[ \nonumber\\
&+&
\left[ C_{i+\frac{1}{2},j,k} F_{i+\frac{1}{2},j,k}^{high}
+ \left( 1 - C_{i+\frac{1}{2},j,k}\right) F_{i+\frac{1}{2},j,k}^{low}\right]\nonumber\\
&-&
\left[ C_{i-\frac{1}{2},j,k} F_{i-\frac{1}{2},j,k}^{high}
+ \left( 1 - C_{i-\frac{1}{2},j,k}\right) F_{i-\frac{1}{2},j,k}^{low}\right]\nonumber\\
&+&
\left[ C_{i,j+\frac{1}{2},k} F_{i,j+\frac{1}{2},k}^{high}
+ \left( 1 - C_{i,j+\frac{1}{2},k}\right) F_{i,j+\frac{1}{2},k}^{low}\right]\nonumber\\
&-&
\left[ C_{i,j-\frac{1}{2},k} F_{i,j-\frac{1}{2},k}^{high}
+ \left( 1 - C_{i,j-\frac{1}{2},k}\right) F_{i,j-\frac{1}{2},k}^{low}\right]\nonumber\\
&+&
\left[ C_{i,j,k+\frac{1}{2}} F_{i,j,k+\frac{1}{2}}^{high}
+ \left( 1 - C_{i,j,k+\frac{1}{2}}\right) F_{i,j,k+\frac{1}{2}}^{low}\right]\nonumber\\
&-&
\left[ C_{i,j,k-\frac{1}{2}} F_{i,j,k-\frac{1}{2}}^{high}
+ \left( 1 - C_{i,j,k-\frac{1}{2}}\right) F_{i,j,k-\frac{1}{2}}^{low}\right]\nonumber\\
\big]
\label{eq:1step_integ_tracer}
\end{eqnarray}
where
\begin{eqnarray}
  F_{i+\frac{1}{2},j,k}^{high,low} &=& \Delta t \Delta y \Delta z (\rho u)_{i+\frac{1}{2},j,k} q_{i+\frac{1}{2},j,k}^{high,low}\\
  F_{i,j+\frac{1}{2},k}^{high,low} &=& \Delta t \Delta z \Delta x (\rho u)_{i,j+\frac{1}{2},k} q_{i,j+\frac{1}{2},k}^{high,low}\\
  F_{i,j,k+\frac{1}{2}}^{high,low} &=& \Delta t \Delta x \Delta y (\rho u)_{i,j,k+\frac{1}{2}} q_{i,j,k+\frac{1}{2}}^{high,low}
\end{eqnarray}
The anti-diffusive flux are defined as
\begin{eqnarray}
  A_{i+\frac{1}{2},j,k} &=& F_{i+\frac{1}{2},j,k}^{high}-F_{i+\frac{1}{2},j,k}^{low}\\
  A_{i,j+\frac{1}{2},k} &=& F_{i,j+\frac{1}{2},k}^{high}-F_{i,j+\frac{1}{2},k}^{low}\\
  A_{i,j,k+\frac{1}{2}} &=& F_{i,j,k+\frac{1}{2}}^{high}-F_{i,j,k+\frac{1}{2}}^{low}
\end{eqnarray}

Equation (\ref{eq:1step_integ_tracer}) can be rewritten as
\begin{eqnarray}
%%
\left(\rho q\right)^{n+1}_{i,j,k}
&=& \left(\rho q\right)^{n}_{i,j,k}
- \frac{1}{\Delta x \Delta y \Delta z}
\big[ \nonumber\\
&+&
\left[ F_{i+\frac{1}{2},j,k}^{low}
+ C_{i+\frac{1}{2},j,k}  A_{i+\frac{1}{2},j,k}\right]\nonumber\\
&-&
\left[ F_{i-\frac{1}{2},j,k}^{low}
+ C_{i-\frac{1}{2},j,k}  A_{i-\frac{1}{2},j,k}\right]\nonumber\\
&+&
\left[ F_{i,j+\frac{1}{2},k}^{low}
+ C_{i,j+\frac{1}{2},k}  A_{i,j+\frac{1}{2},k}\right]\nonumber\\
&-&
\left[ F_{i,j-\frac{1}{2},k}^{low}
+ C_{i,j-\frac{1}{2},k}  A_{i,j-\frac{1}{2},k}\right]\nonumber\\
&+&
\left[ F_{i,j,k+\frac{1}{2}}^{low}
+ C_{i,j,k+\frac{1}{2}}  A_{i,j,k+\frac{1}{2}}\right]\nonumber\\
&-&
\left[ F_{i,j,k-\frac{1}{2}}^{low}
+ C_{i,j,k-\frac{1}{2}}  A_{i,j,k-\frac{1}{2}}\right]\nonumber\\
\big]
\label{eq:1step_integ_tracer2}
\end{eqnarray}
In practice, we calculate Eq.(\ref{eq:1step_integ_tracer2}) by the
following steps:
\begin{enumerate}
%
\item The tentative values are calculated by using the low order flux:
\begin{eqnarray}
\left(\rho q\right)^{\dagger}_{i,j,k}
&=& \left(\rho q\right)^{n}_{i,j,k}\nonumber\\
&-& \frac{1}{\Delta x \Delta y \Delta z}
\left[
+ F_{i+\frac{1}{2},j,k}^{low}-F_{i-\frac{1}{2},j,k}^{low}
+ F_{i,j+\frac{1}{2},k}^{low}-F_{i,j-\frac{1}{2},k}^{low}
+ F_{i,j,k+\frac{1}{2}}^{low}-F_{i,j,k-\frac{1}{2}}^{low}
\right]
\end{eqnarray}
%
\item Allowable maximum and minimum values are calculated:
\begin{eqnarray}
\left(\rho q\right)^{\max}_{i,j,k}
&=& \max [\nonumber\\
&&\max( \left(\rho q\right)^{\dagger}_{i,j,k},\left(\rho q\right)^{n}_{i,j,k} ),\nonumber\\
&&\max( \left(\rho q\right)^{\dagger}_{i-1,j,k},\left(\rho q\right)^{n}_{i-1,j,k} ),\nonumber\\
&&\max( \left(\rho q\right)^{\dagger}_{i+1,j,k},\left(\rho q\right)^{n}_{i+1,j,k} ),\nonumber\\
&&\max( \left(\rho q\right)^{\dagger}_{i,j-1,k},\left(\rho q\right)^{n}_{i,j-1,k} ),\nonumber\\
&&\max( \left(\rho q\right)^{\dagger}_{i,j+1,k},\left(\rho q\right)^{n}_{i,j+1,k} ),\nonumber\\
&&\max( \left(\rho q\right)^{\dagger}_{i,j,k-1},\left(\rho q\right)^{n}_{i,j,k-1} ),\nonumber\\
&&\max( \left(\rho q\right)^{\dagger}_{i,j,k+1},\left(\rho q\right)^{n}_{i,j,k+1} ) \nonumber\\
&&]\\
\left(\rho q\right)^{\min}_{i,j,k}
&=& \min [\nonumber\\
&&\min( \left(\rho q\right)^{\dagger}_{i,j,k},\left(\rho q\right)^{n}_{i,j,k} ),\nonumber\\
&&\min( \left(\rho q\right)^{\dagger}_{i-1,j,k},\left(\rho q\right)^{n}_{i-1,j,k} ),\nonumber\\
&&\min( \left(\rho q\right)^{\dagger}_{i+1,j,k},\left(\rho q\right)^{n}_{i+1,j,k} ),\nonumber\\
&&\min( \left(\rho q\right)^{\dagger}_{i,j-1,k},\left(\rho q\right)^{n}_{i,j-1,k} ),\nonumber\\
&&\min( \left(\rho q\right)^{\dagger}_{i,j+1,k},\left(\rho q\right)^{n}_{i,j+1,k} ),\nonumber\\
&&\min( \left(\rho q\right)^{\dagger}_{i,j,k-1},\left(\rho q\right)^{n}_{i,j,k-1} ),\nonumber\\
&&\min( \left(\rho q\right)^{\dagger}_{i,j,k+1},\left(\rho q\right)^{n}_{i,j,k+1} ) \nonumber\\
&&]
\end{eqnarray}
\item Several values for the flux limiter are calculated:
\begin{eqnarray}
P_{i,j,k}^{+} &=&
-\min ( 0, A_{i+\frac{1}{2},j,k} ) + \max( 0, A_{i-\frac{1}{2},j,k} )\nonumber\\
&&
-\min ( 0, A_{i,j+\frac{1}{2},k} ) + \max( 0, A_{i,j-\frac{1}{2},k} )\nonumber\\
&&
-\min ( 0, A_{i,j,k+\frac{1}{2}} ) + \max( 0, A_{i,j,k-\frac{1}{2}} )\\
P_{i,j,k}^{-} &=&
-\max ( 0, A_{i+\frac{1}{2},j,k} ) + \min( 0, A_{i-\frac{1}{2},j,k} )\nonumber\\
&&
-\max ( 0, A_{i,j+\frac{1}{2},k} ) + \min( 0, A_{i,j-\frac{1}{2},k} )\nonumber\\
&&
-\max ( 0, A_{i,j,k+\frac{1}{2}} ) + \min( 0, A_{i,j,k-\frac{1}{2}} )\\
\end{eqnarray}
\begin{eqnarray}
Q_{i,j,k}^{+} &=&
\left[\left(\rho q\right)^{\max}_{i,j,k} - \left(\rho q\right)^{\dagger}_{i,j,k} \right]
\Delta x \Delta y \Delta z\\
Q_{i,j,k}^{-} &=&
\left[\left(\rho q\right)^{\dagger}_{i,j,k}-\left(\rho q\right)^{\min}_{i,j,k} \right]
\Delta x \Delta y \Delta z
\end{eqnarray}
\begin{eqnarray}
R_{i,j,k}^{+} &=&
\begin{cases}
        \min( 1, Q_{i,j,k}^{+}/P_{i,j,k}^{+} ) & {\rm if~} P_{i,j,k}^{+}>0\\
        0 & {\rm if~} P_{i,j,k}^{+}=0\\
\end{cases}
\\
R_{i,j,k}^{-} &=&
\begin{cases}
        \min( 1, Q_{i,j,k}^{-}/P_{i,j,k}^{-} ) & {\rm if~} P_{i,j,k}^{-}>0\\
        0 & {\rm if~} P_{i,j,k}^{-}=0\\
\end{cases}
\end{eqnarray}
\item The flux limters at the cell wall are calculated:
\begin{eqnarray}
C_{i+\frac{1}{2},j,k} &=&
\begin{cases}
        \min( R_{i+1,j,k}^{+}, R_{i,j,k}^{-} ) & {\rm if~} A_{i+\frac{1}{2},j,k}^{-}\geq 0\\
        \min( R_{i,j,k}^{+}, R_{i+1,j,k}^{-} ) & {\rm if~} A_{i+\frac{1}{2},j,k}^{-}<0\\
\end{cases}
\\
C_{i,j+\frac{1}{2},k} &=&
\begin{cases}
        \min( R_{i,j+1,k}^{+}, R_{i,j,k}^{-} ) & {\rm if~} A_{i,j+\frac{1}{2},k}^{-}\geq 0\\
        \min( R_{i,j,k}^{+}, R_{i,j+1,k}^{-} ) & {\rm if~} A_{i,j+\frac{1}{2},k}^{-}<0\\
\end{cases}
\\
C_{i,j,k+\frac{1}{2}} &=&
\begin{cases}
        \min( R_{i,j,k+1}^{+}, R_{i,j,k}^{-} ) & {\rm if~} A_{i,j,k+\frac{1}{2}}^{-}\geq 0\\
        \min( R_{i,j,k}^{+}, R_{i,j,k+1}^{-} ) & {\rm if~} A_{i,j,k+\frac{1}{2}}^{-}<0\\
\end{cases}
\end{eqnarray}
\end{enumerate}