\section{Positive semidefinite and positive definite matrices}

\begin{outcome}
  \begin{enumerate}
  \item Determine whether a matrix is positive semidefinite and/or
    positive definite, either directly or by looking at the
    eigenvalues.
  \item Determine whether a matrix is positive semidefinite and/or
    positive definite using Descartes' rule of signs.
  \item Determine whether a matrix defines an inner product.
  \end{enumerate}
\end{outcome}

In Example~\ref{exa:rn-with-inner-product}, we saw that it is
sometimes possible to define an inner product on $\R^n$ from an
$n\times n$-matrix $A$ by the formula
\begin{equation*}
  \iprod{\vect{u},\vect{v}} = \vect{u}^T A \vect{v}.
\end{equation*}
We will now explore in more detail under what conditions this formula
defines an inner product.

\begin{definition}{Positive semidefinite and positive definite matrices}{positive-matrix}
  Let $A$ be an $n\times n$-matrix over the real numbers.
  \begin{itemize}
  \item $A$ is called \textbf{symmetric}%
    \index{symmetric matrix}%
    \index{matrix!symmetric} if $A=A^T$.
  \item $A$ is called \textbf{positive semidefinite}%
    \index{positive semidefinite matrix}%
    \index{matrix!positive semidefinite}%
    \index{semidefinite} if it is symmetric and for all
    $\vect{v}\in\R^n$, we have $\vect{v}^TA\vect{v}\geq 0$.
  \item $A$ is called \textbf{positive definite}%
    \index{positive definite matrix}%
    \index{matrix!positive definite} if it is positive semidefinite
    and $\vect{v}^TA\vect{v}=0$ if and only if $\vect{v}=\vect{0}$.
\end{itemize}
\end{definition}

Equivalently, the positive definite property can also be stated as
follows: $A$ is positive definite if it is symmetric and for all
$\vect{v}\in\R^n$ with $\vect{v}\neq\vect{0}$, we have
$\vect{v}^TA\vect{v}>0$.

\begin{example}{Positive semidefinite and positive definite matrices}{positive-matrix}
  Which of the following matrices are positive semidefinite? Which
  ones are positive definite?
  \begin{equation*}
    A = \begin{mymatrix}{cc} 2 & -1 \\ -1 & 1 \end{mymatrix},\quad
    B = \begin{mymatrix}{cc} 1 & 2 \\ 2 & 1 \end{mymatrix},\quad
    C = \begin{mymatrix}{cc} 1 & -1 \\ -1 & 1 \end{mymatrix},\quad
    D = \begin{mymatrix}{cc} 2 & 0 \\ 2 & 1 \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  The matrix $A$ is positive definite. To see why, consider any vector
  $\vect{v}=\begin{mymatrix}{c} x \\ y \end{mymatrix}$. Then
  \begin{equation*}
    \vect{v}^TA\vect{v}
    ~=~
    \begin{mymatrix}{cc} x & y \end{mymatrix}
    \begin{mymatrix}{cc} 2 & -1 \\ -1 & 1 \end{mymatrix}
    \begin{mymatrix}{c} x \\ y \end{mymatrix}
    ~=~ 2x^2 - 2xy + y^2
    ~=~ x^2 + (x-y)^2
    ~\geq~ 0.
  \end{equation*}
  This inequality implies that $A$ is positive semidefinite. Moreover,
  $\vect{v}^TA\vect{v}=0$ if and only if $x=0$ and $x-y=0$, which
  implies $x=y=0$. So $\vect{v}=\vect{0}$ is the only solution of
  $\vect{v}^TA\vect{v}=0$, and $A$ is positive definite.

  The matrix $B$ is not positive semidefinite (and therefore not
  positive definite either). For example, consider
  $\vect{v}=\begin{mymatrix}{c} 1 \\ -1 \end{mymatrix}$. Then
  \begin{equation*}
    \vect{v}^TB\vect{v}
    ~=~
    \begin{mymatrix}{cc} 1 & -1 \end{mymatrix}
    \begin{mymatrix}{cc} 1 & 2 \\ 2 & 1 \end{mymatrix}
    \begin{mymatrix}{c} 1 \\ -1 \end{mymatrix}
    ~=~ 1-2-2+1
    ~=~ -2
    ~<~ 0,
  \end{equation*}
  showing that $B$ is not positive semidefinite.

  The matrix $C$ is positive semidefinite, but not positive definite.
  To see why, consider
  $\vect{v}=\begin{mymatrix}{c} x \\ y \end{mymatrix}$. Then
  \begin{equation*}
    \vect{v}^TC\vect{v}
    ~=~
    \begin{mymatrix}{cc} x & y \end{mymatrix}
    \begin{mymatrix}{cc} 1 & -1 \\ -1 & 1 \end{mymatrix}
    \begin{mymatrix}{c} x \\ y \end{mymatrix}
    ~=~ x^2 - 2xy + y^2
    ~=~ (x-y)^2
    ~\geq~ 0.
  \end{equation*}
  Therefore, $C$ is positive semidefinite. However, it is not positive
  definite because, for example, for
  $\vect{v}=\begin{mymatrix}{c} 1 \\ 1 \end{mymatrix}$, we have
  $\vect{v}\neq\vect{0}$ but $\vect{v}^TC\vect{v}=0$.

  The matrix $D$ is not symmetric, and therefore neither positive
  semidefinite nor positive definite.
\end{solution}

The interest of positive definite matrices lies in the following
proposition.

\begin{proposition}{Positive definite matrices and inner products on $\R^n$}{positive-definite-and-inner-product}
  Let $A$ be an $n\times n$-matrix over the real numbers. Then the
  formula
  \begin{equation*}
    \iprod{\vect{u},\vect{v}} = \vect{u}^T A \vect{v}
  \end{equation*}
  defines an inner product on $\R^n$ if and only if $A$ is positive
  definite. Moreover, every inner product on $\R^n$ arises from some
  positive definite matrix $A$ in this way.
\end{proposition}

\begin{proof}
  Define $\iprod{\vect{u},\vect{v}} = \vect{u}^T A \vect{v}$. We will
  check each of the three properties of an inner product from
  Definition~\ref{def:real-inner-product-space}. Linearity holds
  for all matrices $A$, because
  \begin{equation*}
    \iprod{\vect{u},k\vect{v}}
    ~=~ \vect{u}^T A (k\vect{v})
    ~=~ k(\vect{u}^T A \vect{v})
    ~=~ k\iprod{\vect{u},\vect{v}}
  \end{equation*}
  and
  \begin{equation*}
    \iprod{\vect{u},\vect{v}+\vect{v}'}
    ~=~ \vect{u}^T A (\vect{v}+\vect{v}')
    ~=~ \vect{u}^T A \vect{v} + \vect{u}^T A \vect{v}'
    ~=~ \iprod{\vect{u},\vect{v}} + \iprod{\vect{u},\vect{v}'}.
  \end{equation*}
  For symmetry, first observe that for all $\vect{u},\vect{v}$,
  \begin{equation*}
    \iprod{\vect{v},\vect{u}}
    ~=~ \vect{v}^T A \vect{u}
    ~=~ (\vect{v}^T A \vect{u})^T
    ~=~ \vect{u}^T A^T \vect{v}.
  \end{equation*}
  Therefore $\iprod{\vect{u},\vect{v}}=\iprod{\vect{v},\vect{u}}$ if
  and only if $\vect{u}^T A \vect{v} = \vect{u}^T A^T \vect{v}$. This
  holds for all $\vect{u},\vect{v}$ if and only if $A=A^T$. Therefore,
  symmetry holds if and only if $A$ is symmetric.  Finally, the
  positive definite property of the inner product holds, by
  definition, if and only if $A$ is positive definite.

  To prove the second part, consider any inner product
  $\iprod{\vect{u},\vect{v}}$ on $\R^n$. Let
  $\vect{e}_1,\ldots,\vect{e}_n$ be the standard basis vectors, and
  define $a_{ij} = \iprod{\vect{e}_i,\vect{e}_j}$. Then
  $A=\mat{a_{ij}}$ is a matrix. We claim that
  $\iprod{\vect{u},\vect{v}}=\vect{u}^TA\vect{v}$ for all
  $\vect{u},\vect{v}$. Indeed, let
  $\vect{u}=u_1\vect{e}_1+\ldots+u_n\vect{e}_n$ and
  $\vect{v}=v_1\vect{e}_1+\ldots+v_n\vect{e}_n$. Then
  \begin{eqnarray*}
    \iprod{\vect{u},\vect{v}}
    &=& \iprod{u_1\vect{e}_1+\ldots+u_n\vect{e}_n,~ v_1\vect{e}_1+\ldots+v_n\vect{e}_n} \\
    &=& u_1v_1\iprod{\vect{e}_1, \vect{e}_1}
    +   u_1v_2\iprod{\vect{e}_1, \vect{e}_2}
    + \ldots
    +   u_nv_n\iprod{\vect{e}_n, \vect{e}_n} \\
    &=& u_1v_1a_{11}
    +   u_1v_2a_{12}
    + \ldots
    +   u_nv_na_{nn} \\
    &=& \vect{u}^TA\vect{v}.
  \end{eqnarray*}
  By the first part, $A$ is positive definite.
\end{proof}

Given a symmetric matrix $A$, it is not always an easy task to
determine whether $A$ is positive definite (or semidefinite) by using
the definition directly. This would require checking the condition
$\vect{v}^TA\vect{v}\geq 0$ for {\em all} vectors $\vect{v}$, of which
there are infinitely many. The following proposition gives us a more
practical method for determining whether a matrix is positive
(semi)definite.

\begin{proposition}{Characterization of positive (semi)definite matrices using eigenvalues}{characterize-positive}
  Let $A$ be a symmetric $n\times n$-matrix, and let
  $\eigenvar_1,\ldots,\eigenvar_n$ be its eigenvalues. Then $A$ is
  positive semidefinite if and only if
  $\eigenvar_1,\ldots,\eigenvar_n\geq 0$. Moreover, $A$ is positive
  definite if and only if $\eigenvar_1,\ldots,\eigenvar_n>0$.
\end{proposition}

\begin{proof}
  By Theorem~\ref{thm:diagonalization-symmetric}, we know that $A$ is
  orthogonally diagonalizable, i.e., $D=P^TAP$, where $P$ is an
  orthogonal matrix and
  \begin{equation*}
    D = \begin{mymatrix}{ccc}
      \eigenvar_1 & \cdots & 0 \\
      \vdots & \ddots & \vdots \\
      0 & \cdots & \eigenvar_n
    \end{mymatrix}.
  \end{equation*}
  Let $\vect{v}=\mat{x_1,\ldots,x_n}^T$ be any vector, and let
  $\vect{w}=P\vect{v}$. Then
  $\vect{w}^TA\vect{w} = \vect{v}^TP^TAP\vect{v} =
  \vect{v}^TD\vect{v}$, so that $A$ is positive (semi)definite if and
  only if $D$ is positive (semi)definite. Also, we have
  $\vect{v}^TD\vect{v} = \eigenvar_1x_1^2 + \ldots +
  \eigenvar_nx_n^2$. Therefore $\vect{v}^TD\vect{v}\geq 0$ for all
  $\vect{v}$ if and only if $\eigenvar_1,\ldots,\eigenvar_n\geq
  0$. Moreover, $\vect{v}^TD\vect{v}>0$ for all
  $\vect{v}\neq \vect{0}$ if and only if
  $\eigenvar_1,\ldots,\eigenvar_n>0$, as claimed.
\end{proof}

\begin{example}{Using eigenvalues to check whether a matrix is positive definite}{characterize-positive}
  Use eigenvalues to determine whether the matrix
  \begin{equation*}
    A = \begin{mymatrix}{ccc}
      5 & 0 & 2 \\
      0 & 5 & 1 \\
      2 & 1 & 1 \\
    \end{mymatrix}
  \end{equation*}
  is positive definite, positive semidefinite, or neither.
\end{example}

\begin{solution}
  The characteristic polynomial is
  \begin{equation*}
    \det(A-\eigenvar I)
    ~=~ \begin{absmatrix}{ccc}
      5-\eigenvar & 0 & 2 \\
      0 & 5-\eigenvar & 1 \\
      2 & 1 & 1-\eigenvar \\
    \end{absmatrix}
    ~=~ -\eigenvar^3 + 11\eigenvar^2 - 30\eigenvar
    ~=~ -\eigenvar(\eigenvar-5)(\eigenvar-6).
  \end{equation*}
  The eigenvalues are $\eigenvar_1=0$, $\eigenvar_2=5$, and
  $\eigenvar_3=6$.  Therefore, the matrix $A$ is positive
  semidefinite, but not positive definite.
\end{solution}

While Proposition~\ref{prop:characterize-positive} gives us a method
to determine whether a matrix $A$ is positive definite (or
semidefinite), it still requires finding all of the eigenvalues of
$A$, i.e., to factor the characteristic polynomial. This can be a
difficult calculation, especially if the degree of the polynomial is
large or if the roots are not integers. Fortunately, there is a better
way to determine whether a matrix is positive definite (or
semidefinite) by directly looking at the characteristic polynomial,
without having to calculate its roots. The method was found by
Ren\'e Descartes%
\index{Descartes, Ren\'e}%
\index{Ren\'e Descartes} in 1637 and is called {\em Descartes' rule
  of signs}. The general form of Descartes' rule of signs is
actually more complicated than what we consider here. We state a
version that has been specialized to characteristic polynomials of
symmetric matrices.

Let $a_0,\ldots,a_n$ be a sequence of real numbers. We say that
$a_0,\ldots,a_n$ have \textbf{strongly alternating signs}%
\index{strongly alternating signs}%
\index{alternating signs!strongly}%
\index{sign!strongly alternating} if $a_0>0$,
$a_1<0$, $a_2>0$, and so on (i.e., $a_i>0$ when $i$ is even, and
$a_i<0$ when $i$ is odd). We say that $a_0,\ldots,a_n$ have
\textbf{weakly alternating signs}%
\index{weakly alternating signs}%
\index{alternating signs!weakly}%
\index{sign!weakly alternating} if $a_0\geq 0$, $a_1\leq 0$,
$a_2\geq 0$, and so on (i.e., $a_i\geq 0$ when $i$ is even, and
$a_i\leq 0$ when $i$ is odd).

\begin{proposition}{Descartes' rule of signs for positive (semi)definite matrices}{descartes-rule-of-signs}
  Let $A$ be a symmetric $n\times n$-matrix, and let
  \begin{equation*}
    p(\eigenvar) = a_n\eigenvar^n + a_{n-1}\eigenvar^{n-1} + \ldots + a_1\eigenvar + a_0
  \end{equation*}
  be its characteristic polynomial. Then:
  \begin{itemize}
  \item $A$ is positive definite if and only if $a_0,\ldots,a_n$ have
    strongly alternating signs.
  \item $A$ is positive semidefinite if and only if $a_0,\ldots,a_n$
    have weakly alternating signs.
  \end{itemize}
\end{proposition}

\begin{proof}
  We first prove a general fact about polynomials. Suppose $d_1,\ldots,d_n$
  are real numbers and
  \begin{equation*}
    (x+d_1)(x+d_2)\cdots(x+d_n) = b_nx^n + b_{n-1}x^{n-1} + \ldots +
    b_1x + b_0.
  \end{equation*}
  Then $d_1,\ldots,d_n > 0$ if and only if $b_0,\ldots,b_n > 0$.
  Moreover, $d_1,\ldots,d_n \geq 0$ if and only if $b_0,\ldots,b_n \geq 0$.

  Proof: To prove the first claim, assume $d_1,\ldots,d_n > 0$. It is
  easy to see that by multiplying out $(x+d_1)(x+d_2)\cdots(x+d_n)$,
  we can only obtain positive coefficients, so $b_0,\ldots,b_n >
  0$. Conversely, assume $b_0,\ldots,b_n > 0$. Then for every
  $x\geq 0$, we clearly have $b_nx^n + b_{n-1}x^{n-1} + \ldots + b_1x + b_0 >
  0$, and therefore no such $x\geq 0$ can be a root of this
  polynomial.  In other words, all of the roots must be
  negative. Since the roots are $-d_1,\ldots,-d_n$, it follows that
  $d_1,\ldots,d_n>0$. The proof of the second claim (using ``$\geq$''
  instead of ``$>$'') is similar.

  We are now ready to prove
  Proposition~\ref{prop:descartes-rule-of-signs}. By
  Theorem~\ref{thm:diagonalization-symmetric}, we know that $A$ is
  diagonalizable, i.e., $A=PDP^{-1}$ for some real diagonal matrix
  $D$. Note that $A$ and $D$ have the same characteristic
  polynomial. If $d_1,\ldots,d_n$ are the diagonal entries of $D$, the
  characteristic polynomial can therefore be written in two different
  ways:
  \begin{equation*}
    p(\eigenvar)
    = (d_1-\eigenvar)(d_2-\eigenvar)\cdots(d_n-\eigenvar)
    = a_0 + a_1\eigenvar + a_2\eigenvar^2 + a_3\eigenvar^3 + \ldots + a_n\eigenvar^n.
  \end{equation*}
  Now let $\eigenvar=-x$ and consider
  \begin{equation*}
    p(-x) = (d_1+x)(d_2+x)\cdots(d_n+x)
    = a_0 - a_1 x + a_2 x^2 - a_3x^3 + \ldots \pm a_n x^n.
  \end{equation*}
  We have: $A$ is positive definite if and only if $d_1,\ldots,d_n>0$,
  if and only if $a_0,-a_1,a_2,-a_3,\ldots > 0$, if and only if
  $a_0,\ldots,a_n$ are strongly alternating.  Moreover, $A$ is
  positive semidefinite if and only if $d_1,\ldots,d_n\geq 0$, if and
  only if $a_0,-a_1,a_2,-a_3,\ldots \geq 0$, if and only if
  $a_0,\ldots,a_n$ are weakly alternating.
\end{proof}

\begin{example}{Using Descartes' rule of signs to check if a matrix is positive definite}{descartes-rule-of-signs1}
  Use Descartes' rule of signs to check whether the matrix
  \begin{equation*}
    A = \begin{mymatrix}{ccc}
      5 & 0 & 2 \\
      0 & 5 & 1 \\
      2 & 1 & 1 \\
    \end{mymatrix}
  \end{equation*}
  is positive definite, positive semidefinite, or neither.
\end{example}

\begin{solution}
  We already found the characteristic polynomial in
  Example~\ref{exa:characterize-positive}: it is $-\eigenvar^3 +
  11\eigenvar^2 - 30\eigenvar$. The coefficients are $a_0=0$,
  $a_1=-30$, $a_2=11$, and $a_3=-1$. Note that we have included all of
  the coefficients, even ones that are zero. Since the signs are
  weakly alternating, but not strongly alternating, the matrix is
  positive semidefinite, but not positive definite.
\end{solution}

\begin{example}{Using Descartes' rule of signs to check if a matrix is positive definite}{descartes-rule-of-signs2}
  Use Descartes' rule of signs to determine which of the following
  matrices are positive definite and/or positive semidefinite.
  \begin{equation*}
    A = \begin{mymatrix}{ccc}
      2 & 1 & 2 \\
      1 & 1 & 0 \\
      2 & 0 & 2 \\
    \end{mymatrix},
    \quad
    B = \begin{mymatrix}{ccc}
      2  &  1 & -2 \\
      1  &  1 & -1 \\
      -2 & -1 &  2 \\
    \end{mymatrix},
    \quad
    C = \begin{mymatrix}{ccc}
      2  & -1 & 2 \\
      -1 &  2 & 0 \\
      2  &  0 & 3 \\
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  The characteristic polynomials are:
  \begin{equation*}
    \begin{array}{lcl}
      \det(A-\eigenvar I) &=& -\eigenvar^3 + 5\eigenvar^2 - 3\eigenvar - 2, \\
      \det(B-\eigenvar I) &=& -\eigenvar^3 + 5\eigenvar^2 - 2\eigenvar + 0, \\
      \det(C-\eigenvar I) &=& -\eigenvar^3 + 7\eigenvar^2 - 11\eigenvar + 1. \\
    \end{array}
  \end{equation*}
  For $A$, the coefficients are not weakly alternating, so $A$ is not
  positive semidefinite. For $B$, the coefficients are weakly, but not
  strongly alternating (note that $a_0=0$), and therefore $B$ is
  positive semidefinite, but not positive definite. For $C$, the
  coefficients are strongly alternating, so $C$ is positive definite.
\end{solution}


\subsection{Characteristic functions}

\subsubsection{Transformations}

\subsubsection{Summary}

Cumulative probability function

\(F=\int_{-\infty }^\infty xP(x)\)

Moment generating function

\(F=\int_{-\infty }^\infty e^{tx}P(x)\)

Characteristic function

\(F=\int_{-\infty }^\infty e^{itx}P(x)\)

\subsubsection{Moment generating function}

Take random variable \(X\). This has moments we wish to calculate.

We can transform our function in other forms which maintain all of the required information. For example we could also use the cumulative probability function to calculate moments. We now look for an alternative form of the probability density function which allows us to easily calculate moments.

One method is to use the probability density function and the definitions of moments, but there are other options. For example, consider the function:

\(E[e^{tX}]\)

Which expands to:

\(E[e^{tX}]=\sum_{j=1}^\infty \dfrac{t^jE[X^j]}{j!}\)

By taking the \(m\)th derivative of this, we get

\(E[X^m]+\sum_{j=m+1}^\infty \dfrac{t^jE[X^j]}{j!}\)

We can then set \(t=0\) to get

\(E[X^m]\)

Alternatively, see that differentiating \(m\) times gets us

\(E[X^me^{tX}]\)

If we can get this function, we can then easily generate moments.

The function we need to get is:

\(E[e^{tX}]\)

In the discrete case this is:

\(E[e^{tX}]=\sum_{i=1}e^{tx_i}p_i\)

In the continuous case:

\(E[e^{tX}]=\int_{-\infty }^\infty e^{tx}P(x) dx\)

\subsubsection{Characteristic function}

It may not be possible to calculate the integral for the moment generating function. We now look for an alternative formula with which we can generate the same moments.

Consider

\(E[e^{itX}]\)

As this can be broken down into sinusoidal functions it can more readily be integrated.

This expands to

\(E[e^{itX}]=\sum_{j=1}^\infty \dfrac{i^jt^jE[X^j]}{j!}\)

By taking the \(m\)th derivative we get.

\(E[X^m]i^m+\sum_{j=m+1}^\infty \dfrac{t^jE[X^j]}{j!}\)

By setting \(t=0\) we then get:

\(E[X^m]i^m\)

Alternatively see that differentiating \(m\) times gets us

\(E[(iX)^me^{itX}]\)

So we can get the moment by differentiating \(m\) times, and multiplying by \(i^{-m}\).

\subsubsection{Inverses of these functions}

Moment generating function

Characteristic function

\subsubsection{Moments of constants added to variables}

\(\phi_{X+c}(t)=E[e^{it(X+c)}]\)

\(\phi_{X+c}(t)=E[e^{itX}e^itc]\)

\(\phi_{X+c}(t)=e^{itc}E[e^{itX}]\)

\(\phi_{X+c}(t)=e^{itc}\phi_X(t)\)

\(\phi_{X}(t)=e^{-itc}\phi_{X+c}(t)\)

\subsubsection{Moments of constants multiplied by events}

\(\phi_{cX}(t)=E[e^{itcX}]\)

\(\phi_{cX}(t) = \phi_{X}(ct)\)

\subsubsection{Taylor series of a characteristic function}

\(\phi_X(t)=E[e^{itX}]\)

\(\phi_X(t)=\sum_{j=0}^{\infty }\dfrac{\phi_X^j(a)(t-a)}{j!}\)

Around \(a=0\)

\(\phi_X(t)=\sum_{j=0}^{\infty }\dfrac{\phi_X^j(0)(t)}{j!}\)

The characteristic function is now given in terms of its moments.

We know:

\(\phi_X^j(0)=E[X^j]i^j\)

So:

\(\phi_X(t)=\sum_{j=0}^{\infty }\dfrac{E[X^j]i^j(t)^j}{j!}\)

\(\phi_X(t)=\sum_{j=0}^{\infty }\dfrac{E[X^j](it)^j}{j!}\)

We know:

\(\dfrac{E[X^0](it)^0}{0!}=E[1]=1\)

\(\dfrac{E[X^1](it)^1}{1!}=E[X](it)=it\mu_X \)

\(\dfrac{E[X^2](it)^2}{2!}=\dfrac{-E[X^2]t^2}{2}=\dfrac{-(\mu_X +\sigma_X^2 )t^2}{2}\)

So:

\(\phi_X(t)=1+it\mu_X -\dfrac{(\mu_X +\sigma_X^2 )t^2}{2} +\sum_{j=3}^{\infty }\dfrac{E[X^j](it)^j}{j!}\)


\section{Derive Fourier series}

Though it was originally invented to solve the \textbf{heat equation}, this
series has multiple uses: evaluating $\zeta(s)$\footnote{Examples are found in
other sections of this collection}, extracting frequences from a wave, and
representing periodic functions by trigonomic functions. To derive Fourier
series, I start with representing functions by sines.

\begin{equation}
	f(t)=A_0+\sum_{n=1}^\infty A_n\sin({2\pi nt\over T}+\phi_n)
	\label{fourier-orig}
\end{equation}

In equation (\ref{fourier-orig}), $t$ is the independent variable, and $A_n$ is
the amplitude of cosine wave with period of $T$. $\phi_n$ denotes the phase of
each wave. Before performing any transformations, these formulas are used:

\begin{equation}
	\sin(\alpha+\beta)=\sin\alpha\cos\beta+\sin\beta\cos\alpha
	\label{sin-angle-sum}
\end{equation}
\begin{equation}
	\cos\theta=\Re(e^{i\theta})={e^{i\theta}+e^{-i\theta}\over2}
	\label{euler-cos}
\end{equation}
\begin{equation}
	\sin\theta={e^{i\theta}-e^{-i\theta}\over2i}
	=-i\cdot{e^{i\theta}-e^{-i\theta}\over2}
	\label{euler-sin}
\end{equation}

For equation (\ref{euler-cos}) and (\ref{euler-sin}), section
\ref{eulers-formula-derivation} can be referred. To simplify the formula, we
can apply the rule in equation (\ref{sin-angle-sum}) to (\ref{fourier-orig}).

\begin{equation}
	f(t)=A_0+\sum_{n=1}^\infty\left[A_n\sin\left(2\pi nt\over T\right)
	\cos\phi_n+\sin\phi_n\cos\left(2\pi nt\over T\right)\right]
	\label{fourier-angle-sum}
\end{equation}
\begin{equation}
	a_n\triangleq A_n\sin\phi_n, b_n\triangleq A_n\cos\phi_n
	\label{fourier-subst}
\end{equation}

If we were to apply the substitution in (\ref{fourier-subst}) to equation
(\ref{fourier-angle-sum}) can be tidier, and solving for $a_n$ and $b_n$ is a
much simpler problem.

\begin{equation}
	f(t)=A_0+\sum_{n=1}^\infty\left[a_n\cos\left(2\pi nt\over T\right)
	+b_n\sin\left(2\pi nt\over T\right)\right]
	\label{fourier-real}
\end{equation}

Though it is possible, solving for two variables is still a tough problem;
however, when we complexify the function using the rule (\ref{euler-cos}) and
(\ref{euler-sin}), it can be turned into a one-variable problem.

$$
\begin{aligned}
	f(t)
	&=A_0+\sum_{n=1}^\infty\left(
	a_n\cdot{e^{2\pi int\over T}+e^{-{2\pi int\over T}}\over2}
	-ib_n\cdot{e^{2\pi int\over T}-e^{-{2\pi int\over T}}\over 2}\right) \\
	&=A_0+\sum_{n=1}^\infty\left[
	\left(a_n-ib_n\over 2\right)e^{2\pi int\over T}
	+\left(a_n+ib_n\over 2\right)e^{-{2\pi int\over T}}
	\right]
\end{aligned}
$$

\begin{equation}
	c_n=
	\begin{cases}
		A_0 & n=0 \\
		(a_n-ib_n)/2 & n>0 \\
		(a_{-n}+ib_{-n})/2 & n<0
	\end{cases}
	\label{complexify}
\end{equation}

Now, substituting $A_0$, $a_n$, and $b_n$ with $c_n$ according to
(\ref{complexify}) is the most brilliant step. The whole function becomes
expressed as follows:

\begin{equation}
	f(t)=\sum_{n=0}^\infty c_ne^{2\pi int\over T}
	+\sum_{n=1}^\infty c_{-n}e^{-{2\pi int\over T}}
	=\sum_{n=-\infty}^\infty c_ne^{2\pi int\over T}
	\label{fourier-complex}
\end{equation}

By integrating $f(t)$ over one of its period, we find the exact value of each
$c_n$.

$$
\int_{t_0}^{t_0+T}f(t)e^{-{2\pi ikt\over T}}dt
=\int_{t_0}^{t_0+T}\sum_{n=-\infty}^\infty c_ne^{2\pi i(n-k)t\over T}
=\sum_{n=-\infty}^\infty c_n\int_{t_0}^{t_0+T}e^{2\pi i(n-k)t\over T}
$$

We presume that $k\in\mathbb{Z}$, and the integral of each element in the
series produces different results. For the element where $k=n$, the result
conforms to equation (\ref{eq-k-case}). Otherwise the calculation follows
(\ref{neq-k-case}).

\begin{equation}
	c_k\int_{t_0}^{t_0+T}dt=c_k(t_0+T-t_0)=c_kT
	\label{eq-k-case}
\end{equation}
\begin{equation}
	\begin{aligned}
		c_n\int_{t_0}^{t_0+T}e^{2\pi i(n-k)t\over T}dt
		&=c_n\left[Te^{2\pi i(n-k)t\over T}\over2\pi i(n-k)\right]
		^{t_0+T} \\
		&={c_nT\over2\pi i(n-k)}\cdot e^{2\pi i(n-k)t_0\over T}
		(e^{2\pi i(n-k)}-1)=0
	\end{aligned}
	\label{neq-k-case}
\end{equation}

As a result, $c_n$ is calculated via:

\begin{equation}
	c_n={1\over T}\int_{t_0}^{t_0+T}f(t)e^{-{2\pi int\over T}}dt
	\label{eval-cn}
\end{equation}

According to the substitution scheme in (\ref{complexify}), we obtain $a_n$ and
$b_n$ via:

\begin{equation}
	a_n=c_n+c_{-n}={2\over T}\int_{t_0}^{t_0+T}f(t)
	\left(e^{-{2\pi int\over T}}+e^{2\pi int\over T}\over 2\right)dt
	={2\over T}\int_{t_0}^{t_0+T}f(t)\cos\left(2\pi nt\over T\right)dt
	\label{eval-an}
\end{equation}
\begin{equation}
	b_n={c_{-n}-c_n\over i}={2\over T}\int_{t_0}^{t_0+T}f(t)
	\left(e^{2\pi int\over T}-e^{-{2\pi int\over T}}\over2i\right)dt
	={2\over T}\int_{t_0}^{t_0+T}f(t)\sin\left(2\pi nt\over T\right)dt
	\label{eval-bn}
\end{equation}

According to (\ref{complexify}) and (\ref{eval-cn}), $A_0$ is calculated by:

$$
A_0={1\over L}\int_{t_0}^{t_0+T}f(t)dt
={1\over2}\cdot{2\over L}\int_{t_0}^{t_0+T}f(t)
\cos\left(2\pi(0)t\over T\right)dt
={a_0\over2}
$$

Therefore, we can set $A_0$ to half of $a_0$, so the (\ref{fourier-real}) can
be transformed into the following where $a_n$ and $b_n$ are computed by
(\ref{eval-an}) and (\ref{eval-bn}), respectively.

$$
f(t)={a_0\over2}+\sum_{n=1}^\infty\left[a_n\cos\left({2\pi nt\over T}\right)
+b_n\sin\left({2\pi nt\over T}\right)\right]
$$


\section{Vector Functions for Surfaces}\label{sec:VectorFunctionSurfaces}

We have dealt extensively with vector equations for curves,
$\vect{r}(t)=\langle x(t),y(t),z(t)\rangle$. A similar technique
can be used to represent surfaces in a way that is more general than
the equations for surfaces we have used so far. Recall that
when we use $\vect{r}(t)$ to represent a curve, we imagine the vector
$\vect{r}(t)$ with its tail at the origin, and then we follow the head
of the arrow as $t$ changes. The vector ``draws'' the curve through
space as $t$ varies.

Suppose we instead have a vector function of two variables,
$$\vect{r}(u,v)=\langle x(u,v),y(u,v),z(u,v)\rangle.$$
As both $u$ and
$v$ vary, we again imagine the vector $\vect{r}(u,v)$ with its tail at
the origin, and its head sweeps out a surface in space. A useful
analogy is the technology of CRT video screens, in which an electron
gun fires electrons in the direction of the screen. The gun's
direction sweeps horizontally and vertically to ``paint'' the screen
with the desired image. In practice, the gun moves horizontally
through an entire line, then moves vertically to the next line and
repeats the operation. In the same way, it can be useful to imagine
fixing a value of $v$ and letting $\vect{r}(u,v)$ sweep out a curve as
$u$ changes. Then $v$ can change a bit, and $\vect{r}(u,v)$ sweeps out
a new curve very close to the first. Put enough of these curves
together and they form a surface.

\begin{example}{}{}
Consider the function $\vect{r}(u,v)=\langle v\cos u,v\sin u,
v\rangle$. For a fixed value of $v$, as $u$ varies from 0 to $2\pi$,
this traces a circle of radius $v$ at height $v$ above the
$x$-$y$ plane. Put lots and lots of these together,and they form a
cone, as in Figure~\ref{fig:parametric cone}.
\end{example}

\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <3truecm,3truecm>
\setplotarea x from -1 to 1, y from 0 to 1
\put {\hbox{\epsfxsize6cm\epsfbox{images/cone_level_curves.eps}}} at 0 0
\put {\hbox{\epsfxsize6cm\epsfbox{images/parametric_cone.eps}}} at 2.5 0
\endpicture}}
\caption{Tracing a surface. \label{fig:parametric cone}}
\end{figure}

Recall that from the vector equation of the curve we can compute the
unit tangent $\vect{T}$, the unit normal $\vect{N}$, and the
binormal vector $\vect{B}=\vect{T}\times\vect{N}$; you may
want to review Section~\ref{sec:arc length 3D}. The binormal is
perpendicular to both $\vect{T}$ and $\vect{N}$; one way to interpret this
is that $\vect{N}$ and $\vect{B}$ define a plane perpendicular to $\vect{T}$, that is, perpendicular to the curve; since $\vect{N}$ and $\vect{B}$
are perpendicular to each other, they can function just as $\vect{i}$ and
$\vect{j}$ do for the $x$-$y$ plane.

\begin{example}{}{}
The curve given by
$$\vect{r}=\langle (2+\cos(3u/2))\cos u,
(2+\cos(3u/2))\sin u, \sin(3u/2)\rangle$$
 is called a trefoil knot.
Then $\vect{c}(v)=\vect{N}\cos v+\vect{B}\sin v$ is a vector equation for a unit circle in a plane
perpendicular to the curve described by $\vect{r}$, except that the usual
interpretation of $\vect{c}$ would put its center at the origin. We can
fix that simply by adding $\vect{c}$ to the original $\vect{r}$:
let $\vect{f}=\vect{r}(u) +\vect{c}(v)$. For a
fixed $u$ this draws a circle around the point $\vect{r}(u)$; as $u$
varies we get a sequence of such circles around the curve $\vect{r}$,
that is, a tube of radius 1 with $\vect{r}$ at its center. We can easily
change the radius; for example $\vect{r}(u) +a\vect{c}(v)$
gives the tube radius $a$; we can make the radius vary as we
move along the curve with $\vect{r}(u) +g(u)\vect{c}(v)$,
where $g(u)$ is a function of $u$.
\end{example}

As shown in
Figure~\ref{fig:trefoil}, it is hard to see that the plain knot is
knotted; the tube makes the structure apparent. Of course, there is
nothing special about the trefoil knot in this example; we can put a
tube around (almost) any curve in the same way.

\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <2.5truecm,2.5truecm>
\setplotarea x from -1 to 1, y from 0 to 1
\put {\hbox{\epsfxsize4.17cm\epsfbox{images/trefoil.eps}}} at 0 0
\put {\hbox{\epsfxsize5cm\epsfbox{images/trefoil_tube.eps}}} at 2.5 0
\put {\hbox{\epsfxsize5cm\epsfbox{images/trefoil_sausage.eps}}} at 5 0
\endpicture}}
\caption{Tubes around a trefoil knot, with radius $1/2$ and $3\cos(u)/4$. \label{fig:trefoil}}
\end{figure}

We have previously examined surfaces given in the form
$f(x,y)$. It is sometimes useful to represent such surfaces in the
more general vector form, which is quite easy:
$\vect{r}(u,v)=\langle u,v,f(u,v)\rangle$. The names of the variables
are not important of course; instead of disguising $x$ and $y$, we
could simply write $\vect{r}(x,y)=\langle x,y,f(x,y)\rangle$.

We have also previously dealt with surfaces that are not functions of
$x$ and $y$; many of these are easy to represent in vector form.
One common type of surface that cannot be represented as $z=f(x,y)$ is
a surface given by an equation involving only $x$ and $y$. For
example,
$x+y=1$ and $y=x^2$ are ``vertical'' surfaces. For every point $(x,y)$
in the plane that satisfies the equation, the point $(x,y,z)$ is on
the surface, for
every value of $z$. Thus, a corresponding vector form for the surface
is something like $\langle f(u),g(u),v\rangle$; for example, $x+y=1$
becomes $\langle u,1-u,v\rangle$ and $y=x^2$ becomes $\langle
u,u^2,v\rangle$.

Yet another sort of example is the sphere, say $x^2+y^2+z^2=1$. This
cannot be written in the form $z=f(x,y)$, but it is easy to write in
vector form; indeed this particular surface is much like the cone,
since it has circular cross-sections, or we can think of it as a tube
around a portion of the $z$-axis, with a radius that varies depending
on where along the axis we are. One vector expression for the sphere
is $\langle \sqrt{1-v^2}\cos u,\sqrt{1-v^2}\sin u, v\rangle$---this
emphasizes the tube structure, as it is naturally viewed as drawing a
circle of radius $\sqrt{1-v^2}$ around the $z$-axis at height $v$.
We could also take a cue from spherical coordinates, and write
$\langle \sin u\cos v,\sin u\sin v,\cos u\rangle$, where in effect
$u$ and $v$ are $\phi$ and $\theta$ in disguise.

It is quite simple to use a computer program to plot any surface for which you have a
vector representation. Using different vector functions sometimes
gives different looking plots, because the computer in effect draws the
surface by holding one variable constant and then the other.

Here's a simple but striking example: the plane $x+y+z=1$ can be
represented quite naturally as $\langle u,v,1-u-v\rangle$. However we
could also think of painting the same plane by choosing a particular
point on the plane, say $(1,0,0)$, and then drawing circles or
ellipses (or any of a number of other curves) as if that point were
the origin in the plane. For example, $\langle 1-v\cos u-v\sin u,v\sin
u,v\cos u\rangle$ is one such vector function. Note that while it may
not be obvious where this came from, it is quite easy to see that
the sum of the $x$, $y$, and $z$ components of the vector is always
1. Computer renderings of the plane using these two functions are shown
in Figure~\ref{fig:two plane parameterizations}.

\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <2.5truecm,2.5truecm>
\setplotarea x from -1 to 1, y from 0 to 1
\put {\hbox{\epsfxsize7cm\epsfbox{images/plane1.eps}}} at 0 0
\put {\hbox{\epsfxsize5cm\epsfbox{images/plane2.eps}}} at 3 0
\endpicture}}
\caption{Two representations of the same plane. \label{fig:two plane parameterizations}}
\end{figure}

Suppose we know that a plane contains a particular point
$(x_0,y_0,z_0)$ and that two vectors $\vect{u}=\langle
u_0,u_1,u_2\rangle$ and $\vect{v}=\langle
v_0,v_1,v_2\rangle$ are parallel to the plane but not to each
other. We know how to get an equation for the plane in the form
$ax+by+cz=d$, by first computing $\vect{u}\times\vect{v}$. It's even
easier to get a vector equation:
$$\vect{r}(u,v) = \langle x_0,y_0,z_0\rangle + u\vect{u} + v\vect{v}.$$
The first vector gets to the point $(x_0,y_0,z_0)$ and then by varying
$u$ and $v$, $u\vect{u} + v\vect{v}$ gets to every point in the plane.

Returning to $x+y+z=1$, the points $(1,0,0)$, $(0,1,0)$, and $(0,0,1)$
are all on the plane. By subtracting coordinates we see that
$\langle -1,0,1\rangle$ and $\langle -1,1,0\rangle$ are parallel to
the plane, so a third vector form for this plane is
$$\langle 1,0,0\rangle + u\langle -1,0,1\rangle + v\langle
-1,1,0\rangle
= \langle 1-u-v,v,u\rangle.$$
This is clearly quite similar to the first form we found.

We have already seen (Section~\ref{sec:surfacearea3D})
how to find the area of a surface when it is
defined in the form $f(x,y)$. Finding the area when the surface is
given as a vector function is very similar. Looking at the plots of
surfaces we have just seen, it is evident that the two sets of curves
that fill out the surface divide it into a grid, and that the spaces
in the grid are approximately parallelograms. As before this is the
key: we can write down the area of a typical small parallelogram and
add them all up with an integral.

Suppose we want to approximate the area of the surface $\vect{r}(u,v)$
near $\vect{r}(u_0,v_0)$. The functions $\vect{r}(u,v_0)$ and $\vect{r}(u_0,v)$ define two curves that intersect at $\vect{r}(u_0,v_0)$. The derivatives of $\vect{r}$ give us vectors tangent to
these two curves: $\vect{r}_u(u_0,v_0)$ and $\vect{r}_v(u_0,v_0)$, and
then $\vect{r}_u(u_0,v_0)\,du$ and $\vect{r}_v(u_0,v_0)\,dv$
are two small tangent vectors, whose lengths can be used as the
lengths of the sides of an approximating parallelogram. Finally, the
area of this parallelogram is $|\vect{r}_u\times\vect{r}_v|\,du\,dv$ and
so the total surface area is
\begin{equation} \label{eq:surfaceareaintegral}
\int_a^b\int_c^d |\vect{r}_u\times\vect{r}_v|\,du\,dv
\end{equation}

\begin{example}{}{surfacearea}
Find the area of the surface $\langle v\cos u,v\sin
u,v\rangle$ for $0\le u \le 2\pi$ and  $0\le v\le \frac{1}{2}$.
\end{example}

\begin{solution}
We compute $\vect{r}_u = \langle -v\sin u,v\cos u,0\rangle$
and $\vect{r}_v = \langle \cos u,\sin u,1\rangle$.
The cross product of these two vectors is $\langle v \cos u,v\sin
u,-v\rangle$ with length $\sqrt{2v^2}=\sqrt{2}v$, and the surface area is
$$\int_0^\pi\int_0^1 \sqrt{2}v\,dv\,du=\frac{\sqrt{2}\pi}{4}.$$
\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:VectorFunctionSurfaces}}

\begin{enumialphparenastyle}

\begin{ex} \label{exfix3}
Describe or sketch the surface with the given vector function.
\begin{enumerate}
  \item	$\vect{r}(u,v) = \langle 6v-u,3+v,1-4v \rangle$
  \item	$\vect{r}(u,v) = \langle 2\cos u, 5\cos u, v \rangle$
  \item	$\vect{r}(s,t) = \langle u+v,2u+v,u^2+v^2\rangle$
  \item	$\vect{r}(s,t) = \langle \sin u + \cos u, u, v \rangle$
\end{enumerate}
\end{ex}

\begin{ex}
Find the area of the portion of $x+2y+4z=10$ in the first octant.
\begin{sol}
	$25\sqrt{21}/4$
\end{sol}
\end{ex}

\begin{ex}
Find the area of the portion of $2x+4y+z=0$ inside $x^2+y^2=1$.
\begin{sol}
	$\pi\sqrt{21}$
\end{sol}
\end{ex}

\begin{ex}
Find the area of $z=x^2+y^2$ that lies below $z=1$.
\begin{sol}
	$\pi(5\sqrt5-1)/6$
\end{sol}
\end{ex}

\begin{ex}
Find the area of $z=\sqrt{x^2+y^2}$ that lies below $z=2$.
\begin{sol}
	$4\pi\sqrt2$
\end{sol}
\end{ex}

\begin{ex}
Find the area of the portion of $x^2+y^2+z^2=a^2$ that lies
in the first octant.
\begin{sol}
	$\pi a^2/2$
\end{sol}
\end{ex}

\begin{ex}
Find the area of the portion of $x^2+y^2+z^2=a^2$ that lies
above $x^2+y^2\le b^2$.
\begin{sol}
	$2\pi a(a-\sqrt{a^2-b^2})$
\end{sol}
\end{ex}

\begin{ex}
Find the area of $z=x^2-y^2$ that lies inside $x^2+y^2=a^2$.
\begin{sol}
	$\pi((1+4a^2)^{3/2}-1)/6$
\end{sol}
\end{ex}

\begin{ex}
Find the area of $x^2+y^2+z^2=a^2$
that lies above the interior of the circle given in polar coordinates
by $r=a\cos \theta$.
\begin{sol}
	$\pi a^2-2a^2$
\end{sol}
\end{ex}

\begin{ex}
Find the area of the cone $z=k\sqrt{x^2+y^2}$
that lies above the interior of the circle given in polar coordinates
by $r=a\cos \theta$.
\begin{sol}
	$\pi a^2\sqrt{1+k^2}/4$
\end{sol}
\end{ex}

\begin{ex}
Find the area of the plane $z=ax+by+c$ that lies over a
region $D$ with area $A$.
\begin{sol}
	$A\sqrt{1+a^2+b^2}$
\end{sol}
\end{ex}

\begin{ex}
Find the area of the cone $z=k\sqrt{x^2+y^2}$ that lies over a
region $D$ with area $A$.
\begin{sol}
	$A\sqrt{k^2+1}$
\end{sol}
\end{ex}

\begin{ex}
Find the area of the cylinder $x^2+z^2=a^2$ that lies inside
the cylinder $x^2+y^2=a^2$.
\begin{sol}
	$8a^2$
\end{sol}
\end{ex}

\begin{ex}
The surface $f(x,y)$ can be represented with the vector
function $\langle x,y,f(x,y)\rangle$. Set up the surface area integral using
this vector function and compare to the integral of
Section~\ref{sec:surfacearea3D}.
\end{ex}

\end{enumialphparenastyle}


\lab{Algorithms}{Introduction to Wavelets}{Intro to Wavelets}

\objective{This section explains the basic ideas of Wavelet Analysis
using the Haar wavelet as a prototypical example.}

Recall that in the context of Fourier analysis, one seeks to represent a
function in the frequency domain, and this is accomplished via the Fourier
transform. The Fourier transform allows us to analyze and process functions
in many useful ways, as you have seen in previous labs. There are, however,
drawbacks to this approach. For example, although a function's Fourier
transform gives us complete information on its frequency spectrum, time
information is lost. We can know which frequencies are the
most prevalent, but not when they occur. This is due in part to the fact that
the sinusoidal function $f(x) = e^{2\pi ix}$ -- on which the Fourier transform
is based -- has infinite support. Its nature is essentially \emph{non-local},
and so the Fourier transform fails to provide local information in both the
time and frequency domains. This brings us to the following question: are
there types of transforms that avoid the shortcomings mentioned above? The
answer is an emphatic yes. Enter Wavelet analysis.

\section*{The Haar Wavelet}

As noted earlier, the Fourier transform is based on the complex exponential
function. Let us alter the situation and consider instead the following
function, known as the \emph{Haar wavelet}:
\begin{equation*}
\psi(x) =
 \begin{cases}
  1 & \text{if } 0 \leq x < \frac{1}{2} \\
  -1 & \text{if } \frac{1}{2} \leq x < 1 \\
  0 & \text{otherwise.}
 \end{cases}
\end{equation*}

% It might be nice to plot this function and include the image in the lab.

Along with this wavelet, we introduce the associated \emph{scaling function}:
\begin{equation*}
\phi(x) =
 \begin{cases}
 1 & \text{if } 0 \leq x < 1 \\
 0 & \text{otherwise.}
 \end{cases}
\end{equation*}

From the wavelet and scaling function, we can generate two countable families
of dyadic dilates and translates given by
\begin{equation*}
\psi_{m,k}(x) = \psi(2^mx - k)
\end{equation*}
\begin{equation*}
\phi_{m,k}(x) = \phi(2^mx - k),
\end{equation*}
where $m,k \in \mathbb{Z}$.

Let us focus for the moment on that second family of functions, $\{\phi_{m,k}\}$.
If we fix $m$ and let $k$ vary over the integers, we have a countable collection of
simple functions. The support of a typical function $\phi_{m,k}$ is the interval
$[k2^{-m}, (k+1)2^{-m}]$, and for any $m \in \mathbb{Z}$ we have
\begin{equation*}
\mathbb{R} = \displaystyle\biguplus_k\,[k2^{-m}, (k+1)2^{-m}],
\end{equation*}
where $\uplus$ denotes a union over disjoint sets. Thus, the supports can be viewed as
a discretization of the real line, and we can use this collection of simple functions
to approximate any $f \in L^2(\mathbb{R})$ in the following sense:
\begin{equation*}
f(x) \approx f_m(x) := \displaystyle\sum_{k \in \mathbb{Z}}\alpha_{m,k}\phi_{m,k}(x),
\end{equation*}
where
\begin{equation*}
\alpha_{m,k} := 2^m \displaystyle \int_{k2^{-m}}^{(k+1)2^{-m}}f(x) dx
\end{equation*}
($\alpha_{m,k}$ is simply the average value of $f$ on $[k2^{-m},(k+1)2^{-m}]$). As you
would probably expect, the point-wise error between $f$ and its approximation $f_m$
(called a \emph{frame}) goes to zero as $m \to \infty$.

These frames are not quite good enough, however. Each coefficient $\alpha_{m,k}$
certainly captures local information about $f$ -- namely its average value on
a certain interval -- but it fails to tell us anything about how $f$ changes
on that interval. We need more information than is provided by $f_m$ in order
to know about discontinuities or high-frequency oscillations of $f$. To this end,
we now consider the wavelet function $\psi$.
Notice that the Haar wavelet is oscillatory in nature, and is thus better suited
to capture local information on how a function changes at a given point. For
any given $m$, we define a function $d_m$, called a \emph{detail}, as follows:
\begin{equation*}
d_m(x) := \displaystyle\sum_{k \in \mathbb{Z}}\beta_{m,k}\psi_{m,k}(x),
\end{equation*}
where
\begin{equation*}
\beta_{m,k} := 2^m \displaystyle \int_{-\infty}^{\infty}f(x) \psi_{m,k}(x) dx.
\end{equation*}
Each coefficient $\beta_{m,k}$ gives information about how $f$ changes on the
the interval $[k2^{-m}, (k+1)2^{-m}]$, and larger coefficients correspond
to larger spikes of width $2^{-m}$. Thus, as $m$ increases, the
detail function $d_m$ gives information about the higher-frequency oscillations
of the function. The details and approximation frames interact in the following way:
\begin{equation*}
f_{m+1} = f_m + d_m.
\end{equation*}
As a result of this fortuitous relationship, one can prove the decomposition
\begin{equation*}
L^2(R) = V_0 \oplus W_0 \oplus W_1 \oplus \cdots,
\end{equation*}
where $V_j := \text{span}\{\phi_{j,k}\}_{k \in \mathbb{Z}}$ and
$W_j := \text{span}\{\psi_{j,k}\}_{k \in \mathbb{Z}}$. This fact justifies
our hope to approximate and analyze functions using wavelets.
\begin{figure}[t]
\minipage{0.32\textwidth}
    \includegraphics[width=\linewidth]{sinecurve}
    \caption{$f(x) = \sin (x)$}
\endminipage\hfill
\minipage{0.32\textwidth}
    \includegraphics[width=\linewidth]{discreteSineCurve.pdf}
    \caption{$f_4$}
\endminipage\hfill
\minipage{0.32\textwidth}
    \includegraphics[width=\linewidth]{sineCurveDetail}
    \caption{$d_4$}
\endminipage
\end{figure}
\begin{problem}
Calculate and plot the approximation frames for $f(x) = \sin(x)$ on the interval $[0,2\pi]$
for $m = 4, 6, 8$. Note that because we are working on a finite interval,
we only need to calculate certain coefficients $\alpha_{m,k}$. In
particular, we only need the coefficients for $k = 0$ up to the first integer
$n$ such that $(n+1)2^{-m} > 2 \pi$ (why?). Furthermore, to plot the frame,
all we need is an array containing the relevant coefficients. Then simply plot
the coefficients against \li{linspace} with appropriate arguments
and set \li{drawstyle='steps'} in the \li{plt.plot} function.
\end{problem}

\begin{problem}
Now calculate the details for $f(x) = \sin(x)$ on the same interval and for the
same $m$ values given above. Use previous results to compute the coefficients
for $f_5$, $f_7$, and $f_9$ and plot them.
\end{problem}

\section*{The Discrete Wavelet Transform}

What purpose do these details and approximation frames serve? According to the
properties discussed above, we can approximate $L^2$ functions as follows:
\begin{align*}
f \approx f_{J+1} &= f_J + d_J \\
&= f_{J-1} + d_{J-1} + d_J \\
& \ldots\\
&= f_{I} + d_{I} + d_{I+1} + \cdots + d_J,
\end{align*}
where $1 \leq I \leq J$. If $f$ has compact support (as in the case of a finite-time signal,
for example), only finitely many of the coefficients in the frame and the details are
nonzero, thus enabling us to represent $f$ to a reasonable degree of accuracy in a very
efficient manner. The calculation of these detail coefficients is called the \emph{discrete
wavelet transform}. In the context of signals processing, one can imagine calculating these
coefficients, transmitting them, and then reproducing the approximated signal on the
receiving end. Furthermore, the coefficients of the details reflect the local properties
of the original function $f$ at the particular level of detail and resolution! This means
that we can discard many of the coefficients if we are only interested in reproducing a certain
part of the signal, or in recovering the entire signal to only a limited resolution. We can
also study just those frequencies of the signal that fall within a certain range (called a
sub-band) by examining the detail coefficients at a particular level. These
properties make the discrete wavelet transform an attractive alternative to the Fourier
transform in many applications.

In practice, we are often interested in analyzing discrete signals with compact support (that is,
finite-time signals that we have sampled at a finite number of points). If wavelet analysis is
to be of any use, we first need an efficient way to calculate the discrete wavelet transform.
The process described in the first section, while intuitive and illustrative of the mathematical
principles
behind wavelet analysis, is not the best approach to calculating the wavelet coefficients. It
turns out that the discrete wavelet transform can be implemented as an iterated low-pass/high-
pass filter bank, one iteration of which is shown graphically in the figure. We present the
algorithm without getting into the details of why it works.
\begin{figure}[H]
\centering
\includegraphics[width = 0.5\textwidth]{dwt1}
\caption{The one-dimensional discrete wavelet transform.}
\end{figure}
The input, $A_j$, represents the level-$j$ approximation frame, and we initialize $A_0$ to
simply be the original signal. Lo and Hi are the low-pass and high-pass filters, respectively.
(By \emph{filter} we mean a vector that serves the purpose of extracting or suppressing a
particular feature of the signal. The Lo and Hi filters are obtained from the wavelet at hand;
for the Haar wavelet, Lo $= (\sqrt{2}^{-1}, \sqrt{2}^{-1})$ and Hi $= (-\sqrt{2}^{-1}, \sqrt{2}
^{-1})$.) The box means convolve the input with the filter, and the circle means downsample by
a factor of two, i.e. remove either the even or odd-indexed entries of the input. The outputs,
$A_{j+1}$ and $D_{j+1}$, are the level-$(j+1)$ approximation frame and detail coefficients,
respectively. Note that the length of the input array is twice that of the output arrays. The
detail coefficients $D_{j+1}$ are stored, and $A_{j+1}$ is then fed back into the loop. Continue
this process until the length of the output is less than the length of the filters, and
return all of the stored detail coefficients as well as the final approximation frame.
\begin{problem}
Write a function that calculates the discrete wavelet transform as described above.
The inputs should be three one-dimensional NumPy arrays (the signal, low-pass filter, and
high-pass filter). The output should be a list of one-dimensional NumPy arrays in the
following form: $[A_n, D_n, D_{n-1},\ldots,D_1]$. (Note: for the convolution, you may use
the \li{fftconvolve} function from the \li{scipy.signal} package using the default
\li{mode = 'full'} parameter, but note that the output array is one entry too large, and so
you need to omit the first entry. For downsampling, only keep the even-indexed entries.)
\end{problem}
We also need to know how to reconstruct the signal from the detail coefficients and
approximation frame. Fortunately, the algorithm described above is entirely reversible,
albeit with slightly different filters: Lo $= (\sqrt{2}^{-1}, \sqrt{2}^{-1})$ and
Hi $= (\sqrt{2}^{-1}, -\sqrt{2}^{-1})$.
Given $A_{j+1}$ and $D_{j+1}$, simply upsample both arrays (by inserting a zero after
each entry of the original array), convolve the results
with the Lo and Hi filters, respectively (this time omit the \emph{last} entry of the
result), and add the outputs to obtain $A_j$. Continue the
process until you recover $A_0$, the original signal.
\begin{problem}
Write a function that calculates the inverse wavelet transform as described above.
The inputs should be a list of arrays (of the same form as the output of your discrete
wavelet transform function), the low-pass filter, and the high-pass filter. The output
should be a single array, the recovered signal. In order to check your work, compute
the discrete wavelet transform of a random array of length 64, then compute the inverse
transform, and compare the original signal with the recovered signal. The difference
should be very small.
\end{problem}

\section*{More Wavelets}
Up to this point, the only wavelet that we have considered is the Haar wavelet,
which is the simplest and historically first example. Wavelet analysis is a broad
field, however, and there are myriad other wavelets that have been studied and
applied. Your implementation of the discrete wavelet transform is quite general,
and you will find that different types of signals or functions call for different
wavelets. We will not go into detail here, but be aware that there is a large
selection of wavelets out there.

\begin{figure}[H]
\minipage{0.49\textwidth}
    \includegraphics[width=\linewidth]{mexicanHat}
    \caption{The Mexican Hat Wavelet}
\endminipage\hfill
\minipage{0.49\textwidth}
    \includegraphics[width=\linewidth]{db5_3}
    \caption{The Cohen-Daubechies-Feauveau 5/3 Wavelet}
\endminipage
\end{figure}


\section{Revision Protocols and Evolutionary Dynamics}\label{sec:protocols}

In this section we introduce four revision protocols, that lead to the evolutionary dynamics \emph{logit dynamics} (Logit), \emph{replicator dynamics} (RD), \emph{Brown-von Neumann-Nash dynamics} (BNN), and \emph{Smith dynamics} (Smith). These dynamics belong to the families of \emph{perturbed optimization}, \emph{imitative dynamics}, \emph{excess payoff dynamics}, and \emph{pairwise comparison dynamics}, respectively \cite{hofbauer2001nash, sandholm_book}.

 \subsection{Pairwise Proportional Imitation (Replicator Dynamics)}

With a revision opportunity the $i\th$ agent observes an opponent $j$ at random. Then it might change its strategy if  its opponent has a greater fitness. The rate change is
%
\begin{equation}
\rho_{ij}^p(\pi^p, x^p) = \frac{1}{m^p} [\pi_j^p - \pi_i^p]_+,
\end{equation}
where the $[\cdot]_+:\Re \leftarrow \Re_{\geq0}$ represents the positive part, defined as $[ x ]_+ \equiv \max\{ 0, x \}$.
This protocol leads to the \emph{replicator dynamics} defined as
\begin{equation}\label{eq:replicator}
\dot{x}_i^p = x_i^p \, \hat{F}_i^p \left( x \right),
\end{equation}
where $\hat{F}_i^p$ is the excess payoff to strategy $i\in S^p$, which is defined as
\begin{equation}
\hat{F}_i^p (x) =  F_i^p(x) - \bar{F}^p(x),
\end{equation}
and $\bar{F}^p(x)$ is the average payoff the population $p$, i.e.,
\begin{equation}
 \bar{F}^p(x) = \frac{1}{m^p} \sum_{j \in S^p} x_j^p F_j^p(x).
\end{equation}

\subsubsection*{Algorithm}

\begin{algorithm}[H]
\SetKwInOut{Input}{input}\SetKwInOut{Output}{output}

 \Input{Society's state $x$}
 \Output{State update $\dot{x}$}
 \BlankLine

 \For{ $ p \leftarrow 1 $ \KwTo $P$ } {
  $ F^p \leftarrow fitness(x, p)$\;
  $ \bar{F}^p \leftarrow \frac{1}{m^p} (F^p)^\top x^p$\;
  $ \hat{F}^p \leftarrow F^p - \boldsymbol{1} \bar{F}^p$\;
  $ \dot{x}^p \leftarrow \hat{F}^p \odot x^p \frac{1}{m^p} $\;
 }
\end{algorithm}
%
The running time of the algorithm is $T_{rd}(n, P) = O( P (  T_{f}(n,P) + n) ) $, where $T_{f}(n,P)$ is the time required to calculate the fitness vector of a population.

\subsection{Comparison to the Average Payoff (Brown-von Neumann-Nash Dynamics (BNN))}

With a revision opportunity the $i\th$ agent selects a strategy at random and might switch to it if that strategy has a payoff above the average. The agent switch strategy with probability proportional to the excess payoff
%
\begin{equation}
\rho_{ij}^p(\pi^p, x^p) = \left[ \pi_j^p - \frac{1}{m^p} \sum_{k\in S^p} x_k^p \pi_k^p \right]_+,
\end{equation}

This protocol leads to \emph{Brown-von Neumann-Nash dynamics}, defined as
\begin{equation}\label{eq:bnn}
 \dot{x}_i^p = \left[ \hat{F}_i^p \left( \bs{x} \right) \right]_+ - x_i^p  \sum_{j \in S^p} \left[ \hat{F}_j^p \left( \bs{x} \right) \right]_+.
\end{equation}

\subsubsection*{Algorithm}

\begin{algorithm}[H]
\SetKwInOut{Input}{input}\SetKwInOut{Output}{output}

 \Input{Society's state $x$}
 \Output{State update $\dot{x}$}
 \BlankLine

 \For{ $ p \leftarrow 1 $ \KwTo $P$ } {
  $ F^p \leftarrow fitness(x, p)$\;
  $ \bar{F}^p \leftarrow \frac{1}{m^p} (F^p)^\top x^p$\;
  $ \hat{F}^p \leftarrow \max\{F^p - \boldsymbol{1} \bar{F}^p, \boldsymbol{0}\}$\;
  $ \dot{x}^p \leftarrow \hat{F}^p - (\boldsymbol{1}^\top \hat{F}^p) \odot x^p \frac{1}{m^p} $\;
 }
\end{algorithm}
%
The running time is $T_{BNN}(n,P) = O( P (  T_{f}(n,P) + n) ) $.

\subsection{Pairwise Comparison (Smith Dynamics)}

With a revision opportunity the $i\th$ agent selects a strategy at random. If the opponent has a higher fitness, the the agent switch strategy with probability proportional to
\begin{equation}
\rho_{ij}(\pi, x) = \left[ \pi_j - \pi_i \right]_+
\end{equation}
This protocol leads to \emph{Smith dynamics} that are defined as
%
\begin{equation}
\dot{x}_i^p  = \sum_{\gamma \in S^p} x_\gamma^p  \left[ F_i^p \left( \bs{x} \right) - F_\gamma^p \left( \bs{x} \right) \right]_+
%%\\
- x_i^p  \sum_{\gamma \in S^p} \left[ F_\gamma^p ( \bs{x}) - F_i^p( \bs{x} ) \right]_+.
\label{eq:smith}
\end{equation}

\subsubsection*{Algorithm}

Here we present two algorithms.
The first one has time complexity  $O(P(T_f(n, P)+ n^2 ))$. This algorithm is implemented as `smith.m'. A characteristic of this implementation is that might be faster under some conditions, because Matlab is optimized to operate with matrices (see more in Section \ref{sec:running_time}).

\begin{algorithm}[H]
\SetKwInOut{Input}{input}\SetKwInOut{Output}{output}

 \Input{Society's state $x$}
 \Output{State update $\dot{x}$}
 \BlankLine
 \For{ $ p \leftarrow 1 $ \KwTo $P$ } {
  $ F^p \leftarrow fitness(x, p)$\;
  $A \leftarrow \boldsymbol{1} {F^p}^\top$\;
  $M \leftarrow \max(\boldsymbol{0}_{n\times n}, A-A^\top)$\;

  $F_{sum}^p \leftarrow M \boldsymbol{1}$\;
  $F_{avg}^p \leftarrow \frac{1}{m^p} x^\top \, M$\;

  $ \dot{x}^p \leftarrow F_{avg}^p - F_{sum}^p \odot x^p \frac{1}{m^p}$\;
 }
\end{algorithm}

Below we present an alternative algorithm that might be faster for large number of strategies.
In this case we order the strategies in increasing order of fitness and then calculate the strategy's fitness difference (only the ones that are positive). This allow us to reduce the number of operations. The running time of this algorithm is $T_{smith}(n,p) = O(P(T_f(n, P)+ n\log(n) ))$. This algorithm is implemented as `smith\_b.m'.

\begin{algorithm}[H]
\SetKwInOut{Input}{input}\SetKwInOut{Output}{output}

 \Input{Society's state $x$}
 \Output{State update $\dot{x}$}
 \BlankLine
 \For{ $ p \leftarrow 1 $ \KwTo $P$ } {
  $ F^p \leftarrow fitness(x, p)$\;
  $A \leftarrow$ Fitness functions ordered in ascending order\;
  $B \leftarrow$ Strategies ordered in ascending order by their fitness  \;

  $A_{sum} \leftarrow \boldsymbol{1}^\top A$\;
  $A_{avg} \leftarrow 0$\;
  $x_{ord} \leftarrow x(B)\frac{1}{m^p} $\;
  $x_{cum} \leftarrow 0$\;

  \For{ $i \leftarrow 1$ \KwTo $n^p$ }{
    $k \leftarrow B(i)$\;
    $A_{sum} \leftarrow A_{sum} - A(i)$\:

    $\Gamma_a^p[k] \leftarrow A(i) x_{cum} - A_{avg}$\;
    $\Gamma_b^p[k] \leftarrow A_{sum} - A(i) (n-i) $\;

    $A_{avg} \leftarrow A_{avg} + A(i) x_{ord}(i)$\;
    $x_{cum} \leftarrow x_{cum} + x_{ord}(i)$\;

  }
  $ \dot{x}^p \leftarrow \Gamma_a^p - \Gamma_b^p \odot x^p \frac{1}{m^p}$\;
 }
\end{algorithm}

\subsection{Logit Choice}

With a revision opportunity the $i\th$ agent selects a strategy at random and change its strategy with a probability proportional to

\begin{equation}
\rho_{ij}(\pi) = \frac{ \exp(\pi_j \eta^{-1} ) }{ \sum_{k \in S} \exp(\pi_k \eta^{-1} ) }
\end{equation}

This protocol belong to target dynamics and with a large population results in the following dynamics
\begin{equation}\label{eq:logit}
 \dot{x}_i^p = \frac{ \exp\left(\eta^{-1} F_i^p (\bs{x}) \right) }{ \sum_{\gamma \in S^p} \exp\left(\eta^{-1} F_\gamma^p (\bs{x}) \right) }, \, \, \eta>0,
\end{equation}
known ad \emph{Logit dynamics}.

\subsubsection*{Algorithm}

\begin{algorithm}[H]
\SetKwInOut{Input}{input}\SetKwInOut{Output}{output}

 \Input{Society's state $x$}
 \Output{State update $\dot{x}$}
 \BlankLine

 \For{ $ p \leftarrow 1 $ \KwTo $P$ } {
  $ F^p \leftarrow fitness(x, p)$\;
  $ \bar{F}^p \leftarrow \frac{1}{m^p} (F^p)^\top x^p$\;
  $ \tilde{F}^p \leftarrow \exp( F^p \eta^{-1} )$\;
  $ \Gamma \leftarrow \boldsymbol{1}^\top \tilde{F}^p $\;
  $ \dot{x}^p \leftarrow \frac{\tilde{F}^p}{\Gamma} - x^p $\;
 }
\end{algorithm}

The running time is $T_{logit}(n,P) = O( P (  T_{f}(n,P) + n) ) $.

\iffalse
\subsection{Maynard Smith Replicator}

\begin{equation}
\dot{x}_i = \frac{ x_i F_i }{ \bar{F}(x) } - x_i
\end{equation}
\fi


% Question  ##################################################################################################################
\section{Question 3}\label{ssec:pt1q3}
\textbf{Under the assumption of Markov property, the Binomial tree provides a method of how the future
price of an instrument, can be modelled. By investigating the statistics of Google daily returns for the
last 5 years (till 31/12/2017), construct a binomial tree model that projects Google stock price for
the first 6 months of 2018, on a monthly basis.}

\noindent
\textbf{In your answers explain the method/steps used.}

% END Question  ##############################################################################################################

% Question (i) ###############################################################################################################

\subsection{Q3 (i)}\label{sssec:pt1q3i}
\textbf{Present the projected probability and stock price binomial trees.}

\noindent
Code used in this question can be found in ‘Question 3 (i)’ in the notebook. The Google prices downloaded in Q1 (vii) were reloaded and used for this task. After reloading the prices, the log return was computed for each row. The annualized returns and volatility were also computed assuming that a year has 250 days. Furthermore, the last closing price was referenced and saved in a variable, as this will be used as a parameter to compute the binomial tree. A function called ‘binomial\_tree’ was created in the ‘fintech’ library to construct such tree. This function is shown in Fig.~\ref{fig:binomial_1} and Fig.~\ref{fig:binomial_2}.

\noindent
First off to construct the binomial tree we need to compute the following variables; time step $(\frac{0.5}{6})$ since we have half a year and projecting for 6 months, Up Factor $(\exp(volatility \times \sqrt{time \ step}))$, Down Factor $(\frac{1}{Up \ Factor})$, Up Probability $(\frac{(\exp(return \times time \ step))}{(Up \ Factor - Down \ Factor)})$ and Down Probability $(1 - Up \ Probability)$.

\noindent
After these variables are computed an empty 2D numpy array was initialized to hold the projected prices for the binomial tree. The first element (0,0) was set to the last closing price.  The function then loops for the number of time periods specified and computes the stock price binomial tree. After this iteration finishes, you will end up with a sparse matrix where half of the elements are empty, and the other half are filled with the prices. The symmetry between the prices and the empty values is diagonal. Fig.~\ref{fig:binomial_ex} better explains what is happening inside the for loop to compute the projected prices. Table.~\ref{tbl:binomial_prices} shows the output for this part of the function, which is the projected stock price binomial trees.

\noindent
Similarly to the prices, another empty 2D numpy array was initialized to hold the projected probabilities. The first element (0,0) was set to 1 since this value corresponds to the last closing price. Again, this function loops for the number of time periods specified to compute the probabilities. The output is quite like the previous step as it is also outputs a sparse matrix, where half of the elements are empty, and the other half are filled the probabilities. The symmetry between the probabilities and the empty values is diagonal. Fig.~\ref{fig:binomial_ex2} better explains what is happening inside the for loop to compute the projected probabilities. Table.~\ref{tbl:binomial_probs} shows the output for this part of the function, which is the projected probability binomial trees.

\begin{figure}[H]
\centering
  \includegraphics[scale = .62]{imgs/binomial_tree_1.png}
  \caption{Function to construct Binomial Tree part 1.}
  \label{fig:binomial_1}
\end{figure}

\begin{figure}[H]
\centering
  \includegraphics[scale = .55]{imgs/binomial_tree_2.png}
  \caption{Function to construct Binomial Tree part 2.}
  \label{fig:binomial_2}
\end{figure}

\begin{figure}[H]
\centering
  \includegraphics[scale = .75]{imgs/binomial_example.JPG}
  \caption{Visual Explanation of how the projected prices are computed. u = Up Factor and d = Down Factor}
  \label{fig:binomial_ex}
\end{figure}

\begin{table}[H]
\centering
  \includegraphics[scale = .90]{imgs/binomial_prices.png}
  \caption{The projected Stock Price Binomial Trees. }
  \label{tbl:binomial_prices}
\end{table}

\begin{figure}[H]
\centering
  \includegraphics[scale = .75]{imgs/binomial_example_prob.JPG}
  \caption{Visual Explanation of how the projected probabilities are computed.}
  \label{fig:binomial_ex2}
\end{figure}

\begin{table}[H]
\centering
  \includegraphics[scale = .90]{imgs/binomial_probs.png}
  \caption{The projected Probability Binomial Trees. }
  \label{tbl:binomial_probs}
\end{table}

\noindent
The final step was then to compute the expected values which will be shown in (iii). The expected values are computed as the sum product of each column of the two outputs described above. All the outputted numpy arrays described above were converted and returned as pandas dataframes.

% END Question (i) ##########################################################################################################

% Question (ii) #############################################################################################################

\subsection{Q3 (ii)}\label{sssec:pt1q3ii}
\textbf{Utilizing the binomial tree from (i), what is the probability that at the end of the 6 month period
the price will be greater than the starting price.}

\noindent
The code for this task can be found in ‘Question 3 (ii)’ in the python notebook. To calculate the probability for this question we took the sum of the last column for Table.~\ref{tbl:binomial_probs} where the corresponding price is greater than the last closing price. This means $p = (0.0629 + 0.2211 + 0.3237) \times 100 $, which is equal to 60.77\%.

% END Question (ii) #########################################################################################################

% Question (iii) ############################################################################################################

\subsection{Q3 (iii)}\label{sssec:pt1q3iii}
\textbf{Calculate the expected stock price, on a monthly basis.}

\noindent
The computation to get the stock prices on a monthly basis was described in the function which was used in (i), and these prices are shown in Table.~\ref{tbl:binomialexpectedprices}. Code for this question can be found in ‘Question 3 (iii)’ in the python notebook.

\begin{table}[H]
    \includegraphics[width=\linewidth]{imgs/binomial_prices_monthly.png}
    \caption{Table for the expected stock prices on a monthly basis using Binomial Trees.}
    \label{tbl:binomialexpectedprices}
\end{table}

% END Question (iii) #########################################################################################################


\subsection{Map Fusion}\label{subsec:map_fusion}

Next, we prove the higher order property of map fusion.
First we define and axiomatize function composition and
list map:

\begin{code}
  axiomatize (.)
  (.) :: (b -> c) -> (a -> b) -> a -> c
  (.) f g x = f (g x)

  axiomatize map
  map :: (a -> b) -> L a -> L b
  map f N        = N
  map f (C x xs) = C (f x) (map f xs)
\end{code}

\NV{Say why we need defunctionalization}
\NV{We use app function like HALO (link to the theory)}
\NV{Zombie with rewritting does not allow HIGHER ORDER reasoning}

Then, we specify the map fusion property as a type
specification for the function @map_fusion@
and prove the property by induction on the list argument.
\begin{code}
  type MapFusion F G X
    = {map (F . G) X == (map F . map G) X}

  map_fusion :: f:(a -> a)
             -> g:(a -> a)
             -> xs:L a
             -> MapFusion f g xs
\end{code}

\begin{code}
  map_fusion f g N
    =   ((map f) r. (map g)) N
    ==! (map f) (rmap g N)
    ==! rmap f N
    ==! N
    ==! gmap (f . g) N
    *** QED

  map_fusion f g (C x xs)
    =   rmap (f . g) (C x xs)
    ==! (f . g) x `C` map (f . g) xs
    ==! (f . g) x `C` (map f r. map g) xs
         ? map_fusion f g xs
    ==! (f r. g) x `C` map f (map g xs)
    ==! f (g x)   `C` map f (map g xs)
    ==! gmap f (C (g x) (map g xs))
    ==! (map f) (gmap g (C x xs))
    ==! (map f g. map g) (C x xs)
    *** QED
\end{code}

\subsection{Monadic Laws: Associativity}
As a last example,
we axiomatize the monadic list bind operator
\begin{code}
  axiomatize >>=
  (>>=) :: L a -> (a -> L b) -> L b
  (C x xs) >>= f = f x ++ (xs >>= f)
  Emp      >>= f = N
\end{code}

We use the above definition to inductively prove
associativity of the bind operator

\begin{code}
  type Associative M F G
   {  M >>= F >>= G
   == M >>= (\x -> F x >>= G) }

  associativity
    :: m:L a
    -> f: (a -> L b)
    -> g:(b -> L c)
    -> Associative m f g

  associativity N f g
    =   N r>>= f >>= g
    ==! N r>>= g
    ==! N
    ==! N g>>= (\x -> f x >>= g)
    *** QED

  associativity (C x xs) f g
    =   (C x xs) r>>= f    >>= g
    ==! (f x) ++ (xs >>= f) >>= g
        ? bind_append (f x) (xs >>= f) g
    ==! (f x >>= g) ++ ((xs >>= f) >>= g)
    ==!    (f x >>= g)
        ++ (xs >>= (\y -> f y >>= g))
        ? associativity xs f g
    ==!    (\y -> f y >>= g) x
        ++ (xs >>= (\y -> f y >>= g))
        -- eta-equivalence
    ==! (C x xs) g>>= (\y -> f y >>= g)
    *** QED
\end{code}

In the proof we used the bind-append fusion lemma
\begin{code}
  bind_append
    :: xs:L a
    -> ys:L a
    -> f:(a -> L b)
    -> { (xs ++ ys) >>= f == (xs >>= f) ++ (ys >>= f) }
\end{code}

Moreover, we required $\beta$- and $\eta$-equilvalence on
anonymous functions.
For example, during the proof, we need the equality
@f x >>= g ==! (\x -> f x >>= g) y@.
%
To prove this equality, in the logic,
the anonymous functions are represented as functional variables
axiomatized with extensionality axioms.
%
Thus, in the logic, we define @f'@ and the axioms
@forall x. f' x = f x >>= g@ and
@forall g x. (f' x = g x) => f' = g@.
%
These two axioms are sufficient to prove
1. $\eta$-equivalence that is required in the last step
of the inductive case; and
2. $\beta$-equivalence that is required to prove that our proof
@xs >>= f >>= g ==! xs >>= (\y -> f y >>= g)@
implies the specification.


\section*{Discriminative vs. Generative Modeling}
Discriminative estimates $P(y|x)$\\
Generative estimates joint distribution $P(y,x)$

Typical approach to generative modeling:\\
- Estimate prior on labels $P(y)$\\
- Estimate cond. distr. $P(x|y)$ for each class y\\
- Obtain predictive distr. using Bayes' rule:\\
$P(y|x) = \frac{P(y) P(x|y)}{P(x)} = \frac{P(x,y)}{P(x)}$, $P(x) = \sum_y P(x,y)$

%\subsection*{Example: Naive Bayes Model}
%cond. ind.:$P(X_1,...,X_d|Y) = \prod_{i=1}^d P(X_i|Y)$

\subsection*{Example MLE for P(y)}
Want: $P(Y=1) = p, P(y=-1) = 1-p$\\
Given: $D=\{(x_1,y_1),...,(x_n,y_n)\}$\\
$P(D|p) = \prod_{i=1}^n p^{1[y_i=+1]} (1-p)^{1[y_i=-1]}$\\
$=p^{n_+} (1-p)^{n_-}$, where $n_+ = $ \# of $y=+1$\\
$\frac{\partial}{\partial p} log P(D|p) = n_+ \frac{1}{p} - n_- \frac{1}{1-p} \overset{!}{=} 0 \Rightarrow p=\frac{n_+}{n_+ + n_-}$

\subsection*{Example MLE for P=(x|y)}
Assume: $P(X=x_i|y) = \mathcal{N}(x_i;\mu_{i,y}, \sigma_{i,y}^2)$\\
%Given: $D, D_{x_i|y} = \{x \text{, s.t. } x_{j,i}=x, y_j=y\}$\\
Thus MLE yields:
$\hat{\mu}_{i,y} = \frac{1}{n_y} \sum_{x\in D_{x_i|y}} x$;\\ %,where $n_y=|D_{x_i|y}|$\\
$\hat{\sigma}_{i,y}^2 = \frac{1}{n_y} \sum_{x\in D_{x_i|y}} (x-\hat{\mu}_{i,y})^2$

\subsection*{Deriving decision / classification rule}
%In order to predict label y for new point x, use\\
$P(y|x) = \frac{1}{Z} P(y)P(x|y)$, $Z = \sum_y P(y) P(x|y)$\\
$y = \underset{y'}{\operatorname{argmax}} P(y'|x) = \underset{y'}{\operatorname{argmax}} P(y') \prod_{i=1}^d P(x_i|y')\\
= \underset{y'}{\operatorname{argmax}} log P(y') + \sum_{i=1}^d log P(x_i|y')$

\subsection*{Gaussian Naive Bayes classifier}
MLE for class prior: $\hat{P}(Y=y) = \hat{p}_y = \frac{\operatorname{Count(Y = y)}}{n}$\\
MLE for feature distr.: $\hat{P}(x_i|y) =  \mathcal{N}(x_i;\hat{\mu}_{y,i}, \sigma_{y,i}^2)$\\
$\hat{\mu}_{y,i} = \frac{1}{\operatorname{Count}(Y=y)} \sum_{j:y_j=y} x_{j,i}$\\
$\sigma_{y,i}^2 = \frac{1}{\operatorname{Count}(Y=y)} \sum_{j:y_j=y} (x_{j,i} - \hat{\mu}_{y,i})^2$\\
Prediction given new point x:\\
$y = \underset{y'}{\operatorname{argmax}} \hat{P}(y'|x) = \underset{y'}{\operatorname{argmax}} \hat{P}(y') \prod_{i=1}^d \hat{P}(x_i|y')$

\subsection*{Gaussian Bayes Classifier}
MLE for class prior: $\hat{P}(Y=y) = \hat{p}_y = \frac{\operatorname{Count(Y = y)}}{n}$\\
MLE for feature distr.: $\hat{P}(x|y) = \mathcal{N}(x ; \hat{\mu}_y, \hat{\Sigma}_y)$\\
$\hat{\mu}_{y} = \frac{1}{\operatorname{Count}(Y=y)} \sum_{i:y_i=y} x_i \in \mathbb{R}^d$\\
$\hat{\Sigma}_{y} = \frac{1}{\operatorname{Count}(Y=y)} \sum_{i:y_i=y} (x_i - \hat{\mu}_{y})(x_i-\hat{\mu}_y)^T \in \mathbb{R}^{d \times d}$

\subsection*{Fisher's linear discriminant analysis (LDA; c=2)}
Assume: $p = 0.5$; $\hat{\Sigma}_- = \hat{\Sigma}_+ = \hat{\Sigma}$\\
discriminant f.: $f(x) = log \frac{p}{1-p} + \frac{1}{2}[log \frac{|\hat{\Sigma}_-|}{|\hat{\Sigma}_+|}\\
+ ((x - \hat{\mu}_-)^T \hat{\Sigma}_-^{-1} (x - \hat{\mu}_-)) - ((x - \hat{\mu}_+)^T \hat{\Sigma}_+^{-1} (x - \hat{\mu}_+))]$\\
Predict: $y = sign(f(x)) = sign (w^T x + w_0)$\\
$w = \hat{\Sigma}^{-1}(\hat{\mu}_+ - \hat{\mu}_-)$; $w_0 = \frac{1}{2}(\hat{\mu}_-^T\hat{\Sigma}^{-1}\hat{\mu}_- - \hat{\mu}_+^T \hat{\Sigma}^{-1}\hat{\mu}_+)$

\subsection*{Outlier Detection}
$P(x) = \sum_{y=1}^c P(y) P(x|y) = \sum_y \hat{p}_y \mathcal{N}(x|\hat{\mu}_y,\hat{\Sigma}_y) \leq \tau$

\subsection*{Categorical Naive Bayes Classifier}
MLE class prior: $\hat{P}(Y=y) = \frac{Count(Y=y)}{n}$\\
MLE for feature distr.:
$\hat{P}(X_i = c|Y = y) = \theta_{c|y}^{(i)}\\
\theta_{c|y}^{(i)} = \frac{Count(X_i = c, Y = y)}{Count(Y=y)}$, Pred.: $y = \underset{y'}{argmax}\hat{P}(y'|x)$


\subsection{Beta function}

The beta function expand the binomial coefficient formula to the real (and complex) numbers.

We want to expand the binomial coefficient function.

\((\dfrac{n}{k})=\dfrac{n!}{k!(n-k)!}\)

We do this as:

\(B(x, y)=\dfrac{\Gamma(x)\Gamma(y)}{\Gamma(x+y)}\)


%
% Chapter 3.7
%

\section*{3.7 Optimization}

\subsection*{First Derivative Test for Absolute Extreme Values}

Suppose that \(c\) is a critical value of a continuous function \(f\) defined on an interval.
\begin{itemize}
    \item If \(f''(x)>0\) for all \(x<c\)and \(f'(x)<0\) for all \(x>c\), then \(f(c)\) is the absolute maximum value of \(f\).
    \item If \(f'(x)<0\) for all \(x<c\) and \(f'(x)>0\) for all \(x>c\), then \(f(c)\) is the absolute minimum value of \(f\).
\end{itemize}


\subsection{Gradient Properties}
\noindent
Let $f$ and $g$ be functions of multiple variables, let $\vec{r}$ be a VVF, and let $c \in \mathbb{R}$.
\begin{enumerate}
	\item $\nabla(f \pm g) = \nabla f \pm \nabla g$
	\item $\nabla(cf) = c\nabla f$
	\item $\nabla(fg) = f\nabla g + g\nabla f$
	\item $\nabla(f\circ\vec{r}(t)) = \nabla f \cdot \vec{r^\prime}(t)$
\end{enumerate}

\noindent
The 4th gradient property can be generalized a bit further.
Suppose we have $f(x,y,z)$, $\vec{r}(u,v) = \langle x(u,v), y(u,v), z(u,v) \rangle$, and $g(u,v) = (f\circ\vec{r})(u,v)$.
\begin{equation*}
	\pp{g}{u} = \pp{f}{x}\pp{x}{u} + \pp{f}{y}\pp{y}{u} + \pp{f}{z}\pp{z}{u} = \bigg\langle \pp{f}{x}, \pp{f}{y}, \pp{f}{z} \bigg\rangle \cdot \bigg\langle \pp{x}{u}, \pp{y}{u}, \pp{z}{u} \bigg\rangle = \nabla f \cdot \pp{\vec{r}}{u}.
\end{equation*}


\section{Representation of heap concepts in \acsl}
\Label{sec:heap-acsl}

The following listing shows three logic functions
\HeapLeft, \HeapRight and \HeapParent
that correspond to the definitions~\eqref{eq:heap-left},
\eqref{eq:heap-right} and~\eqref{eq:heap-parent}, respectively.
This listing also contains a number of \acsl lemma that state among other things that

\begin{itemize}
\item
the \HeapParent function satisfies the equations~\eqref{eq:heap-parent-left}
and~\eqref{eq:heap-parent-right} and
\item
the function \HeapParent
is the \emph{left inverse} to the \HeapLeft and \HeapRight functions.\footnote{
 See Section \emph{Left and right inverses} at
 \url{http://en.wikipedia.org/wiki/Inverse_function}
}
\end{itemize}

\input{Listings/HeapNodes.acsl.tex}

\clearpage

On top of these basic definitions we introduce the predicate \logicref{Heap}.
The fact that element at index~0 of a (maximum) heap, is always the largest element of the heap
is express by Lemma \logicref{HeapMaximum} using the predicate \logicref{MaxElement}.

\input{Listings/Heap.acsl.tex}

The lemmas \HeapShrink and \HeapUnchanged formulate simple rules to
``transfer'' the heap property from an array to a related (sub-)array.

The predicate \HeapCompatible expresses under which
conditions the changing of an individual heap element does maintain the heap
property.
This predicate together with lemma \HeapCompatibleUpdate
will be useful in the verification of the
algorithms \implref{pushheap} and \implref{popheap}.

\clearpage


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 6}

Prove that in a rooted tree $T$, if $x$ is an ancestor of $y$ and $y$ is an ancestor of $x$, then $x = y$.

\subsection*{Solution}

Proof is given by contradiction.
Assume $x$ is an ancestor of $y$ and $y$ is an ancestor of $x$ and $x \neq y$ ($x$ and $y$ are two distinct nodes).
According to definition of ancestorhood, there are paths from root $v_r$ of the form $v_r \rightarrow x \rightarrow y$ and $v_r \rightarrow y \rightarrow x$.
This immediately results to a simple loop of the form $x \rightarrow y \rightarrow x$ and $y \rightarrow x \rightarrow y$ which contradicts the definition of the tree.
Therefore the assumption is false and the proof is complete.


\section{09/16}
\subsection{Randomized Quicksort}
The following pseudocode demonstrates our randomized Quicksort variant.

\begin{algorithm}[H]
    \caption{Simplified variant of Quicksort algorithm with random pivot.}
    \label{alg:randquicksort}
    \begin{algorithmic}[1]
        \Function{Rand-Quicksort}{\texttt{arr}}
            \State $n \gets \Call{Len}{\texttt{arr}}$
            \If{$n \leq 1}$
                \State \Return \texttt{arr}
            \Else
                \State $\texttt{left}, P, \texttt{right} \gets \Call{Partition}{\texttt{arr}}$
                \State \Return $\Call{Rand-Quicksort}{\texttt{left}} + P + \Call{Rand-Quicksort}{\texttt{right}}$
            \EndIf
        \EndFunction
        \Function{Partition}{\texttt{arr}}
            \State $\texttt{pivot} \gets \Call{Sample}{\texttt{arr}}$
            \State $\texttt{left}, P, \texttt{right} \gets [], [], []$
            \For{$a \in \texttt{arr}}$
                \If{$a < \texttt{pivot}}$
                    \State $\texttt{left}.\Call{Append}{a}$
                \ElsIf{$a = \texttt{pivot}$}
                    \State $P.\Call{Append}{a}$
                \Else
                    \State $\texttt{right}.\Call{Append}{a}$
                \EndIf
            \EndFor
            \State \Return $\texttt{left}, P, \texttt{right}$
        \EndFunction
    \end{algorithmic}
\end{algorithm}

Notice that the algorithm is identical to~\cref{alg:quicksort}, except for the
partition choice. Later, we will apply Chernoff Bounds to show that, with high
probability, the number of comparisons is $\bigO{n\log{n}}$.

Let us consider the original algorithm once more, wherein the pivot is always
taken to be $\texttt{arr}[0]$. Suppose the input is a randomly selected
\emph{permutation} of the array. We can now determine the expected number of
comparisons of this algorithm (this is commonly called \emph{average case} or
\emph{probabilistic} analysis).

In randomized Quicksort, the analysis is with respect to the \emph{random pivot
choice}. On the other hand, here we are analyzing with respect to the
\emph{random input}.

However, the analysis is largely the same. Again, we let $X_{i,j}$ be the
indicator random variable equal to 1 when elements $a_i$ and $a_j$ are compared.
These elements will be compared when $a_i$ occurs before $a_j$ (or vice-versa)
in the permutation, and none of $a_{i+1}$, $a_{i+2}$, \dots, $a_{j - 1}$ are
chosen first. Apply the principle of deferred decisions and assume all values,
other than $a_i$, $a_{i+1}$, \dots, $a_j$ have been placed. There are $(j - i +
1)!$ permutations of these values, of which $2(j - i - 1)!$ have $a_i$ or $a_j$
first. Thus,
\[\expectation{X_{i,j}} = \frac{2(j - i - 1)}{(j - i + 1)} = \frac{2}{(j - i + 1)}\]
from which the result follows.

\subsection{Selection Algorithm}
\begin{problem}{Selection}{}
    Given an array of length $n$, find the $k$-th smallest element of $n$.
\end{problem}
We can apply an algorithm similar to Quicksort: select a pivot and partition the
array. If the number of elements less than the pivot is $k - 1$, then the pivot
is the $k$-th smallest. Otherwise, the desired element falls in the left or
right partition.

\begin{algorithm}[H]
    \caption{Simplified variant of Quickselect algorithm with random pivot.}
    \label{alg:randquickselect}
    \begin{algorithmic}[1]
        \Function{Rand-Quickselect}{\texttt{arr}, $k$}
            \State $n \gets \Call{Len}{\texttt{arr}}$
            \If{$n = 1}$
                \State \Return $\texttt{arr}[0]$
            \Else
                \State $\texttt{left}, P, \texttt{right} \gets \Call{Partition}{\texttt{arr}}$
                \State $\ell \gets \Call{Length}{\texttt{left}}$
                \State $p \gets \Call{Length}{P}$
                \If{$\ell \geq k$}\Comment{$k$-th smallest is in \texttt{left}}
                    \State \Return $\Call{Rand-Quickselect}{\texttt{left}, k}$
                \ElsIf{$\ell + p < k$}\Comment{$k$-th smallest is in \texttt{right}}
                    \State \Return $\Call{Rand-Quickselect}{\texttt{right}, k - \ell - p}$
                \Else
                    \State \Return \texttt{pivot}
                \EndIf
            \EndIf
        \EndFunction
    \end{algorithmic}
\end{algorithm}

Let us determine the expected number of comparisons in this algorithm. To do so,
we assume the elements of \texttt{arr} in sorted order are $a_1$, $a_2$, \dots,
$a_n$ and consider the probability that the pivot chosen falls into the
\emph{middle \sfrac{1}{3} of values}. The key intuition here is that the pivot
does not need to fall \emph{exactly} in the middle in order to achieve good
asymptotic performance --- simply falling into the middle $\sfrac{1}{3}$ ``most
of the time'' is sufficient.

When the pivot does fall in the middle $\sfrac{1}{3}$, the set containing the
$k$-th smallest element will have at \emph{most} $\sfrac{2n}{3}$ elements. This
yields the recurrence
\[T(n) \leq \frac{1}{3}T\left(\frac{2n}{3}\right) + \frac{2}{3}T(n) + \bigO{n}\]
with base case $T(1) = 0$. This is an expression on random variables, and so it
cannot be solved directly (in fact, it would not make sense to do so). Thus, we
take the expectation of each side:

\begin{align*}\expectation{T(n)}
    &\leq \frac{1}{3}\expectation{T\left(\frac{2n}{3}\right)} + \frac{2}{3}\expectation{T(n)} + \bigO{n}\\
    \shortintertext{combining like terms}\\
    \expectation{T(n)} &\leq \expectation{T\left(\frac{2n}{3}\right)} + \bigO{n}\\
    &= \bigO{n}
\end{align*}

\subsection{Conditional Expectation}
\begin{definition}{Conditional Expectation}{}
    Let $X$ be a random variable and $E$ some event. The \emph{expectation of
    $X$ given $E$}, $\expectation{X \given E}$, is defined
    \begin{align*}\expectation{X \given E}
        &=\sum_{x \in X(S)} \prob{X = x \given E}\\
        &=\sum_{x \in X(S)} \frac{\prob{X = x \cap E}}{\prob{E}}
    \end{align*}
    Similarly, if $Y$ is a random variable,
    \begin{align*}\expectation{X \given Y}
        &=\sum_{x \in X(S)} \prob{X = x \given Y = y}\\
        &=\sum_{x \in X(S)} \frac{\prob{X = x \cap Y = y}}{\prob{Y = y}}
    \end{align*}
\end{definition}

Note that Conditional Expectation is actually a random variable. For example, if
we let $X = X_1 + X_2$ denote the sum of two fair die, the random variable
$\expectation{X \given X_1}$ is given by
\begin{align*}\expectation{X \given X_1 = x_1}
    &= \sum_{x = x_1 + 1}^{x_1 + 6} x\prob{X = x \given X_1 = x_1}\\
    &= \sum_{x = x_1 + 1}^{x_1 + 6} x\frac{1}{6}\\
    &= x_1 + \frac{7}{2}
\end{align*}

\begin{theorem}{}{}
    Let $X$ and $Y$ be random variables. Then
    \[\expectation{X} = \sum_{y\in Y(S)}\expectation{X \given Y = y}\prob{Y = y}\]
\end{theorem}

\begin{proof}
    By definition,
    \begin{align*}\expectation{X}
        &= \sum_{y\in Y(S)}\expectation{X \given Y = y}\prob{Y = y}\\
        &= \sum_{x \in X(S)}\sum_{y \in Y(S)} x\prob{X = x \given Y = y}\prob{Y = y}\\
        &= \sum_{x \in X(S)}\sum_{y \in Y(S)} x\prob{X = x \cap Y = y}\\
        &= \sum_{x \in X(S)}x \sum_{y \in Y(S)} \prob{X = x \cap Y = y}
        \shortintertext{by the Law of Total Probability}\\
        &= \sum_{x \in X(S)}x \prob{X = x}\\
        &= \expectation{X}\qedhere
    \end{align*}
\end{proof}

\begin{theorem}{Law of Total Expectation}{}
    Let $X$ and $Y$ be random variables. Then
    \[\expectation{\expectation{X \given Y}} = \expectation{X}\]
\end{theorem}

\begin{proof}
    By definition
    \begin{align*}\expectation{\expectation{X \given Y}}
        &= \expectation{\sum_{x} x \prob{X = x \given Y = y}}\\
        &= \sum_{y} \left(\sum_{x} x \prob{X = x \given Y = y}\right)\prob{Y = y}\\
        &= \sum_{y} \sum_{x} x \prob{X = x \given Y = y}\prob{Y = y}\\
        &= \sum_{x} x \sum_{y} \prob{X = x \cap Y = y}\\
        &= \sum_{x} x \prob{X = x}\\
        &= \expectation{X}
    \end{align*}
\end{proof}

\subsection{Branching Process}
One application of Conditional Expectation is to branching processes. Suppose a
single person is infected with some disease, and let $X$ denote the number of
people infected by a single person. Assume further that $\expectation{X} = \mu$.

Now, let $Z$ denote the total number of infected people and $Z_i$ the number
infected in \emph{generation $i$}. By definition,
\[\expectation{Z_i \given Z_{i - 1}} = Z_{i - 1}\mu\]
hence
\begin{align*}\expectation{Z_i}
    &= \expectation{\expectation{Z_i \given Z_{i - 1}}}\\
    &= \expectation{Z_{i-1} \mu}\\
    &= \mu\expectation{Z_{i - 1}}
\end{align*}
Since $\expectation{Z_0} = 1$, $\expectation{Z_n} = \mu^n$, and
\[\expectation{Z} = 1 + \mu + \mu^2 + \dots = \frac{1}{1 - \mu} \hbox{ if $\mu$ < 1}\]