\documentclass{ximera}
\input{../preamble}
\title{Exercises: Taylor Series Applications}
%%%%%\author{Philip T. Gressman}

\begin{document}
\begin{abstract}
Various exercises relating to the application of Taylor Series to other problems of interest.
\end{abstract}
\maketitle

\begin{exercise}
Compute the limit
\[ \lim_{n \rightarrow \infty} n^3 \left[ \arctan \frac{1}{n} - \frac{1}{n} \right]. \]
\begin{itemize}
\item The first few terms of the Maclaurin series for $\arctan x$ are
\[ \cos x = \answer{x} - \answer{\frac{x^3}{3}} + \answer{\frac{x^5}{5}} + \cdots \]
\item Plug in $x = \frac{1}{n}$ and subtract $1/n$:
\[ \arctan \frac{1}{n} - \frac{1}{n} = - \frac{1}{3n^3} + \answer{\frac{1}{5n^5}} + \cdots \]
\item Multiply by $n^3$, neglect all but the dominant term, and conclude
\[ \lim_{n \rightarrow \infty} n^3 \left[ \arctan \frac{1}{n} - \frac{1}{n} \right] = \answer{-\frac{1}{3}} \]
\end{itemize}
\end{exercise}

\begin{exercise}
Compute the limit
\[ \lim_{n \rightarrow \infty} \frac{\ln (n+3) - \ln n}{\ln (n+1) - \ln n}. \]
\begin{itemize}
\item First combine logarithms to simplify a bit:
\[  \ln (n+3) - \ln n  =  \ln \answer{1 + \frac{3}{n}} \ \ \text{ and } \ \ \ln (n+1) - \ln n  =  \ln \answer{1 + \frac{1}{n}}. \]
\item Write out the first few terms of the Maclaurin series for $\ln (1+x)$ and substitute in the appropriate expression involving $n$:
\[ \ln (1 + x) = \answer{x} - \answer{\frac{x^2}{2}} + \answer{\frac{x^3}{3}} - \cdots \]
\[ \ln (n+3) - \ln n = \answer{ \frac{3}{n}} - \answer{\frac{9}{2n^2}} + \answer{\frac{9}{n^3}} - \cdots \]
\[ \ln (n+1) - \ln n = \answer{\frac{1}{n}} - \answer{\frac{1}{2n^2}} + \answer{\frac{1}{3n^3}} - \cdots \]
\item Multiply by $n$, neglect all but the dominant term (as $n \rightarrow \infty)$ to conclude
\[ \lim_{n \rightarrow \infty} \frac{\ln (n+3) - \ln n}{\ln (n+1) - \ln n} = \answer{3}. \]
\end{itemize}
\end{exercise}

\begin{exercise}
Compute the limit
\[ \lim_{x \rightarrow 1}  \frac{\ln x - x + 1}{\sin \pi x}. \]
\begin{itemize}
\item Compute the first few terms of the Taylor series of numerator and denominator centered at $x = 1$:
\[ \ln x - x + 1 = \answer{0} + \answer{0} (x-1) + \answer{-\frac{1}{2}} (x-1)^2 + \answer{\frac{1}{3}} (x-1)^3 + \cdots \]
\[ \sin \pi x = \answer{0} + \answer{-\pi}(x-1) + \answer{0} (x-1)^2 + \answer{\frac{\pi^3}{6}}(x-1)^3 + \cdots \]
\item Neglect all but the dominant terms in numerator and denominator. Note that dominant here means as $x \rightarrow 1$. Then take the limit $x \rightarrow 1$:
\[ \lim_{x \rightarrow 1}  \frac{\ln x - x + 1}{\sin \pi x} = \answer{0}. \]
\end{itemize}
\end{exercise}

\begin{exercise}
Use the remainder formula for Taylor series to determine which partial sums of the series
\[ \sum_{n=0}^\infty \frac{1}{n!} \]
differ from $e$ by at most $1/100$?
\begin{itemize}
\item The remainder formula says that
\[ e^1 - \sum_{n=0}^{N} \frac{1}{n!} = \frac{e^{\xi}}{(N+1)!} (1-0)^{N+1} \]
for some point $\xi$ which depends on $N$ and is somewhere in the interval $(\answer{0},\answer{1})$. The upper bound for the magnitude of $e^\xi$ on this interval is $M = \answer{e}$.
\item For which values of $N$ do we have
\[  \frac{M}{(N+1)!} \leq \frac{1}{50}? \]
(Check manually for small values of $N$; use the fact that $e$ is between $2$ and $3$.)
\[ N \geq \answer{4}. \]
\end{itemize}
\end{exercise}

\begin{exercise}
In this exercise, we will investigate two different ways of numerically approximating the value of $\ln 2$ using infinite series.
\begin{itemize}
\item The Maclaurin series of the function $\ln 1 + x$ converges conditionally at $x = 1$ to $\ln 2$. Compute the Maclaurin series:
\[ \ln (1 + x) = \sum_{n=0}^\infty \answer{\frac{(-1)^n}{n+1}} x^{n+1} \]
(Reindex your answer to match the template above if your answer doesn't work as-is.)
\item  What degree Taylor polynomial would you need to use to approximate the value of $\ln 2$ to an error strictly less than $10^{-3}$? Answer: Taylor polynomial used should have degree $N = \answer{1000}$ or greater.
\item A similar but distinct strategy would be to compute $\ln (1/2)$ instead because we know $- \ln (1/2)  = \ln 2$. Evaluating the Maclaurin series at $x=-1/2$ and doing a little simplification, we see from the above series that
\[ \ln 2 = \sum_{n=1}^\infty \answer{\frac{1}{n 2^n}}. \]
(Once again, reindex if your answer does not already start at $n=1$.)
\item The presence of a factor exponential in $n$ suggests that this second series converges to $\ln 2$ \wordChoice{\choice{much slower}\choice{slightly slower}\choice{slightly faster}\choice[correct]{much faster}} than the first series. Of the two series, then, the \wordChoice{\choice{first}\choice[correct]{second}} series presents a more efficient way to compute $\ln 2$ numerically.
\end{itemize}
\begin{hint}
For the first estimation, use the series estimation features of the Alternating Series Test.
\begin{hint}
For an alternating series, the error of approximating the series by a partial sum is never greater than the magnitude of the \textit{first term omitted}.
\end{hint}
\end{hint}
\end{exercise}

\begin{exercise}
Use the remainder formula for Taylor series to determine which partial sums of the series
\[ \sum_{n=1}^\infty \frac{1}{n 3^n} \]
differ from $\ln (3/2)$ by at most $10^{-2}$?
\begin{itemize}
\item Guided by the example above, we will use the Maclaurin series
\[ - \ln (1-x) = \sum_{n=1}^\infty \frac{x^n}{n}. \]
First, compute the derivatives of $- \ln (1-x)$. Find a pattern which holds for all $N$:
\[ \frac{d^{N+1}}{dx^{N+1}} \left( - \ln (1 - x) \right)  = \answer{ \frac{N!}{(1-x)^{N+1}}}. \]
When $x$ is between $0$ and $1/3$, the largest value of the $(N+1)$-st derivative is what?
\[ \left| \frac{d^{N+1}}{dx^{N+1}} \left( - \ln (1 - x) \right) \right| \leq \answer{N! 3^{N+1}/2^{N+1}}. \]
\item Using this upper bound for the $(N+1)$-st derivative, we know that
\[ \left| \ln \frac{3}{2} - \sum_{n=0}^N \frac{1}{n 3^n} \right| \leq \answer{ \frac{1}{(N+1) 2^{N+1}}} \]
(use the upper bound and the remainder formula.)
\item For which values of $N$ is the expression you just found less than $1/100$? Check by hand for smallish values of $N$ to find the smallest one which works.
\[ N \geq \answer{4}. \]
\end{itemize}
\end{exercise}

\section*{Sample Quiz Questions}

\begin{question}%%%%%[TAp0]

Use Taylor series to estimate the value of
\[\sqrt[3]{\frac{11}{10}}\] to within an error of at most \(1/900\).
(Hints will not be revealed until you choose a response.)
\begin{multiplechoice}
\choice[correct]{\(\displaystyle \frac{31}{30}\)}
\choice{\(\displaystyle \frac{47}{45}\)}
\choice{\(\displaystyle \frac{19}{18}\)}
\choice{\(\displaystyle \frac{16}{15}\)}
\choice{\(\displaystyle \frac{97}{90}\)}
\choice{\(\displaystyle \frac{49}{45}\)}
\end{multiplechoice}
\begin{feedback}
We may use the remainder formula for Taylor series to approach this problem.
Suppose \(p_n(x)\) is the degree \(n\) Taylor polynomial of the function
\[ f(x) = \sqrt[3]{1+x}\]
with center \(a=0\). Then the error \(E_n(x)\), i.e., the difference between the polynomial and the function, does not exceed \(\frac{f^{(n+1)}(\xi)}{(n+1)!}x^{n+1}\), where \(\xi\) is some unknown point  in the range \(0 \leq \xi \leq x\). \begin{hint} In this case one should take \(x = 1/10\) and determine how many derivatives are required to make this error estimate less than the given threshold. \begin{hint} This means checking by hand for small numbers of derivatives. For the specific problem at hand, if we approximate \(f(x)\) by the Taylor polynomial of degree \( n = 1\), we have
\[ \left| \frac{x^{n+1} }{(n+1)!} f^{(n+1)}(\xi) \right| = \left| \frac{x^{n+1}}{(n+1)!} \left( -\frac{2}{9} (\xi+1)^{-5/3} \right) \right| \leq \left| \frac{x^{n+1}}{(n+1)!} \left( \frac{2}{9} \right) \right| = \frac{1}{900} \] when \(x = 1/10\). \begin{hint}
We conclude that the correct Taylor approximation is
\[ p_n \left( \frac{1}{10}\right) =  \left(\frac{1}{10}\right)^{0} + \frac{1}{3}\left(\frac{1}{10}\right)^{1} =  1 + \frac{1}{30} = \frac{31}{30}.\]
\end{hint} \end{hint} \end{hint}
\end{feedback}

\end{question}

\begin{question}%%%%%[TAp1]

Use Taylor series to estimate the value of
\[e^{-\frac{1}{3}}\] to within an error of at most \(1/162\).
\begin{multiplechoice}
\choice{\(\displaystyle \frac{5}{9}\)}
\choice[correct]{\(\displaystyle \frac{13}{18}\)}
\choice{\(\displaystyle \frac{8}{9}\)}
\choice{\(\displaystyle \frac{19}{18}\)}
\choice{\(\displaystyle \frac{11}{9}\)}
\choice{\(\displaystyle \frac{25}{18}\)}
\end{multiplechoice}
\begin{feedback}
We may use the remainder formula for Taylor series to approach this problem.
Suppose \(p_n(x)\) is the degree \(n\) Taylor polynomial of the function
\[ f(x) = e^{-x}\]
with center \(a=0\). Then the error \(E_n(x)\), i.e., the difference between the polynomial and the function, does not exceed \(\frac{f^{(n+1)}(\xi)}{(n+1)!}x^{n+1}\), where \(\xi\) is some unknown point  in the range \(0 \leq \xi \leq x\). \begin{hint}  In this case one should take \(x = 1/3\) and determine how many derivatives are required to make this error estimate less than the given threshold. \begin{hint} This means checking by hand for small numbers of derivatives. For the specific problem at hand, if we approximate \(f(x)\) by the Taylor polynomial of degree \( n = 2\), we have
\[ \left| \frac{x^{n+1} }{(n+1)!} f^{(n+1)}(\xi) \right| = \left| \frac{x^{n+1}}{(n+1)!} \left( e^{-\xi} \right) \right| \leq \left| \frac{x^{n+1}}{(n+1)!} \left( 1 \right) \right| = \frac{1}{162} \] when \(x = 1/3\). \begin{hint}
We conclude that the correct Taylor approximation is
\[ p_n \left( \frac{1}{3}\right) =  \left(\frac{1}{3}\right)^{0} -1\left(\frac{1}{3}\right)^{1} + \frac{1}{2}\left(\frac{1}{3}\right)^{2} =  1 -\frac{1}{3} + \frac{1}{18} = \frac{13}{18}.\] \end{hint} \end{hint} \end{hint}
\end{feedback}

\end{question}

\end{document}


\chapter{Operations}
\label{ch:operations}
%% FIXME: \vskip -4em \citep[chapter 2]{pinter_2016}

\begin{enumerate}[label={\Alph*.},font={\bfseries}]
\item {\bf Examples of Operations}
  \begin{enumerate}[label={\arabic*},font={\bfseries}]
  \item $a * b = \sqrt{\abs{ab}}$ is not an operation on $\mathbb{Q}$, because
    $2 * 1 = \sqrt{\abs{2}}$, but $\sqrt{\abs{2}} \not\in \mathbb{Q}$.
  \item $a * b = a \ln b$ is not an operation on $\mathbb{R}_{>0}$, because
    $\forall a,b\in\mathbb{R}_{>0} (b \le 1 \to a \ln b \not\in \mathbb{R}_{>0})$
  \item If $a * b$ is a root of the equation $x^2 - a^2b^2=0$,
    $*$ is not an operation on $\mathbb{R}$, because
    $\forall a,b \in \mathbb{R} (a \ne 0 \land b \ne 0 \to x = \pm ab)$
  \item Subtraction is an operation on $\mathbb{Z}$, because
    $\forall a,b \in \mathbb{Z} (a-b \in \mathbb{Z})$.
  \item Subtraction is not an operation on $\mathbb{Z}_{\ge 0}$, because
    e.g. $0-1 \not\in \mathbb{Z}_{\ge 0}$.
  \item $a * b = \abs{a-b}$ is an operation on $\mathbb{Z}_{\ge 0}$,
    because $\forall a,b \in \mathbb{Z}_{\ge 0} (\abs{a-b} \in
    \mathbb{Z}_{\ge 0})$.
  \end{enumerate}
\item {\bf Properties of Operations}
  \begin{enumerate}[label={\arabic*},font={\bfseries}]
  \item $x*y=x+2y+4$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is not commutative.
      \begin{align*}
        x*y &= x+2y+4 \\
        y*x &= y+2x+4 \\
        x*y &\neq y*x
      \end{align*}
    \item $*$ is not associative.
      \begin{align*}
        x*(y*z) &= x*(y+2z+4) \\
        &= x+2(y+2z+4)+4 \\
        &= x+2y+4z+12 \\
        (x*y)*z &= (x+2y+4)*z \\
        &= x+2y+4+2z+4 \\
        &= x+2y+2z+8 \\
        x+2y+4z+12 &\neq x+2y+2z+8
      \end{align*}
    \item $\mathbb{R}$ does not have an identity element with respect to $*$.
      \begin{align*}
        x*e &= x \\
        x+2e+4 &= x \\
        2e+4 &= 0 \\
        e &= -2 \\
        e*x &= x \\
        e+2x+4 &= x \\
        e &= -x-4 \neq -2
      \end{align*}
    \item Since there is no identity element, there can be no inverses.
    \end{enumerate}
  \item $x*y=x+2y-xy$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is not commutative.
      \begin{align*}
        x*y &= x+2y-xy \\
        y*x &= y+2x-yx \\
        x*y &\neq y*x
      \end{align*}
    \item $*$ is not associative.
      \begin{align*}
        x*(y*z) &= x*(y+2z-yz) \\
        &= x+2(y+2z-yz)-x(y+2z-yz) \\
        &= x+2y+4z-2yz-xy-2xz+xyz \\
        (x*y)*z &= (x+2y-xy)*z \\
        &=(x+2y-xy)+2z-(x+2y-xy)z \\
        &=x+2y+2z-2yz-xy-xz+xyz \\
        x*(y*z) &\neq (x*y)*z
      \end{align*}
    \item $\mathbb{R}$ does not have an identity element with respect to $*$.
      \begin{align*}
        x*e &= x \\
        x+2e-xe &= x \\
        2e-xe &= 0 \\
        e(2-x) &= 0 \\
        e &=0 \\
        e*x &= x \\
        e+2x-ex &= x \\
        e+x-ex &= 0 \\
        e(1-x) &= -x \\
        e &= -x(1-x) \neq 0
      \end{align*}
    \item Since there is no identity element, there can be no inverses.
    \end{enumerate}
  \item $x*y=\abs{x+y}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative.
      \begin{align*}
        x*y &= \abs{x+y} \\
        y*x &= \abs{y+x} = \abs{x+y} \\
        x*y &= y*x
      \end{align*}
    \item $*$ is not associative.
      \begin{align*}
        x*(y*z) &= x*\abs{y+z} = \abs{x+\abs{y+z}} \\
        (x*y)*z &= \abs{x+y}*z = \abs{\abs{x+y}+z} \\
        x=0,y<0 \to x*(y*z) &= \abs{y+z} \\
        (x*y)*z &= \abs{\abs{y}+z} \\
        y<0 \to y \neq \abs{y} \to \abs{y+z} &\neq \abs{\abs{y}+z}\\
        x*(y*z) &\neq (x*y)*z
      \end{align*}
    \item $\mathbb{R}$ has an identity element with respect to $*$.
      \begin{align*}
        x*e &= x \\
        \abs{x+e} &= x \\
        e &= 0 \\
        e*x &= x \\
        \abs{e+x} &= x \\
        e &= 0
      \end{align*}
    \item Every $x\in\mathbb{R}$ has an inverse with respect to $*$.
      \begin{align*}
        x*x^\prime &= 0 \\
        \abs{x+x^\prime} &= 0 \\
        x^\prime = -x \\
        x*(-x) &= \abs{x-x} = 0 \\
        (-x)*x &= \abs{-x+x} = 0 \\
        x*x^\prime &= x^\prime*x
      \end{align*}
    \end{enumerate}
  \item $x*y=\abs{x-y}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative.
      \begin{align*}
        x*y &= \abs{x-y} \\
        y*x &= \abs{y-x} \\
        x=y \to x*y &= 0 \\
        y*x &= 0 \\
      \end{align*}
      If $x<y$ then $x=y+k$, and:
      \begin{align*}
        x*y &= \abs{(y+k)-y} = \abs{k} \\
        y*x &= \abs{y-(y+k)} = \abs{-k} = \abs{k} \\
        x*y &= y*x
      \end{align*}
      If $x=y$:
      \begin{align*}
        x*y &= \abs{y-y} = 0 \\
        y*x &= \abs{y-y} = 0 \\
        x*y &= y*x
      \end{align*}
      If $x>y$ then $y=x+k$, and:
      \begin{align*}
        x*y &= \abs{x-(x+k)} = \abs{-k} = \abs{k}\\
        y*x &= \abs{(x+k)-x} = \abs{k} \\
        x*y &= y*x
      \end{align*}
    \item $*$ is not associative.
      \begin{align*}
        x*(y*z) &= x*\abs{y-z} \\
        &= \abs{x-\abs{y-z}} \\
        (x*y)*z &= \abs{x-y}*z \\
        &= \abs{\abs{x-y}-z} \\
      \end{align*}
      If $x=0$ and $y<0$:
      \begin{align*}
        x*(y*z) = \abs{-\abs{y-z}} = \abs{y-z} &= \sqrt{(y-z)^2} \\
        (x*y)*z = \abs{\abs{-y}-z} = \abs{\abs{y}-z} &= \sqrt{(\abs{y}-z)^2} \\
        \abs{y} &\neq y \\
        x*(y*z) &\neq (x*y)*z
      \end{align*}
    \item $\mathbb{R}$ does not have an identity element with respect to $*$.
      \begin{align*}
        x*e &= x \\
        \abs{x-e} &= x \\
        e &= 2x
      \end{align*}
    \item Since there is no identity element, there can be no inverses.
    \end{enumerate}
  \item $x*y=xy+1$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative.
      \begin{align*}
        x*y &= xy+1 \\
        y*x &= yx+1 = xy+1 \\
        x*y &= y*x
      \end{align*}
    \item $*$ is not associative.
      \begin{align*}
        x*(y*z) &= x*(yz+1) \\
        &= x(yz+1)+1 = xyz+x +1 \\
        (x*y)*z &= (xy+1)*z \\
        &= (xy+1)z+1 = xyz+z+1 \\
        x*(y*z) &\neq (x*y)*z
      \end{align*}
    \item $\mathbb{R}$ does not have an identity element with respect to $*$.
      \begin{align*}
        x*e &= x \\
        xe+1 &= x \\
        xe &= x-1 \\
        x &= 1-\frac{1}{x}
      \end{align*}
    \item Since there is no identity element, there can be no inverses.
    \end{enumerate}
  \item $x*y=\text{max}\Set{x,y}=\text{the larger of the two numbers $x$ and $y$}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative.
      \begin{align*}
        x*y &= \text{max}\Set{x,y} \\
        y*x &= \text{max}\Set{y,x} = \text{max}\Set{x,y} \\
        x*y &= y*x
      \end{align*}
    \item $*$ is associative.
      \begin{align*}
        x*(y*z) &= x*\text{max}\Set{y,z} \\
        &= \text{max}\Set{x,\text{max}\Set{y,z}} = \text{max}\Set{x,y,z} \\
        (x*y)*z &= (\text{max}\Set{x,y})*z \\
        &= \text{max}\Set{\text{max}\Set{x,y},z} = \text{max}\Set{x,y,z} \\
        x*(y*z) &= (x*y)*z
      \end{align*}
    \item $\mathbb{R}$ does not have an identity element with respect to $*$.
      \begin{align*}
        x*e &= x \\
        \text{max}\Set{x,e} &= x \\
        e &= \{ n\in\mathbb{R} : n \leq x \}
      \end{align*}
    \item Since there is no identity element, there can be no inverses.
    \end{enumerate}
  \item $x*y=\frac{xy}{x+y+1}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative.
      \begin{align*}
        x*y &= \frac{xy}{x+y+1} \\
        y*x &= \frac{yx}{y+x+1} = \frac{xy}{x+y+1} \\
        x*y &= y*x
      \end{align*}
    \item $*$ is associative.
      \begin{align*}
        x*(y*z) &= x*(\frac{yz}{y+z+1}) \\
        &= \frac{\frac{xyz}{y+z+1}}{x+\frac{yz}{y+z+1}+1} \\
        &= \frac{xyz}{x(y+z+1)+yz+(y+z+1)} \\
        &= \frac{xyz}{xy+xz+yz+x+y+z+1} \\
        (x*y)*z &= \left(\frac{xy}{x+y+1}\right)*z \\
        &= \frac{\frac{xyz}{x+y+1}}{\frac{xy}{x+y+1}+z+1} \\
        &= \frac{xyz}{xy+z(x+y+1)+z+(x+y+1)} \\
        &= \frac{xyz}{xy+xz+yz+x+y+z+1} \\
        x*(y*z) &= (x*y)*z
      \end{align*}
    \item $\mathbb{R}$ does not have an identity element with respect to $*$.
      \begin{align*}
        x*e &= x \\
        \frac{xe}{x+e+1} &= x \\
        xe &= x(x+e+1) \\
        e &= e+x+1
      \end{align*}
    \item Since there is no identity element, there can be no inverses.
    \end{enumerate}
  \end{enumerate}
\item {\bf Operations on a Two-Element Set} \\
  Let $A$ be the two-element set $A=\Set{a,b}$.
  \begin{enumerate}[label={\arabic*},font={\bfseries}]
  \item
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_1$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $a$ \\
        $(a,b)$ & $a$ \\
        $(b,a)$ & $a$ \\
        $(b,b)$ & $a$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_2$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $a$ \\
        $(a,b)$ & $a$ \\
        $(b,a)$ & $a$ \\
        $(b,b)$ & $b$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_3$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $a$ \\
        $(a,b)$ & $a$ \\
        $(b,a)$ & $b$ \\
        $(b,b)$ & $a$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_4$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $a$ \\
        $(a,b)$ & $a$ \\
        $(b,a)$ & $b$ \\
        $(b,b)$ & $b$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_5$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $a$ \\
        $(a,b)$ & $b$ \\
        $(b,a)$ & $a$ \\
        $(b,b)$ & $a$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_6$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $a$ \\
        $(a,b)$ & $b$ \\
        $(b,a)$ & $a$ \\
        $(b,b)$ & $b$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_7$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $a$ \\
        $(a,b)$ & $b$ \\
        $(b,a)$ & $b$ \\
        $(b,b)$ & $a$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_8$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $a$ \\
        $(a,b)$ & $b$ \\
        $(b,a)$ & $b$ \\
        $(b,b)$ & $b$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_9$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $b$ \\
        $(a,b)$ & $a$ \\
        $(b,a)$ & $a$ \\
        $(b,b)$ & $a$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_{10}$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $b$ \\
        $(a,b)$ & $a$ \\
        $(b,a)$ & $a$ \\
        $(b,b)$ & $b$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_{11}$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $b$ \\
        $(a,b)$ & $a$ \\
        $(b,a)$ & $b$ \\
        $(b,b)$ & $a$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_{12}$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $b$ \\
        $(a,b)$ & $a$ \\
        $(b,a)$ & $b$ \\
        $(b,b)$ & $b$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_{13}$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $b$ \\
        $(a,b)$ & $b$ \\
        $(b,a)$ & $a$ \\
        $(b,b)$ & $a$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_{14}$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $b$ \\
        $(a,b)$ & $b$ \\
        $(b,a)$ & $a$ \\
        $(b,b)$ & $b$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_{15}$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $b$ \\
        $(a,b)$ & $b$ \\
        $(b,a)$ & $b$ \\
        $(b,b)$ & $a$
      \end{tabular}
    \end{minipage}
    \begin{minipage}[h]{.25\textwidth}
      \captionof{table}{$0_{16}$}
      \begin{tabular}{ r | l }
        $(x,y)$ & $x*y$ \\
        \hline
        $(a,a)$ & $b$ \\
        $(a,b)$ & $b$ \\
        $(b,a)$ & $b$ \\
        $(b,b)$ & $b$
      \end{tabular}
    \end{minipage}
  \item Commutativity
    \begin{itemize}
    \item $0_1$ is commutative: $a*b=a=b*a$
    \item $0_2$ is commutative: $a*b=a=b*a$
    \item $0_3$ is not commutative: $a*b=a \neq b=b*a$
    \item $0_4$ is not commutative: $a*b=a \neq b=b*a$
    \item $0_5$ is not commutative: $a*b=b \neq a=b*a$
    \item $0_6$ is not commutative: $a*b=b \neq a=b*a$
    \item $0_7$ is commutative: $a*b=b=b*a$
    \item $0_8$ is commutative: $a*b=b=b*a$
    \item $0_9$ is commutative: $a*b=a=b*a$
    \item $0_{10}$ is commutative: $a*b=a=b*a$
    \item $0_{11}$ is not commutative: $a*b=a \neq b=b*a$
    \item $0_{12}$ is not commutative: $a*b=a \neq b=b*a$
    \item $0_{13}$ is not commutative: $a*b=b \neq a=b*a$
    \item $0_{14}$ is not commutative: $a*b=b \neq a=b*a$
    \item $0_{15}$ is commutative: $a*b=b=b*a$
    \item $0_{16}$ is commutative: $a*b=b=b*a$
    \end{itemize}
  \item Associativity
    \begin{itemize}
    \item $0_1$ is associative:
      $$\forall x,y \in A (x*y=a \to x*(y*z)=x*a=a=a*z=(x*y)*z)$$
    \item $0_2$ is associative.
      \begin{align*}
        a*(a*a) = a*a &= (a*a)*a \\
        a*(a*b) = a*a &= a*b = (a*a)*b \\
        a*(b*a) = a*a &= (a*b)*a \\
        a*(b*b) = a*b &= (a*b)*b \\
        b*(a*a) = b*a &= a*a = (b*a)*a \\
        b*(a*b) = b*a &= a*b = (b*a)*b \\
        b*(b*a) = b*a &= (b*b)*a \\
        b*(b*b) = b*b &= (b*b)*b
      \end{align*}
    \item $0_3$ is not associative: $b*(a*b)=b*a=b \neq a=b*b=(b*a)*b$
    \item $0_4$ is associative.
      \begin{align*}
        a*(a*a) = a*a &= (a*a)*a \\
        a*(a*b) = a*a &= a*b = (a*a)*b \\
        a*(b*a) = a*b &= a*a = (a*b)*a \\
        a*(b*b) = a*b &= (a*b)*b \\
        b*(a*a) = b*a &= (b*a)*a \\
        b*(a*b) = b*a &= b*b = (b*a)*b \\
        b*(b*a) = b*b &= b*a = (b*b)*a \\
        b*(b*b) = b*b &= (b*b)*b
      \end{align*}
    \item $0_5$ is not associative: $b*(a*b)=b*b=a \neq b=a*b=(b*a)*b$
    \item $0_6$ is associative.
      \begin{align*}
        a*(a*a) = a*a &= (a*a)*a\\
        a*(a*b) = a*b &= (a*a)*b\\
        a*(b*a) = a*a &= b*a = (a*b)*a \\
        a*(b*b) = a*b &= (a*b)*b \\
        b*(a*a) = b*a &= (b*a)*a \\
        b*(a*b) = b*b &= (b*a)*b \\
        b*(b*a) = b*a &= (b*b)*a \\
        b*(b*b) = b*b &= (b*b)*b
      \end{align*}
    \item $0_7$ is associative.
      \begin{align*}
        a*(a*a) = a*a &= (a*a)*a \\
        a*(a*b) = a*b &= (a*a)*b \\
        a*(b*a) = a*b &= b*a = (a*b)*a \\
        a*(b*b) = a*a &= b*b = (a*b)*b \\
        b*(a*a) = b*a &= (b*a)*a \\
        b*(a*b) = b*b &= (b*a)*b \\
        b*(b*a) = b*b &= a*a = (b*b)*a \\
        b*(b*b) = b*a &= a*b = (b*b)*b
      \end{align*}
    \item $0_8$ is associative.
      \begin{align*}
        a*(a*a) = a*a &= (a*a)*a \\
        a*(a*b) = a*b &= (a*a)*b \\
        a*(b*a) = a*b &= b*a = (a*b)*a \\
        a*(b*b) = a*b &= b*b = (a*b)*b \\
        b*(a*a) = b*a &= (b*a)*a \\
        b*(a*b) = b*b &= (b*a)*b \\
        b*(b*a) = b*b &= b*a = (b*b)*a \\
        b*(b*b) = b*b &= (b*b)*b
      \end{align*}
    \item $0_9$ is not associative: $a*(a*b)=a*a=b \neq a=b*b=(a*a)*b$
    \item $0_{10}$ is associative.
      \begin{align*}
        a*(a*a) = a*b &= b*a = (a*a)*a \\
        a*(a*b) = a*a &= b*b = (a*a)*b \\
        a*(b*a) = a*a &= (a*b)*a \\
        a*(b*b) = a*b &= (a*b)*b \\
        b*(a*a) = b*b &= a*a = (b*a)*a \\
        b*(a*b) = b*a &= a*b = (b*a)*b \\
        b*(b*a) = b*a &= (b*b)*a \\
        b*(b*b) = b*b &= (b*b)*b
      \end{align*}
    \item $0_{11}$ is not associative: $a*(a*a)=a*b=a \neq b=b*a=(a*a)*a$
    \item $0_{12}$ is not associative: $a*(b*a)=a*b=a \neq b=a*a=(a*b)*a$
    \item $0_{13}$ is not associative: $a*(a*a)=a*b=b \neq a=b*a=(a*a)*a$
    \item $0_{14}$ is not associative: $a*(b*a)=a*a=b \neq a=b*a=(a*b)*a$
    \item $0_{15}$ is not associative: $a*(a*a)=a*b=b \neq a=b*b=(a*a)*b$
    \item $0_{16}$ is associative:
      $$\forall x,y \in A (x*y=b \to x*(y*z)=x*b=b=b*z=(x*y)*z)$$
    \end{itemize}
  \item Identity
    \begin{itemize}
    \item $A$ does not have an identity element with respect to $0_1$.
    \item $A$ has an identity element with respect to $0_2$.
      \begin{align*}
        x*e &= x \\
        a*b &= a \\
        b*b &= b \\
        e &= b \\
        e*x &= x \\
        b*a &= a \\
        b*b &= b \\
        e &= b
      \end{align*}
    \item $A$ does not have an identity element with respect to $0_3$.
    \item $A$ does not have an identity element with respect to $0_4$.
    \item $A$ does not have an identity element with respect to $0_5$.
    \item $A$ does not have an identity element with respect to $0_6$.
    \item $A$ does not have an identity element with respect to $0_7$.
    \item $A$ has an identity element with respect to $0_8$.
      \begin{align*}
        x*e &= x \\
        a*a &= a \\
        b*a &= b \\
        e &= a \\
        e*x &= x \\
        a*a &= a \\
        a*b &= b \\
        e &= a
      \end{align*}
    \item $A$ does not have an identity element with respect to $0_9$.
    \item $A$ has an identity element with respect to $0_{10}$.
      \begin{align*}
        x*e &= x \\
        a*b &= a \\
        b*b &= b \\
        e &= b \\
        e*x &= x \\
        b*a &= a \\
        b*b &= b \\
        e &= b
      \end{align*}
    \item $A$ does not have an identity element with respect to $0_{11}$.
    \item $A$ does not have an identity element with respect to $0_{12}$.
    \item $A$ does not have an identity element with respect to $0_{13}$.
    \item $A$ does not have an identity element with respect to $0_{14}$.
    \item $A$ does not have an identity element with respect to $0_{15}$.
    \item $A$ does not have an identity element with respect to $0_{16}$.
    \end{itemize}
  \item Since $A$ only has identity elements with respect to $0_2$, $0_8$,
    and $0_{10}$, the rest cannot have inverses. As it turns out, with
    respect to those three operations, it is not the case that every $x \in
    A$ has an inverse.
  \end{enumerate}
\item {\bf Automata: The Algebra of Input/Output Sequences} \\
  Let $A$ be an alphabet and $A^*$ be the set of all sequences of symbols in the alphabet $A$. There is an operation on $A^*$ called {\em concatenation}: If {\bf a} and {\bf b} are in $A^*$, say $\textbf{a} = a_1a_2...a_n$ and $\textbf{b} = b_1b_2...b_m$, then $$\textbf{ab} = a_1a_2...a_nb_1b_2...b_m$$
  The symbol $\lambda$ denotes the empty sequence.
  \begin{enumerate}[label={\arabic*},font={\bfseries}]
  \item Concatenation is associative.
    %% $$a = a_1a_2...a_n; b = b_1b_2...b_m; c = c_1c_2...c_k$$
    \begin{align*}
      a(bc) &= a(b_1b_2...b_mc_1c_2...c_k) = a_1a_2...a_nb_1b_2...b_mc_1c_2...c_k \\
      (ab)c &= (a_1a_2...a_nb_1b_2...b_m)c = a_1a_2...a_nb_1b_2...b_mc_1c_2...c_k \\
      a(bc) &= (ab)c
    \end{align*}
  \item Concatenation is not commutative.
    %% $$a = a_1a_2...a_n; b = b_1b_2...b_m$$
    \begin{align*}
      ab &= a_1a_2...a_nb_1b_2...b_m \\
      ba &= b_1b_2...b_ma_1a_2...a_n \\
      ab &\neq ba
    \end{align*}
  \item $\lambda$ is the identity element for concatenation:
    $x\lambda = \lambda{}x = x$
  \end{enumerate}
\end{enumerate}


\section{Statistic: Statistics}
\subsection*{describe}
This analysis follows from NR92, pp 610--613.  However, the basic definitions
are common to all statistics books.  The key insight from NR92 (and not
original there), is the 2-pass corrected calculation of variance.

To rehearse the equations:
\begin{eqnarray}
  \mbox{mean} & = & \mbox{avg} = \overline{x} =
                    \frac{1}{N}\sum_{j=1}^{N}x_j\\
  \mbox{avg dev} & = & \frac{1}{N}\sum_{j=1}^{N}|x_j-\overline{x}|\\
  \mbox{var} & = & \frac{1}{N}\left [ \sum_{j=1}^{N}(x_j-\overline{x})^2
                   - \frac{1}{N} \left [ \sum_{j=1}^{N}(x_j
                       -\overline{x})\right]^2\right]\\
  \mbox{std dev} & =& \sigma  =  \sqrt{\mbox{var}}\\
  \mbox{skew} & = & \frac{1}{N}\sum_{j=1}^{N}
                    \left [\frac{x_j-\overline{x}}{\sigma}\right]^3\\
  \mbox{kurt} & = & \left\{\frac{1}{N}\sum_{j=1}^{N}
                    \left [\frac{x_j-\overline{x}}{\sigma}\right]^4
                    \right\} - 3
\end{eqnarray}
We also pick up min and max values during the first pass.

\subsection*{ttest}
This is Student's t-test.  The idea is to find if the means of
two samples are the same and how likely that is to be due to chance.

Again, we follow NR92, pg 616, but the formula
is common:
\begin{eqnarray}
s_D & =  & \sqrt{\frac{\sum_{i\in A}(x_i-\overline{x_A})^2
                      +\sum_{i\in B}(x_i-\overline{x_B})^2}
                      {N_A+N_B-2}
                 \left(\frac{1}{N_A}+\frac{1}{N_B}\right)
                }\\
t & = & \frac{\overline{x_A}-\overline{x_B}}{s_D}
\end{eqnarray}

There are a number of ways
to compute the probability that this was due to chance.  Following
NR92, we use:
\begin{eqnarray}
    I_x(a,b) & = & \mbox{incomplete Beta function}
               = \frac{B_x(a,b)}{B(a,b)}
               = \frac{1}{B(a,b)}\int_0^xt^{a-1}(1-t)^{b-1}dt\\
    \nu & = & \mbox{degrees of freedom}\\
    A(t|\nu) & = & \mbox{probability that t could be that small by chance}\\
             & = & 1- I_{\frac{\nu}{\nu_t^2}}(\frac{\nu}{2},\frac{1}{2})
\end{eqnarray}

\subsection*{ftest}
This is the F-test.  The idea is to find if the variances of two
samples are the same, and how likely that is to be due to chance.
\begin{equation}
 F = \mbox{var1} / \mbox{var2}
\end{equation}
var1 and var2 are chosen so the ratio comes out  $>1$.  There are a number of ways
to compute the probability that this was due to chance.  Following
NR92, we use:
\begin{eqnarray}
    \nu_1,\nu_2 & = & \mbox{degrees of freedom}\\
    Q(F|\nu_1,\nu_2) & = & \mbox{probability that F could be that small by chance}\\
             & = & I_{\frac{\nu_2}{\nu_2+\nu_1F}}
                     (\frac{\nu_2}{2},\frac{\nu_1}{2})
\end{eqnarray}

\subsection*{chi\_sqr1}
$\chi^2$ tests whether or not a binned distribution is as expected.
NR92 (pg 621, eqn 14.3.1 and pg 221, eqn 6.2.19) gives:
\begin{eqnarray}
  N_i & = & \mbox{number of items in bin i}\\
  n_i & = & \mbox{number of items expected in bin i}\\
  \chi^2 & = & \sum_i\frac{(N_i-n_i)^2}{n_i}\\
  Q(\chi^2|\nu) & = & Q(\frac{\nu}{2},\frac{\chi^2}{2})
         = \mbox{incomplete gamma function P}
\end{eqnarray}

Krey88, pg 1280 says the bins should have $\ge 5$ items each.  We'll
raise an exception if that is not met.

\subsection*{chi\_sqr2}
This version of $\chi^2$ checks similarlity between two actual
collecitons of bins (not just an actual and an expected).
NR92, pg 622, give sthe equaiton eqn 14.3.2:
\begin{eqnarray}
  R_i & = & \mbox{number of items in R's bin i}\\
  S_i & = & \mbox{number of items in S's bin i}\\
  \chi^2 & = & \sum_i\frac{(R_i-S_i)^2}{R_i+S_i}\\
  Q(\chi^2|\nu) & = & Q(\frac{\nu}{2},\frac{\chi^2}{2})
         = \mbox{incomplete gamma function P}
\end{eqnarray}


\subsection{Motivation}
The paradigmatic problem of Fast Multipole Methods (\textbf{\gls{FMM}})
\footnote{The first usage of a technical term or abbreviation listed in the
glossary is highlighted throughout the text for ease of reference.} is the so
called $N$-Body problem. This classic problem refers to the calculation of the pairwise
interactions between $N$ particles over a potentially long-range, for example in
gravitational or electrostatic problems. The straightforward calculation can be
written in the form of the following sum,

\begin{equation}
\Phi(x_j) = \sum_{i=1}^N w_i K(x_i, x_j)
\label{eq:n_body_problem}
\end{equation}

Where $i, j \in [1,N]$ and $K(x, y)$ is called the Green's function, or equivalently
a `kernel function', where one is generally concerned with coordinates of particles in an
$n=2 \> \> \text{or} \> \> 3$ dimensional Hilbert space taking  $\ x_i \in \mathbb{R}^n$.
Additionally, each summand is weighted by $w_i$. For solving for electrostatic
potential in three dimensions, which is used as the model problem throughout this
thesis, this goes to,

\begin{equation}
\Phi(x_j) = \sum_{i=1}^N q_iK(x_i, x_j)
\label{eq:electrostatic_paradigm}
\end{equation}

where $q_i$ refers to a charge density with the kernel function,

\begin{equation}
    K(x, y) = \frac{1}{4\pi \epsilon_0}\frac{1}{| x - y |}
\label{eq:laplace_kernel}
\end{equation}

the constant $\epsilon_0$ is the permittivity of free space. For practical
purposes, we define $K(x,x) = 0$. It's easy to see
how a naive direct application of this equation over $N$ particles
results in an algorithm of $O(N^2)$ complexity, therefore it's only practicable
for systems of moderate size, whereas in realistic systems, one may be interested in
interactions involving $10^{6}$ to $10^8$ particles.

This chapter introduces the analytic \gls{FMM}, the kernel-independent
version, which is the main focus of this thesis, is presented later.
Though substantially different in implementation, the analytic FMM will provide the opportunity
to exposit many of the key ideas behind all FMM-based algorithms, and provides
a good starting point for understanding and developing upon these algorithms.
First presented by Greengard \cite{Greengard:1987:JCP},
the analytic \gls{FMM} represented a sea change for $N$-Body simulation. By
trading off computations for error, it manages to achieve an asymptotic complexity
 of just $O(N)$. Additionally, it comes equipped with rigorous error bounds,
making fast and accurate massive $N$-Body simulations feasible on available
computing hardware. It's success has been such that it is regarded as one of
the key developments in numerical algorithms in the twentieth century \cite{Cipra:2000:SN}.

The original analytic FMM solves the electrostatic problem
in two and three dimensions, this is equivalently known as the Poisson problem,
represented by the differential equation,

\begin{equation}
    \nabla^2 \phi =f
\label{eq:poisson}
\end{equation}

Where $\phi$ is some scalar potential to be determined, and $f$ is a scalar source
term which is usually known. For electrostatics the corresponding formulation
can be derived from Gauss' law as \cite{Griffiths:2017:CUP},

\begin{equation}
  \nabla^2 \phi = - \frac{q}{\epsilon_0}
\label{eq:electrostatic_poisson}
\end{equation}

where $\phi$ is the electrostatic potential, $q$ is the charge density and
$\epsilon_0$ is the permittivity of free space. It can therefore be seen that
the \gls{FMM} is actually solving the Poisson problem by reformulating it as an
integral equation \cite{Epton:1995:SIAM}. The ubiquity of problems of the form (\ref{eq:n_body_problem})
in computational science has lead to diverse applications of the FMM. For example,
in the modeling the electrostatic interactions of charged particles in complex
biological molecules at biologically relevant length scales \cite{Board:1992:CPL}.
The extension of FMMs to Helmholtz equations \cite{Rokhlin:1990:JCP}, has lead
to even more applications, such as in seismic and acoustic scattering
\cite{Hwu:2011:MKP}. Though as the focus of this thesis is on solving the Poisson
model problem for electrostatics, this is mentioned only for completeness.

The key insight that leads to the \gls{FMM}'s asymptotic complexity is the idea
that if the field created by a distribution of charge (or mass) density is approximated
to be relatively smooth in the \textbf{\gls{far-field}}, then it should be possible to apply
some form of compression for the evaluation of contribution to local potentials due
to particles in the \gls{far-field}. The FMM performs this compression by encoding
the field contributions of particles in the \gls{far-field} using a multipole
expansion.

For simple kernel functions and charge distributions, such as the model problem
of this thesis, one can easily derive the expression for this multipole expansion
by finding an series expansion of the system's Green's function.
In order to generalise the discussion, an arbitrary continuous
distribution of charge is considered as shown in figure \ref{fig:1_1_continuous_charge_distribution},
for which the potential is evaluated at some other evaluation point outside of
the distribution. This can be written as follows \cite{Griffiths:2017:CUP},

\begin{flalign}
    \Phi(\mathbf{r}) = \frac{1}{4 \pi \epsilon_0} \int \frac{1}{d}\rho(\mathbf{r}')d\tau'
    \label{eq:1_1_continuous_integral_formulation}
\end{flalign}

where $\rho(\mathbf{r}')$ is a charge density, and the other symbols take their
meanings from figure (\ref{fig:1_1_continuous_charge_distribution}).

\begin{figure}[!h]
    \centering
    {\includegraphics[width=0.45\textwidth]{introduction/continuous_charge.png}}
  \vspace{0pt}
  \caption{An arbitrary charge distribution, with an orange point to mark a point
  where the potential is being evaluated. Here, $\mathbf{r}$ is the the vector
  between the centre of the multipole expansion and the evaluation point, $\mathbf{r}'$
  is the vector between the centre of expansion and a given volume element $d\tau'$, and
  $d$ is a vector between the volume element $d\tau'$ and the evaluation point.}
  \label{fig:1_1_continuous_charge_distribution}
\end{figure}

From law of the cosines,

\begin{flalign}
    d^2 &= r^2 + (r')^2 - 2rr'\cos \alpha = r^2 \left [ 1 + \left ( \frac{r'}{r} \right)^2 - 2 \left (\frac{r'}{r} \right)\cos \alpha \right]\\
    d &= r \sqrt{1+\epsilon},
    \label{eq:1_1_law_of_cosines}
\end{flalign}

where,

\begin{flalign}
    \epsilon \equiv \left ( \frac{r'}{r} \right) \left (\frac{r'}{r} - 2 \cos \alpha \right)
\end{flalign}

As $\epsilon$ is small far away from charge distribution one can expand $1/d$ binomially,

\begin{flalign}
    \frac{1}{d} &= \frac{1}{r}(1+\epsilon)^{-1/2} = \frac{1}{r}\left (1 - \frac{1}{2}\epsilon + \frac{3}{8}\epsilon^2 - ... \right) \\
    \frac{1}{d} &= \frac{1}{r} \sum_{n=0}^{\infty} \left(\frac{r'}{r} \right)^n P_n(\cos \alpha),
\end{flalign}

where $P_n(\cos \alpha)$ are Legendre polynomials. Using this, the exact multipole
expansion for this charge distribution is,

\begin{flalign}
    \Phi(\mathbf{r}) = \frac{1}{4 \pi \epsilon_0}\sum_{n=0}^{\infty}\frac{1}{r^{n+1}}\int (r')^nP_n(\cos \alpha)\rho(\mathbf{r'}) d \tau'
\end{flalign}

If instead one considers a charge distribution composed of $N$ discrete charges
$q_i$ at positions $r_i$, this goes to,

\begin{flalign}
    \Phi(\mathbf{r}) = \frac{1}{4 \pi \epsilon_0}\sum_{i=1}^N\sum_{n=0}^{\infty}\frac{(r_i)^n q_i}{r^{n+1}}P_n(\cos \alpha)
\end{flalign}

Using the addition theorem for Legendre polynomials \cite{Greengard:1987:Yale},

\begin{flalign}
    P_n(\cos \gamma) = \sum_{m=-n}^n Y_n^{-m}(\alpha, \beta) Y_n^m(\theta, \phi),
\end{flalign}

where the Legendre polynomial is written in terms of spherical harmonics,
where $(r, \theta, \phi)$ and $(\rho, \alpha, \beta)$ define two spherical coordinates,
and $\gamma$ is the angle subtended between them. The multipole expansion goes to,

\begin{flalign}
    \Phi(\mathbf{r}) &= \frac{1}{4 \pi \epsilon_0}\sum_{i=1}^N\sum_{n=0}^{\infty}\frac{(r_i)^n q_i}{r^{n+1}}P_n(\cos \alpha)\\
    & = \frac{1}{4 \pi \epsilon_0}\sum_{i=1}^N\sum_{n=0}^{\infty}\sum_{m=-n}^n \frac{(r_i)^n q_i Y_n^{-m}(\alpha_i, \beta_i) }{r^{n+1}}Y_n^m(\theta, \phi)\\
    & = \frac{1}{4 \pi \epsilon_0}\sum_{n=0}^{\infty}\sum_{m=-n}^n M_n^m \cdot \frac{Y_n^m(\theta, \phi)}{r^{n+1}}  ,
    \label{eq:1_1_multipole_expansion}
\end{flalign}

where,

\begin{equation}
    M_n^m = \sum_{i=1}^N (r_i)^n q_i Y_n^{-m}(\alpha_i, \beta_i)
\end{equation}

This is an exact expansion, and it converges for $\frac{r_i}{r} < 1$. This convergence
condition means that estimating the potential at a given evaluation point
using the multipole expansion is only possible in the far-field, the
boundary of which is often tuned empirically for different systems as it's user
defined. If instead the expansion is taken centered at the evaluation point,
one can rewrite as the multipole expansion as a `local' expansion,

\begin{flalign}
    \Phi(\mathbf{r}) & = \frac{1}{4 \pi \epsilon_0}\sum_{i=1}^N\sum_{n=0}^{\infty}\sum_{m=-n}^n \frac{(r)^n q_i Y_n^{-m}(\alpha_i, \beta_i) }{(r_i)^{n+1}}Y_n^m(\theta, \phi)\\
    & = \frac{1}{4 \pi \epsilon_0} \sum_{n=0}^{\infty} \sum_{m=-n}^n L_n^m \cdot  Y_n^m(\theta, \phi) \cdot r^n,
    \label{eq:1_1_local_expansion}
\end{flalign}

where,

\begin{equation}
    L^m_n = \sum_{i=1}^N \frac{q_i Y^{-m}_n(\alpha_i, \beta_i)}{(r_i)^{n+1}},
\end{equation}

which converges when $\frac{r}{r_i} < 1$. The region of convergence for both types
of expansions are shown in figure (\ref{fig:1_1_multipole_local_expansions}).

The key point to note is that the multipole and local expansions are exact, and
can be truncated as required to ensure that the asymptotic complexity of evaluating
a multipole or local expansion at an evaluation point is bounded by $O(N)$. A
rigorous error analysis of the analytic FMM is outside the scope of this thesis, and
we defer to the literature for exact expressions for these truncation errors \cite{Greengard:1987:JCP},
Furthermore, there exist exact operations for shifting the center of these expansions,
as well as for translating multipole expansion coefficients into equivalent local expansion
coefficients. These are crucial in providing the improvements to asymptotic complexity\footnote{
    Expressions for these shift operators for the three dimensional Laplace kernel
    considered as our model problem are provided in Appendix \ref{app:3d_laplace}.
}.

\begin{figure}[!h]
    \centering
    {\includegraphics[width=0.33\textwidth]{introduction/multipole_expansion.png}}
    \hfill
  {\includegraphics[width=0.4\textwidth]{introduction/local_expansion.png}}
  \vspace{0pt}
  \caption{(A) A multipole expansion centered on charge distribution. (B) A local
  expansion, centered around a point of evaluation. Regions in which the expansions
  converge are shaded in grey. For the multipole expansion this is the entire domain
  outside of the region for $r>r_i, \> \forall i \in [1, N]$, and for the local expansion this is the region
  for which $r < r_i, \>  \forall i \in [1, N]$}

  \label{fig:1_1_multipole_local_expansions}
\end{figure}

\hspace{10pt}

\subsection{Algorithm Structure \& Analysis}

The convergence condition of the multipole expansion prohibits the compression
of charges from particles in the \textbf{\gls{near-field}}. Therefore the FMM
makes use of a tree structure in a recursive algorithm, this structure is known
as an Octree in three dimensions and a Quadtree in two dimensions\footnote{The usage
of trees naturally leads to biological adjectives to describe their structure, for example
the coarsest level of a tree is referred to as the `root' level, and the finest
level as the `leaf' level. Confusingly these are often combined with familial adjectives
to describe the relationship between tree nodes, for example parent and child nodes to describe
the relationship between a given node and the nodes that occupy the same domain
a level deeper in the tree. Each node has exactly one parent, except the root node which
has no parents.}. This structure
hierarchically partitions space such that each level, $l$, of the tree is equally partioned into
$(2^d)^l$ boxes\footnote{Regardless of the spatial dimension, partitions of a domain are invariably
referred to as boxes.} over the domain of the tree, where $d$ is the dimension,
i.e. $d=3$ in three dimensions. If one were to simply traverse the tree from
the coarsest, or `root', level to the finest, or `leaf', level and find the multipole expansion of source
particles in each box of each level, one could then evaluate these multipole expansions
at each particle to solve the $N$-Body problem. As there are $O(\log(N))$
boxes in the tree, and $N$ particles, this results in a $O(N\log(N))$ asymptotic
complexity.

However the FMM reduces computational complexity further by making use of local
expansions. Using the exact expressions to shift the multipole expansion
coefficients\footnote{Expressions for these shift operators for the three dimensional
Laplace kernel considered as our model problem are provided in Appendix \ref{app:3d_laplace}.}
$M_n^m$ to local expansion coefficients $L_n^m$ one can approximate the the interaction
between two boxes in the tree. Roughly speaking, because of the hierarchical
nature of the tree, each box only needs to consider the interaction with a
constant number of neighbouring boxes. Because the number of boxes is $O(N)$,
the FMM is bound by an $O(N)$ asymptotic complexity \cite{Hwu:2011:MKP}.

Using the above analysis, one can then describe the FMM algorithm in terms of
two basic steps,

\begin{enumerate}
    \item \textbf{\textit{Upward Pass}}: The tree is traversed \textbf{\gls{post-order}}.
    Beginning at the leaf boxes, a multipole expansion is computed for each box
    due to the \textbf{\gls{source-particles}} it contains. This is also referred to as the particle-to-multipole
    operation, or \textbf{\gls{P2M}}. Then as one moves up the tree hierarchy, the
    multipole expansions of a each box's parent box is computed by shifting the
    expansion centers of the multipole expansion of a given child box, to the center
    of the parent box, in a multipole-to-multipole operation or \textbf{\gls{M2M}},
    and summing together all the coefficients. Following the upward pass, one
    obtains the multipole expansion for each box containing source particles at
    all levels of the hierarchical tree.
    \item \textbf{\textit{Downward Pass}}: The tree is now traversed in
    \textbf{\gls{pre-order}}, and the local expansion of each box is computed.
    This local expansion is the sum of two parts: (1) the local expansion of the
    parent box of a given box, if it exists, which is a compression of the potential due to boxes
    non-adjacent to a given box's parent. The parent box's local expansion centre
    is shifted to the center of the child box, this is also known as the local-to-local
    operation or \textbf{\gls{L2L}}. (2) The multipole expansion of boxes which
    are the children of the \textbf{\gls{near-neighbours}} of a given box's
    parent but are not adjacent to the box itself. Such `source' boxes are described
    as being in the \textbf{\gls{interaction-list}} of a given box. These multipole
    expansions for each source box in a given box's interaction list are translated into local expansions centered at
    the given box, this is also known as
    the multipole-to-local operation, or \textbf{\gls{M2L}}. Notice that the
    \gls{M2L} operation is only available from $l=2$ of the tree, as in coarser
    levels the \gls{interaction-list} for all boxes is empty. The coefficients found
    from (1) and (2) are summed for each box. Operations (1) and (2) are repeated for each
    box until the leaf level. At this point, the local expansion
    of each leaf box is evaluated at all the \textbf{\gls{target-particles}} it contains.
    This local-to-particle, or \textbf{\gls{L2P}}, operation encodes all the \gls{far-field}
    interaction of the target particles in this leaf box. This is then combined with
    a \gls{near-field} interaction, due to the source particles in the leaf box,
    as well as in the \gls{near-neighbours}, which are computed directly. As the
    tree is refined to the point where the leaf levels contain only a small constant
    number of particles, this final direct computation is of low cost.
\end{enumerate}

With this specification, a more detailed analysis of the algorithm is possible,
though we defer to \cite{Greengard:1987:JCP} for a rigorous discussion.
Firstly, as mentioned above, the tree must be refined such that the leaf boxes
contain only a small constant number of particles, $\kappa$. The level of refinement $n$
is therefore approximately taken to be $n \approx \log(N)$, where $N$ is the number
of \gls{source-particles} in the tree. Beginning with upward pass, at the leaf level each particle contributes to one
multipole expansion. If this expansion is truncated to contain $p$ multipole terms, the
\gls{P2M} operation has a complexity of $O(Np^2)$, which can be seen from
(\ref{eq:1_1_multipole_expansion}), as well as the fact that the nature of a
hierarchical tree means that are $O(N)$ boxes at the leaf level. The shift operators\footnote{
    Expressions for these shift operators for the three dimensional Laplace kernel
    considered as our model problem are provided in Appendix \ref{app:3d_laplace}
} \gls{M2L}, \gls{L2L} and \gls{M2M} require $p^4$ operations with this truncation,
so the computation of all of these are bounded by\footnote{A more precise
bound depends on the size of a given box's \gls{interaction-list}.} $O(Np^4)$. Finally,
evaluating the $p^{th}$ degree local expansions at each target particle in the
\gls{L2P} operation, is bounded by $O(Np^2)$. The choice for level of refinement
$n$, leads to $O(\kappa N)$ complexity for direct calculations at the
leaf level. The whole algorithm is therefore bounded by $O(N)$.
In practice, the number of expansion terms $p$ is chosen for a prescribed relative error $\epsilon$, using
\footnote{There are optimal choices for $c$, with the authors of \cite{Ying:2004:JCP}
specifying $c=\frac{4-\sqrt{3}}{\sqrt{3}}$ for three dimensional problems.}
$p=\log_c \epsilon$. The algorithm is illustrated in figure (\ref{fig:1_1_main_loop})
in the two dimensional case, which is direct analogue of the three dimensional case
which is the focus of this thesis, and a full pseudo-code specification is
provided in Appendix \ref{app:analytic_fmm}.

\begin{figure}[!h]
    \centering
    {\includegraphics[width=1.1\textwidth]{introduction/main_loop.png}}
  \caption{
      The main FMM loop in two dimensions, encapsulating the upward and downward
      pass. The same set of particles are used for the sources and targets, and
      are shown in green. Local expansions are illustrated with red stars, and
      multipole expansions are illustrated with blue stars. For the P2M step the
      grey region indicates the \gls{near-field}, where this multipole expansion
      does not converge. For the M2L and P2P steps the grey region indicates
      box interactions already compressed and available via the translation or
      direct usage of local expansions. The larger and smaller stars in the M2M and
      L2L steps correspond to the parent and child expansions respectively.
  }
  \label{fig:1_1_main_loop}
\end{figure}

\subsection{Summary}

The main practical challenge in implementing software to solve the analytic FMM
is the requirement of kernel-specific code to calculate the expansion coefficients of
(\ref{eq:1_1_multipole_expansion}) and (\ref{eq:1_1_local_expansion}). For example,
multiple different implementations already exist for Poison problems alone \cite{Greengard:1996:JCP, Etheridge:2001:SIAM}.
In software engineering terms this is inconvenient for the purposes of studying
the applications of the FMM in multiple different problem settings, as it leads to the requirement
to implement problem-specific solvers for each kernel one may encounter. This
leads to a productivity overhead in either creating a single, extensible
library, which by definition will be complex to design. Or to otherwise
maintain multiple problem-specific libraries.

The algorithm thus far described referred to as the analytic FMM is more correctly
called the non-adaptive analytic FMM. The non-adaptivity refers to the assumption that
all boxes at the leaf level are refined to the same degree, however as mentioned
in the above analysis of the algorithm, the degree of refinement is chosen such
that the number of particles in a leaf box is \textit{constant}. Therefore, If
the distribution of the particles of interest is not uniform over the computational
domain, one may be needlessly refining the boxes in some regions which may even be
empty. Therefore, although the asymptotic complexity of
the FMM is $O(N)$, it is clear that practical implementations will suffer unless
care is taken to use efficient vectorised data structures for the creation of an
appropriate hierarchical tree as required by the algorithm.

Additionally, there is significant scope for multiple levels of
parallelism in practical implementations. For example, the computations
for the local and multipole coefficients as well as the application of M2L, L2L
and M2M operators at a given level, are candidates for an implementation of
\textbf{\gls{task-level-parallelism}}. In addition to parallelising of each operator
application as a task, there is scope for implementing
\textbf{\gls{data-level-parallelism}} to find the expansion coefficients. For example,
Yokota and Barba \cite{Hwu:2011:MKP} demonstrate how the calculation of the P2P,
and M2L operators can be transferred to \textbf{\gls{GPU}}s using \textbf{\gls{CUDA}}.
The M2L, and P2P operators represent the largest computational bottlenecks due to
the number of such interactions in the FMM algorithm, therefore are a priority for
acceleration. For example, in three dimensions each target box has potentially
up to 189 source boxes in its \gls{interaction-list} with which to compute the
M2L operation, and for any particularly deep tree there will be roughly $O(N)$
leaf boxes, for which interactions are calculated directly. This leads to both of
these operations dominating the run-time of any \gls{FMM} implementation.

In summary, the implementation of the analytical FMM is complicated by the
fact that it is problem specific - which will also apply to any parallel
optimisation code. This in itself provides the main motivation for developing
an implementation that does not rely on explicit kernel expansions. Furthermore,
the desire for developer productivity, at the expense of hyper-optimised
implementations, is realised in this thesis by making use of Python,
a \textbf{\gls{high-level-interpreted-language}}, for our software implementation.


\chapter{The Informal Notions of Algorithms}

\emph{“形式语言与自动机课程速成。”}

\section{Algorithms}

\begin{definition}[Algorithms (Informal)]
    \label{def:Algorithm}
    An algorithm is a \textbf{finite ordered list} of instructions.
\end{definition}

Possible outcomes of running an algorithm
\begin{itemize}
    \item The algorithm does not halt
    \item The algorithm halts
    \begin{itemize}
        \item In an erroneous state (fails)
        \item Gives valid outputs
    \end{itemize}
\end{itemize}
Cases other than the algorithm giving valid outputs are collectively identified as ``no output''.

\subsection{Algorithms for Determining Membership}
\label{sub:AlgoForDeterminingMembership}

\begin{definition}[Algorithms for Determining Membership]
    \label{def:AlgoForDeterminingMembership}
    An algorithm for \emph{determining membership} in a set $A \subseteq \mathbb{N}$ has an input, and two possible outputs ``yes'' and ``no''. If the algorithm is run on input $n$, it will halt in finite steps with output ``yes'' if $n \in A$ and ``no'' if $n \notin A$.
\end{definition}

\begin{definition}[Effectively Decidable Sets]
    \label{def:EffectivelyDecidableSet}
    Let $A$ be a subset of $\mathbb{N}$. $A$ is \textbf{effectively decidable} if there is an algorithm for determining membership of $A$.
\end{definition}

\begin{itemize}
    \item $\mathbb{N}$ is effectively decidable. (Always ``yes'')
    \item $\emptyset$ is effectively decidable. (Always ``no'')
\end{itemize}

\begin{theorem}
    If $A$ and $B$ are effectively decidable subsets of $\mathbb{N}$, then $\mathbb{N}\backslash A$, $A \cap B$ and $A \cup B$ are all effectively decidable.
\end{theorem}

Algorithms can have different kinds of outputs and inputs.

\begin{definition}[Diophantine Equations]
    Consider polynomials with integer coefficients (and any number of variables), a \textbf{diophantine equation} is an equation of the form $p=0$, where $p$ is sunch a polynomial. (e.g., $3x^2 + 5xy - 2z^4 +3 = 0$)
\end{definition}

\paragraph{Hilbert's 10th Problem.} Is there an algorithm for determining whether or not diophantine equations have integer solutions?

\subsection{Algorithm for Listing Members of Sets}
\label{sub:AlgoForListingMembersOfSets}

\begin{definition}[Algorithm for Listing Members of Sets]
    \label{def:AlgoForListingMembersOfSets}
    Let $A \subseteq \mathbb{N}$. An algorithm for \textbf{listing the members of $A$} prints (or enumerates) a list of numbers $a_0,a_1,\dots$ s.t.
    \begin{itemize}
        \item $a_n \in A$ for all $n \in \mathbb{N}$
        \item If $a \in A$, then $a = a_n$ for some $n$
    \end{itemize}
\end{definition}
\begin{remark} ~{}
    \begin{itemize}
        \item No input.
        \item Repetitions in the listing are permitted.
        \item If $A$ is finite, the algorithm may terminate or run forever (by iterating endlessly over the finite set).
        \item If $A$ is infinite, the algorithm should run forever.
    \end{itemize}
\end{remark}

\begin{definition}[Effectively Enumerable Sets]
    \label{def:EffectivelyEnumerableSet}
    A set $A \subseteq \mathbb{N}$ is \textbf{effectively enumerable} if there is an algorithm for listing its members.
\end{definition}

\subsection{Effectively Computable Functions}
\label{sub:EffectivelyComputableFunction}

\begin{definition}[Partial Functions]
    \label{def:PartialFunction}
    A function $f:\mathbb{N}\mapsto\mathbb{N}$ is a \textbf{partial number-theoretic function} if its domain is a subset of $\mathbb{N}$.
\end{definition}
\begin{definition}[Total Functions]
    \label{def:TotalFunction}
    A partial function whose domain is all of $\mathbb{N}$ is a total function.
\end{definition}

\begin{definition}[Notation for Definability]
    For a partial function $f$
    \begin{itemize}
        \item $n \in \textrm{dom}(f)$, then $f(n)\downarrow$
        \item $n \notin \textrm{dom}(f)$, then $f(n)\uparrow$
    \end{itemize}
\end{definition}

\begin{definition}[Effectively Computable Functions]
    \label{def:EffectivelyComputableFunction}
    The partial function $f$ is \textbf{effectively computable} if there is an algorithm $\mathcal{A}$ s.t. on input $n$
    \begin{itemize}
        \item Prints output $f(n)$ if $f(n)\downarrow$
        \item No output if $f(n)\uparrow$
    \end{itemize}
    $\mathcal{A}$ is called an algorithm for computing $f$.
\end{definition}

\begin{remark}
    The definition of effectively decidable and enumerable sets, and effectively computable functions can be extended to $\mathbb{N}^k$.
\end{remark}

\subsection{Relations among Effectively Decidable Sets, Effectively Enumerable Sets and Effectively Computable Functions}

\begin{proposition}
    Let $A \subseteq \mathbb{N}$. If $A$ is effectively decidable then $A$ is effectively enumerable.
\end{proposition}

\begin{proposition}
    Let $A \subseteq \mathbb{N}$. $A$ is effectively decidable iff the characteristic function (\ref{def:CharacteristicFunctions}) $C_A$ is effectively computable.
\end{proposition}

\begin{proposition}
    Let $f:\mathbb{N}\mapsto\mathbb{N}$. If $f$ is an effectively computable function, then $\textrm{rng}(f)$ is an effectively enumerable set.
\end{proposition}

\begin{proposition}
    Combining the above two propositions, we have: Let $A$ be a non-empty subsect of $\mathbb{N}$, $A$ is effectively enumerable iff there is an effectively computable function whose range is $A$.
\end{proposition}

\begin{proposition}
    If $A\neq\emptyset$ is an effectively enumerable subset of $\mathbb{N}$, then there is an effectively computable function $f$ with $\range(f)=A$.
\end{proposition}

\section{Enumerability of Algorithms}
\label{sec:EnumerabilityOfAlgos}

\begin{theorem}
    There are only \emph{enumerably many} algorithms.
\end{theorem}
\begin{corollary}
    There are only enumerably many partial effectively computable functoins. Since there are uncountably many total number-theoretic functions, there must exist total number-theoretic functions that are not effectively computable.
\end{corollary}
\begin{corollary}
    There are only enumerably many effectively enumerable subsets of $\mathbb{N}$. Since there are uncountably many subsets of $\mathbb{N}$, there must exist subsets of $\mathbb{N}$ that are not effectively enumerable.
\end{corollary}

\subsubsection{Listing of Effectively Computable Total Function}

Suppose $f_0,\dots$ is a listing of all effectively computable total function (this listing is unknown, we only know a listing of all computable partial functions). Let $g$ be a function such that

\[ g(m,n) = f_m(n) \]

$g$ is not effectively computable.

\begin{proof}
    By diagonal argument. Let $h(n) = g(n,n) + 1$. If $g$ is effectively computable, then so is $h$. Then $h$ must be in the listing $\{f_n\}$. Suppose $f_k = h$. Then
    \[ f_k(k) = h(k) = g(k,k)+1 = f_k(k) + 1 \]
    噔噔咚。
\end{proof}

\section{The Halting Problem}

Is there an algorithm for deciding whether any algorithm halts on given input or not?

\begin{theorem}
    \label{thm:HaltingProblem}
    Let $\mathcal{A}_0,\dots,\mathcal{A}_n,\dots$ be a listing of algorithm with space for only one input. There does not exist an algorithm $\mathcal{H}$ with space for two number inputs s.t. on input $(m,n)$
    \begin{itemize}
        \item Halts with output ``yes'' if $\mathcal{A}_m$ halts on input $n$.
        \item Halts with output ``no'' if $\mathcal{A}_m$ does not halt on input $n$
    \end{itemize}
\end{theorem}
\begin{proof}
    Construct $\mathcal{C}(n)$ by:
    \begin{itemize}
        \item Run $\mathcal{H}(n,n)$
        \item If ``yes'' then loop forever
        \item If ``no'' then print ``yes'' and return
    \end{itemize}

    Then $\mathcal{C}$ must be in the listing $\{\mathcal{A}_n\}$. Suppose $\mathcal{A}_k = \mathcal{C}$.
    \begin{itemize}
        \item $\mathcal{C}(k)$ halts $\Leftrightarrow$ $\mathcal{H}(k,k)$ prints ``no'' $\Leftrightarrow$ $\mathcal{A}_k(k) = \mathcal{C}(k)$ does not halt.
    \end{itemize}
    噔噔咚。
\end{proof}


%!TEX root=report.tex

\subsection{OLS with autocorrelated residuals}

In OLS it is assumed that the residuals between different observations are uncorrelated. However since our data is a time series this assumption does most likely not hold. This can also be confirmed with the Durbin-Watson test statistic \cite[p.~173]{autocorrelation-kousgaard}. This test statistic is calculated using
\begin{equation}
d = \frac{\sum_{i=2}^n \left( \hat{\epsilon}_i - \hat{\epsilon}_{i-1} \right)^2}{ \sum_{i=1}^n\hat{\epsilon}_i^2 }
\end{equation}

which lies between 0 and 4. If $d$ is close to 0 the residuals $\hat{\epsilon}_i$ and $\hat{\epsilon}_{i-1}$ are correlated positively, if close to 4 the residuals are negatively correlated. The purpose of this method is to reduce the correlation between the residuals, the Durbin-Watson test statistic should then become 2.

When the residuals are correlated, one gets bad estimation of the variance of the residuals $\hat{\sigma}_\epsilon^2$. This in turn leads to bad interference statistics, such as misleading p-values.

To correct for the correlated residuals, one can use a general least squares (GLS) regression. This uses a $\Sigma$ matrix there will correct for the correlated residuals
\begin{equation}
\min_{\beta, \rho}\ (Y-X\beta)^T \Sigma^{-1}(Y-X\beta),
\label{eq:theory-olsar-min}
\end{equation}

where $ \Sigma^{-1}$ is given as
\begin{equation}
\Sigma^{-1}  = \begin{bmatrix}
1         & -\rho         & 0               & \cdots & 0              & 0         \\
-\rho   & 1+\rho^2 & -\rho         & \cdots & 0               & 0         \\
0         & -\rho         & 1+\rho^2 & \cdots &0                & 0         \\
\vdots & \vdots      & \vdots       & \ddots & \vdots      & \vdots \\
0         & 0               &0                & \cdots & 1+\rho^2 & -\rho    \\
0         & 0               &0                & \cdots &-\rho          & 1
\end{bmatrix}
\end{equation}

The optimization problem \eqref{eq:theory-olsar-min} is nonlinear and there is no closed form solution to this problem. However when keeping $\rho$ constant the $\beta$ parameters can be estimated using  General Least Squared, which is similar to OLS but with a constant $\Sigma$ matrix and have the solution \cite[p.~38]{time-series-analysis}
\begin{equation}
\hat{\beta} = (X^T \Sigma^{-1} X)^{-1} X^T \Sigma^{-1} Y.
\end{equation}

This should then be rewritten using SVD, to account for a near singular $X^T \Sigma^{-1} X$ matrix.

Similarly when $\beta$ is kept constant, $\rho$ can be estimated with  \cite[p.~178]{autocorrelation-kousgaard}
\begin{equation}
\hat{\rho} = \frac{ \sum_{i=2}^n \hat{\epsilon_i}\hat{\epsilon}_{i-1} }{ \sum_{i=2}^{n-1} \hat{\epsilon_i}^2 }.
\label{eq:theory-olsar-rho}
\end{equation}

A practical way of solving the problem with respect to both $\beta$ and $\rho$, looks as follows:
\begin{enumerate}
\item initialize by letting $\rho=0$
\item Iterate: \begin{enumerate}
	\item Keep $\hat{\rho}$ constant and estimate $\beta$ using as a WLS problem.

	\item Keep $\hat{\beta}$ constant and estimate $\rho$ using \eqref{eq:theory-olsar-rho}.

	\item Repeat until convergence of $\rho$ or until some predetermined upper iteration boundary is met.
\end{enumerate}
\end{enumerate}


\chapter{Expectation}

\begin{ex}
  Let $X_i$ take on the values $\left\{\frac{1}{2}, 2\right\}$ depending on
  whether the $i$th turn of the game resulted in a halving or a doubling
  respectively. Since both outcomes occur with equal probability,
  \[
    \E{X_i} = \frac{1}{2}\cdot \frac{1}{2} + 2\cdot \frac{1}{2}=\frac{5}{4}.
  \]

  Let $Y_n$ be the amount of money we have after the $n$th turn, and note that
  \[
    Y_n = c\prod_{i=1}^nX_i,
  \]
  where $X_i$ is independent of $X_j$ whenever $i\neq j$. Therefore,
  \begin{align*}
    \E{Y_n}
    =\E{c\prod_{i=1}^nX_i}
    =c\prod_{i=1}^n\E{X_i}
    =c\left(\frac{5}{4} \right)^{n}.
  \end{align*}
\end{ex}

\begin{ex}
  Suppose that there is a constant $c$ such that $\P{X=c}=1$. It follows that
  $X$ is a discrete random variable, and therefore
  \[
    \E{X}=\sum_{x\in \supp(X)} x\P{X=x}=c,
  \]
  since $\P{X=x}=0$ for $x\neq c$. Likewise,
  \[
    \E{X^2}=\sum_{x\in \supp(X)} x^2\P{X=x}=c^2,
  \]
  and so
  \[
    \var{X}=\E{X^2}-\E{X}^2=c^2-c^2=0.
  \]

  For the converse, we begin by proving a lemma. Let $Y$ be a non-negative
  random variable such that $\E{Y}=0$. We want to prove that $\P{Y=0}=1$.

  Let $B_n=[1/n,\infty)$ and note that
  \[
    0=\E{Y}\geq \frac{1}{n}\P{X\in B_n},
  \]
  and hence $\P{B_n}=0$. However,
  \[
    1 = \P{X\in \{0\}\cup\bigcup_{n=1}^\infty B_n}=\P{X=0}+\lim_{n\to\infty}
    \P{X\in B_m}= \P{X=0}.
  \]

  For the converse of the original claim, suppose we are given a random variable
  $X$ such that $\var{X}=0$. Note that we assume that $X$ has a mean in our
  definition of variance, and therefore can let $\E{X}=c$. Hence,
  \[
    \E{(X-c)^2}=0,
  \]
  or, by our previous lemma,
  \[
    \P{(X-c)^2= 0}=1.
  \]
  But this is equivalent to $\P{X=c}=1$.
\end{ex}

\begin{ex}
  Let $X_1,\ldots,X_n\sim\text{Uniform}(0, 1)$ and let
  $Y_n=\max\{X_1,\ldots,X_n\}$. Then
  \[
    \P{Y_n\leq y}y
    =\P{X_1\leq y}\cdots \P{X_n\leq y}
    =\begin{cases}
      0   & y<0,          \\
      y^n & 0\leq y\leq1, \\
      1   & y>1,
    \end{cases}
  \]
  and therefore
  \[
    f_{Y_n}(y)=\begin{cases}
      ny^{n-1} & 0\leq y\leq1      \\
      0        & \text{otherwise}.
    \end{cases}
  \]

  Hence,
  \[
    \E{Y_n}
    =\int_0^1\! ny^n\,\d{y}
    =\frac{n y^{n+1}}{n+1}\,\bigg\rvert_{y=0}^1
    =\frac{n}{n+1}.
  \]
\end{ex}

\begin{ex}
  Let $J_i$ be the outcome of the $i$th jump: $-1$ if it is a jump to the left
  and $1$ if it is a jump to the right. Note that
  \begin{align*}
    \E{J_i}   & =-1\cdot p+1\cdot (1-p)=1-2p,        \\
    \E{J_i^2} & =1\cdot p+1\cdot (1-p)=1\text{, and} \\
    \var{J_i} & =1-(1-2p)^2=4p-4p^2.
  \end{align*}

  Let $X_n$ be the position of the particle after $n$ time units. Note that
  $X_n=\sum_{i=1}^n J_i$, where all the $J_i$'s are independent. Therefore,
  \[
    \E{X_n}
    = \E{\sum_{i=1}^n J_i}
    = \sum_{i=1}^n \E{J_i}
    = n(1-2p) \text{, and}
  \]
  \[
    \var{X_n}
    = \var{\sum_{i=1}^n J_i}
    = \sum_{i=1}^n \var{J_i}
    = n(4p-4p^2).
  \]
\end{ex}

% 5
\begin{ex}
  Let $X$ be the number of fair coin tosses until a head is obtained. Note that
  since the only way to obtain the first head in $n$ tosses is to toss tails
  $n-1$ times and then toss a head, it follows that
  $\P{X=n}=\left(\frac{1}{2}\right)^n$.
  Hence,
  \[
    \E{X}
    = \sum_{n=1}^\infty n\cdot \left(\frac{1}{2}\right)^n
    = 2
  \]
  by comparison to the Taylor series of $x/(1-x)^2$.
\end{ex}

\begin{ex}
  Note that
  \[
    \E{r(X)}
    =\sum_{y\in\supp{Y}}y\P{r(X)=y}
    =\sum_{y\in\supp{Y}}y\sum_{\{x\in\supp{X}\mid r(x)=y\}}\P{X=x},
  \]
  where the double summation $\sum_y\sum_{\{x\mid r(x)=y\}}$ is equivalent to
  the single summation $\sum_{x}$ by replacing $y$ with $r(x)$. Hence,
  \[
    \E{r(X)}
    =\sum_{x\in\supp{X}}r(x)f_X(x).
  \]
\end{ex}

\begin{ex}
  Let $X$ be a continuous random variable with CDF $F$, such that $\P{X>0}=1$
  and $\E{X}$ exists. Note that then $\P{X\leq 0}=1-\P{X>0}=0$. Then
  \begin{align*}
    \int_{0}^\infty\! \P{X>x}\,\d{x}
     & =\int_{0}^\infty\! 1-F(x)\,\d{x}                                                                                \\
     & =x(1-F(x))\,\bigg\rvert_{x=0}^\infty+\int_{0}^\infty\! xF'(x)\,\d{x} &  & (\text{integration by parts})         \\
     & =\int_{0}^\infty\! xF'(x)\,\d{x}                                                                                \\
     & =\int_{-\infty}^\infty\! xF'(x)\,\d{x}                               &  & (\text{since $F(x)=0$ for $x\leq 0$}) \\
     & =\E{X}.
  \end{align*}

\end{ex}

\begin{ex}
  Let $X_1,\ldots,X_n$ be \iid with $\mu=\E{X_i}$ and $\sigma^2=\var{X_i}$. Then
  \begin{align*}
    \E{\overline{X}_n}
     & =\E{\frac{\sum_{i=1}^nX_i}{n}}                                      \\
     & =\frac{1}{n}\sum_{i=1}^n\E{X_i} & \text{(linearity of expectation)} \\
     & =\frac{n\mu}{n}                                                     \\
     & =\mu.
  \end{align*}

  Likewise,
  \begin{align*}
    \var{\overline{X}^2_n}
     & =\var{\frac{\sum_{i=1}^nX_i}{n}}           \\
     & =\frac{1}{n^2}\sum_{i=1}^n\var{X_i}        \\
     & =\frac{n\sigma^2}{n^2}=\frac{\sigma^2}{n}.
  \end{align*}

  Finally, observe that
  \[
    \E{X_i^2}
    =\var{X_i}+\E{X_i}^2
    =\sigma^2+\mu^2,
  \]
  \[
    \E{\overline{X}^2_n}
    =\var{\overline{X}_n}+\E{\overline{X}_n}^2
    =\frac{\sigma^2}{n}+\mu^2,
  \]
  and
  \begin{align*}
    \E{X_i\overline{X}_n}
     & =\frac{1}{n}\E{X_i\sum_{j=1}^nX_j}                                           \\
     & =\frac{1}{n}\E{X^2_i+\sum_{\substack{j=1                                     \\ j\neq i}}^n X_iX_j} \\
     & =\frac{1}{n}\left(\sigma^2+\mu^2+(n-1)\mu^2\right)=\frac{\sigma^2}{n}+\mu^2,
  \end{align*}
  and that therefore
  \begin{align*}
    \E{S^2_n}
     & =\E{\frac{1}{n-1}\sum_{i=1}^n\left(X_i-\overline{X}_n)^2\right)}                               \\
     & =\frac{1}{n-1}\sum_{i=1}^n\E{\left(X_i-\overline{X}_n)^2\right)}                               \\
     & =\frac{1}{n-1}\sum_{i=1}^n\E{X_i^2-2X_i\overline{X_n}+\overline{X}_n^2}                        \\
     & =\frac{n}{n-1}\left(\sigma^2+\mu^2-\frac{2\sigma^2}{n}-2\mu^2+\frac{\sigma^2}{n}+\mu^2 \right) \\
     & =\frac{n}{n-1}\left(\sigma^2-\frac{\sigma^2}{n}\right)                                         \\
     & =\sigma^2.
  \end{align*}
\end{ex}

\begin{ex}
  We use the following Python code to graph the sample means:

  \inputminted{python}{../code/03-09a.py}

  \begin{figure}[H]
    \centering
    \includegraphics[scale=0.9]{../images/03-09a}
    \caption{Graph of $\overline{X}_n$ for a standard normal distribution.}
  \end{figure}

  \inputminted{python}{../code/03-09b.py}

  \begin{figure}[H]
    \centering
    \includegraphics[scale=0.9]{../images/03-09b}
    \caption{Graph of $\overline{X}_n$ for a Cauchy distribution.}
  \end{figure}

  The standard normal distribution has a defined mean of $0$ and, as expected,
  the sample mean seems to be converging to it. The Cauchy distribution does not
  have a mean, and that is why the sample mean is not converging to a particular
  value.
\end{ex}

% 10
\begin{ex}
  Let $X\sim N(0, 1)$ and let $Y=e^X$. Then
  \begin{align*}
    \E{Y}
     & =\int_{-\infty}^\infty\!\frac{e^x}{\sqrt{2\pi}}e^{-\frac{1}{2}x^2}\,\d{x}                    \\
     & =\int_{-\infty}^\infty\!\frac{1}{\sqrt{2\pi}}e^{-\frac{1}{2}(x^2-2x)}\,\d{x}                 \\
     & =\int_{-\infty}^\infty\!\frac{e^{\frac{1}{2}}}{\sqrt{2\pi}}e^{-\frac{1}{2}(x^2-2x+1)}\,\d{x} \\
     & =e^{\frac{1}{2}}\int_{-\infty}^\infty\!\frac{1}{\sqrt{2\pi}}e^{-\frac{1}{2}(x-1)^2}\,\d{x}   \\
     & =e^{\frac{1}{2}}.
  \end{align*}

  Recall that $\var{Y}=\E{Y^2}-\E{Y}^2$, and note that
  \begin{align*}
    \E{Y^2}
     & =\int_{-\infty}^\infty\!\frac{e^{2x}}{\sqrt{2\pi}}e^{-\frac{1}{2}x^2}\,\d{x}     \\
     & =\int_{-\infty}^\infty\!\frac{1}{\sqrt{2\pi}}e^{-\frac{1}{2}(x^2-4x)}\,\d{x}     \\
     & =\int_{-\infty}^\infty\!\frac{e^2}{\sqrt{2\pi}}e^{-\frac{1}{2}(x^2-4x+4)}\,\d{x} \\
     & =e^2\int_{-\infty}^\infty\!\frac{1}{\sqrt{2\pi}}e^{-\frac{1}{2}(x-2)^2}\,\d{x}   \\
     & =e^2.
  \end{align*}
  Therefore,
  \[
    \var{Y}=e^2-e.
  \]
\end{ex}

\begin{ex}
  \begin{enumerate}[(a)]
    \item Note that $X_n$ is identical to the $X_n$ of Exercise 3.4 with
          $p=\frac{1}{2}$. Therefore, $\E{X_n}=0$ and $\var{X_n}=n$.
    \item We use the following Python code.
          \inputminted{python}{../code/03-11.py}

          \begin{figure}[H]
            \centering
            \includegraphics[scale=0.8]{../images/03-11a}
            \caption{First simulation of $X_n$.}
          \end{figure}

          \begin{figure}[H]
            \centering
            \includegraphics[scale=0.8]{../images/03-11b}
            \caption{Second simulation of $X_n$.}
          \end{figure}

          \begin{figure}[H]
            \centering
            \includegraphics[scale=0.8]{../images/03-11c}
            \caption{Third simulation of $X_n$.}
          \end{figure}

          The expected value of $X_n$ is $0$, but none of the simulations show
          $X_n$ converging to $0$. This is explainable by the fact that the
          variance is increasing with $n$.
  \end{enumerate}
\end{ex}

\begin{ex}
  % Bernoulli
  Let $B$ be a $\text{Bernoulli}(p)$ random variable. Then
  \begin{align*}
    \E{B}   & =(1-p)\cdot 0 + p\cdot 1 = p,     \\
    \E{B^2} & =(1-p)\cdot 0^2 + p\cdot 1^2 = p, \\
    \var{B} & = \E{B^2}-\E{B}^2=p-p^2=p(1-p).
  \end{align*}

  % Poisson
  Let $P$ be a $\text{Poisson}(\lambda)$ random variable. Then
  \begin{align*}
    \E{P}
     & =\sum_{k=1}^\infty k\frac{e^{-\lambda}\lambda^{k}}{k!}              \\
     & =e^{-\lambda}\lambda \sum_{k=1}^\infty \frac{\lambda^{k-1}}{(k-1)!} \\
     & =e^{-\lambda}\lambda \sum_{k=0}^\infty \frac{\lambda^{k}}{k!}       \\
     & = \lambda,
  \end{align*}
  \begin{align*}
    \E{P^2}
     & =\sum_{k=1}^\infty k^2\frac{e^{-\lambda}\lambda^{k}}{k!}                                                        \\
     & =e^{-\lambda}\lambda\sum_{k=1}^\infty k\frac{\lambda^{k-1}}{(k-1)!}                                             \\
     & =e^{-\lambda}\lambda\sum_{k=0}^\infty (k+1)\frac{\lambda^k}{k!}                                                 \\
     & =e^{-\lambda}\lambda\left[\sum_{k=0}^\infty k\frac{\lambda^k}{k!}+\sum_{k=0}^\infty \frac{\lambda^k}{k!}\right] \\
     & =e^{-\lambda}\lambda\left[\lambda e^{\lambda} + e^{\lambda}\right]                                              \\
     & =\lambda^2+\lambda,
  \end{align*}
  and therefore
  \[
    \var{P}
    =\E{P^2}-\E{P}^2
    =\lambda^2+\lambda-\lambda=\lambda.
  \]

  % Uniform
  Let $U$ be a $\text{Uniform}(a, b)$ random variable. Then
  \begin{align*}
    \E{U}   & = \int_a^b\!\frac{x}{b - a}\,\d{x}
    =\frac{1}{2}\frac{b^2-a^2}{b-a}
    =\frac{a + b}{2},                              \\
    \E{U^2} & = \int_a^b\!\frac{x^2}{b - a}\,\d{x}
    =\frac{1}{3}\frac{b^3-a^3}{b-a}
    =\frac{a^2+ab+b^2}{3},\text{ and hence}        \\
    \var{U} & = \E{U^2}-\E{U}^2
    =\frac{a^2+ab+b^2}{3}-\frac{a^2+2ab+b^2}{4}
    =\frac{(b-a)^2}{12}.
  \end{align*}

  % Exponential
  Let $X$ be an $\text{Exponential}(\beta)$ random variable. Then
  \begin{align*}
    \E{X}
     & =\int_0^\infty\!\frac{x}{\beta}e^{-x/\beta}\,\d{x}                                            \\
     & =\beta\int_0^\infty\!ue^{-u}\,\d{u}                                             & (u=x/\beta) \\
     & =\beta\left[-ue^{-u}\bigg\rvert_{u=0}^\infty-\int_0^\infty e^{-u}\,\d{u}\right]               \\
     & =\beta,
  \end{align*}
  \begin{align*}
    \E{X^2}
     & =\int_0^\infty\!\frac{x^2}{\beta}e^{-x/\beta}\,\d{x}                                  \\
     & =\beta^2\int_0^\infty\!u^2e^{-u}\,\d{u}                                               \\
     & =\beta^2\left[u^2e^{-u}\bigg\rvert_{u=0}^\infty-2\int_0^\infty ue^{-u}\,\d{u} \right] \\
     & =2\beta^2,
  \end{align*}
  and hence
  \[
    \var{X}
    =\E{X^2}-\E{X}^2
    =2\beta^2-\beta^2=\beta^2.
  \]

  % Gamma
  Let $G$ be an $\text{Gamma}(\alpha, \beta)$ random variable. Then
  \begin{align*}
    \E{G}
     & =\int_0^\infty\!\frac{x}{\beta^\alpha\Gamma(\alpha)}x^{\alpha-1}e^{-x/\beta}\,\d{x}                                                 \\
     & =\frac{\beta\Gamma(\alpha+1)}{\Gamma(\alpha)}\int_0^\infty\!\frac{1}{\beta^{\alpha+1}\Gamma(\alpha+1)}x^{\alpha}e^{-x/\beta}\,\d{x} \\
     & =\alpha\beta,
  \end{align*}
  \begin{align*}
    \E{G^2}
     & =\int_0^\infty\!\frac{x^2}{\beta^\alpha\Gamma(\alpha)}x^{\alpha-1}e^{-x/\beta}\,\d{x}                                                   \\
     & =\frac{\beta^2\Gamma(\alpha+2)}{\Gamma(\alpha)}\int_0^\infty\!\frac{1}{\beta^{\alpha+2}\Gamma(\alpha+2)}x^{\alpha+1}e^{-x/\beta}\,\d{x} \\
     & =\alpha(\alpha+1)\beta^2,
  \end{align*}
  and thus
  \[
    \var{G}
    =\E{G^2}-\E{G}^2
    =\alpha(\alpha+1)\beta^2-\alpha^2\beta^2
    =\alpha\beta^2.
  \]

  % Beta
  Finally, let $Y$ be a $\text{Beta}(\alpha, \beta)$ random variable. Then
  \begin{align*}
    \E{Y}
     & =\int_0^\infty\!x\cdot\frac{\Gamma(\alpha+\beta)}{\Gamma(\alpha)\Gamma(\beta)}x^{\alpha-1}(1-x)^{\beta-1} \,\d{x}                                                                             \\
     & =\frac{\Gamma(\alpha+\beta)\Gamma(\alpha+1)}{\Gamma(\alpha)\Gamma(\alpha+\beta+1)}\int_0^\infty\!\frac{\Gamma(\alpha+\beta+1)}{\Gamma(\alpha+1)\Gamma(\beta)}x^{\alpha}(1-x)^{\beta-1}\,\d{x} \\
     & =\frac{\alpha}{\alpha+\beta},
  \end{align*}
  \begin{align*}
    \E{Y^2}
     & =\int_0^\infty\!x^2\cdot\frac{\Gamma(\alpha+\beta)}{\Gamma(\alpha)\Gamma(\beta)}x^{\alpha-1}(1-x)^{\beta-1} \,\d{x}                                                                             \\
     & =\frac{\Gamma(\alpha+\beta)\Gamma(\alpha+2)}{\Gamma(\alpha)\Gamma(\alpha+\beta+2)}\int_0^\infty\!\frac{\Gamma(\alpha+\beta+2)}{\Gamma(\alpha+2)\Gamma(\beta)}x^{\alpha+1}(1-x)^{\beta-1}\,\d{x} \\
     & =\frac{\alpha(\alpha+1)}{(\alpha+\beta+1)(\alpha+\beta)},
  \end{align*}
  and so
  \begin{align*}
    \var{Y}
     & =\E{Y^2}-\E{Y}^2                                         \\
     & =\frac{\alpha(\alpha+1)}{(\alpha+\beta+1)(\alpha+\beta)}
    -\frac{\alpha^2}{(\alpha+\beta)^2}                          \\
     & =\frac{\alpha\beta}{(a+\beta)^2(\alpha+\beta+1)}.
  \end{align*}
\end{ex}

\begin{ex}
  Let $Z$ be a uniform discrete random variable on $\{0, 1\}$,
  $X_1\sim\text{Uniform}(0, 1)$ and $X_2\sim\text{Uniform}(3, 4)$. Then
  $X=ZX_1 + (1-Z)X_2$.
  \begin{itemize}[(a)]
    \item We have
          \begin{align*}
            \E{X} & =\E{ZX_1 + (1-Z)X_2}           \\
                  & =\E{ZX_1} + \E{(1-Z)X_2}       \\
                  & =\E{Z}\E{X_1} + \E{1-Z}\E{X_2} \\
                  & =\frac{1}{4}+\frac{7}{4}       \\
                  & =2.
          \end{align*}
    \item[(b)] We begin by finding the variance:
          \begin{align*}
            \var{X}
             & =\var{ZX_1 + (1-Z)X_2}                           \\
             & =\var{ZX_1}+\var{(1-Z)X_2}+2\cov{ZX_1, (1-Z)X_2} \\
             & =\frac{5}{48}+\frac{149}{48}-\frac{7}{8}         \\
             & =\frac{7}{3},
          \end{align*}
          since
          \begin{align*}
            \var{ZX_1}
             & =\E{Z^2}\E{X_1^2}-\frac{1}{16}
            =\frac{1}{2}\cdot\frac{1}{3}-\frac{1}{16}
            =\frac{5}{48},                            \\
            \var{(1-Z)X_2}
             & =\E{(1-Z)^2}\E{X_2^2}-\frac{49}{16}
            =\frac{1}{2}\cdot\frac{37}{3}-\frac{49}{16}
            =\frac{149}{48},\text{ and}               \\
            \cov{ZX_1, (1-Z)X_2}
             & =\E{Z(1-Z)X_1X_2}-\E{ZX_1}\E{(1-Z)X_2}
            =-\frac{1}{4}\cdot\frac{7}{4}
            =-\frac{7}{16}.
          \end{align*}
          Therefore, the standard deviation is $\sqrt{7/3}$.
  \end{itemize}
\end{ex}

\begin{ex}
  Let $X_1,\ldots, X_m$ and $Y_1,\ldots,Y_n$ be random variables and
  $a_1,\ldots,a_m,b_1,\ldots,b_n$ be constants. Then
  \begin{align*}
    \cov{\sum_{i=1}^m a_iX_i, \sum_{j=1}^nb_jY_j }
     & =\E{\left(\sum_{i=1}^m a_iX_i\right)\left(\sum_{j=1}^nb_jY_j\right)}
    -\E{\sum_{i=1}^m a_iX_i}\E{\sum_{j=1}^nb_jY_j}                             \\
     & =\E{\sum_{i=1}^m \sum_{j=1}^n a_ib_jX_iY_j}
    -\left[\sum_{i=1}^m a_i\E{X_i}\right] \left[\sum_{j=1}^n b_j\E{Y_j}\right] \\
     & =\sum_{i=1}^m\sum_{j=1}^n a_ib_j \E{X_iY_j}
    -\sum_{i=1}^m\sum_{j=1}^n a_ib_j \E{X_i}\E{Y_j}                            \\
     & =\sum_{i=1}^m\sum_{j=1}^n a_ib_j\left[\E{X_iY_j}-\E{X_i}\E{Y_j}\right]  \\
     & =\sum_{i=1}^m\sum_{j=1}^n a_ib_j\cov{X_i, Y_j}.
  \end{align*}
\end{ex}

% 15
\begin{ex}
  Let $X,Y$ be random variables with joint PDF
  \[
    f_{X, Y}(x,y)
    =\begin{cases}
      \frac{1}{3}\left(x+y\right) & 0\leq x\leq 1, 0\leq y\leq 2 \\
      0                           & \text{otherwise}.
    \end{cases}
  \]

  Note that
  \begin{align*}
    f_X(x)
     & =\int_0^2\! f_{X,Y}(x,y)\,\d{y}                      \\
     & =\int_0^2\! \frac{1}{3}x+\frac{1}{3}y\,\d{y}         \\
     & =\left[ \frac{1}{3}xy+\frac{1}{6}y^2 \right]_{y=0}^2 \\
     & =\frac{2}{3}\left(x+1\right),
  \end{align*}
  and that therefore
  \[
    \E{X}
    =\int_0^1\!x\cdot\frac{2}{3}\left(x+1\right)\,\d{x}
    =\left[ \frac{2}{9}x^3+\frac{1}{3}x^2 \right]_{x=0}^1
    =\frac{5}{9},
  \]
  \[
    \E{X^2}
    =\int_0^1\!x^2\cdot\frac{2}{3}\left(x+1\right)\,\d{x}
    =\left[ \frac{2}{12}x^4+\frac{2}{9}x^3 \right]_{x=0}^1
    =\frac{7}{18},
  \]
  and
  \begin{align*}
    \var{X}=\E{X^2}-\E{X}^2=\frac{7}{18}-\frac{25}{81}=\frac{13}{162}.
  \end{align*}

  Similarly,
  \begin{align*}
    f_Y(y)
     & =\int_0^1\! f_{X,Y}(x,y)\,\d{x}                    \\
     & =\int_0^1\! \frac{1}{3}x+\frac{1}{3}y\,\d{x}       \\
     & =\left[\frac{1}{6}x^2+\frac{1}{3}xy\right]_{x=0}^1 \\
     & =\frac{1}{6}+\frac{1}{3}y,
  \end{align*}
  and so
  \begin{align*}
    \E{Y}
    =\int_0^2\!y\left(\frac{1}{6}+\frac{1}{3}y\right)\,\d{y}
    =\left[ \frac{1}{12}y^2+\frac{1}{9}y^3 \right]_{y=0}^2
    =\frac{11}{9},
  \end{align*}
  \begin{align*}
    \E{Y^2}
    =\int_0^2\!y^2\left(\frac{1}{6}+\frac{1}{3}y\right)\,\d{y}
    =\left[ \frac{1}{18}y^3+\frac{1}{12}y^4 \right]_{y=0}^2
    =\frac{16}{9},
  \end{align*}
  and
  \begin{align*}
    \var{Y}=\E{Y^2}-\E{Y}^2=\frac{16}{9}-\frac{121}{81}=\frac{23}{81}.
  \end{align*}

  Note that
  \begin{align*}
    \E{XY}
    =\int_0^2\!\int_0^1\!\frac{1}{3}x^2y+\frac{1}{3}xy^2\,\d{x}\,\d{y}
    =\int_0^2\!\frac{1}{6}y^2+\frac{1}{9}y\,\d{y}
    =\frac{2}{3},
  \end{align*}
  and so
  \begin{align*}
    \cov{X,Y}=\E{XY}-\E{X}\E{Y}=\frac{2}{3} - \frac{55}{81}=-\frac{1}{81}.
  \end{align*}

  Therefore,
  \begin{align*}
    \var{2X-3Y+8}
    =4\var{X}+9\var{Y}-12\cov{X,Y}
    =\frac{26}{81}+\frac{207}{81} +\frac{12}{81}
    =\frac{245}{81}.
  \end{align*}

\end{ex}

\begin{ex}
  We will assume that $X$ and $Y$ are continuous random variables and note that
  the proof in the discrete case proceeds similarly. Then
  \begin{align*}
    \cE{r(X)s(Y)}{X=x}
     & =\int_\R\!r(x)s(y)f_{X|Y}(x|y)\,\d{y} \\
     & =r(x)\int_\R\!s(y)f_{X|Y}(x|y)\,\d{y} \\
     & =r(x)\cE{s(y)}{X=x},
  \end{align*}
  and we note that if we let $g(y)=1$,
  \[
    \cE{r(X)s(Y)}{X=x}
    =\cE{r(X)}{X=x}
    =r(x)\cE{1}{X=x}=r(x).
  \]
\end{ex}

\begin{ex}
  We have
  \begin{align*}
    \var{Y}
     & =\E{Y^2}-\E{Y}^2                                   \\
     & =\E{\cE{Y^2}{X}}-\E{\cE{Y}{X}}^2                   \\
     & =\E{\var{Y\,|\,X}}+\E{\cE{Y^2}{X}}-\E{\cE{Y}{X}}^2 \\
     & =\E{\var{Y\,|\,X}}+\E{\cE{Y^2}{X}-\cE{Y}{X}^2}     \\
     & =\E{\var{Y\,|\,X}}+\var{\cE{Y}{X}}.
  \end{align*}
\end{ex}

\begin{ex}
  Note that
  \[
    \E{XY}
    =\E{\cE{XY}{Y}}
    =\E{Y\cE{X}{Y}}
    =c\E{Y},
  \]
  \[
    \E{X}
    =\E{\cE{X}{Y}}
    =c,
  \]
  and that therefore
  \[
    \cov{X,Y}
    =\E{XY}-\E{X}\E{Y}
    =c\E{Y}-c\E{Y}
    =0.
  \]
\end{ex}

\begin{ex}
  Note that $f_X(x)=I_{[0,1]}(x)$.
  \begin{figure}[H]
    \centering
    \includegraphics[scale=0.8]{../images/03-19a}
    \caption{Graph of $f_X$.}
  \end{figure}

  We have
  \[
    \E{\overline{X}_n}
    =\E{\frac{1}{n}\sum_{i=1}^n X_i}
    =\E{X_1}=\frac{1}{2}, \text{ and}
  \]
  \[
    \var{\overline{X}_n}
    =\frac{n}{n^2}\var{X_1}
    =\frac{1}{12n}.
  \]

  \inputminted{python}{../code/03-19b.py}

  \begin{figure}[H]
    \centering
    \includegraphics[scale=0.8]{../images/03-19b}
    \caption{Graph of $\E{\overline{X}_n}$ as a function of $n$.}
  \end{figure}

  \inputminted{python}{../code/03-19c.py}

  \begin{figure}[H]
    \centering
    \includegraphics[scale=0.95]{../images/03-19c}
    \caption{Graph of $\var{\overline{X}_n}$ as a function of $n$.}
  \end{figure}

  \inputminted{python}{../code/03-19d.py}
  \inputminted{text}{../output/03-19d.txt}

  \begin{figure}[H]
    \centering
    \includegraphics[scale=0.535]{../images/03-19d}
    \caption{Sampling distribution of $\overline{X}_n$ for different $n$.}
  \end{figure}

  Notice that as $n$ increases, the variance of the sampling distribution of
  $\overline{X}_n$ decreases, i.e.\ it becomes more narrow and sharply peaked
  around the true mean of the underlying distribution.
\end{ex}

% 20
\begin{ex}
  We begin by extending the definition of expectation to a random matrix, by
  defining the expected value of a random $m\times n$ matrix $X$ to be the
  $m\times n$ matrix whose $(i, j)$th entry is the expected value of $X_{i,j}$.

  Suppose that $X$ is a random matrix of dimension $m\times n$ with mean $\mu$,
  and let $A$ be a $p\times m$ matrix. Then
  \[
    \E{AX}_{i,j}
    =\E{\sum_{k=1}^m A_{i,k} X_{k,j}}
    =\sum_{i=1}^m A_{i,k}\E{X_{k,j}}
    =\sum_{i=1}^m A_{i,k}\mu_{k,j}
    =A\mu,
  \]
  and, likewise, if we let $B$ be an $n\times p$ matrix,
  \[
    \E{XB}_{i,j}
    =\E{\sum_{k=1}^n X_{i,k} B_{k,j}}
    =\sum_{i=1}^n \E{X_{i,k}}B_{k,j}
    =\sum_{i=1}^n \mu_{i,k}B_{k,j}
    =\mu B.
  \]

  The two claims about the expected value of $\E{a^TX}$ and $\E{AX}$ in Lemma
  3.21 are then special cases of this property for $X$ an $n\times 1$ random
  matrix.

  For the rest of the exercise we restrict $X$ to a random vector with mean
  $\mu$ and variance-covariance matrix $\Sigma$, and continue by noting that
  \begin{align*}
     & \E{(X-\mu)(X-\mu)^T}            \\
     & =\E{\begin{pmatrix}
        (X_1-\mu_1)^2          & (X_1-\mu_1)(X_2-\mu_2) & \cdots & (X_1-\mu_1)(X_n-\mu_n) \\
        (X_2-\mu_2)(X_1-\mu_1) & (X_2-\mu_2)^2          & \cdots & (X_2-\mu_2)(X_n-\mu_n) \\
        \vdots                 & \vdots                 & \ddots & \vdots                 \\
        (X_n-\mu_n)(X_1-\mu_1) & (X_n-\mu_n)(X_2-\mu_2) & \cdots & (X_n-\mu_n)^2          \\
      \end{pmatrix}} \\
     & =\begin{pmatrix}
      \var{X_1}      & \cov{X_1, X_2} & \cdots & \cov{X_1,X_n} \\
      \cov{X_2,X_1}  & \var{X_2}      & \cdots & \cov{X_2,X_n} \\
      \vdots         & \vdots         & \ddots & \vdots        \\
      \cov{X_n, X_1} & \cov{X_n, X_2} & \cdots & \var{X_n}     \\
    \end{pmatrix}     \\
     & =\var{X},
  \end{align*}
  and that therefore,
  \begin{align*}
    \var{AX}
     & =\E{(AX-A\mu)(AX-A\mu)^T} \\
     & =\E{A(X-\mu)(A(X-\mu))^T} \\
     & =\E{A(X-\mu)(X-\mu)^TA^T} \\
     & =A\E{(X-\mu)(X-\mu)^T}A^T \\
     & =A\Sigma A^T.
  \end{align*}

  This is the second claim about variances of random vectors in Lemma 3.21, but
  we conclude by noting that the first one is a special case of the second for
  matrices of the form $1\times n$.
\end{ex}

\begin{ex}
  Recall that by the rule of iterated expectations,
  \[
    \E{\cE{Y}{X}}=\E{Y},
  \]
  and therefore we have $\E{X}=\E{Y}$.

  Therefore,
  \begin{align*}
    \cov{X, Y}
     & =\E{XY} - \E{X}\E{Y} \\
  \end{align*}

  Likewise,
  \begin{align*}
    \E{XY}
     & =\E{\cE{XY}{X}}                     \\
     & =\int\! \cE{XY}{X=x} f_X(x) \,\d{x} \\
     & =\int\! \cE{Y}{X=x} xf_X(x) \,\d{x} \\
     & =\int\! x^2f_X(x) \,\d{x}           \\
     & =\E{X^2},
  \end{align*}
  and thus
  \begin{align*}
    \cov{X, Y}
     & =\E{XY} - \E{X}\E{Y} \\
     & =\E{X^2} - \E{X}^2   \\
     & = \var{X}.
  \end{align*}
\end{ex}

\begin{ex}
  \begin{enumerate}[(a)]
    \item[]
    \item No, since $\P{(Y, Z)=(1, 1)}=b-a$, while
          $\P{Y=1}=b$ and $\P{Z=1}=1-a$.
    \item We have
          \[
            \cE{Y}{Z=0}
            =\cP{Y=1}{Z=0}
            =\cP{Y=1}{X<a}
            =1,
          \]
          \[
            \cE{Y}{Z=1}
            =\cP{Y=1}{Z=1}
            =\cP{Y=1}{X>a}
            =\frac{b-a}{1-a}.
          \]
  \end{enumerate}
\end{ex}

\begin{ex}
  Suppose that $X\sim\text{Poisson}(\lambda)$. Then
  $f_X(x)=e^{-\lambda}\frac{\lambda^x}{x!}$ and thus
  \begin{align*}
    \psi_X(t)
     & =\E{e^{tX}}                                               \\
     & =\sum_{n=0}^\infty e^{tn}e^{-\lambda}\frac{\lambda^n}{n!} \\
     & =e^{-\lambda}\sum_{n=0}^\infty \frac{(\lambda e^t)^n}{n!} \\
     & =e^{\lambda(e^t-1)}.
  \end{align*}

  Suppose that $Y\sim N(\mu, \sigma^2)$. Then
  $f_Y=\frac{1}{\sqrt{2\pi\sigma^2}}
    \exp\left(-\frac{1}{2}\frac{(x-\mu)^2}{\sigma^2}\right)$,
  and therefore
  \begin{align*}
    \psi_Y(t)
     & =\E{e^{tY}}                                                  \\
     & =\int_{-\infty}^\infty\! e^{ty}\frac{1}{\sqrt{2\pi\sigma^2}}
    \exp\left(-\frac{1}{2}\frac{(y-\mu)^2}{\sigma^2}\right)\,\d{y}  \\
     & =\int_{-\infty}^\infty\! \frac{1}{\sqrt{2\pi\sigma^2}}
    \exp\left(-\frac{1}{2}\frac{(y-\mu)^2+ty\sigma^2}{\sigma^2}\right)\,\d{y}.
  \end{align*}
  However, since
  \begin{align*}
    -\frac{1}{2}(y-\mu)^2+ty\sigma^2
     & =-\frac{1}{2}(y^2-2y\mu+\mu^2-2ty\sigma^2)                                 \\
     & =-\frac{1}{2}(y^2-2y(\mu+t\sigma^2)+(\mu+t\sigma)^2-(\mu+t\sigma)^2+\mu^2) \\
     & =-\frac{1}{2}(y-\mu-t\sigma^2)^2+\frac{1}{2}((\mu+t\sigma^2)^2-\mu^2)      \\
     & =-\frac{1}{2}(y-\mu-t\sigma^2)^2+(t\mu\sigma^2+t^2\sigma^4/2),
  \end{align*}
  it follows that
  \begin{align*}
    \psi_Y(t)
     & =\exp(t\mu+t^2\sigma^2/2)\int_{-\infty}^\infty\! \frac{1}{\sqrt{2\pi\sigma^2}}
    \exp\left(-\frac{1}{2}\frac{(y-\mu-t\sigma^2)^2}{\sigma^2}\right)\,\d{y}          \\
     & =\exp(t\mu+t^2\sigma^2/2),
  \end{align*}
  since the integrand is the PDF for an $N(m+t\sigma^2, \sigma^2)$ distribution
  being integrated over its support.

  Suppose that $Z\sim\text{Gamma}(\alpha, \beta)$. Then
  \[
    f_Z(z)=\frac{1}{\beta^\alpha \Gamma(\alpha)}z^{\alpha-1}e^{-z/\beta}
  \]
  and so
  \begin{align*}
    \psi_Z(t)
     & =\E{e^{tZ}}                                                                                \\
     & =\int_{-\infty}^\infty e^{tz}\frac{1}{\beta^\alpha \Gamma(\alpha)}z^{\alpha-1}e^{-z/\beta}\,\d{z}                                             \\
     & =\int_{-\infty}^\infty \frac{1}{\beta^\alpha \Gamma(\alpha)}z^{\alpha-1}e^{-z(1/\beta-t)}\,\d{z}                                              \\
     & =\int_{-\infty}^\infty \frac{(1/\beta-t)^\alpha}{(1/\beta-t)^\alpha}\frac{1}{\beta^\alpha \Gamma(\alpha)}z^{\alpha-1}e^{-z(1/\beta-t)}\,\d{z} \\
     & =\frac{1}{(1/\beta-t)^\alpha\beta^\alpha}
    \int_{-\infty}^\infty \frac{1}{(1/\beta-t)^{-\alpha}\Gamma(\alpha)}z^{\alpha-1}e^{-z(1/\beta-t)}\,\d{z}                                          \\
     & =(1-\beta t)^{-\alpha},
  \end{align*}
  since the integrand is the PDF for a $\text{Gamma}(\alpha, (1/\beta-t)^{-1})$
  distribution integrated over its support.
\end{ex}

\begin{ex}
  Note that $f_{X_i}(x)=\frac{1}{\beta} e^{-x/\beta}$ and that therefore
  \begin{align*}
    \psi_{X_i}(t)
     & =\E{e^{tX_i}}                                                                                \\
     & =\int_{0}^\infty e^{tx}\frac{1}{\beta} e^{-x/\beta}\,\d{x}                                              \\
     & =\int_0^\infty\!\frac{1}{\beta}\exp\left(-x/(\beta/(1-\beta t))\right)\,\d{t}                           \\
     & =(1-\beta t)^{-1}\int_0^\infty\! \frac{1-\beta t}{\beta} \exp\left(-x/(\beta/(1-\beta t))\right)\,\d{t} \\
     & =(1-\beta t)^{-1}
  \end{align*}
  since the integrand is the PDF for an exponential distribution integrated over
  its support.

  Let $Y=\sum_{i=1}^n X_i$. Then
  \[
    \psi_Y(t)
    =\prod_{i=1}^n \psi_{X_i}(t)
    =\left(1-\beta t\right)^{-n},
  \]
  which by the previous problem is the MGF of a $\text{Gamma}(n,\beta)$
  distribution.
\end{ex}