\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{physics}
\usepackage{amsmath}
\usepackage{showlabels}
\title{Statistical Computing for Scientists and Engineers\\[1em] Homework 1}
\author{Jiale Shi}
\date{September/11/2018}

\usepackage{natbib}
\usepackage{graphicx}

\begin{document}

\maketitle

\newpage
\section{Problem 1}
Consider $n$ samples $D_{n} = (x_{1},...,x_{n})$, independently drawn from normal distribution, $N_{\mu, \sigma^{2}})$, with known variance and unknown mean, $\sigma^{2}$ and $\mu$, respectively. Assume that the mean has a prior given by the normal distribution:

\begin{equation}
p(\mu|m,v^{2})= N(\mu|m,v^{2}) \propto \exp(\frac{1}{2v^{2}} (\mu-m)^2)
%\pdv[2]{u}{x}+\pdv[2]{u}{y} = 0
\end{equation}

Derive the posterior:

\begin{equation}
p(\mu|D) \sim N(\mu_{n},\sigma^{2})
\end{equation}

\begin{equation}
\mu_{n} = \sigma^{2}\left(\frac{m}{v^{2}}+\frac{n\Bar{x}}{\sigma^{2}}\right), \sigma_{n}^{2} = \frac{1}{\frac{n\Bar{x}}{\sigma^{2}}+\frac{1}{v^2}}
\end{equation}

in which $\Bar{x}$ is the empirical mean $(\Bar{x}=\sum^n_{i=1}x_{i})$.

Answer:
\begin{equation}
\begin{aligned}
    p(\mu|D) &= \prod_{i=1}^{n} f(x_{i}|\mu) p(\mu) \\
    &\propto \exp{-\frac{\sum_{i=1}^{n}(x_{i}-\mu)^2}{2\sigma^{2}} - \frac{(\mu-m)}{2v^2}} \\
     &\propto \exp{-\frac{\mu^2}{2}(\frac{n}{\sigma^2}+\frac{1}{v^2})+\mu(\frac{\sum_{i}^{n}x_i}{\sigma^2}+\frac{m}{v^2})} \\
    &\propto \exp{-\frac{\mu^2}{2}(\frac{n}{\sigma^2}+\frac{1}{v^2})+\mu(\frac{n\Bar{x}}{\sigma^2}+\frac{m}{v^2})} \\
    &\propto \exp{-\frac{1}{2\sigma_{n}^{2}}(\mu-\mu_{n})}
\end{aligned}
\end{equation}
\begin{equation}
\frac{1}{\sigma_{n}^{2}} = \frac{1}{v^2}+\frac{n}{\sigma^2} \Rightarrow{} \sigma_{n}^{2} = \frac{1}{\frac{1}{v^2}+\frac{n}{\sigma^2}}
\end{equation}
\begin{equation}
    \mu_n = \sigma_{n}^2\left(\frac{m}{v^2}+\frac{n\Bar{x}}{\sigma^2}\right) = \left(\frac{1}{\frac{1}{v^2}+\frac{n}{\sigma^2}}\right)\left(\frac{m}{v^2}+\frac{n\Bar{x}}{\sigma^2}\right)
\end{equation}

\newpage
\section{Problem 2}
Consider a univariate Gaussian $N(X|\mu, \lambda^{-1})$ and a dataset $X = \{x_{1}, x_{2}, x_{3},...,x_{N}\}$, $x_{i}\sim N(X|\mu,\lambda^{-1})$ of i.i.d observations. Write down the likelihood for this model and show that a conjugate prior is of the form:

\begin{equation}
    p(\mu,\lambda)\propto \left[\lambda^{1/2} \exp(-\frac{\lambda \mu^{2}}{2})\right]^{\beta} \exp(c\lambda \mu - d\lambda)
\end{equation}

where $c$, $d$, $\beta$ are constants. Given $p(\mu,\lambda) = p(\mu\mid\lambda) p(\lambda)$, we can recognize that the above prior is the product of a Gaussian $p(\mu\mid\lambda)$ whose precision is a linear function of $\lambda$ and of the gamma distribution $p(\lambda)$. As a result the normalized prior take the form:
\begin{equation}
p(\mu, \lambda) = N \left( \mu\mid \mu_{0},(\beta \lambda)^{-1}\right) Gam(\lambda\mid a,b)
\end{equation}
where $\mu_{0}=c/\beta, \alpha = 1+\beta/2$ and $b = d-c^2/2\beta$. Show that the posterior $p(\mu, \lambda \mid X)$ is also a Gaussian-gamma distribution of the same functional form as thr prior:
\begin{equation}
    N(\mu \mid \mu_N, ((N+\beta)\lambda)^-1) Gam(\lambda \mid a_N, b_N)
\end{equation}
write down expressions for the parameters of this posterior distribution $\mu_N$, $a_N$ and $b_N$

Answer:
the likelihood takes the form:
\begin{equation}
\begin{aligned}
    p(X|\mu, \lambda) & = \prod_{n=1}^{N} f(x_{n}|\mu) \propto \lambda^{N/2}\exp{-\frac{1}{2}\lambda \sum_{n=1}^N(x_{n}-\mu)^2}\\ & \propto \left[\lambda^{1/2} \exp{-\frac{\lambda \mu^2}{2}}\right]^{N} \exp{\lambda\mu\sum_{n=1}^{N}x_{n}-\frac{1}{2}\lambda\sum_{n=1}^{N}x_{n}^{2}}
\end{aligned}
\end{equation}

we need a prior that has a similar functional form in terms of $\lambda$ and $\mu$.

\begin{equation}
\begin{aligned}
&\beta = N \\
&c = \sum_{n=1}^{N}x_{n} \\
&d = \frac{1}{2}\sum_{n=1}^{N}x_{n}^{2}\\
&p(\mu,\lambda)\propto \left[\lambda^{1/2} \exp(\frac{\lambda \mu^{2}}{2})\right]^{\beta} \exp(c\lambda \mu - d\lambda)
\end{aligned}
\end{equation}

\begin{equation}
\begin{aligned}
p(\mu,\lambda) &\propto \left[\lambda^{1/2} \exp(\frac{\lambda \mu^{2}}{2})\right]^{\beta} \exp(c\lambda \mu - d\lambda) \\
& = (\beta\lambda)^{1/2} \exp{-\frac{\beta \lambda}{2}\left(\mu - \frac{c}{\beta}\right)^{2}} \lambda^{(\beta-1)/2} \exp{-\left(d-\frac{c^2}{2\beta}\right)\lambda} \\
& = p(\mu|\lambda) p(\lambda) \\
& = N\left(\mu| \mu_{0}=\frac{c}{d}, \sigma = (\beta \lambda)^{-1}\right) Gamma\left(\lambda|a = \frac{1+\beta}{2},b = d-\frac{c^2}{2\beta}\right)
\end{aligned}
\end{equation}

\begin{equation}
\begin{aligned}
p(\mu,\lambda|X) &\propto \lambda^{N/2}\lambda^{a-1}\exp\left[-\left(b+\frac{1}{2}\sum_{n=1}^{N}x_{n}^{2}+\frac{\beta}{2}\mu_{0}^{2}\right)\lambda\right]\times \\
&[\lambda(N+\beta)]^{1/2}\exp{-\frac{\lambda(N+\beta)}{2}\left[\mu^{2}-\frac{2}{N+\beta}(\beta\mu_{0}+\sum_{n=1}^{N}x_{n})\mu\right]} \\
& = N(\mu | \mu_{N},[\lambda(N+\beta)]^{-1}) Gam(\lambda | a_{N},b_{N}) \\
\end{aligned}
\end{equation}

\begin{equation}
\begin{aligned}
& \mu_{N}= -\frac{\beta\mu_{0}+\sum_{n=1}^{N}x_{n}}{N+\beta}\\
& a_{N} = \frac{N}{2}+a\\
& b_{N} = -\left(b+\frac{1}{2}\sum_{n=1}^{N}x_{n}^{2}+\frac{\beta}{2}\mu_{0}^{2}-\frac{(\beta\mu_{0}+\sum_{n=1}^{N}x_{n})^{2}}{2(N+\beta)}\right)
\end{aligned}
\end{equation}

\newpage
\section{Problem 3}
The Wishart distribution distribution $W_m(n,\Sigma)$.

1. To prove $E[X|n,\Sigma]=n\Sigma$.

$X = \sum_{i=1}^{n} z_{i}z_{i}^{T} \sim W_{m}(n,\Sigma)$, where $z_{1},...,z_{n} \sim N_{m}(0,\Sigma)$, and $z_{1},...,z_{n}$ are independent with each other.
\begin{equation}
    E[X|n,\sum] =E[\sum_{1}^{n}z_{i}z_{i}^{T}] = \sum_{1}^{n}E[z_{1}z_{1}^{T}] = \sum_{1}^{n} Var(z_{1}) = n\Sigma
\end{equation}

Therefore,$E[X|n,\Sigma]=n\Sigma$.

2. To prove $Cov[X] = 2n\Sigma \bigotimes \Sigma$.

\begin{equation}
\begin{aligned}
    Cov(X) &= Cov\left(\sum_{i=1}^{n} z_{i}z_{i}^{T}\right) \\
    &= \sum_{1}^{n}Cov\left(z_{1}z_{1}^{T}\right) \\
    &= n Cov\left(z_{1}z_{1}^{T}\right)  \\
    &= n Cov\left(z_{1} \boxed{ } z_{1}\right) \\
    &= n Cov((\Sigma \bigotimes \Sigma)(x \boxed{ } x)) \\
    &= 2n\Sigma \bigotimes \Sigma
\end{aligned}
\end{equation}

\newpage
\section{Problem 4}
(a) Derive the log Gamma distribution $Y \sim log[X]$, $X= G(\alpha, \beta)$ and show that it belongs to both families above.

Answer:
$Y \sim log[X]$, $X= G(\alpha, \beta)$, therefore, $X \sim \exp[Y]$.
\begin{equation}
\begin{aligned}
    f_{Y}(x) & = f_{X=e^Y} (e^x) \frac{d e^x}{d x} = \frac{1}{\Gamma(\alpha)} e^{-\beta e^x}(\beta e^{x})^{\alpha} \\
    & = \frac{\beta^{\alpha}}{\Gamma(\alpha)} e^{-\beta e^x}(e^{x})^{\alpha}
\end{aligned}
\end{equation}

Check whether it belongs to exponential family.
\begin{equation}
\begin{aligned}
    \frac{\beta^{\alpha}}{\Gamma(\alpha)} e^{-\beta e^x}(e^{x})^{\alpha} & = \frac{\beta^{\alpha}}{\Gamma(\alpha)} \exp{-\beta e^x+x\alpha} \\
    & = \frac{\beta^{\alpha}}{\Gamma(\alpha)} \exp{(-\beta,\alpha)(e^x,x)^{T}} \\
    & = \frac{\beta^{\alpha}}{\Gamma(\alpha)} \exp{\eta(\alpha,\beta)^{T}u(x)}
\end{aligned}
\end{equation}
where $u(x)=(e^x,x)^T, \eta(\alpha,\beta)=(-\beta,\alpha)^{T}, h(x)=1,g(\alpha,\beta)=\frac{\beta^{\alpha}}{\Gamma(\alpha)} $. Therefore, the log Gamma distribution $Y$ belongs to the exponential family.
\newline
(b) Provide examples of two distributions that part of the exponential family.

Answer:
Consider the Bernoulli Distribution
\begin{equation}
\begin{aligned}
  p(x|\mu) & =  \mu^{x}(1-\mu)^(1-x) \\
  & = \exp{x\ln{\mu}+(1-x)\ln\left(1-\mu\right)} \\
  & = (1-\mu)\exp{\ln\left(\frac{\mu}{1-\mu}\right)x}
\end{aligned}
\end{equation}

\begin{equation}
\begin{aligned}
&\eta = \ln\left(\frac{\mu}{1-\mu}\right) \\
&\mu = \sigma(\eta)= \frac{1}{1+e^{-\eta}} \\
& g(\eta) = 1-\mu = \frac{1}{1+e^{\eta}} \\
& p(x|\mu) = g(\eta)\exp{\eta x}, \mu(x)= x, h(x)=1,g(\eta) = 1-\mu = \frac{1}{1+e^{\eta}}
\end{aligned}
\end{equation}

Consider the Beta distribution

\begin{equation}
\begin{aligned}
  p(x|a,b) & =  \frac{\Gamma(a+b)}{\Gamma(a)\Gamma(b)} x^{a-1}(1-x)^{b-1} \\
  & = \frac{\Gamma(a+b)}{\Gamma(a)\Gamma(b)} \exp[(a-1)\ln(x) + (b-1)\ln(1-x)] \\
  & = h(x)g(\eta)\exp{\eta^{T}u(x)}
\end{aligned}
\end{equation}

\begin{equation}
\begin{aligned}
&\eta = (a-1,b-1)^{T} \\
&\mu(x) = (\ln(x),\ln(1-x))^{T} \\
& g(a,b) = \frac{\Gamma(a+b)}{\Gamma(a)\Gamma(b)} \\
& h(x)=1
\end{aligned}
\end{equation}
\newline
(c) Provide examples of two distributions that are part of the local-scale family.

Answer:

Consider the Cauchy distribution:
take $\Psi(x) = \frac{1}{\pi}\frac{1}{1+x^2}$
\begin{equation}
\begin{aligned}
p(x|a,b) = \frac{1}{b}\Psi\left(\frac{x+a}{b}\right) = \frac{1}{\pi b}\frac{1}{1+\left(\frac{x+a}{b}\right)^2}
\end{aligned}
\end{equation}
defines the Cauchy L-S family.

Consider the Uniform distribution:
take $\Psi(x) = I_{(0,1)}(x)$
\begin{equation}
\begin{aligned}
    I_{(0,1)}(x) = 1, if 0<x<1
    I_{(0,1)}(x) = 0, otherwise
\end{aligned}
\end{equation}
$\Psi(x)$ is the pdf of the Unif(0,1) distribution
\begin{equation}
\begin{aligned}
p(x|a,b) = \frac{1}{b}\Psi\left(\frac{x+a}{b}\right) = \frac{1}{b}I_{(0,1)}\left(\frac{x+a}{b}\right) = \frac{1}{b}I_{(-a,-a+b)}(x)
\end{aligned}
\end{equation}
This is the pdf of the Unif$(-a,-a+b)$
\newpage
\section{Problem 5}
(a) Start by plotting a histogram of the camera's max resolution. Fit a normal distribution to this data and plot the fitted distribution on top of histogram. Repeat for the camera weight.

Answer:
the camera's max resolution
\begin{figure}[ht]
\centering
\includegraphics[scale=0.5]{w5a1.jpg}
\caption{Histogram and normal distribution fitting of the camera's max resolution}
%\label{fig:universe}
\end{figure}

the camera's weight
\begin{figure}[ht]
\centering
\includegraphics[scale=0.5]{w5a2.jpg}
\caption{Histogram and normal distribution fitting of the camera's weight}
%\label{fig:universe}
\end{figure}
\newline
(b) Standardize the camera max resolution data using the fitted mean, $\Bar{x_{i}}$, and standard deviation, $\sigma_{i}$. This can be completed by computing $(x_{ij}-\Bar{x_{i}})/\sigma_{i}$. Plot a standard Gaussian $N(0,1)$ on top of the normalized data. Comment on why we would be interested in doing this.

Answer:
from (a) the normal distribution fitting for the max resolution
$\Bar{x_i} = 2491.7618497109806, \sigma_{i} = 752.5256308818603$.
Then, we standardize the camera max resolution data by computing $(x_{i}-\Bar{x_{i}})/\sigma_{i}$.

\begin{figure}[ht]
\centering
\includegraphics[scale=0.5]{w5b1.jpg}
\caption{Standard Gaussian $N(0,1)$ fitting of the normalized camera's max resolution}
%\label{fig:universe}
\end{figure}

The reason that we would be interested in doing this is that the standardized data is easier to compare with each other.
\newline
(c) Fit a Gamma, $G(\alpha, \beta)$, distribution to the histogram of the camera's weight through MLE. Plot both fitted distributions on top of the histogram. Comment on which distribution provides a better fit.

start by finding the log-likelihood of Gamma distribution
\begin{equation}
  f=\ln(T) = (\alpha-1) \sum \ln(x)  - n\ln(\Gamma(\alpha)) - n\alpha \ln(\beta)-\frac{1}{\beta}\sum x
\end{equation}
\begin{equation}
  \frac{d\ln(T)}{d\beta} =  -\frac{n\alpha}{\beta}+\frac{1}{\beta^2}\sum x =0
\end{equation}
\begin{equation}
  \beta = \frac{\sum x}{n \alpha} = \frac{\Bar{x}}{\alpha}
\end{equation}
\begin{equation}
\begin{aligned}
  f(\alpha) &= (\alpha-1)n \Bar{\ln(x)} - n \ln(\Gamma(\alpha))-n\alpha \ln(\frac{\Bar{x}}{\alpha})-n\alpha
\end{aligned}
\end{equation}
Using the minka-newton method from the paper provided.
\begin{equation}
\begin{aligned}
  f^{'}(\alpha) &= n \Bar{\ln(x)} - n \Psi(\alpha)-n \ln(\Bar{x})+n\ln(\alpha)
\end{aligned}
\end{equation}
\begin{equation}
\begin{aligned}
  f^{''} (\alpha)&=  - n \Psi^{'}(\alpha)+\frac{n}{\alpha}
\end{aligned}
\end{equation}
\begin{equation}
\begin{aligned}
  \frac{1}{\alpha^{new}} = \frac{1}{\alpha^{old}}+ \frac{f^{'}(\alpha)}{\alpha^2 f^{''}(\alpha)} = \frac{1}{\alpha^{old}}+ \frac{ \Bar{\ln(x)} -  \Psi(\alpha)- \ln(\Bar{x})+\ln(\alpha)}{-  \alpha^{2}\Psi^{'}(\alpha)+\alpha}
\end{aligned}
\end{equation}

\begin{figure}[ht]
\centering
\includegraphics[scale=0.5]{alpha.jpg}
\caption{ alpha value with minka-newton method}
%\label{fig:universe}
\end{figure}
From Figure 4, we find that the value of $\alpha$ converges very quickly. $\alpha \approx 1.84$ and $\beta \approx 166.6387422858897$.

Then we use the $\alpha$ and $\beta$ to plot the fitted Gamma distribution and compare with the Gaussian in Figure 2.
\begin{figure}[ht]
\centering
\includegraphics[scale=0.5]{h5c.jpg}
\caption{ Gamma distribution and Gaussian distribution camera's weight}
%\label{fig:universe}
\end{figure}
After comparing with the Gaussian distribution fitting, we find that the Gamma distribution is better than the Gaussian distribution for the camera's weight.
\newline
(d)Plot the camera's release year vs. max-resolution. Fit a 2D Gaussian distribution to this data through MLE. Plot the fitted Gaussian on top of the data.
\begin{figure}[ht]
\centering
\includegraphics[scale=0.5]{w5d.jpg}
\caption{ 2D Gaussian distribution camera's release year vs. max-resolution}
%\label{fig:universe}
\end{figure}
\newline

%\bibliographystyle{plain}
%\bibliography{references}
\end{document}


\setlength{\abovedisplayskip}{2pt}
\setlength{\belowdisplayskip}{2pt}
\section{Bayesian Thinking and Estimation}
\label{sec:bayesian_thinking}
In this section we introduce the core topics of Bayesian statistics and, whenever possible, compare proposed methods and results to their classicist counterpart.

We use standard notation wherever possible, nonetheless we make one exception in that we write $p(Z)$ for the probability density function of the random variable $Z$, where $Z$ may be scalar-valued or vector-valued.
If it is clear from the context we will also write $p(z)$ for the density of $Z$ evaluated at $z$.

\subsection{Probabilistic Modeling}
We begin by introducing a formal notion of stochastic modeling and continue with a taxonomic description of different schools of thought.

Say we observe data $\mathcal{D} = \{(y_i, x_i) : i=1,\mydots,n\}$ for which we have some intuition about the relationship between $X$ and $Y$ ---this intuition might come from (economic) theory for example.
We formalize this by writing the data generating process as a (possibly algorithmic) mathematical model $Y = \mathcal{M}(X; \epsilon, \theta)$, where $\theta$ denotes the model parameters and $\epsilon$ an explicitly modeled error term which corrects for uncertainty in the model, e.g. in the case of non-observables.

In the rest of this paper we assume that we know the parametric structure of $\mathcal{M}$ and that our goal lies in learning about the parameters after observing the data $\mathcal{D}$.
An important aspect here is to postulate the existence of a \textit{true data generating process}, which we do by assuming that there is some (fixed) $\theta_0$, in the parameter space $\Theta$, so that $Y = \mathcal{M}(X; \epsilon, \theta_0)$ describes reality sufficiently accurate and better than for any other parameter.
The subsequent goal is then to learn about $\theta_0$.
Note that for everything that follows we need to assume that our \emph{true model} actually describes reality accurately; if not we enter the realm of model misspecification which can render any analysis useless.

Next we compare two different schools of thought present in the statistical domain, which consider the estimation of $\theta_0$ and the quantification of uncertainty in the estimate.

\paragraph{Frequentist.}
In the literature the so called frequentist methods constitute the most widely used approaches.
A particular method ---which we choose here as it lends itself nicely to a comparison--- is the maximum likelihood approach.
There we use the distributional assumptions on our model to construct the likelihood function $\mathcal{L}(\theta; \mathcal{D}) = p(\mathcal{D}; \theta)$, which is simply the joint density of the data evaluated at the observed data points for varying parameter $\theta$.
We can then find an estimator $\hat{\theta}$ for $\theta_0$ as the maximizer of this function, i.e. $\hat{\theta} \in \argmax{\theta \in \Theta} \mathcal{L}(\theta; \mathcal{D})$.
There has been published an extensive amount of research on the properties of this estimator, for example on sufficient conditions for the uniqueness of the maximization or large-sample normal approximations.
In particular, under some regularity conditions we can find a matrix $\widehat{V}$ so that $\sqrt{n}\widehat{V}^{-1/2}(\hat{\theta} - \theta_0) \overset{d}{\longrightarrow} \normal{0, \mathrm{I}}$.
This result we can use to quantify uncertainty in $\hat{\theta}$ by computing standard errors and confidence intervals, as well as to formulate tests.

One fundamental idea which stretches over all methods in the frequentist world is the interpretation of probability as the limit of an infinite sequence of relative frequencies of events---hence the name.
Probability then just counts how many times an event happened or not; for example if we toss a coin an infinite number of times, the relative frequency of heads converges to the probability of heads.
We do expect the outcome of an experiment (e.g. coin flip) to vary, however, we do not assume the true parameter to vary.
For instance, we may interpret the probability of a coin landing on heads as the true model parameter.
In this sense, it would be absurd to let this object vary for different experiments.
Therefore any hypothesis on $\theta_0$ is either true or false.
And it is this binarity which makes hypothesis testing (interpretation of confidence intervals) awkward in the frequentist setting.
By testing some hypothesis $H_0: \theta_0 = \theta^*$ we do not directly compute the probability of the hypothesis being true --hypothesis are either false or true--- but we compute if the observed data $\mathcal{D}$ is more likely to have orginated under the null hypothesis or the alternative.

\paragraph{Bayesian.}
The main difference of the Bayesian mindset is the understanding of probability as a subjective quantification of uncertainty.
We may still believe that $\theta_0$ is fixed, nevertheless, in the Bayesian paradigm we build uncertainty about the true location of the parameter into the model by allowing for probability distributions to be defined on $\Theta$ ---which, as we saw above, is nonsensical in a frequentist worldview.
We can see the direct utility of this liberation by considering Bayes theorem applied to densities on our model
\begin{flalign*}
  && p(\theta \mid \mathcal{D}) &= \frac{p(\mathcal{D} \mid \theta) p(\theta)}{p(\mathcal{D})} \propto p(\mathcal{D} \mid \theta) p(\theta) \,,\tag{Bayes theorem}&\\[0.7em] \text{which reads} &&
  posterior &= \frac{likelihood \times prior}{evidence} \propto likelihood \times prior \,.
\end{flalign*}
The posterior distribution is the object of interest for any subsequent Bayesian analysis, it describes the distribution of the parameter of interest given the observed data $\mathcal{D}$.
From a naive standpoint this is too good to be true.
And in fact it is.
To give Bayes theorem any meaning we have to define the prior $p(\theta)$, a probability distribution of the model parameter on $\Theta$.
The prior may be used to incorporate knowledge about the parameter into the analysis that existed prior to observing the data.
But this can be highly subjective and can lead to \emph{two} different researchers having \emph{two} different priors which would result in \emph{two} different posteriors.
This is where the main criticism of Bayesian statistics is focused on: where does the prior distribution come from?
With the scientific goal of objectivity in mind, many feel at unease having results dependent on subjective choices of the prior.
In what follows we will embark on the Bayesian idea without providing much more fundamental criticism, nonetheless, when adequate we will consider the influence of different priors on the posterior.

In comparison to the maximum likelihood approach, in a Bayesian analysis there is no need for one specific point estimator or confidence interval.
The result of such an analysis is a complete probability distribution from which we can compute, in principle, any quantity we like.
Being clear on all prior assumptions and giving up (some) \emph{objectivity} we gain the possibility to formulate answers to more natural questions, as for example: $\prob{\theta \in \Theta_0 \mid \mathcal{D}} = \int_{\Theta_0} p(\theta \mid \mathcal{D}) \mathrm{d}\theta$.

\subsection{Solving for the posterior analytically}
In this subsection we present an analytical derivation of the posterior distribution of mean and variance parameters in a univariate normal model for two priors.
We compare the results to the maximum likelihood estimator.
As it will be of major importance in the subsequent sections, we have included the definition of the scaled inverse $\chi^2$ probability distribution in the appendix (see definition \ref{def:scaledInverseChi}).

Let us assume that we observe a sample $y = (y_1, \mydots, y_n)$
with $y_i \mid \theta \overset{\text{iid}}{\sim} \normal{\mu, \sigma^2}$, where $\theta = (\mu, \sigma^2)$.
Our interest lies in solving for the marginal posteriors $p(\mu \mid y)$ and $p(\sigma^2 \mid y)$.

\paragraph{Noninformative Prior.}
We start our analysis with a common prior choice in settings where we have little prior information and sufficent data.
In these cases we can use noninformative priors to model complete ignorance of any prior information, in particular, here we use \emph{flat priors}, which assign equal weight to every region in the parameter space.
Let us go with the common assumption that $\mu$ and $\sigma^2$ are independent a priori.
Mathematically we can write a flat prior as $p(\mu) \propto 1$.
We note that this does not define a proper proability distribution, which will not matter in this case but can lead to problems in others; see for example section 4.2 in \citet{kass1996}.
Since the variance is restricted to be positive we impose a flat prior on the log-transform thereof, i.e. $p(\log \sigma) \propto 1$.
Using that $x \mapsto \text{exp}^2(x)$ is one-to-one we get the density of the transformed variable $p(\mu, \sigma^2) = p(\mu)p(\sigma^2) \propto p(\sigma^2) \propto (\sigma^2)^{-1}$.

The likelihood is given by $p(y \mid \mu, \sigma^2) \propto (\sigma^2)^{-n/2} \EXP{-\sum_i (y_i - \mu)^2/2\sigma^2}$, where we dropped all proportionality constants.
Application of Bayes theorem yields $p(\mu, \sigma^2 \mid y) \propto p(y \mid \mu, \sigma^2) p(\mu, \sigma^2) \propto (\sigma^2)^{-(n+2)/2} \EXP{-\sum_i (y_i - \mu)^2/2\sigma^2}$.
Integrating over the respective parameter yields the marginal posteriors.

\begin{proposition}\label{prop:posterior_uniform}
  Under the above setup and a flat prior on $\mu$ and $\log \sigma$ we find $\mu \mid y \sim t_{n-1}(\bar{y}, s^2/n)$ and $\sigma^2 \mid y \sim \scaledInvChi{n-1, s^2}$, where $s^2 = \sum_i (y_i - \bar{y})^2 / (n-1)$ denotes the (unbiased) sample variance and $\bar{y} = \frac{1}{n} \sum_i y_i$ the sample mean, respectively.
\end{proposition}
\begin{proof}
See appendix.
\end{proof}

We compare the marginal posteriors to their maximum likelihood counterpart by reporting summary statistics of the distributions in table \ref{tab:comp_uniform_bay_ml}.
We focus on the mean and variance of the posterior, as well as the \emph{maximum a posteriori} (MAP) estimate $\left(\argmax{\theta \in \Theta} p(\theta \mid y)\right)$, but withhold from a discussion as we consider the more general results of the next paragraph in more detail.

\begin{table}[ht]
\centering
\def\arraystretch{1.3}
{\small
 \begin{tabular}{c | c c c c c}
 Parameter & ML Estimate & ML Variance & MAP & Posterior Mean & Posterior Variance\\[0.5ex]
 \hline
 $\mu$ & $\bar{y}$ & $\sigma^2/n$ & $\bar{y}$ & $\bar{y}$ & $s^2 / n$\\
 $\sigma^2$ & $\frac{n-1}{n} s^2$ & $2 \sigma^4 /n$ & $\frac{n-1}{n+1} s^2$ & $\frac{n-1}{n-3} s^2$ & $\frac{2 (n-1)^2}{(n-3)^2 (n-5)} s^4$\\
 \end{tabular}
 }
\caption{\small{Comparison of Bayesian estimates using a flat prior to ML estimates. See appendix for derivation.}}
\label{tab:comp_uniform_bay_ml}
\end{table}

\vspace{-15pt}
\paragraph{Conjugate Prior.}
In case substantial information on the parameters is available a priori, we can model this information properly to gain more stable results.
However, not every product of prior and likelihood results in a sensible posterior.
As we are interested in analytical results in this section we seek priors that guarantee posteriors of known form.
The class of \emph{conjugate priors} (see definition \ref{def:conjugate_prior} in the appendix) plays an important part in Bayesian statistics as they are able to provide such assurance.

Consider again the likelihood but written dependent on the sufficient statistics $\bar{y}$ and $s^2$
\begin{align}
  p(y \mid \mu, \sigma^2) \propto (\sigma^2)^{n/2} \EXP{-\frac{1}{2\sigma^2} \left[ n(\mu - \bar{y})^2 + (n-1)s^2 \right]} \,,
  \label{eq:likelihood2}
\end{align}
where $s^2$ again denotes the (unbiased) sample variance.
We want to construct a two dimensional conjugate prior for $(\mu, \sigma^2)$ such that multiplying the prior by the likelihood does not change its structure, as this assures the posterior to be of known form.
Note that we have $p(\mu, \sigma^2) = p(\mu \mid \sigma^2) p(\sigma^2)$.
Looking at equation (\ref{eq:likelihood2}) we see that in order to \emph{not} change the inherent structural dependence on the parameters we must have $\mu \mid \sigma^2 \sim \normal{\mu_0, \sigma^2 / \kappa_0}$, with \emph{hyperparameters} $\mu_0$ and $\kappa_0 > 0$.
Similarly, we observe that we must have $\sigma^2 \sim \scaledInvChi{\nu_0, \sigma_0^2}$, with hyperparameters $\nu_0$ and $\sigma_0^2 > 0$.
This becomes apparent when considering the respective densities.
Following \citet{gelmanbda04} we write $(\mu, \sigma^2) \sim \NormalscaledInvChi{\mu_0, \sigma_0^2 / \kappa_0; \nu_0, \sigma_0^2}$, with corresponding density function $p(\mu, \sigma^2) = p(\mu \mid \sigma^2) p(\sigma^2) \propto (\sigma^2)^{\frac{3 + \nu_0}{2}} \EXP{-\frac{1}{2\sigma^2} \left[\nu_0 \sigma_0^2 + \kappa_0(\mu_0 - \mu)^2 \right]}$.
Multiplying the likelihood with our constructed prior we get the joint posterior
\begin{align}
  p(\mu, \sigma^2 \mid y) \propto& (\sigma^2)^{-\frac{3 + \nu_0 + n}{2}} \EXP{-\frac{1}{2 \sigma^2} \left[\nu_0\sigma_0^2 + \kappa_0(\mu - \mu_0)^2 + (n-1)s^2 + n(\bar{y} - \mu)^2 \right]} \,.
  \label{eq:conjugate_posterior}
\end{align}

\begin{proposition}
  The (posterior) distribution of $(\mu, \sigma^2) \mid y$ , as given by the conditional density in equation (\ref{eq:conjugate_posterior}), is $\NormalscaledInvChi{\mu_n, \sigma_n^2/\kappa_n; \nu_n, \sigma_n^2}$, where
    $\nu_n = \nu_0 + n$, $\kappa_n = \kappa_0 + n$, $\mu_n =\frac{\kappa_0}{\kappa_n}\mu_0 + \frac{n}{\kappa_n}\bar{y}$ and $\sigma_n^2 = \left[\nu_0 \sigma_0^2 + (n-1)s^2 + \frac{\kappa_0 n}{\kappa_n} (\bar{y} - \mu_0)^2\right] /\nu_n$.
  \label{prop:posterior_conjugate}
\end{proposition}
\begin{proof}
  See appendix.
\end{proof}

Since the prior and the posterior are both normal scaled inverse $\chi^2$ distributed, we indeed constructed a conjugate prior.
Using the intermediate finding from proposition \ref{prop:posterior_conjugate} we can derive the main result of this section.

\begin{proposition}\label{prop:marginal_posterior}
  The marginal posterior distributions are given by $\mu \mid y \sim t_{\nu_n}(\mu_n, \sigma_n^2 / \kappa_n)$ and $\sigma^2 \mid y \sim \scaledInvChi{\nu_n, \sigma_n^2}$, where $\nu_n, \sigma_n^2, \mu_n$ and $\kappa_n$ are as in proposition \ref{prop:posterior_conjugate}.
\end{proposition}
\begin{proof}
  See appendix.
\end{proof}

\begin{table}[ht]
\centering
\def\arraystretch{1.3}
{\small
 \begin{tabular}{c | c c c c c}
 Parameter & ML Estimate & ML Variance & MAP & Posterior Mean & Posterior Variance\\
 \hline
 $\mu$ & $\bar{y}$ & $\sigma^2 / n$ & $\mu_n$ & $\mu_n$ & $\sigma_n^2 / \kappa_n$\\
 $\sigma^2$ & $\frac{n-1}{n}s^2$ & $2 \sigma^4 / n$ & $\frac{\nu_n}{\nu_n + 2} \sigma_n^2$ & $\frac{\nu_n}{\nu_n - 2} \sigma_n^2$ & $\frac{2 \nu_n^2}{(\nu_n - 2)^2(\nu_n - 4)} \sigma_n^4$
 \end{tabular}
 }
\caption{{\small Comparison of Bayesian estimates using a conjugate prior to ML estimates. See appendix for derivation.}}
\label{tab:comp_conjugate_bay_ml}
\end{table}

Next we consider the results of Proposition \ref{prop:marginal_posterior}, of which some summary statistics are tabulated in table \ref{tab:comp_conjugate_bay_ml}.
We focus on the analysis of the mean parameter $\mu$.

As the t-distribution is parameterized over its mean and variance (and degrees of freedom) we can directly deduce the posterior mean as $\mu_n =\frac{\kappa_0}{\kappa_0 + n}\mu_0 + \frac{n}{\kappa_0 + n}\bar{y}$ and the posterior variance as $\sigma_n^2 / \kappa_n$.
We see that the posterior mean is simply a convex combination of the prior $\mu_0$ and the sample average $\bar{y}$, with weights determined by the sample size and $\kappa_0$.
For any fixed $n$ this pulls our estimate of the posterior mean away from $\bar{y}$ and closer to $\mu_0$ (and vice versa), which can be helpful if we have insufficient data and believe that the parameter should be around $\mu_0$ ---we can use $\kappa_0$ then to express our degree of believe in the prior.
As $n$ grows to infinity the information in the data overwhelms all prior information and the posterior mean is dominated by the sample mean.

Similarly we can use the hyperparameters $\sigma_0^2, \nu_0$ and $\kappa_0$ to model our prior knowledge of the variance parameter, which propagates to the posterior variance of the mean parameter.
Considering the variance of the posterior mean as a function in $n$ we can use the \emph{Landau notation} to write $\sigma_n^2 / \kappa_n =  \frac{n}{(\nu_0 + n)(\kappa_0 + n)} s^2 + \mathcal{O}(1/n^2) = \mathcal{O}(1/n)$, which resembles the usual $1/n$ convergence rate.

As $v_n = v_0 + n$ tends to infinity the distribution of the posterior mean tends to a normal distribution with parameters behaving (asymptotically) similar to the maximum likelihood estimators.
In this sense, informative Bayesian priors can be appropriate if the data contains insufficient information \emph{and} we have reasonable knowledge a priori, where we use the prior to stabilize the results.
But they are also reasonable if we consider large samples, where the prior is simply dominated by the likelihood.
We refrain from an analogous analysis for $\sigma^2$ here and only note that similar results hold, as can be seen from table \ref{tab:comp_conjugate_bay_ml}.

Above we considered a simple model, as this allowed us to derive the results analytically.
Closed form solutions allow us to fully investigate the influence of the prior on our results.
However, we have also seen that Bayesian analyses are far from trivial and depend critically on the complexities of the model structure.
If we want to consider more realistic models we have to make ever more restrictive assumptions to yield analytical results.
For this reason among others, in the next section we discuss methods which trade off the clarity of an analytical result for the generality of being able to combine near arbitrary priors with complex, possibly high-dimensional likelihoods.

\subsection{Sampling From The Posterior}
In this section we consider approaches that allow us to characterize the posterior distribution in complex settings using sampling methods.

For the rest of this section let us assume we observed data $\mathcal{D}$ for which we have a (possibly algorithmic) model in mind, which can be represented by the likelihood $p(\mathcal{D} \mid \theta)$.
We also assume that a prior distribution $p(\theta)$ has been constructed, so that the posterior is again given by $p(\theta \mid \mathcal{D}) \propto p(\mathcal{D} \mid \theta) p(\theta)$.
Unlike before however, we now consider more general settings in which we do not restrict $p(\theta \mid \mathcal{D})$ to be available in analytical form.
This may occur in many settings, for example when using priors that do not mix well with the likelihood or more apparent when using computational models which produce likelihood evaluations based on algorithms.

To motivate the following, say we are able to draw independent samples $\theta^{(1)}, \mydots, \theta^{(n)}$ from $p(\theta \mid \mathcal{D})$.
By the law of large numbers we get $1 / n \sum_i h\left(\theta^{(i)}\right) \overset{a.s.}{\longrightarrow} \Exp{h(\theta) \mid \mathcal{D}}$, under some regularity conditions on $h$ and $p(\theta \mid \mathcal{D})$, with similar results holding for sample quantiles.
Hence, to learn something about $p(\theta \mid \mathcal{D})$ we can formulate questions using quantiles or general expectations and rely on the statement above.

In the subsequent paragraphs we discuss methods to sample from the posterior that work under the general assumption that we can evaluate the posterior at arbitrary points up to an integration constant.
We will see that these methods do \emph{not} produce independent samples but instead create \emph{Markov chains} whose realizations can be seen as autocorrelated samples.

With this in mind, we first consider what properties these chains must fulfill in order to create equivalent results as motivated above for independent samples.
We end this section by introducing an algorithm that accomplishes the above.

\paragraph{Markov Chain Monte Carlo.}
Say we are able to construct a Markov chain with unique invariant distribution equal to the posterior distribution we want to sample from.
Assume also that the distribution of the chain at time $n$ converges to this invariant distribution no matter where we initialize the chain.
Then, in principle, we could run the chain \emph{long enough} until it converged to the invariant distribution and then consider all subsequent realizations as draws from the stationary distribution.
This is the core idea of Markov chain Monte Carlo (MCMC).
In practice, however, we do not know when a chain is run \emph{long enough}.
In part 3 we present some measures that can be of help with this problem during the application.

Under some regularity conditions, similar but not as strict as in Theorem \ref{thm:mcclt}, we get a law of large numbers for such Markov chains (see e.g. \citet{roberts2004}, Fact 5).
This tells us that if we run the chain forever, our average will eventually converge to the number we seek.
However, forever is usually too long.
That is why we focus on assumptions which admit a central limit theorem with the usual $\sqrt{n}$ convergence rate, as it allows for more rigorous statements about our confidence in the whereabouts of the estimator for large samples.

\begin{remark}
  \emph{(i)} Having a central limit theorem in the background does \emph{not} imply that the asymptotic distribution provides a good approximation for finite samples.
  We still do not know when the asymptotics `kicks in' (\citet{Jackman2009}).
  But under assumptions that allow for a CLT we can be more confident in our results than under assumption that only allow of a LLN.
 \emph{(ii)} As is often the case, there are many different sets of assumptions that allow for a CLT.
  The following theorem presents a particular set of assumptions which will be seen to have favorable properties when also considering the creational process of the Markov chain.
  We remark that we will \emph{not} formally introduce all concepts and will provide only a heuristic explaination.
  This is due to the fact that Markov chain theory on general state spaces requires a good understanding of measure theory, which we do not want to assume as a prerequisite.
  We refer to \citet{roberts2004} for a survey on recent advances with application to MCMC and \citet{meynandtweedie09} for a comprehensive treatment of Markov chain theory.
\end{remark}

\begin{theorem}{(A Central Limit Theorem for Markov Chains).}\label{thm:mcclt}
  Let $\{X_n \}$ be a (discrete time) Markov chain and $\pi$ a probability distribution on the same space.
  Consider some measurable function $h$ with $\Expwrt{\pi}{h^2} < \infty$.
  Define $\sigma^2(h) := \varwrt{\pi}{h}\tau := \varwrt{\pi}{h} \sum_{k \in \mathrm{Z}} \text{Corr}\left(h(X_0), h(X_k)\right)$.
  Assume the Markov chain is $\phi$-irreducible, aperiodic, reversible with respect to $\pi$ and that $\sigma^2(h) < \infty$. Then $\pi$ is stationary for the chain and\footnote{In the original paper by \citet{roberts2004} the statement of this theorem differs in that they write $\tau = \sum_{k \in \mathrm{Z}} \text{Corr}\left(X_0, X_k\right)$.
  We believe that this is an error as \citet{haggstrom2007} state in their comparison of different ways of writing the asymptotic variance that $\sigma^2(h) = \sum_{k \in \mathrm{Z}} \Cov{h(X_0), h(X_k)}$.
  Now if we use that $X_0 \sim \pi$ we get $\sigma^2(h) = \sum_{k \in \mathrm{Z}} \Cov{h(X_0), h(X_k)}= \var{h(X_0)} + \sum_{k \neq 0} \Cov{h(X_0), h(X_k)} = \varwrt{\pi}{h}(1 + \sum_{k \neq 0} \Cov{h(X_0), h(X_k)}) / \var{h(X_0)} =\varwrt{\pi}{h}(1 + \sum_{k \neq 0} \text{Corr}{(h(X_0), h(X_k))}) = \varwrt{\pi}{h} \sum_{k\in\mathrm{Z}} \text{Corr}{(h(X_0), h(X_k))}$.}
  \begin{align}
    \sqrt{n}\left(\frac{1}{n}\sum_{i=1}^n h(X_i) - \Expwrt{\pi}{h} \right) \overset{d}{\longrightarrow} \normal{0, \sigma^2(h)} \,.
  \end{align}
\end{theorem}
\begin{proof}
  See \citet{kipnis1986} for a complete proof of the second claim; see \citet{roberts2004} Proposition 1 for the first claim and Theorem 27 for the second.
\end{proof}

\noindent
We end this paragraph by discussing the assumptions of Theorem \ref{thm:mcclt} on an intuitive level.

\textbf{$\phi$-irreducibility} assumes that we can find a measure $\phi$ such that no matter where the chain starts, we eventually reach every region of the state space which has positive measure with respect to $\phi$.
In the next paragraph we will see that this condition can be satisfied by construction of the chain.

\textbf{Aperiodicity} assumes that we cannot find disjoint regions on which the chain jumps from one region to another in a cyclical predictable fashion.
It seems intuitive that such a behavior will prevent the chain from actually converging to its stationary distribution.

\textbf{Reversibility with respect to $\bm{\pi}$} is a technical assumption which is best explained by its implications.
In particular, it implies that the Markov chain has $\pi$ as its stationary distribution (which is unique by the other assumptions).
Again, in the next paragraph we will see that this condition can be satisfied by construction with $\pi$ being the posterior distribution from which we want to sample.

\textbf{Finite variance} ($\sigma^2(h) < \infty$) implies that the integrated correlation time $\tau$ must be finite, as we assumes square integrability of $h$.
The integrated correlation time is finite if the correlation function decreases fast enough to zero.
Heuristically speaking, for a CLT to work we need more information as would be available in a sample for which the integrated autocorrelation time is infinite.
If it is finite we get the usual large sample variance approximation $\sigma^2(h) / n = \varwrt{\pi}{h} / (n/\tau)$.
In this sense we might say that $n/\tau$ denotes the \emph{effective sample size}, which corrects for the fact that we are not drawing independent samples and therefore (in most cases) need more samples to yield the same amount of information as in the independent case.

\paragraph{Metropolis-Hastings Algorithm.}
Here we consider one method which implicitly defines a Markov chain with the desired properties, the Metropolis-Hastings algorithm (\citet{Metropolis1953}, \citet{hastings70}).
For other approaches and more involved algorithms see for example \citet{liang10}.

\begin{algorithm}
\caption{Metropolis-Hastings}\label{alg:metropolis-hastings}
\begin{algorithmic}[1]
  \Require $(\pi, q, T) =$ (target density, proposal density, number of samples to draw)
\State initialize $x_0$ with an arbitrary point from the support of $q$
\For{$t = 0,\mydots,T-1$}
  \State sample a candidate: $y \sim q(\cdot \mid x_t)$
  \State compute the acceptance probability: $\mathcal{A} \gets \min\left\{ \ddfrac{\pi(y)}{\pi(x_t)} \ddfrac{q(y \mid x_t)}{q(x_t \mid y)}, 1\right\}$
  \State update the chain: $x_{t+1} \gets \begin{cases} y &\mbox{,with probability } \mathcal{A}\\ x_t &\mbox{,with remaining probability} \end{cases}$
\EndFor{}
\State \textbf{return} $\{x_t : t = 1,\mydots,T\}$
\end{algorithmic}
\end{algorithm}

\noindent
Algorithm \ref{alg:metropolis-hastings} displays the Metropolis-Hastings algorithm.
The main idea is that we start with an initial value and iteratively propose new values of the chain, however, we do not accept every proposal, but we do so only with a certain probability ---this probability can be thought of as high if the proposal has a high relative density compared to the last chain link.
Here we also see why we do not care about any integration constant, as the target density only appears as a ratio (line 4).

Clearly the results depend on the choice of the proposal density.
A common pick are so called \emph{random walk} proposals, which add some random number to the current position of the chain; for example a gaussian random walk proposal is given by $q(\cdot \mid x_t) = \normal{\cdot \mid x_t, \sigma^2}$ or equivalently stated $y = x_t + \normal{0, \sigma^2}$.
See \citet{sherlock2010} for a recent survey on random walk proposals.

The simplicity of the algorithm is remarkable, but the main question is if the resulting Markov chain inherits favorable properties.
And indeed this is the case.
By construction the algorithm creates Markov chains which are reversible with respect to $\pi$ and aperiodic, and if additionally the proposal density is positive and continuous and $\pi$ is finite then the chain is $\pi$-irreducible; see for example \citet{roberts2004}.
This tells us that under these regularity conditions a CLT holds for chains created using the Metropolis-Hastings algorithm.
To avoid using samples that do not come from the stationary distribution, in practice we choose $T$ very large and discard the first few (\emph{burn-in}) samples.

The ability to sample draws in complex settings using the Metropolis-Hastings algorithm (and other Markov chain Monte Carlo methods for that matter) made Bayesian statistics applicable for real problems.
Still, \citet{Au2001EstimationOS} show that the classical Metropolis-Hastings algorithm is highly dependent on the proposal density and fails in higher dimensions; \citet{zuev08} provide a geometric intution.
Numerous novel methods which deal with dimensionality problems have been published.
A particular promising route seems to be \emph{Hamiltonian Monte Carlo} (\citet{duane87}), which is one of the algorithms used in the probabilistc programming language STAN (\citet{standev2018stancore}) with ongoing research on theoretical properties, see for example \citet{betancourt2014geometric}.

\paragraph{Volume in Higher Dimensions.}
Classical MCMC methods can have too slow convergence rates; In higher dimensions this might be due to probability mass being distributed very far from where it is expected (\citet{betancourt2017convergence}).
In this paragraph we motivate this phenomenon and in the following we present methods which utilize it.

Let $B_d$ denote the unit ball in $\reals^d$ and define $C_d$ as the smallest cube containing $B_d$.
We consider two questions.
First, how does the ratio $\nicefrac{\vol{B_d}}{\vol{C_d}}$ change as $d$ increases.
And second, how does the ratio of probability mass distributed by a standard gaussian on these regions change as $d$ increases.
Since closed form expressions of volumina of geometrical objects exist the first questions needs little work.
Similarly we can easily compute $\prob{X \in C_d} = \left[\Phi(1) - \Phi(-1)\right]^d$, where $\Phi$ denotes the one-dimensional gaussian cumulative distribution function.
However, to compute $\prob{X \in B_d}$ we need to integrate over the unit ball with respect to a gaussian distribution, which is non-trivial.
For this reason we decide to report an upper bound, as this is sufficient for our motivation.
In particular we compute $\overline{\prob{X \in B_d}} := \sup_{\bm{x} \in B_d} \phi(\bm{x}) \vol{B_d} = \sup_{\bm{x} \in B_d} \phi(\bm{x}) \int_{B_d}1 \mathrm{d}\bm{x} \geqslant \int_{B_d} \phi(\bm{x}) \mathrm{d}\bm{x} = \prob{X \in B_d}$.
The results of these computations are depicted in table \ref{tab:vol_high_dim}.
We note that both ratios tend to zero very fast as $d$ increases.
With this phenomenon in mind one has to be cautious when working in high-dimensional spaces, since the regions of interest, that is the regions containing non-negligible probability mass, might not be located where our low-dimensional intution says.
This idea is formalized by the \emph{Gaussian Annulus Theorem} (\citet{blum2017foundations}; theorem 2.9) which states, inter alia, that most probability mass lies within an annulus centered at the origin with an average distance to the origin of $\sqrt{d}$.

\begin{table}[ht]
\def\arraystretch{1.3}
\centering
 \begin{tabular}{c | c c c c c c c}
 $d$ & 1 & 2 & 3 & 5 & 7 & 10 & 15\\
 \hline
 ${\vol{B_d}} / {\vol{C_d}}$ & 1.00000 & 0.78540 & 0.52360 & 0.16449 & 0.03691 & 0.00249 & 0.00001\\
 $\overline{\prob{X \in B_d}} / {\prob{X \in C_d}}$ & 1.16874 & 1.07281 & 0.83589 & 0.35870 & 0.10995 & 0.01184 & 0.00012
 \end{tabular}
 \caption{Comparison of volume ratio of unit ball and cube, and probability ratio of gaussian falling in unit ball and cube for varying dimension $d$. Numbers are rounded to five decimal places.}
\label{tab:vol_high_dim}
\end{table}

We have seen some unintuitive behavior in higher dimensions which might explain why regular methods do not work or only work very slowly.
The next paragraph presents one method which utilizes this behavior to efficiently produce samples.

\paragraph{Hamiltonian Monte Carlo.}
We conclude our digression on Bayesian thinking by presenting \emph{Hamiltonian Monte Carlo} (HMC), an innovative Markov chain Monte Carlo method from the statistical physics literature which works in higher dimensions  \citet{duane87}.
There are of course multiple MCMC algorithms which work in higher dimensions with many more being actively developed.
Here we focus on HMC as it is the main algorithm used in the probabilistic programming language STAN (\citet{standev2018stancore}), which we will be using in our Monte Carlo study and application part.

We note that it is impossible to provide a rigorous introduction to HMC here, which is why we will focus on the general intution and refer to a series of papers by Michael Betancourt and several coauthors on, the geometric foundations of HMC (\citet{betancourt2014geometric}); geometric ergodicity of HMC (\citet{livingstone2016geometric}) and HMC for hiearchical models (\citet{betancourt2013hamiltonian}).\footnote{Besides doing theoretical research on HMC, Michael Betancourt worked for STAN on integrating the HMC algorithm and runs an educational blog where he presents his research using modern tools, see \url{https://betanalpha.github.io/}.}

One reason why ordinary MCMC methods might converge only very slowly in higher dimensions is that the proposal distribution used in the Metropolis-Hastings algorithm does not properly capture the geometry of the high-dimensional space which leads to many rejected proposals and therefore an inefficient exploration of the parameter space.
The main idea of HMC is to extend the parameter space by constructing a specific vector field on it, which moves the chain from one proposed point to another in a way so that we consider points that lie in regions with high probability mass and we regulary jump to faw away points as to explore the space as quickly as possible.
But how do we construct this vector field?
Note that when considering differentiable posterior densities the gradient defines a vector field.
However, this vector field points to the modes of the posterior and as we saw in the last paragraph, in higher dimensions we will find little to no probability mass near the modes.
This is where Hamiltonian mechanics comes into play by providing a set of equations (Hamilton's equations) that describe the time-evolution of the interplay of kinetic and potential energy of a system.
What this means for our case is best explained by imagining the mode as the center of gravity, with gravity pulling harder as we get closer to the mode ---this can be thought of as the gradient vector field.
But as most probability mass is spread around an annulus around the mode we do not want to move closer to the center of gravity, we want to move around an orbit around the mode.
The distance of the orbit to the mode and exact shape depend of course on the dimensionality and posterior distribution of the problem.
Hamilton's equations provide us with a way to construct a vector field so that moving along the field drifts the Markov chain into this orbit.
Once reached the chain explores the relevant space quickly.

Why is the above important?
Bayesian statistics and its application to real world problems has gained immense popularity with the invention of Markov chain Monte Carlo methods.
It's naive application to general complex high-dimensional problems is not computationally feasible however.
These problems are under active development and new approaches on the algorithmic and theoretical side, as the one presented above, prove fruitfull.


\chapter{Mathematics}
Digital signal processing is in general a very mathematical subject and audio- and musical signal processing is no exception to that rule. So it shouldn't come as a surprise, that many of the RAPT library functions rely on underlying algorithms that implement a considerable mathematical machinery. This chapter discusses some of the mathematical concepts that are relevant in the context of audio processing and shows how they are implemented in RAPT. The purpose here is mostly to explain, \emph{what} the respective classes and functions do, not \emph{how} they work internally. Most of the algorithms discussed in this chapter fall under the broad umbrella of numerical analysis.
Detailed explanations and derivations can be found in the vast literature, specifically [Reference: Numerical Recipies] is a first class reference for such things.

\section{Complex Numbers}
For representing complex numbers, RAPT uses \texttt{std::complex} from the C++ standard library.

\section{Functions}
For representing functions, RAPT uses \texttt{std::function} from the C++ standard library ....maybe...this has to be decided

\section{Sequences}
Mathematically, a sequence is an ordered set of elements of some given type.

\section{Polynomials}
A polynomial of a variable $x$ is a weighted sum of integer powers of $x$, starting at $x^0 = 1$ and going up to $x^N$. Polynomials play a central role in the theory of analog and digital filters and they occur in the context of interpolation, curve fitting and function approximation - that's reason enough for RAPT to devote them a dedicated class - which is unsurprisingly called \texttt{Polynomial}. Mathematically, a polynomial $p(x)$ of order $N$ is defined as:
\begin{equation}
	p(x) = \sum_{n=0}^N a_n x^n
\end{equation}
where the $a_n, n=0,\ldots,N$ are called the coefficients. An $N$th order polynomial has $N+1$ coefficients, ranging from index $0$ to index $N$. This is also the convention in which polynomial coefficient arrays are represented in RAPT - as length \texttt{N+1} arrays \texttt{a[0]}...\texttt{a[N]}.  Consider the following code for multiplying the 2nd order polynomial $p(x) = 2 - 3 x + x^2$ by the 1st order polynomial $q(x) = -3 + 4x$ and evaluating the resulting polynomial $r(x)$ at $x = 2$:
\begin{lstlisting}
Polynomial<int> p(2, 2, -3, 1), q(1, -3, 4);  // allocation, initialization
Polynomial<int> r = p * q;                    // multiplication
int y = r(2);                                 // evaluation
\end{lstlisting}
This code uses the high-level interface of the RAPT library. The first line creates the two polynomials (with integer coefficients). The first parameter of the constructor of the \texttt{Polynomial} class is the order of the polynomial followed by an appropriate number of coefficients. The 2nd line multiplies the two polynomials which results in another polynomial which is stored in the variable \texttt{r}. The third line produces a numeric output by applying the product polynomial $r$ to the value $x=2$. The code looks reasonably straightforward and readable by C/C++ standards. However, if you have to do polynomial multiplication in a context where performance is critical, for example inside a realtime audio callback, you probably don't want to have any dynamic memory allocations. But that's what the constructors of the \texttt{Polynomial} class do. For performance and predictability reasons, you may want to operate on pre-allocated arrays, even if that means to uglify your code. In this case, your code could look something like this:
\begin{lstlisting}
// pre-allocation and initialization:
int p[3] = { 2, -3, 1};                  // 2nd order, 3 coeffs,  2 - 3x + x^2
int q[2] = {-3,  4};                     // 1st order, 2 coeffs, -3 + 4x
int r[4];                                // 3rd order, 4 coeffs

// multiplication and evaluation:
Polynomial<int>::mul(&p, 2, &q, 1, &r);  // multiply p, q, store result in r
int y = Polynomial<int>::eval(&r, 3, 2); // evaluate polynomial r at 2
\end{lstlisting}
First, we allocate the three arrays \texttt{p, q, r} of lengths \texttt{3, 2, 4} respectively. The order of the result polynomial \texttt{r} is given by the sum of the orders of the factors $2+1=3$, so we need space for $4=3+1$ coefficients for the \texttt{r} polynomial. The \texttt{p, q} arrays are also initialized in this process. Then we call static member functions \texttt{mul} and \texttt{eval} of the \texttt{Polynomial} class that operate on pre-allocated arrays of polynomial coefficients to perform the multiplication and evaluation. Arguably, the 2nd version is much less readable but these static member functions can be used whenever it is undesirable to have any dynamic memory allocations. Internally, the operators \texttt{*} and \texttt{()} of the \texttt{Polynomial} class call the low-level routines on internally managed coefficient array member variables. This idiom of static member functions that are internally called from the non-static functions and operators on the instance variables of an object will be used throughout the library. It allows the low- and high-level interface to share the same code.

\section{Root Finding}
Root finding is the problem of finding one or more solutions of the equation:
\begin{equation}
	f(x) = 0
\end{equation}
where $f(x)$ is, in general, an arbitrary function. Any value of $x$ that turns the equation into a true statement, when substituted for $x$, is called a solution or 'root' of the equation. In this problem setting, it is assumed that we have a means to evaluate the function at arbitrary values of $x$. In RAPT, the root finding algorithms must get such a means by passing them either function pointer to the respective function or a function object of class \texttt{Functor}. Functors are objects that provide a means to evaluate a function at some point $x$. What makes them more flexible than plain functions is the fact that they can have internal instance variables which may affect the function evaluation. They can be seen as a general way to define families of functions with an arbitrary set of parameters. A polynomial is an example for such a functor where the parameters are the coefficients.

\subsection{Bisection}
Suppose that, for two particular values $x=a$ and $x=b$, we know that $f(a)<0$ and $f(b)>0$. By assuming that the function is continuous between $a$ and $b$, that means that a root, i.e. a zero crossing, must be present somewhere between $a$ and $b$. The same is true, if $f(a)>0$ and $f(b)<0$. We say, the root is bracketed by the values $a$ and $b$. We may now evaluate the function at the midpoint between $a$ and $b$. If the value is equal to $0$, we have found our root, if it's less than $0$ or greater than $0$, we have found a new bracketing interval either to the left or to the right of the midpoint. The new bracketing interval is only half as long as the old one. Iterating that procedure, the brackets move closer and closer to the root. When their difference reaches a threshold, typically related to the numeric precision of the floating point datatype, we stop the iteration. It is then converged to the root (up to an error given by our threshold). Since the interval halves at each iteration, the number of correct binary digits in our approximate solution (given by the current midpoint) increases by one. Because the number of correct digits is a linear function of the iteration number, we say that the algorithm has a linear order of convergence. In RAPT, root finding by bisection is implemented in \texttt{RootFinder::bisection}. As a subtle sidenote, it may make sense to choose the new evaluation point not exactly at the midpoint of the bracketing interval, but at [... tbc. Reference NR].

\subsection{Newton Iteration}
Newton iteration is a method to iteratively improve an initial guess $x_0$ for the solution by iteratively applying the update rule:
\begin{equation}
	x_{n+1} = x_n - \frac{f(x_n)}{f'(x_n)}
\end{equation}
This formula results from approximating the function $f(x)$ as a straight line that goes through the point $(x_0, y_0=f(x_0))$. This straight line is constructed to match the slope of the function $f(x_0)$ given by the derivative $f'(x_0)$ in the point $(x_0, y_0)$. The resulting line equation is then solved for the zero crossing of the line, which becomes our new root estimate $x_{n+1}$. Thus, to actually implement it, we also need a means to evaluate the derivative of $f(x)$ for any $x$ in addition to being able to evaluate the function $f(x)$ itself. The convergence of this iteration is typically quadratic, which means that in each iteration, the number of correct digits doubles [verify this] but in certain pathological cases it may be slower and in some cases, the iteration may even diverge. If a reasonable initial guess for the solution is available that is close to the true solution and the function is well behaved near the solution and we know how evaluate the derivative, Newton iteration is one of the methods of choice to converge to a true solution. In RAPT, Newton iteration is available in the function \texttt{RootFinder::newton} which takes two function pointers or functors - one for the function itself and one for the derivative and also takes an initial guess and returns the refined value.

\subsection{Secant Method}
The secant method works similar to the bisection method, just that we choose the new evaluation point not as midpoint of our current bracketing interval $[a, b]$, but instead, we fit a straight line to the points $(a, f(a)), (b, f(b))$ and solve for the root of this straight line. The order of convergence is given by the golden ratio: $\alpha = (1+\sqrt{5})/2 \approx 1.618$. So, it converges faster than the bisection method but not quite as fast as the Newton iteration. It can be seen as an approximation to Newton iteration, where the derivative is replaced by a finite difference approximation. On the plus side, it does not need to evaluate the derivative, which might be expensive or sometimes even not feasible. The method is available in \texttt{RootFinder::secant}. Note that it may also diverge, since the new evaluation point may fall outside the initial bracketing interval.

\subsection{False Position}
Also known as 'regula falsi', this method is similar to the secant method, but it applies...

%\subsection{Fixpoint Iteration}
%\subsection{Brent's Method}
%\subsection{Ridders' Method}

\subsection{Roots of Polynomials}
If the function $f(x)$ is a polynomial, there are specialized algorithms for finding its roots. One such algorithm is the Laguerre algorithm which is implemented in \texttt{Polynomial::roots(T *a, int N, complex<T> *r)}. It takes an array of \texttt{N+1} coefficients as parameter and returns the roots in the length \texttt{N} array \texttt{r}. The fundamental theorem of algebra tells us that each polynomial of order $N$ has exactly $N$ roots such that it can be expressed as:
\begin{equation}
  \label{Eq:PolynomialProductForm}
	p(x) = k \prod_{n=1}^N (x - r_n)
\end{equation}
that is, a scaled product of the differences between $x$ and the respective root $r_n$. Remember that a product is zero, if and only if (at least) one of its factors is zero, so if $x$ is equal to one of the roots $r_n$, the corresponding factor $x - r_n$ in the product is zero, making the whole product zero - which verifies that $r_n$ is a root, indeed. These roots, however, may not be unique - some of them may have a multiplicity, i.e. occur more than once in the above product. Some of the roots may also be complex. If the coefficients of the polynomial are real, complex roots will always come in pairs of complex conjugate numbers. In audio applications, we will mostly deal will polynomials that have real coefficients. For generality, however, there's also a version of the polynomial root finder that takes an array of complex coefficients. Internally, the same algorithm is used anyway, no matter if the coefficients are real or complex. For example, the roots of the $5$th order polynomial $p(x) = 1 + 2 x - 3 x^2 + 3 x^4 - x^5$ can be found via:
\begin{lstlisting}
vector<complex<double>> r(5);
r = Polynomial<double>(5, 1., 2. -3., 0., 3., -1.).roots();
\end{lstlisting}
or, alternatively using the low-level routine that operates on pre-allocated arrays:
\begin{lstlisting}
double a[6] = { 1., 2. -3., 0., 3., -1. };
complex<double> r[5];
Polynomial<double>::roots(a, 5, &r);
\end{lstlisting}
A function for getting from the roots back to the coefficients is also available via the function \texttt{Polynomial::fromRoots}. In addition to the $N$ roots, it also needs the value of the overall scaling constant $k$ in Eq. \ref{Eq:PolynomialProductForm}. This is simply the $N$th coefficient, but in case you forget that, the \texttt{Polynomial} class has also the function \texttt{productFormScaler(int N, T a)} which trivially just returns \texttt{a[N]}. Since the \texttt{Polynomial} class is a subclass of class \texttt{Functor}, the general root finders from above can be applied for polynomial root finding as well. They can, however, find only one root at a time and you need an initial guess or bracketing interval.
% reconstructing the coefficients form the roots

\section{Rational Functions}
A rational function is a quotient or 'ratio' of two polynomials:
\begin{equation}
	r(x) = \frac{\sum_{m=0}^M b_m x^m} {\sum_{n=0}^N a_n x^n}
\end{equation}
where $M$ is the order of the numerator and $N$ is the order of the denominator. All realizable analog and digital filter transfer functions are functions of this very type - which explains the special significance of rational functions, and hence polynomials, in the field of signal processing. The roots of the numerator are called the zeros of the transfer function and the roots of the denominator are called the poles. Since the denominator is zero at these poles, the value of the rational function at these points goes to infinity, unless there's a corresponding zero in the numerator at exactly the same point to cancel the pole. RAPT provides the class \texttt{RationalFunction} to deal with rational functions.

\subsection{Partial Fraction Decomposition}
Any rational function can be expanded into the form of a sum of a polynomial and several first order rational functions, in which the numerator is a  constant (called the residue which is possibly complex) and the denominator is a linear function of $x$. That is:
\begin{equation}
	r(x) = \frac{\sum_{m=0}^M b_m x^m} {\sum_{n=0}^N a_n x^n}
	     = \sum ...
\end{equation}
If the coefficients of the original numerator and denominator polynomial are both real, complex values for the numerators and denominators in the expanded form come in conjugate pairs, such that 1st order terms can be combined pairwise into 2nd order terms with real coefficients. In signal processing applications, such decompositions amount to decomposing a direct form filter into a parallel connection of the respective 1st or 2nd order filters and an FIR part, where the latter results from the polynomial part. Because the individual terms are easier to understand and analyze, partial fraction decompositions play a role in certain filter analysis applications. For example, they are required for finding closed form expressions of a filter's impulse response. This impulse response is given by the inverse $z$-transform of its transfer function, but inverse $z$-transforms are only available for low-order terms - which is precisely what the decomposition delivers. Partial fraction decomposition is available in RAPT via \texttt{RationalFunction::partialFractions}.

\section{Vectors}
\section{Matrices}

\section{Transforms}
Transforms are - together with filters - one of the mainstays in the field of signal processing. Consider an $N$ dimensional vector $\mathbf{x}$ which may represent a (segment of a) signal. A general transform would take that vector as input and produce a new vector $\mathbf{y}$ as output. If the transform is designed well, the transformed vector $\mathbf{y}$ may reveal relevant features of the signal that are hard to see in the original representation. An important concern is invertibility - we want to be able to reconstruct the original vector $\mathbf{x}$ from the transformed vector $\mathbf{y}$ by some kind of associated inverse transform. Given such a pair of a transform and its inverse, we could transform an input vector, modify the transformed representation and apply the inverse transform to get a modified output vector/signal. A good transform in an audio processing setting would be one, that allows for convenient modifications that closely relate to modifications of perceptual, auditory attributes of the signal. We will consider mainly linear transforms. These are the class of operations on our input vector $\mathbf{x}$ that can be expressed as a matrix-vector multiplication of an  $N \times N$ matrix $\mathbf{A}$ with our input vector $\mathbf{x}$:
\begin{equation}
	\mathbf{y} = \mathbf{A} \mathbf{x}
\end{equation}
If the matrix $\mathbf{A}$ is non-singular, there will be an inverse matrix $\mathbf{A}^{-1}$ and hence, there will be an inverse transform, given by:
\begin{equation}
	\mathbf{x} = \mathbf{A}^{-1} \mathbf{y}
\end{equation}
Alternatively to thinking in terms of an $N$ dimensional vector $\mathbf{x}$, we may also consider the inputs and outputs as sequences of length $N$, which, in this context, can be seen as merely a different name for the same thing - a finite, ordered set of $N$ numbers.

\subsection{Discrete Fourier Transform}
The discrete Fourier transform (DFT) of a length $N$ sequence $x_0, \ldots, x_{N-1}$ is another length $N$ sequence $X_0, \ldots, X_{N-1}$ in which each element $X_k$ can be computed by the formula:
\begin{equation}
	X_k = \sum_{n=0}^{N-1} x_n W_N^{k n}, \qquad \text{where }W_N = e^{-j \frac{2 \pi}{N} }
\end{equation}
expressed as matrix-vector product, this looks like:
\begin{equation}
	...
\end{equation}
Computing a product of an $N \times N$ matrix and an $N$ vector in general requires $\mathcal{O}(N^2)$ operations, because we have to form $N^2$ individual products between numbers. In the DFT matrix, however, not all elements are distinct. This redundancy of the matrix elements allows for an algorithmic shortcut, known as the fast Fourier transform (FFT) which requires only $\mathcal{O}(N \cdot \log N)$ operations. Fast Fourier transforms are particularly efficient when $N$ is a power of two - these transforms are known as radix-2 FFTs. There are also $\mathcal{O}(N \cdot \log N)$ algorithms for arbitrary $N$, for example, the Bluestein FFT algorithm which uses a radix-2 algorithm on a zero-padded sequence internally. In RAPT, both radix-2 and Bluestein FFTs are available via the functions \texttt{Transforms::fftRadix2} and \texttt{Transforms::fftBluestein}. There's also the function \texttt{Transforms::fft} which tests, if $N$ is a power of two and then dispatches between the radix-2 and the Bluestein algorithm accordingly.

\subsubsection{Inverse Transform}

\subsection{Short Time Fourier Transforms}

\subsection{Wavelet Transforms}

\section{Statistics}
Statistical signal measures such as the mean value, the autocorrelation sequence or the crosscorrelation sequence (between two signals) play an important role in many signal analysis algorithms and in adaptive filters. ...

\section{Interpolation}
Assume that we have a certain number $N$ of pairs of abscissa/ordinate values $(x_n, y_n), n = 0, \ldots, N-1$ represented as length \texttt{N} arrays \texttt{x} and \texttt{y}. We imagine these data points as a table that represents a continuous function and we assume that the $x$-values are in strictly ascending order. The goal of interpolation is to create a continuous function from the discrete data. That means, interpolation would connect the points by assigning a $y$-value to \emph{any} $x$-value between \texttt{x[0]} and \texttt{x[N-1]}. If we try to find a $y$-value for an $x$-value outside this range, it's called extrapolation. In RAPT, a continuous interpolating (and extrapolating) function is represented as an \texttt{Interpolator} object which itself is subclassed from \texttt{Functor}. This implies that the root-finding algorithms can be applied to interpolating functions in the same way as they are applied to regular functions. In the context of audio processing, interpolation may be required for obtaining signal values between the actual sample values or for creating smooth curves of control-data for which values are available only at certain time instants but not at each sample instant.

\subsection{Linear Interpolation}
A very simple way of interpolating between datapoints is to connect the datapoints with straight lines.
%It's not suitable for high-quality audio interpolation (unless, maybe, in combination with oversampling) but it has its place for interpolating control data. A feature that is sometimes desirable is that is preserves the mean value - i.e. the mean value of the interpolated \texttt{y}-array equals the mean of the original \texttt{y}-array [verify]
%The only form of interpolation that is even more crude would be piecewise constant interpolation which just returns the $y$-value of the nearest neighbour.

\subsection{Lagrange Interpolation}
Lagrange interpolation takes a given number $N$ of datapoints and constructs an $N$th order polynomial that passes exactly through these points. Such an interpolating polynomial tends to oscillate wildly between the actual datapoints, especially if the order of the polynomial is high. In most application, such oscillations are undesirable and that's why Lagrange interpolation is mostly used only with low orders. In certain audio applications, however, these oscillations might actually be exactly what is needed - as the order of the Lagrange interpolator goes up, it approaches the optimal sinc-interpolator [Reference: 'Elephant interpolators']

\subsection{Spline Interpolation}

\subsection{Sinc Interpolation}

\section{Numerical Differentiation}

\section{Numerical Integration}

\section{Differential Equations}
Differential equations are the main tool for describing physical systems mathematically. They relate an unknown function $f(x)$, which is to be found, to its own derivatives $f'(x), f''(x), \ldots$. If the function $f$ is a function of only one independent variable, we have an ordinary differential equation. There are also so called partial differential equations which are differential equations for a function of several independent variables, say $f(x,y,\ldots)$, and an equation in terms of the partial derivatives $\partial f / \partial x, \partial f / \partial y, \ldots$ is given. Sometimes, such differential equations can be solved analytically in the sense that an explicit expression for $f(x)$ in terms of $x$ can be given. Often, however, such an analytic solution is not feasible and we must resort to numerical algorithms to find the behavior of the system in a particular situation. In the context of audio processing, ordinary differential equations are typically used to describe electronic circuits such as analog filters, dynamics processors, etc.. Partial differential equations occur in the modeling of acoustic systems like vibrating strings, plates, air columns, etc..

\subsection{Ordinary Differential Equations}

\subsection{Partial Differential Equations}

\section{Curve Fitting}

\section{Optimization}


\section*{Preliminaries}
\begin{itemize}
  \item [natural number:] $\naturalNumber = 0, 1, 2, 3, \dotsc, \infty$
    \begin{itemize}
    \item $\naturalNumber_0 = 0, 1, 2, 3, \dotsc, \infty$
    \item $\naturalNumber_1 = 1, 2, 3, \dotsc, \infty$
    \end{itemize}

  \item [integer number:] $\integerNumber = -\infty, \dotsc, -2, -1,
    \naturalNumber$
  \item [real number:] $\realNumber = (-\infty, \infty) $
  \item [imaginary number:] $i = \sqrt{-1}$
  \item [complex number:] $\complexNumber = a + ib = Ae^{i\theta}$
    \begin{itemize}
    \item $a, b \in \realNumber$
    \item $A = \sqrt{a^2 + b^2}$
    \item $\theta = \arctan(b/a)$
    \end{itemize}
\end{itemize}


\chapter{Estimation Theory}\label{App:estimation}
The Bayesian definition requires knowledge of the posterior probability on $\theta$, that is, $pr(\theta|g)$.  The Bayesian regards $\theta$ as random, but has no concept of an ensemble of $g$ vectors; the data are fixed.
Estimates are statistical quantities, hence their evaluation requires statistical methods.
Bayesian estimation is the determination of an estimate of a random $\theta$ through minimization of the Bayes risk.  Knowledge of $pr(\theta)$ is assumed, and a cost function $C(\hat{\theta}, \theta)$ must be specified.   The EMSE (Ensemble mean-squared error) is a common form for the cost measure $C(\hat{\theta}, \theta)$ to be minimized in the design of an estimation procedure.

The quantity $pr(\theta|g)$ describes the posterior probability of $\theta$ after the data $g$ are obtained; thus the estimation rule $\hat{\theta} = \arg\max_{\theta} pr(\theta|g)$ is known as maximum a posteriori, or MAP, estimation.  Equivalently, the MAP estimate is the mode of the posterior, or the posterior mode.

Interestingly, MAP estimation is often what is meant in the literature on Bayesian estimation, However, MAP estimation is just a special case of Bayesian estimation in which the particular form of the cost function is a uniform cost function given by:

\begin{equation}
C(\hat{\theta}, \theta) = C(\hat{\theta}-\theta) = 1 - rect \left( \frac{\hat{\theta}- \theta}{2\epsilon}   \right)
\end{equation}

The uniform cost function is when the cost is considered negligible if smaller than some tolerance $\epsilon$, and all estimator errors beyond that tolerance are regarded as equally costly.
Another way of interpreting MAP estimation is that $pr(\theta)$ characterizes the prior uncertainty in the parameter, which is often subjective in nature.  Then $pr(\theta|g)$ is the (presumably reduced) uncertainty after data are collected, hence the term posterior.

Maximum-likelihood or ML estimation uses the following rule to determine the underlying parameters:
\begin{equation}
\hat{\theta}_{ML} \equiv \arg\max_{\theta} pr(g|\theta)
\end{equation}
This procedure can be written equivalently as:
\begin{equation}
\hat{\theta}_{ML} = \arg \max_{\theta} ln \left[ pr)g|\theta \right].
\end{equation}
ML estimation can be considered as a limit to MAP estimation when the prior $pr(\theta)$ is sufficiently broad that $pr(g|\theta) pr(\theta)$ is dominated by $pr(g|\theta)$.  However, ML estimation is much more than a limiting form of MAP estimation.
In ML estimation, $pr(g|\theta)$ is the quantity being maximized, but even in MAP estimation its role has some level importance that depends on the relative weight of the prior.  The quantity $pr(g|\theta)$ means different things to different people.  The frequentist interpretation is that it is a function of $g$ for fixed $\theta$.  Repeated observations can be used to determine the nature of this quantity for a given underlying object (parameter vector).  Alternatively, $pr(g|\theta)$ can be viewed as a function of $\theta$ for fixed $g$.  In this viewpoint $pr(g|\theta)$ is a measure of the likelihood of any $\theta$ once that data are in hand.
\begin{itemize}
\item frequentist approach: what are all $g$'s that is produced by a single $\theta$?
\item viewing $pr(g|\theta)$ as a function of $\theta$ for fixing g; or what are all $\theta$'s that can produce identical $g$.
\end{itemize}

\noindent
\textbf{Score}: \medskip The \textit{score} is a random vector that tells us how sensitive the likelihood is to change in the parameters:
\begin{equation}
s(g) = \frac{\frac{\partial}{\partial \theta} pr(g|\theta}{pr(g|\theta)} = \frac{\partial}{\partial\theta} In \left[ pr(g|\theta) \right]
\end{equation}

In words, the score is the gradient of the log-likelihood. Since the score is a function of the log-likelihood, which is a random variable through its dependence on $g$, the score is also random.  Note that $\langle s \rangle_{g|\theta} = 0$, where $\langle \cdot \rangle_{g|\theta}$ denotes an average with respect to $pr(g|\theta)$. Score is a zero-mean random vector.
As the gradient of the log-likelihood, $s(g, \theta) = 0$ when $\theta = \hat{\theta}(g)$.  The process of finding the ML estimate is thus equivalent to determining the point in parameter space where all components of the score vanish.

ML estimator is efficient, where if the Cramer-Rao lower bound on the variance is attainable, it will be attained by an ML estimator.
ML estimator is sufficient.  In estimation, a sufficient statistic is one that captures all the essential features in the data necessary for optimal performance of a given estimation task.  The maximum-likelihood estimator is a sufficient statistic for estimation; it makes optimal use of the information in the data.  No other estimator can yield more information.
A necessary and sufficient condition for $\hat{\theta}$ to be a sufficient estimate is that the likelihood function must be factorable into the product:
\begin{equation}
pr(g|\theta) = pr(\hat{\theta}|\theta) f(g),
\end{equation}
where $f(g)$ is independent of $\theta$.
A sufficient estimator may exist even when an efficient estimator does not; efficiency is a stricter criteria than sufficiency.  A sufficient estimator is unique; we can use a function of the sufficient statistic because it will also be sufficient, and we can choose it such that the estimate maybe be consistent and unbiased.
In general there can be many solutions to ML estimation equation that give equal likelihood.  This is especially the case in high-dimensional problems, where null space can be very large.
Asymptotic properties of the ML estimator.  It describes the behavior of the estimate as the number of observations approaches infinity.  Cramer proved that, under reasonably general conditions, an ML estimate is consistent.  The density function for a consistent estimate becomes increasingly narrow about the value of the underlying parameter as the number of samples increase.  ML estimate is efficient, if an efficient estimator exists.  ML estimates achieve the equality sign in the bound as the number of samples goes to infinity; that is, ML estimates are asymptotically efficient.
ML estimates are asymptotically unbiased, efficient, normally distributed and consistent.

Summarize: The full Bayesian estimation approach requires that the cost function as well as the prior probability of the random parameters and their likelihood function be completely specified. The ML approach does not require a prior, but it still requires full knowledge of the likelihood function.

\noindent
\textit{Bayesian and frequentist:}
In both pure estimation problems and hybrid detection/estimation problems, we have found that the optimal strategy for handling nuisance parameters is to marginalize (to treat as insignificant or peripheral) rather than to estimate them.  This finding begs the question of the appropriate prior to utilize.  To a frequentist, the prior should be a sampling prior, verifiable by experiment; an example is the exponential absorption law for photons.  To a Bayesian, the prior could incorporate prior beliefs.  Indeed, a Bayesian would say that the problem of nuisance parameters is an example of a fundamental dilemma that arises in any inference problem: we \textit{never} have enough empirical (verifiable by observation or experience rather than theory or pure logic) information to solve the problem at hand, and we must always bring in prior belief.
Yes, one can use prior beliefs in estimation problems in imaging, but the final measure of the efficacy of the belief is a long-run, frequentist measure of task performance.


\lab{Sparse Grids}{spgrid}
\label{lab:spgrid}

\objective{Sparse Grids are an important tool when dealing with high-dimensional problems.  Computers operate in discrete space, not in continuous space.  It is important to choose evaluation points wisely so as to maximize accuracy without sacrificing computation time.  In particular, we explore how to use sparse grids to compute integrals of high dimensionality.}

\section*{Discretization}
At first inspection, our world appears to be nice and continuous.  However, this only works on a macroscopic level.  As we zoom in on matter, we find that it is made of discrete atoms with much empty space between.  Computers likewise work in discrete space.  You have already seen many examples of this.  Consider plotting the function $y=x^2$, such as in Figure \ref{fig:x_squared}.  To do this we take an array of discrete points of $x$ and $y$ values, which are then joined in a linear manner.  As you either zoom in on the function or decrease the number of plotting points, you can see the discrete nature of even this simple function.

In order to get better results, we need to use a larger number of points.  This is true not just for graphing purposes, but also for standard computation.  If we double the number of points, we approximately double the computation time necessary.  This effect, while not irrelevant, pales in comparison to the case when working in multiple dimensions.  Imagine a function that we discretize into $20$ points.  A similar function of two variables, where each variable is discretized into $20$ points, would necessitate $400$ unique points.  Expanding to seven variables would necessitate $1,280,000,000$ unique points!

In general, for $n$ discrete points in $d$ dimensions, this gives $n^d$ points.  This can be visualized as a $d$-dimensional grid of size $n \times n \times \cdots \times n$. In practice, we seldom need to know the value for each and every point.  Instead of using the full grid, we can use a sparse grid.  The main idea of a sparse grid is that it can reduce the order of difficulty for a standard $d$-dimensional problem.

\begin{center}
\begin{figure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{x2.png}
\end{subfigure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{x2a.png}
\end{subfigure}
\caption{Plots of the function $y=x^2$ using $31$ points and $9$ points, respectively.  In the $9$-point plot, the linear, discrete nature of the function is easily visible.}
\label{fig:x_squared}
\end{figure}
\end{center}

\section*{The Hierarchical Basis}
The basic sparse grid is based on the use of the \emph{Hierarchical Basis}.  This basis is composed of piecewise-linear functions known as the \emph{standard hat functions}, and can be used to approximate any function.  The standard hat function in one dimension is defined as:

\begin{equation}
\phi(x) = \left\{
        \begin{array}{ll}
            1-\abs{x} & \quad -1 \leq x \leq 1 \\
            0 & \quad otherwise
        \end{array}
    \right.
\end{equation}

From this equation, we can build the set of basis functions of order $j$

\begin{equation}
\phi_{j,i}(x) = \phi(2^{j-1}(x+1) - 2i+1)
\end{equation}

\begin{center}
where $i=1,2,3,\cdots,2^{j-1}$.
\end{center}

The first few basis functions are listed in Table \ref{table:functions}, and are plotted in Figure \ref{fig:basis_functions}.

\begin{center}
\begin{figure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{j1.png}
\end{subfigure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{j2.png}
\end{subfigure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{j3.png}
\end{subfigure}
\begin{subfigure}{.49\textwidth}
\includegraphics[width=\textwidth]{j4.png}
\end{subfigure}
\caption{Plots of the first few standard hat functions.  Note that the value of $j$ gives the number of "hats" in the domain $[-1,1]$, and $i$ gives which hat, counting from the left-most hat.}
\label{fig:basis_functions}
\end{figure}
\end{center}

\begin{center}
\begin{table}
\begin{center}
\begin{tabular}{|c|c|c|c|c|}
\hline
$j\backslash i$ & $1$ & $2$ & $3$  & max $i$ \\ \hline
$1$ & $\phi_{11}(x) = \phi(x)$ & & & 1 \\ \hline
$2$ & $\phi_{21}(x) = \phi(2x+1)$ & $\phi_{22}(x) = \phi(2x-1)$ & & 2\\ \hline
$3$ & $\phi_{31}(x) = \phi(4x+3)$ & $\phi_{32}(x) = \phi(4x+1)$ & $\phi_{33}(x) = \phi(4x-1)$ & 4 \\ \hline
$4$ & $\phi_{41}(x) = \phi(8x+7)$ & $\phi_{42}(x) = \phi(8x+5)$ & $\phi_{43}(x) = \phi(8x+3)$ & 8 \\ \hline
$5$ & $\phi_{51}(x) = \phi(16x+15)$ & $\phi_{52}(x) = \phi(16x+13)$ & $\phi_{53}(x) = \phi(16x+11)$ & 16 \\ \hline
\end{tabular}
\caption{The first few standard hat functions that make up the hierarchical basis, for $i\leq3$.}
\label{table:functions}
\end{center}
\end{table}
\end{center}

Using these basis functions, we can approximate any function of our choosing.  Suppose we want to solve
\begin{equation*}
\int_{-1}^{1} \sqrt{1-x^2}\: dx .
\end{equation*}
This is the area of a semi-circle of radius $1$, and can easily be computed mathematically as $\pi / 2 \approx 1.57079632679$.  But a computer does not calculate this in the same manner.  Instead, it uses certain points to evaluate the integral numerically.  We can approximate $f(x) = \sqrt{1-x^2}$ by interpolating the basis hat functions.  If we approximate up to order $l$ then this gives
\begin{equation*}
f(x) \approx \sum_{n=1}^l \sum_{m=1}^{2^{n-1}} c_{nm} \phi_{nm}(x)
\end{equation*}
where $c_{nm}$ are the appropriate constants which can be found by evaluating the difference between $f(x)$ and the sum of the basis hat functions of lower order at the peak of each hat:
\begin{equation}
c_{nm} = f(p_{nm}) - \sum_{s=1}^{n-1} \sum_{t=1}^{2^{s-1}} c_{st}*\phi_{st}(p_{nm})
\end{equation}
where
\begin{equation}
p_{nm}=2^{2-n}*(m-0.5)-1
\label{eq:points}
\end{equation}
and $p_{nm}$ is the peak of the $m^{th}$ hat of level $n$.  These points will be important because we will use these to create the basic sparse grid.

\begin{center}
\begin{figure}
\includegraphics[width=.7\textwidth]{HB.png}
\caption{Interpolating a function using the standard hat basis functions.}
\label{fig:HB}
\end{figure}
\end{center}

One benefit of decomposing $f(x)$ into the basic hat functions is that they are extremely simple to integrate.  Each is a triangle, for which $A=bh/2$.  The height is $1$, and the base is $2^{2-n}$, and so
\begin{equation*}
\int_{-1}^1 \phi_{nm} dx= 2^{1-n}
\end{equation*}

One problem with using the basic hat functions to approximate a function arises when the function does not go to zero at the endpoints.  Higher order approximations yield better results, but ultimately we need to decide how accurate we need our answer to be.  When an additional level changes the answer by less that our error tolerance, we have found a good level of approximation.

 \begin{problem}
Use the function declaration below to approximate a given function on $[-1,1]$ with the basic hat functions up to order $l$:
\begin{lstlisting}
def hat_approximation(f,l):
    """This function will return a list of the correct coefficients to approximate the function f up to level l.
    Parameters
    ----------
        f (function) : The function of a single variable to approximate
        l (int) : The order to use for interpolating the hat function

    Returns
    -------
        coeffs (list) : Entry n is an ndarray of len(2^(n-1)) and contains the coefficients for phi_{nm}.  The length of this should be l.
    """
    pass
\end{lstlisting}
\label{prob:one}
\end{problem}

\begin{center}
\begin{figure}
\includegraphics[width=.7\textwidth]{errors.png}
\caption{}
\label{fig:errors}
\end{figure}
\end{center}

\begin{problem}
Write a function that, given the coefficients returned by the function in Problem \ref{prob:one}, returns the value of the integral.
\end{problem}

\begin{problem}
Using the function $f(x)=\sqrt{1-x^2}$, calculate the error of $\int_{-1}^1 f(x)$ in regards to the actual value $\pi/2$.  Do this for the order $l = 1,2,\cdots,10$, and plot the error as a function of $l$.  Your results should match Figure \ref{fig:errors}.  Additionally, time how long it takes to execute the code for each $l$, and plot your results in another graph.
\end{problem}

Keep in mind that thus far we have only used a function of one variable.  For each variable (dimension) the time to compute will increase exponentially.   With multiple dimensions, it becomes infeasible to compute to any degree of accuracy, because the time to compute will increase much too quickly to be reasonable for higher dimensions.  Hence the need for sparse grids.

\section*{Sparse Grids}
Rather than dividing each dimension into $n$ sections, which for $d$ dimensions would give $n^d$ points, sparse grids use only a few of these points.  There are many types of sparse grids, and each is useful for certain purposes.  Which type of sparse grid is used will often depend on the nature of the problem you are trying to solve.  For this lab, we will use the basic sparse grid for purposes of integration.

The basic sparse grid has two main properties, the dimension and the level.  Using these two variables and Equation \ref{eq:points}, we can construct a sparse grid.  Visualizing this in one dimension is simple: we use as our evaluation points each $p_{nm}$.  A level $1$ grid has one point at the center, which is the peak of the level $1$ hat.  A level $2$ grid has the level $1$ point with two level $2$ points, which are the peaks of the two level $2$ hats.  The third level adds its $4$ points, etc.  So for one dimension of level $l$ we have $\sum_{i=1}^l 2^{i-1}$ points.

Understanding a $2$-dimensional grid  is a bit more complicated.  Our level $1$ point goes at the center of the grid.  To expand to level $2$, we add in the peaks of the level $2$ hats \emph{in both dimensions}, for an additional $4$ points.  Level $3$ is where it gets a little bit tricky mathematically.  We can't just keep expanding only along the axes of the grid, because we need points on the interior as well.  Conceptually, what we do is take our current points and divide the space between points and the end of the grid into two, and there we create a new point.  This is synonymous to adding a hat between the peaks of currently existing points.

We can make a lot more sense of that last paragraph by actually plotting the points.  The file \li{pysg.py} contains a sparse grid class with many useful functions.  Using \li{plotGrid}, you can plot the grid points.  You will first need to create a sparse grid object, and then generate the points using the built in function.  Unfortunately, \li{plotGrid} can only plot up to three dimensions.  Note that in this file, our domain is $[0,1]^d$ rather than $[-1,1]^d$.
%Be sure to take note of this when computing the integrals in the following problem.

\begin{center}
\begin{figure}
% \includegraphics[width=.7\textwidth]{gridpoints.png}
\caption{Sparse grid points for 2-D grid of level 5}
\label{fig:gridpoints}
\end{figure}
\end{center}

% THIS FILE IS NOT IN THE REPOSITORY!!

\begin{problem}
Explore the \li{sparseGrid} class contained in the module \li{pysg.py}. Figure out how to create a sparse grid object, generate the points, and plot them. This function should plot the sparse grid points for a grid of dimension 2 and level 5. See the figure above.
\end{problem}

Multi-dimensional integration for functions with independent endpoints (those that are numbers that do not depend on other variables) is simplified greatly by using sparse grids.  Functions must be manipulated appropriately before sparse grid classes like those in \li{pysg.py} can be used to do the integration, but once this has been done, their evaluation becomes rather simple.  Once the coefficients for the hat functions have been computed, the integrals can be evaluated without having to perform any integration.  Sparse grids are powerful tools for fast computation of high-dimension integrals.
%In general, note that
%\begin{equation*}
%\int_{a_1}^{b_1} \int_{a_2}^{b_2} \cdots \int_{a_d}^{b_d} \phi_{sii\cdots i}dx_d dx_{d-1} \cdots dx_1= \frac{\prod_{i=1}^d (b_i-a_i)}{2^{(s-1)d}*(d+1)}
%\end{equation*}

%\begin{problem}
%Using the sparse grid class in \li{pysg.py}, compute the value of the following integrals:
%\begin{enumerate}
%\item $\int_{0}^1 \int_{0}^1 (x^2+\frac{1}{3}xy) \ dx dy \ $ to order 8.
%\item $\int_{.36}^{1.8} \int_{0}^{\pi} y*cos(x) \ dx dy \ $ to order 6.
%\item $\int_{-2}^3 \int_{5}^6 \int_{4.2}^{6.3} (\frac{1}{43}x+\frac{1}{123}x^2 y^3 + %\frac{1}{200}yz^2) \ dx dy dz \ $ to order 5.
%\item $\int_{[0,1]^{18}} \prod_{i=1}^{18} x_i \ dx_i \ $ to order 3.
%\end{enumerate}
%\end{problem}