\chapter{Probability}

\section{Frequentists vs. Bayesians}
what is probability?

One is called the \textbf{frequentist} interpretation. In this view, probabilities represent long run frequencies of events. For example, the above statement means that, if we flip the coin many times, we expect it to land heads about half the time.

The other interpretation is called the \textbf{Bayesian} interpretation of probability. In this view, probability is used to quantify our \textbf{uncertainty} about something; hence it is fundamentally related to information rather than repeated trials (Jaynes 2003). In the Bayesian view, the above statement means we believe the coin is equally likely to land heads or tails on the next toss

One big advantage of the Bayesian interpretation is that it can be used to model our uncertainty about events that do not have long term frequencies. For example, we might want to compute the probability that the polar ice cap will melt by 2020 CE. This event will happen zero or one times, but cannot happen repeatedly. Nevertheless, we ought to be able to quantify our uncertainty about this event. To give another machine learning oriented example, we might have observed a “blip” on our radar screen, and want to compute the probability distribution over the location of the corresponding target (be it a bird, plane, or missile). In all these cases, the idea of repeated trials does not make sense, but the Bayesian interpretation is valid and indeed quite natural. We shall therefore adopt the Bayesian interpretation in this book. Fortunately, the basic rules of probability theory are the same, no matter which interpretation is adopted.

\section{A brief review of probability theory}

\subsection{Basic concepts}
We denote a random event by defining a \textbf{random variable} $X$.

\textbf{Descrete random variable}: $X$ can take on any value from a finite or countably infinite set.

\textbf{Continuous random variable}: the value of $X$ is real-valued.

\subsubsection{CDF}

\begin{equation}
F(x) \triangleq P(X \leq x)=\begin{cases}
\sum_{u \leq x}p(u) & \text{, discrete}\\
\int_{-\infty}^{x} f(u)\mathrm{d}u & \text{, continuous}\\
\end{cases}
\end{equation}

\subsubsection{PMF and PDF}
For descrete random variable, We denote the probability of the event that $X=x$ by $P(X=x)$, or just $p(x)$ for short. Here $p(x)$ is called a \textbf{probability mass function} or \textbf{PMF}.A probability mass function is a function that gives the probability that a discrete random variable is exactly equal to some value\footnote{\url{http://en.wikipedia.org/wiki/Probability_mass_function}}. This satisfies the properties $0 \leq p(x) \leq 1$ and $\sum_{x \in \mathcal{X}} p(x)=1$.

For continuous variable, in the equation $F(x)=\int_{-\infty}^{x} f(u)\mathrm{d}u$, the function $f(x)$ is called a \textbf{probability density function} or \textbf{PDF}. A probability density function is a function that describes the relative likelihood for this random variable to take on a given value\footnote{\url{http://en.wikipedia.org/wiki/Probability_density_function}}.This satisfies the properties $f(x) \geq 0$ and $\int_{-\infty}^{\infty} f(x)\mathrm{d}x=1$.

\subsection{Mutivariate random variables}

\subsubsection{Joint CDF}
We denote joint CDF by $F(x,y) \triangleq P(X \leq x \cap Y \leq y)=P(X \leq x , Y \leq y)$.

\begin{equation}
F(x,y) \triangleq P(X \leq x, Y \leq y)=\begin{cases}
\sum_{u \leq x, v \leq y}p(u,v) \\
\int_{-\infty}^{x}\int_{-\infty}^{y} f(u,v)\mathrm{d}u\mathrm{d}v \\
\end{cases}
\end{equation}

\textbf{product rule}:
\begin{equation}\label{eqn:product-rule}
p(X,Y)=P(X|Y)P(Y)
\end{equation}

\textbf{Chain rule}:
\begin{equation}
p(X_{1:N})=p(X_1)p(X_3|X_2,X_1)...p(X_N|X_{1:N-1})
\end{equation}

\subsubsection{Marginal distribution}
\textbf{Marginal CDF}:
\begin{equation}\begin{split}
& F_X(x) \triangleq F(x,+\infty)= \\
& \begin{cases}
\sum\limits_{x_i \leq x}P(X=x_i)=\sum\limits_{x_i \leq x}\sum\limits_{j=1}^{+\infty}P(X=x_i,Y=y_j) \\
\int_{-\infty}^{x}f_X(u)du=\int_{-\infty}^{x}\int_{-\infty}^{+\infty} f(u,v)\mathrm{d}u\mathrm{d}v \\
\end{cases}
\end{split}\end{equation}

\begin{equation}\begin{split}
& F_Y(y) \triangleq F(+\infty,y)= \\
& \begin{cases}
\sum\limits_{y_j \leq y}p(Y=y_j)=\sum\limits_{i=1}^{+\infty}\sum_{y_j \leq y}P(X=x_i,Y=y_j) \\
\int_{-\infty}^{y}f_Y(v)dv=\int_{-\infty}^{+\infty}\int_{-\infty}^{y} f(u,v)\mathrm{d}u\mathrm{d}v \\
\end{cases}
\end{split}\end{equation}

\textbf{Marginal PMF and PDF}:
\begin{equation} \begin{cases}
P(X=x_i)=\sum_{j=1}^{+\infty}P(X=x_i,Y=y_j) & \text{, descrete}\\
f_X(x)=\int_{-\infty}^{+\infty} f(x,y)\mathrm{d}y & \text{, continuous}\\
\end{cases}\end{equation}

\begin{equation}\begin{cases}
p(Y=y_j)=\sum_{i=1}^{+\infty}P(X=x_i,Y=y_j) & \text{, descrete}\\
f_Y(y)=\int_{-\infty}^{+\infty} f(x,y)\mathrm{d}x & \text{, continuous}\\
\end{cases}\end{equation}

\subsubsection{Conditional distribution}
\textbf{Conditional PMF}:
\begin{equation}
p(X=x_i|Y=y_j)=\dfrac{p(X=x_i,Y=y_j)}{p(Y=y_j)} \text{ if } p(Y)>0
\end{equation}
The pmf $p(X|Y)$ is called \textbf{conditional probability}.

\textbf{Conditional PDF}:
\begin{equation}
f_{X|Y}(x|y)=\dfrac{f(x,y)}{f_Y(y)}
\end{equation}

\subsection{Bayes rule}
\begin{equation}
\begin{split}
p(Y=y|X=x) & =\dfrac{p(X=x,Y=y)}{p(X=x)} \\
           & =\dfrac{p(X=x|Y=y)p(Y=y)}{\sum_{y'}p(X=x|Y=y')p(Y=y')}
\end{split}
\end{equation}

\subsection{Independence and conditional independence}
We say $X$ and $Y$ are unconditionally independent or marginally independent, denoted $X \perp Y$, if we can represent the joint as the product of the two marginals, i.e.,
\begin{equation}
X \perp Y=P(X,Y)=P(X)P(Y)
\end{equation}

We say $X$ and $Y$ are conditionally independent(CI) given $Z$ if the conditional joint can be written as a product of conditional marginals:
\begin{equation}
X \perp Y|Z=P(X,Y|Z)=P(X|Z)P(Y|Z)
\end{equation}

\subsection{Quantiles}
Since the cdf $F$ is a monotonically increasing function, it has an inverse; let us denote this by $F^{-1}$. If $F$ is the cdf of $X$ , then $F^{-1}(\alpha)$ is the value of $x_{\alpha}$ such that $P(X \leq x_{\alpha})=\alpha$; this is called the $\alpha$ quantile of $F$. The value $F^{-1}(0.5)$ is the \textbf{median} of the distribution, with half of the probability mass on the left, and half on the right. The values $F^{-1}(0.25)$ and $F^{−1}(0.75)$are the lower and upper \textbf{quartiles}.

\subsection{Mean and variance}
The most familiar property of a distribution is its \textbf{mean},or \textbf{expected value}, denoted by $\mu$. For discrete rv’s, it is defined as $\mathbb{E}[X] \triangleq \sum_{x \in \mathcal{X}}xp(x)$, and for continuous rv’s, it is defined as $\mathbb{E}[X] \triangleq \int_{\mathcal{X}}xp(x)\mathrm{d}x$. If this integral is not finite, the mean is not defined (we will see some examples of this later).

The \textbf{variance} is a measure of the “spread” of a distribution, denoted by $\sigma^2$. This is defined as follows:
\begin{align}
var[X]& =\mathbb{E}[(X-\mu)^2] \\
      & =\int{(x-\mu)^2p(x)\mathrm{d}x} \nonumber \\
      & =\int{x^2p(x)\mathrm{d}x}+{\mu}^2\int{p(x)\mathrm{d}x}-2\mu\int{xp(x)\mathrm{d}x} \nonumber \\
	  & =\mathbb{E}[X^2]-{\mu}^2
\end{align}

from which we derive the useful result
\begin{equation}
\mathbb{E}[X^2]=\sigma^2+{\mu}^2
\end{equation}

The \textbf{standard deviation} is defined as
\begin{equation}
std[X] \triangleq \sqrt{var[X]}
\end{equation}

This is useful since it has the same units as $X$ itself.

\section{Some common discrete distributions}
In this section, we review some commonly used parametric distributions defined on discrete state spaces, both finite and countably infinite.

\subsection{The Bernoulli and binomial distributions}

\begin{definition}
Now suppose we toss a coin only once. Let $X \in \{0,1\}$ be a binary random variable, with probability of “success” or “heads” of $\theta$. We say that $X$ has a \textbf{Bernoulli distribution}. This is written as $X \sim \text{Ber}(\theta)$, where the pmf is defined as
\begin{equation}
\text{Ber}(x|\theta) \triangleq \theta^{\mathbb{I}(x=1)}(1-\theta)^{\mathbb{I}(x=0)}
\end{equation}
\end{definition}

\begin{definition}
Suppose we toss a coin $n$ times. Let $X \in \{0,1,\cdots,n\}$ be the number of heads. If the probability of heads is $\theta$, then we say $X$ has a \textbf{binomial distribution}, written as $X \sim \text{Bin}(n, \theta)$. The pmf is given by
\begin{equation}\label{eqn:binomial-pmf}
\text{Bin}(k|n,\theta) \triangleq \dbinom{n}{k}\theta^k(1-\theta)^{n-k}
\end{equation}
\end{definition}

\subsection{The multinoulli and multinomial distributions}

\begin{definition}
The Bernoulli distribution can be used to model the outcome of one coin tosses. To model the outcome of tossing a K-sided dice, let $\vec{x} =(\mathbb{I}(x=1),\cdots,\mathbb{I}(x=K)) \in \{0,1\}^K$ be a random vector(this is called \textbf{dummy encoding} or \textbf{one-hot encoding}), then we say $X$ has a \textbf{multinoulli distribution}(or \textbf{categorical distribution}), written as $X \sim \text{Cat}(\theta)$. The pmf is given by:
\begin{equation}
p(\vec{x}) \triangleq \prod\limits_{k=1}^K\theta_k^{\mathbb{I}(x_k=1)}
\end{equation}
\end{definition}

\begin{definition}
Suppose we toss a K-sided dice $n$ times. Let $\vec{x} =(x_1,x_2,\cdots,x_K) \in \{0,1,\cdots,n\}^K$ be a random vector, where $x_j$ is the number of times side $j$ of the dice occurs, then we say $X$ has a \textbf{multinomial distribution}, written as $X \sim \text{Mu}(n, \vec{\theta})$. The pmf is given by
\begin{equation}\label{eqn:multinomial-pmf}
p(\vec{x}) \triangleq \dbinom{n}{x_1 \cdots x_k} \prod\limits_{k=1}^K\theta_k^{x_k}
\end{equation}
where $\dbinom{n}{x_1 \cdots x_k} \triangleq \dfrac{n!}{x_1!x_2! \cdots x_K!}$
\end{definition}

Bernoulli distribution is just a special case of a Binomial distribution with $n=1$, and so is multinoulli distribution as to multinomial distribution. See Table \ref{tab:multinomial-summary} for a summary.

\begin{table}
\caption{Summary of the multinomial and related distributions.}
\label{tab:multinomial-summary}
\centering
\begin{tabular}{llll}
\hline\noalign{\smallskip}
Name & K & n & X \\
\noalign{\smallskip}\svhline\noalign{\smallskip}
Bernoulli & 1 & 1 & $x \in \{0,1\}$ \\
Binomial & 1 & - & $\vec{x} \in \{0,1,\cdots,n\}$ \\
Multinoulli & - & 1 & $\vec{x} \in \{0,1\}^K, \sum_{k=1}^K x_k=1$ \\
Multinomial & - & - & $\vec{x} \in \{0,1,\cdots,n\}^K, \sum_{k=1}^K x_k=n$ \\
\noalign{\smallskip}\hline
\end{tabular}
\end{table}

\subsection{The Poisson distribution}
\begin{definition}
We say that $X \in \{0,1,2,\cdots\}$ has a \textbf{Poisson distribution} with parameter $\lambda>0$, written as $X \sim \text{Poi}(\lambda)$, if its pmf is
\begin{equation}
p(x|\lambda)=e^{-\lambda}\dfrac{\lambda^x}{x!}
\end{equation}
\end{definition}

The first term is just the normalization constant, required to ensure the distribution sums to 1.

The Poisson distribution is often used as a model for counts of rare events like radioactive decay and traffic accidents.

\begin{table*}
\caption{Summary of Bernoulli, binomial multinoulli and multinomial distributions.}
\label{tab:Summary-distribution}
\centering
\begin{tabular}{llllll}
\hline\noalign{\smallskip}
Name & Written as & X & $p(x)$(or $p(\vec{x})$) & $\mathbb{E}[X]$ & $\text{var}[X]$ \\
\noalign{\smallskip}\svhline\noalign{\smallskip}
Bernoulli & $X \sim \text{Ber}(\theta)$ & $x \in \{0,1\}$ & $\theta^{\mathbb{I}(x=1)}(1-\theta)^{\mathbb{I}(x=0)}$ & $\theta$ & $\theta(1-\theta)$ \\
Binomial & $X \sim \text{Bin}(n,\theta)$ & $x \in \{0,1,\cdots,n\}$ & $\dbinom{n}{k}\theta^k(1-\theta)^{n-k}$ & $n\theta$ & $n\theta(1-\theta)$ \\
Multinoulli & $X \sim \text{Cat}(\vec{\theta})$ & $\vec{x} \in \{0,1\}^K, \sum_{k=1}^K x_k=1$ & $\prod\limits_{k=1}^K\theta_j^{\mathbb{I}(x_j=1)}$ & - & - \\
Multinomial & $X \sim \text{Mu}(n,\vec{\theta})$ & $\vec{x} \in \{0,1,\cdots,n\}^K, \sum_{k=1}^K x_k=n$ & $\dbinom{n}{x_1 \cdots x_k} \prod\limits_{k=1}^K\theta_j^{x_j}$ & - & - \\
Poisson & $X \sim \text{Poi}(\lambda)$ & $x \in \{0,1,2,\cdots\}$ & $e^{-\lambda}\dfrac{\lambda^x}{x!}$ & $\lambda$ & $\lambda$ \\
\noalign{\smallskip}\hline
\end{tabular}
\end{table*}

\subsection{The empirical distribution}
The \textbf{empirical distribution function}\footnote{\url{http://en.wikipedia.org/wiki/Empirical_distribution_function}}, or \textbf{empirical cdf}, is the cumulative distribution function associated with the empirical measure of the sample. Let $\mathcal{D}=\{x_1,x_2,\cdots,x_N\}$ be a sample set, it is defined as
\begin{equation}
F_n(x) \triangleq \dfrac{1}{N}\sum\limits_{i=1}^N\mathbb{I}(x_i \leq x)
\end{equation}

\section{Some common continuous distributions}
In this section we present some commonly used univariate (one-dimensional) continuous probability distributions.

\subsection{Gaussian (normal) distribution}

% \begin{table}
% \caption{Summary of Gaussian distribution}
% \centering
% \begin{tabular}{cccccc}
% \hline\noalign{\smallskip}
% Name & Written as & $f(x)$ & $\mathbb{E}[X]$ & mode & $\text{var}[X]$ \\
% \noalign{\smallskip}\svhline\noalign{\smallskip}
% Gaussian distribution & $X \sim \mathcal{N}(\mu,\sigma^2)$ & $\dfrac{1}{\sqrt{2\pi}\sigma}e^{-\frac{1}{2\sigma^2}\left(x-\mu\right)^2}$ & $\mu$ & $\mu$ & $\sigma^2$ \\
% \noalign{\smallskip}\hline
% \end{tabular}
% \end{table}

\begin{table}
\caption{Summary of Gaussian distribution.}
\centering
\begin{tabular}{cccccc}
\hline\noalign{\smallskip}
Written as & $f(x)$ & $\mathbb{E}[X]$ & mode & $\text{var}[X]$ \\
\noalign{\smallskip}\svhline\noalign{\smallskip}
$X \sim \mathcal{N}(\mu,\sigma^2)$ & $\dfrac{1}{\sqrt{2\pi}\sigma}e^{-\frac{1}{2\sigma^2}\left(x-\mu\right)^2}$ & $\mu$ & $\mu$ & $\sigma^2$ \\
\noalign{\smallskip}\hline
\end{tabular}
\end{table}

If $X \sim N(0,1)$,we say $X$ follows a \textbf{standard normal} distribution.

The Gaussian distribution is the most widely used distribution in statistics. There are several reasons for this.
\begin{enumerate}
\item First, it has two parameters which are easy to interpret, and which capture some of the most basic properties of a distribution, namely its mean and variance.
\item Second,the central limit theorem (Section TODO) tells us that sums of independent random variables have an approximately Gaussian distribution, making it a good choice for modeling residual errors or “noise”.
\item Third, the Gaussian distribution makes the least number of assumptions (has maximum entropy), subject to the constraint of having a specified mean and variance, as we show in Section TODO; this makes it a good default choice in many cases.
\item Finally, it has a simple mathematical form, which results in easy to implement, but often highly effective, methods, as we will see.
\end{enumerate}
See (Jaynes 2003, ch 7) for a more extensive discussion of why Gaussians are so widely used.

\subsection{Student's t-distribution}
\begin{table}
\caption{Summary of Student's t-distribution.}
\centering
\begin{tabular}{cccccc}
\hline\noalign{\smallskip}
Written as & $f(x)$ & $\mathbb{E}[X]$ & mode & $\text{var}[X]$ \\
\noalign{\smallskip}\svhline\noalign{\smallskip}
$X \sim \mathcal{T}(\mu,\sigma^2,\nu)$ & $\dfrac{\Gamma(\frac{\nu+1}{2})}{\sqrt{\nu\pi}\Gamma(\frac{\nu}{2})}\left[1+\dfrac{1}{\nu}\left(\dfrac{x-\mu}{\nu}\right)^2\right]$ & $\mu$ & $\mu$ & $\dfrac{\nu\sigma^2}{\nu-2}$ \\
\noalign{\smallskip}\hline
\end{tabular}
\end{table}
where  $\Gamma(x)$ is the gamma function:
\begin{equation}
\Gamma(x) \triangleq \int_0^\infty t^{x-1}e^{-t}\mathrm{d}t
\end{equation}
$\mu$ is the mean, $\sigma^2>0$ is the scale parameter, and $\nu>0$ is called the \textbf{degrees of freedom}. See Figure \ref{fig:pdfs-for-NTL} for some plots.

\begin{figure}[hbtp]
\centering
\subfloat[]{\includegraphics[scale=.70]{pdfs-for-NTL-a.png}} \\
\subfloat[]{\includegraphics[scale=.70]{pdfs-for-NTL-b.png}}
\caption{(a) The pdf’s for a $\mathcal{N}(0,1)$, $\mathcal{T}(0,1,1)$ and $Lap(0,1/\sqrt{2})$. The mean is 0 and the variance is 1 for both the Gaussian and Laplace. The mean and variance of the Student is undefined when $\nu=1$.(b) Log of these pdf’s. Note that the Student distribution is not log-concave for any parameter value, unlike the Laplace distribution, which is always log-concave (and log-convex...) Nevertheless, both are unimodal.}
\label{fig:pdfs-for-NTL}
\end{figure}

The variance is only defined if $\nu>2$. The mean is only defined if $\nu>1$.

As an illustration of the robustness of the Student distribution, consider Figure \ref{fig:robustness}. We see that the Gaussian is affected a lot, whereas the Student distribution hardly changes. This is because the Student has heavier tails, at least for small $\nu$(see Figure \ref{fig:pdfs-for-NTL}).

\begin{figure}[hbtp]
\centering
\subfloat[]{\includegraphics[scale=.70]{robustness-a.png}} \\
\subfloat[]{\includegraphics[scale=.70]{robustness-b.png}}
\caption{Illustration of the effect of outliers on fitting Gaussian, Student and Laplace distributions. (a) No outliers (the Gaussian and Student curves are on top of each other). (b) With outliers. We see that the Gaussian is more affected by outliers than the Student and Laplace distributions.}
\label{fig:robustness}
\end{figure}

If $\nu=1$, this distribution is known as the \textbf{Cauchy} or \textbf{Lorentz} distribution. This is notable for having such heavy tails that the integral that defines the mean does not converge.

To ensure finite variance, we require $\nu>2$. It is common to use $\nu=4$, which gives good performance in a range of problems (Lange et al. 1989). For $\nu \gg 5$, the Student distribution rapidly approaches a Gaussian distribution and loses its robustness properties.

\subsection{The Laplace distribution}
\begin{table}
\caption{Summary of Laplace distribution.}
\centering
\begin{tabular}{cccccc}
\hline\noalign{\smallskip}
Written as & $f(x)$ & $\mathbb{E}[X]$ & mode & $\text{var}[X]$ \\
\noalign{\smallskip}\svhline\noalign{\smallskip}
$X \sim \text{Lap}(\mu,b)$ & $\dfrac{1}{2b}\exp\left(-\dfrac{|x-\mu|}{b}\right)$ & $\mu$ & $\mu$ & $2b^2$ \\
\noalign{\smallskip}\hline
\end{tabular}
\end{table}

Here $\mu$ is a location parameter and $b>0$ is a scale parameter. See Figure \ref{fig:pdfs-for-NTL} for a plot.

Its robustness to outliers is illustrated in Figure \ref{fig:robustness}. It also put mores probability density at 0 than the Gaussian. This property is a useful way to encourage sparsity in a model, as we will see in Section TODO.

\subsection{The gamma distribution}

\begin{table}
\caption{Summary of gamma distribution}
\centering
\begin{tabular}{ccccccc}
\hline\noalign{\smallskip}
Written as & $X$ & $f(x)$ & $\mathbb{E}[X]$ & mode & $\text{var}[X]$ \\
\noalign{\smallskip}\svhline\noalign{\smallskip}
$X \sim \text{Ga}(a,b)$ & $x \in \mathbb{R}^+$ & $\dfrac{b^a}{\Gamma(a)}x^{a-1}e^{-xb}$ & $\dfrac{a}{b}$ & $\dfrac{a-1}{b}$ & $\dfrac{a}{b^2}$ \\
\noalign{\smallskip}\hline
\end{tabular}
\end{table}

Here $a>0$ is called the shape parameter and $b>0$ is called the rate parameter. See Figure \ref{fig:gamma-distribution} for some plots.

\begin{figure}[hbtp]
\centering
\subfloat[]{\includegraphics[scale=.50]{gamma-distribution-a.png}} \\
\subfloat[]{\includegraphics[scale=.50]{gamma-distribution-b.png}}
\caption{Some Ga$(a, b=1)$ distributions. If $a \leq 1$, the mode is at 0, otherwise it is $>0$.As we increase the rate $b$, we reduce the horizontal scale, thus squeezing everything leftwards and upwards. (b) An empirical pdf of some rainfall data, with a fitted Gamma distribution superimposed.}
\label{fig:gamma-distribution}
\end{figure}

\subsection{The beta distribution}

\begin{table*}
\caption{Summary of Beta distribution}\label{tab:beta-distribution}
\centering
\begin{tabular}{ccccccc}
\hline\noalign{\smallskip}
Name & Written as & $X$ & $f(x)$ & $\mathbb{E}[X]$ & mode & $\text{var}[X]$ \\
\noalign{\smallskip}\svhline\noalign{\smallskip}
Beta distribution & $X \sim \text{Beta}(a,b)$ & $x \in [0,1]$ & $\dfrac{1}{B(a,b)}x^{a-1}(1-x)^{b-1}$ & $\dfrac{a}{a+b}$ & $\dfrac{a-1}{a+b-2}$ & $\dfrac{ab}{(a+b)^2(a+b+1)}$ \\
\noalign{\smallskip}\hline
\end{tabular}
\end{table*}

Here $B(a, b)$is the beta function,
\begin{equation}
B(a,b) \triangleq \dfrac{\Gamma(a)\Gamma(b)}{\Gamma(a+b)}
\end{equation}

See Figure \ref{fig:beta-distribution} for plots of some beta distributions. We require  $a, b >0$ to ensure the distribution is integrable (i.e., to ensure $B(a, b)$ exists). If $a=b=1$, we get the uniform distirbution. If $a$ and $b$ are both less than 1, we get a bimodal distribution with “spikes” at 0 and 1; if $a$ and $b$ are both greater than 1, the distribution is unimodal.

\begin{figure}[hbtp]
\centering
    \includegraphics[scale=.60]{beta-distribution.png}
\caption{Some beta distributions.}
\label{fig:beta-distribution}
\end{figure}

\subsection{Pareto distribution}

\begin{table*}
\caption{Summary of Pareto distribution}
\centering
\begin{tabular}{ccccccc}
\hline\noalign{\smallskip}
Name & Written as & $X$ & $f(x)$ & $\mathbb{E}[X]$ & mode & $\text{var}[X]$ \\
\noalign{\smallskip}\svhline\noalign{\smallskip}
Pareto distribution & $X \sim \text{Pareto}(k,m)$ & $x \geq m$ & $km^kx^{-(k+1)}\mathbb{I}(x \geq m)$ & $\dfrac{km}{k-1} \text{ if } k > 1$ & $m$ & $\dfrac{m^2k}{(k-1)^2(k-2)} \text{ if } k>2$ \\
\noalign{\smallskip}\hline
\end{tabular}
\end{table*}

The \textbf{Pareto distribution} is used to model the distribution of quantities that exhibit \textbf{long tails}, also called \textbf{heavy tails}.

As $k \rightarrow \infty$, the distribution approaches $\delta(x-m)$. See Figure \ref{fig:Pareto-distribution}(a) for some plots. If we plot the distribution on a log-log scale, it forms a straight line, of the form $\log p(x)=a\log x+c$ for some constants $a$ and $c$. See Figure \ref{fig:Pareto-distribution}(b) for an illustration (this is known as a \textbf{power law}).

\begin{figure}[hbtp]
\centering
\subfloat[]{\includegraphics[scale=.50]{pareto-distribution-a.png}} \\
\subfloat[]{\includegraphics[scale=.50]{pareto-distribution-b.png}}
\caption{(a) The Pareto distribution Pareto$(x|m, k)$ for $m=1$. (b) The pdf on a log-log scale.}
\label{fig:Pareto-distribution}
\end{figure}

\section{Joint probability distributions}
Given a \textbf{multivariate random variable} or \textbf{random vector} \footnote{\url{http://en.wikipedia.org/wiki/Multivariate_random_variable}} $X \in \mathbb{R}^D$, the \textbf{joint probability distribution}\footnote{\url{http://en.wikipedia.org/wiki/Joint_probability_distribution}} is a probability distribution that gives the probability that each of $X_1, X_2, \cdots,X_D$ falls in any particular range or discrete set of values specified for that variable. In the case of only two random variables, this is called a \textbf{bivariate distribution}, but the concept generalizes to any number of random variables, giving a \textbf{multivariate distribution}.

The joint probability distribution can be expressed either in terms of a \textbf{joint cumulative distribution function} or in terms of a \textbf{joint probability density function} (in the case of continuous variables) or \textbf{joint probability mass function} (in the case of discrete variables).

\subsection{Covariance and correlation}
\begin{definition}
The \textbf{covariance} between two rv’s $X$ and $Y$ measures the degree to which $X$ and $Y$ are (linearly) related. Covariance is defined as
\begin{equation}
\begin{split}
\mathrm{cov}[X,Y] & \triangleq \mathbb{E}\left[(X-\mathbb{E}[X])(Y-\mathbb{E}[Y])\right] \\
         & =\mathbb{E}[XY]-\mathbb{E}[X]\mathbb{E}[Y]
\end{split}
\end{equation}
\end{definition}

\begin{definition}
If $X$ is a $D$-dimensional random vector, its \textbf{covariance matrix} is defined to be the following symmetric, positive definite matrix:
\begin{align}
\mathrm{cov}[X] & \triangleq \mathbb{E}\left[(X-\mathbb{E}[X])(X-\mathbb{E}[X])^T\right] \\
       &  = \left( \begin{array}{cccc}
           \text{var}[X_1] & \text{Cov}[X_1,X_2] & \cdots & \text{Cov}[X_1,X_D] \\
           \text{Cov}[X_2,X_1] & \text{var}[X_2] & \cdots & \text{Cov}[X_2,X_D] \\
		   \vdots & \vdots & \ddots & \vdots \\
           \text{Cov}[X_D,X_1] & \text{Cov}[X_D,X_2] & \cdots & \text{var}[X_D] \end{array} \right)
\end{align}
\end{definition}

\begin{definition}
The (Pearson) \textbf{correlation coefficient} between $X$ and $Y$ is defined as
\begin{equation}
\text{corr}[X,Y] \triangleq \dfrac{\text{Cov}[X,Y]}{\sqrt{\text{var}[X],\text{var}[Y]}}
\end{equation}
\end{definition}

A \textbf{correlation matrix} has the form
\begin{equation}
\mathbf{R} \triangleq \left( \begin{array}{cccc}
           \text{corr}[X_1,X_1] & \text{corr}[X_1,X_2] & \cdots & \text{corr}[X_1,X_D] \\
           \text{corr}[X_2,X_1] & \text{corr}[X_2,X_2] & \cdots & \text{corr}[X_2,X_D] \\
		   \vdots & \vdots & \ddots & \vdots \\
           \text{corr}[X_D,X_1] & \text{corr}[X_D,X_2] & \cdots & \text{corr}[X_D,X_D] \end{array} \right)
\end{equation}

The correlation coefficient can viewed as a degree of linearity between $X$ and $Y$, see Figure \ref{fig:Correlation-examples}.
\begin{figure*}[hbtp]
\centering
    \includegraphics[scale=.80]{Correlation-examples.png}
\caption{Several sets of $(x, y)$ points, with the Pearson correlation coefficient of $x$ and $y$ for each set. Note that the correlation reflects the noisiness and direction of a linear relationship (top row), but not the slope of that relationship (middle), nor many aspects of nonlinear relationships (bottom). N.B.: the figure in the center has a slope of 0 but in that case the correlation coefficient is undefined because the variance of $Y$ is zero.Source:\url{http://en.wikipedia.org/wiki/Correlation}}
\label{fig:Correlation-examples}
\end{figure*}

\textbf{Uncorrelated does not imply independent}. For example, let $X \sim U(-1,1)$ and $Y =X^2$. Clearly $Y$ is dependent on $X$(in fact, $Y$ is uniquely determined by $X$), yet one can show that corr$[X, Y]=0$. Some striking examples of this fact are shown in Figure \ref{fig:Correlation-examples}. This shows several data sets where there is clear dependence between $X$ and $Y$, and yet the correlation coefficient is 0. A more general measure of dependence between random variables is mutual information, see Section TODO.

\subsection{Multivariate Gaussian distribution}
\label{sec:MVN}
The \textbf{multivariate Gaussian} or \textbf{multivariate normal}(MVN) is the most widely used joint probability density function for continuous variables. We discuss MVNs in detail in Chapter 4; here we just give some definitions and plots.

The pdf of the MVN in $D$ dimensions is defined by the following:
\begin{equation}
\mathcal{N}(\vec{x}|\vec{\mu},\Sigma) \triangleq \dfrac{1}{(2\pi)^{D/2}|\Sigma|^{1/2}}\exp\left[-\dfrac{1}{2}(\vec{x}-\vec{\mu})^T\Sigma^{-1}(\vec{x}-\vec{\mu})\right]
\end{equation}
where $\vec{\mu}=\mathbb{E}[X] \in \mathbb{R}^D$ is the mean vector, and $\Sigma=\text{Cov}[X]$ is the $D \times D$ covariance matrix. The normalization constant $(2\pi)^{D/2}|\Sigma|^{1/2}$ just ensures that the pdf integrates to 1.

Figure \ref{fig:2d-Gaussions} plots some MVN densities in 2d for three different kinds of covariance matrices. A full covariance matrix has A $D(D+1)/2$ parameters (we divide by 2 since $\Sigma$ is symmetric). A diagonal covariance matrix has $D$ parameters, and has 0s in the off-diagonal terms. A spherical or isotropic covariance,$\Sigma=\sigma^2\vec{I}_D$, has one free parameter.

\begin{figure}[hbtp]
\centering
\subfloat[]{\includegraphics[scale=.60]{2d-Gaussions-a.png}} \\
\subfloat[]{\includegraphics[scale=.60]{2d-Gaussions-b.png}} \\
\subfloat[]{\includegraphics[scale=.60]{2d-Gaussions-c.png}} \\
\subfloat[]{\includegraphics[scale=.60]{2d-Gaussions-d.png}}
\caption{We show the level sets for 2d Gaussians. (a) A full covariance matrix has elliptical contours.(b) A diagonal covariance matrix is an axis aligned ellipse. (c) A spherical covariance matrix has a circular shape. (d) Surface plot for the spherical Gaussian in (c).}
\label{fig:2d-Gaussions}
\end{figure}

\subsection{Multivariate Student's t-distribution}
A more robust alternative to the MVN is the multivariate Student's t-distribution, whose pdf is given by
\begin{align}
& \mathcal{T}(x|\vec{\mu},\Sigma,\nu) \nonumber \\
& \triangleq \dfrac{\Gamma(\frac{\nu+D}{2})}{\Gamma(\frac{\nu}{2})}\dfrac{|\Sigma|^{-\frac{1}{2}}}{\left(\nu\pi\right)^{\frac{D}{2}}}\left[1+\dfrac{1}{\nu}\left(\vec{x}-\vec{\mu}\right)^T\Sigma^{-1}\left(\vec{x}-\vec{\mu}\right)\right]^{-\frac{\nu+D}{2}} \\
&= \dfrac{\Gamma(\frac{\nu+D}{2})}{\Gamma(\frac{\nu}{2})}\dfrac{|\Sigma|^{-\frac{1}{2}}}{\left(\nu\pi\right)^{\frac{D}{2}}}\left[1+\left(\vec{x}-\vec{\mu}\right)^T\vec{V}^{-1}\left(\vec{x}-\vec{\mu}\right)\right]^{-\frac{\nu+D}{2}}
\end{align}
where $\Sigma$ is called the scale matrix (since it is not exactly the covariance matrix) and $\vec{V}=\nu\Sigma$. This has fatter tails than a Gaussian. The smaller $\nu$ is, the fatter the tails. As $\nu \rightarrow \infty$, the distribution tends towards a Gaussian. The distribution has the following properties
\begin{equation}
\text{mean}=\vec{\mu} \text{ , mode}=\vec{\mu} \text{ , Cov}= \dfrac{\nu}{\nu-2}\Sigma
\end{equation}

\subsection{Dirichlet distribution}
A multivariate generalization of the beta distribution is the \textbf{Dirichlet distribution}, which has
support over the probability simplex, defined by
\begin{equation}
S_K=\left\{\vec{x}:0 \leq x_k \leq 1,\sum\limits_{k=1}^K x_k=1\right\}
\end{equation}

The pdf is defined as follows:
\begin{equation}
\text{Dir}(\vec{x}|\vec{\alpha}) \triangleq \dfrac{1}{B(\vec{\alpha})}\prod\limits_{k=1}^K x_k^{\alpha_k-1}\mathbb{I}(\vec{x} \in S_K)
\end{equation}
where $B(\alpha_1,\alpha_2,\cdots,\alpha_K)$ is the natural generalization of the beta function to $K$ variables:
\begin{equation}
B(\alpha) \triangleq \dfrac{\prod_{k=1}^K \Gamma(\alpha_k)}{\Gamma(\alpha_0)} \text{ where } \alpha_0 \triangleq \sum_{k=1}^K \alpha_k
\end{equation}

Figure \ref{fig:3d-Dirichlet} shows some plots of the Dirichlet when $K=3$, and Figure \ref{fig:5d-Dirichlet} for some sampled probability vectors. We see that $\alpha_0$ controls the strength of the distribution (how peaked it is), and theαkcontrol where the peak occurs. For example, Dir$(1,1,1)$ is a uniform distribution, Dir$(2,2,2)$ is a broad distribution centered at $(1/3,1/3,1/3)$, and Dir$(20,20,20)$ is a narrow distribution centered at $(1/3,1/3,1/3)$.If $\alpha_k < 1$ for all $k$, we get “spikes” at the corner of the simplex.

\begin{figure}[hbtp]
\centering
\subfloat[]{\includegraphics[scale=.50]{3d-Dirichlet-a.png}} \\
\subfloat[]{\includegraphics[scale=.60]{3d-Dirichlet-b.png}} \\
\subfloat[]{\includegraphics[scale=.60]{3d-Dirichlet-c.png}} \\
\subfloat[]{\includegraphics[scale=.60]{3d-Dirichlet-d.png}}
\caption{(a) The Dirichlet distribution when $K=3$ defines a distribution over the simplex, which can be represented by the triangular surface. Points on this surface satisfy $0 \leq \theta_k \leq 1$ and $\sum_{k=1}^K \theta_k=1$. (b) Plot of the Dirichlet density when $\vec{\alpha}=(2,2,2)$. (c) $\vec{\alpha}=(20,2,2)$.}
\label{fig:3d-Dirichlet}
\end{figure}

\begin{figure}[hbtp]
\centering
\subfloat[$\vec{\alpha}=(0.1,\cdots,0.1)$. This results in very sparse distributions, with many 0s.]{\includegraphics[scale=.50]{5d-Dirichlet-a.png}} \\
\subfloat[$\vec{\alpha}=(1,\cdots,1)$. This results in more uniform (and dense) distributions.]{\includegraphics[scale=.50]{5d-Dirichlet-b.png}}
\caption{Samples from a 5-dimensional symmetric Dirichlet distribution for different parameter values.}
\label{fig:5d-Dirichlet}
\end{figure}

For future reference, the distribution has these properties
\begin{equation}\label{eqn:Dirichlet-properties}
\mathbb{E}(x_k)=\dfrac{\alpha_k}{\alpha_0} \text{, mode}[x_k]=\dfrac{\alpha_k-1}{\alpha_0-K} \text{, var}[x_k]=\dfrac{\alpha_k(\alpha_0-\alpha_k)}{\alpha_0^2(\alpha_0+1)}
\end{equation}

\section{Transformations of random variables}
If $\vec{x} \sim P()$ is some random variable, and $\vec{y}=f(\vec{x})$, what is the distribution of $Y$? This is the question we address in this section.

\subsection{Linear transformations}
Suppose $g()$ is a linear function:
\begin{equation}
g(\vec{x})=A\vec{x}+b
\end{equation}

First, for the mean, we have
\begin{equation}
\mathbb{E}[\vec{y}]=\mathbb{E}[A\vec{x}+b]=A\mathbb{E}[\vec{x}]+b
\end{equation}
this is called the \textbf{linearity of expectation}.

For the covariance, we have
\begin{equation}
\text{Cov}[\vec{y}]=\text{Cov}[A\vec{x}+b]=A\Sigma A^T
\end{equation}

\subsection{General transformations}
\label{sec:General-transformations}
If $X$ is a discrete rv, we can derive the pmf for $y$ by simply summing up the probability mass for all the $x$’s such that $f(x)=y$:
\begin{equation}\label{eqn:transformation-discrete}
p_Y(y)=\sum\limits_{x:g(x)=y}p_X(x)
\end{equation}

If $X$ is continuous, we cannot use Equation \ref{eqn:transformation-discrete} since $p_X(x)$ is a density, not a pmf, and we cannot sum up densities. Instead, we work with cdf’s, and write
\begin{equation}
F_Y(y)=P(Y \leq y)=P(g(X) \leq y)=\int\limits_{g(X) \leq y} f_X(x)\mathrm{d}x
\end{equation}

We can derive the pdf of $Y$ by differentiating the cdf:
\begin{equation}\label{eqn:General-transformations}
f_Y(y)=f_X(x)|\dfrac{dx}{dy}|
\end{equation}

This is called \textbf{change of variables} formula. We leave the proof of this as an exercise.

For example, suppose $X \sim U(−1,1)$, and $Y=X^2$. Then $p_Y(y)=\dfrac{1}{2}y^{-\frac{1}{2}}$.

\subsubsection{Multivariate change of variables *}
Let $f$ be a function $f:\mathbb{R}^n \rightarrow \mathbb{R}^n$, and let $\vec{y}=f(\vec{x})$. Then its Jacobian matrix $\vec{J}$ is given by
\begin{equation}
\vec{J}_{\vec{x} \rightarrow \vec{y}} \triangleq \frac{\partial \vec{y}}{\partial \vec{x}} \triangleq \left(\begin{array}{ccc}
\frac{\partial y_1}{\partial x_1} & \cdots & \frac{\partial y_1}{\partial x_n} \\
\vdots & \vdots & \vdots \\
\frac{\partial y_n}{\partial x_1} & \cdots & \frac{\partial y_n}{\partial x_n}
\end{array}\right)
\end{equation}
$|\mathrm{det}(\vec{J})|$ measures how much a unit cube changes in volume when we apply $f$.

If $f$ is an invertible mapping, we can define the pdf of the transformed variables using the Jacobian of the inverse mapping $\vec{y} \rightarrow \vec{x}$:
\begin{equation}\label{eqn:Multivariate-transformation}
p_y(\vec{y})=p_x(\vec{x})|\mathrm{det}(\frac{\partial \vec{x}}{\partial \vec{y}})|=p_x(\vec{x})|\mathrm{det}(\vec{J}_{\vec{y} \rightarrow \vec{x}})|
\end{equation}

\subsection{Central limit theorem}
Given $N$ random variables $X_1,X_2,\cdots,X_N$, each variable is \textbf{independent and identically distributed}\footnote{\url{http://en.wikipedia.org/wiki/Independent_identically_distributed}}(\textbf{iid} for short), and each has the same mean $\mu$ and variance $\sigma^2$, then
\begin{equation}
\dfrac{\sum\limits_{i=1}^n X_i-N\mu}{\sqrt{N}\sigma} \sim \mathcal{N}(0,1)
\end{equation}
this can also be written as
\begin{equation}
\dfrac{\bar{X}-\mu}{\sigma/\sqrt{N}} \sim \mathcal{N}(0,1) \quad \text{, where } \bar{X} \triangleq \dfrac{1}{N}\sum\limits_{i=1}^n X_i
\end{equation}

\section{Monte Carlo approximation}
\label{sec:Monte-Carlo-approximation}
In general, computing the distribution of a function of an rv using the change of variables formula can be difficult. One simple but powerful alternative is as follows. First we generate $S$ samples from the distribution, call them $x_1,\cdots,x_S$. (There are many ways to generate such samples; one popular method, for high dimensional distributions, is called Markov chain Monte Carlo or MCMC; this will be explained in Chapter TODO.) Given the samples, we can approximate the distribution of $f(X)$ by using the empirical distribution of $\left\{f(x_s)\right\}_{s=1}^S$. This is called a \textbf{Monte Carlo approximation}\footnote{\url{http://en.wikipedia.org/wiki/Monte_Carlo_method}}, named after a city in Europe known for its plush gambling casinos.

We can use Monte Carlo to approximate the expected value of any function of a random variable. We simply draw samples, and then compute the arithmetic mean of the function applied to the samples. This can be written as follows:
\begin{equation}
\mathbb{E}[g(X)]=\int g(x)p(x)\mathrm{d}x \approx \dfrac{1}{S}\sum\limits_{s=1}^S f(x_s)
\end{equation}
where $x_s \sim p(X)$.

This is called \textbf{Monte Carlo integration}\footnote{\url{http://en.wikipedia.org/wiki/Monte_Carlo_integration}}, and has the advantage over numerical integration (which is based on evaluating the function at a fixed grid of points) that the function is only evaluated in places where there is non-negligible probability.

\section{Information theory}

\subsection{Entropy}
\label{sec:Entropy}
The entropy of a random variable $X$ with distribution $p$, denoted by $\mathbb{H}(X)$ or sometimes $\mathbb{H}(p)$, is a measure of its uncertainty. In particular, for a discrete variable with $K$ states, it is defined by
\begin{equation}
\mathbb{H}(X) \triangleq -\sum\limits_{k=1}^{K}{p(X=k)\log_2p(X=k)}
\end{equation}

Usually we use log base 2, in which case the units are called \textbf{bits}(short for binary digits). If we use log base $e$ , the units are called \textbf{nats}.

The discrete distribution with maximum entropy is the uniform distribution (see Section XXX for a proof). Hence for a K-ary random variable, the entropy is maximized if $p(x = k)=1/K$; in this case, $\mathbb{H}(X)=\log_2K$.

Conversely, the distribution with minimum entropy (which is zero) is any \textbf{delta-function} that puts all its mass on one state. Such a distribution has no uncertainty.

\subsection{KL divergence}
One way to measure the dissimilarity of two probability distributions, $p$ and $q$ , is known as the \textbf{Kullback-Leibler divergence}(\textbf{KL divergence})or \textbf{relative entropy}. This is defined as follows:
\begin{equation}
\mathbb{KL}(P||Q) \triangleq
\sum\limits_{x}{p(x)\log_2\dfrac{p(x)}{q(x)}}
\end{equation}
where the sum gets replaced by an integral for pdfs\footnote{The KL divergence is not a distance, since it is asymmetric. One symmetric version of the KL divergence is the \textbf{Jensen-Shannon divergence}, defined as $JS(p_1,p_2)=0.5\mathbb{KL}(p_1||q)+0.5\mathbb{KL}(p_2||q)$,where $q=0.5p_1+0.5p_2$}. The KL divergence is only defined if P and Q both sum to 1 and if $q(x)=0$ implies $p(x)=0$ for all $x$(absolute continuity). If the quantity  $0\ln0$ appears in the formula, it is interpreted as zero because $\lim\limits_{x \to 0}x\ln x$. We can rewrite this as
\begin{equation}\begin{split}
\mathbb{KL}(p||q) & \triangleq \sum\limits_{x}{p(x)\log_2p(x)}-\sum\limits_{k=1}^{K}{p(x)\log_2q(x)} \\
    & =\mathbb{H}(p)-\mathbb{H}(p,q)
\end{split}\end{equation}
where $\mathbb{H}(p,q)$ is called the \textbf{cross entropy},
\begin{equation}\label{eqn:cross-entropy}
\mathbb{H}(p,q)=\sum\limits_{x}{p(x)\log_2q(x)}
\end{equation}

One can show (Cover and Thomas 2006) that the cross entropy is the average number of bits needed to encode data coming from a source with distribution $p$ when we use model $q$ to define our codebook. Hence the “regular” entropy $\mathbb{H}(p)=\mathbb{H}(p,p)$, defined in section \S \ref{sec:Entropy},is the expected number of bits if we use the true model, so the KL divergence is the diference between these. In other words, the KL divergence is the average number of \emph{extra} bits needed to encode the data, due to the fact that we used distribution $q$ to encode the data instead of the true distribution $p$.

The “extra number of bits” interpretation should make it clear that $\mathbb{KL}(p||q) \geq 0$, and that the KL is only equal to zero if $q = p$. We now give a proof of this important result.

\begin{theorem}
(\textbf{Information inequality}) $\mathbb{KL}(p||q) \geq 0 \text{ with equality iff } p=q$.
\end{theorem}

One important consequence of this result is that \emph{the discrete distribution with the maximum
entropy is the uniform distribution}.

\subsection{Mutual information}
\label{sec:Mutual-information}
\begin{definition}
\textbf{Mutual information} or \textbf{MI}, is defined as follows:
\begin{equation}\begin{split}
\mathbb{I}(X;Y) & \triangleq \mathbb{KL}(P(X,Y)||P(X)P(X)) \\
    & =\sum\limits_x\sum\limits_yp(x,y)\log\dfrac{p(x,y)}{p(x)p(y)}
\end{split}\end{equation}
We have $\mathbb{I}(X;Y) \geq 0$ with equality if $P(X,Y)=P(X)P(Y)$. That is, the MI is zero if the variables are independent.
\end{definition}

To gain insight into the meaning of MI, it helps to re-express it in terms of joint and conditional entropies. One can show that the above expression is equivalent to the following:
\begin{eqnarray}
\mathbb{I}(X;Y)&=&\mathbb{H}(X)-\mathbb{H}(X|Y)\\
               &=&\mathbb{H}(Y)-\mathbb{H}(Y|X)\\
               &=&\mathbb{H}(X)+\mathbb{H}(Y)-\mathbb{H}(X,Y)\\
               &=&\mathbb{H}(X,Y)-\mathbb{H}(X|Y)-\mathbb{H}(Y|X)
\end{eqnarray}
where $\mathbb{H}(X)$ and $\mathbb{H}(Y)$ are the \textbf{marginal entropies}, $\mathbb{H}(X|Y)$ and $\mathbb{H}(Y|X)$ are the \textbf{conditional entropies}, and $\mathbb{H}(X,Y)$ is the \textbf{joint entropy} of $X$ and $Y$, see Fig. \ref{fig:mi}\footnote{\url{http://en.wikipedia.org/wiki/Mutual_information}}.

\begin{figure}[hbtp]
\centering
    \includegraphics[scale=.25]{mutual-information.png}
\caption{Individual $\mathbb{H}(X),\mathbb{H}(Y)$, joint $\mathbb{H}(X,Y)$, and conditional entropies for a pair of correlated subsystems $X,Y$ with mutual information $\mathbb{I}(X;Y)$.}
\label{fig:mi}
\end{figure}

Intuitively, we can interpret the MI between $X$ and $Y$ as the reduction in uncertainty about $X$ after observing $Y$, or, by symmetry, the reduction in uncertainty about $Y$ after observing $X$.

A quantity which is closely related to MI is the \textbf{pointwise mutual information} or \textbf{PMI}. For two events (not random variables) $x$ and $y$, this is defined as
\begin{equation}
PMI(x,y) \triangleq \log\dfrac{p(x,y)}{p(x)p(y)}=\log\dfrac{p(x|y)}{p(x)}=\log\dfrac{p(y|x)}{p(y)}
\end{equation}

This measures the discrepancy between these events occuring together compared to what would be expected by chance. Clearly the MI of $X$ and $Y$ is just the expected value of the PMI. Interestingly, we can rewrite the PMI as follows:
\begin{equation}
PMI(x,y)=\log\dfrac{p(x|y)}{p(x)}=\log\dfrac{p(y|x)}{p(y)}
\end{equation}

This is the amount we learn from updating the prior $p(x)$ into the posterior $p(x|y)$ , or equivalently, updating the prior $p(y)$ into the posterior $p(y |x)$ .


\subsection{Estimating the Markov chain stochastic matrix}

\subsubsection{Introduction}

Given a sequence: \(x_1,...x_n\).

The likelihood is:

\(L=\prod_{i=2}^n p_{x_{i-1},x_i}\)

If there are \(k\) states we can rewrite this as:

\(L=prod_{i=1}^k\prod_{j=1}^k n_{ij}p_{ij}\)

Where \(p_{ij}\) is the chance of moving from state \(i\) to state \(j\), and \(n_{ij}\) is the number of transtions between \(i\) and \(j\).

The log likelhood is:

\(\ln L=\sum_{i=1}^k\sum_{j=1}^kn_{ij}\ln p_{ij}\)

\subsubsection{Constrained optimisation}

Not all parameters are free. All probabilities must sum to \(1\).

\(\ln L=\sum_{i=1}^k\sum_{j=1}^kn_{ij}\ln p_{ij}-sum_{i=1}\lambda_i (\sum_{j=1}p_{ij}-1)\)

This gives us:

\(\hat p_{ij}=\dfrac{n_{ij}}{\sum_k n_{ik}}\)


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS622: Theory of Formal Languages
% Copyright 2014 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 1}

Let $x, y, z$ be three words from $A^*$ such that $xy = yz$ and $x \neq \lambda$.
Prove that there exist $u, v \in A^*$ such that $x = uv$, $y = (uv)^n u$ and $z = vu$ for some $n \in \mathbb{N}$.

\subsection*{Solution}

Statement is verified using proof of induction on the length of $|xy|$.
Since $x \neq \lambda$, $u$ and $v$ cannot be $\lambda$ at the same time, following that the length of $|xy|$ is never 0 and is always a multiple of 2.

\begin{itemize}

\item
Initial Step: $|xy| = 2$\\
Suppose $u = \lambda$ and $v \in A$. No restrictions are imposed on $n \in \mathbb{N}$.

\begin{equation}\label{1eq1}
xy = uv(uv)^nu = vv^n = v^{n+1}
\end{equation}

\begin{equation}\label{1eq2}
yz = (uv)^nuvu = v^nv = v^{n+1}
\end{equation}

From \eqref{1eq1} and \eqref{1eq2}, it follows that $xy = yz$ and the statement holds true.

\item
Assumption Step: $|xy| = m$\\
It is assumed that $xy = yz$ for some $u,v \in A^*$ and some $n \in \mathbb{N}$.
That is there are some $u, v$ and $n$ that

\begin{equation}\label{1eq3}
xy = uv(uv)^nu = (uv)^nuvu = yz
\end{equation}

\item
Induction Step $|xy| = m + 2$\\
Suppose $u$ and $v$ are same words from $A^*$ that were used in assumption step.
By assuming $n^\prime = n + 1$ where $n$ was the same $n$ that was used in assumption step, we will have:

\begin{equation}\label{1eq4}
xy = uv(uv)^{n^\prime}u = uv(uv)^{n+1}u = uv(uv)^nuvu = (xy)^\prime vu
\end{equation}

\begin{equation}\label{1eq5}
yz = (uv)^{n^\prime}uvu = (uv)^{n+1}uvu = (uv)^nuvuvu = (yz)^\prime vu
\end{equation}

where $(xy)^\prime$ and $(yz)^\prime$ are those in assumption step.
From \eqref{1eq3} we know $(xy)^\prime = (yz)^\prime$. Therefore, \eqref{1eq4} = \eqref{1eq5} and the induction is complete.

\end{itemize}


    \documentclass{article}
    \usepackage[utf8]{inputenc}
    \usepackage{amsmath}
    \begin{document}
 %Title
 \section{Collatz Sequence for \(n=15\) }
    %Collatz Function
    \[
        f(n)=
        \begin{cases}
        \frac{n}{2}, & n \mod 2=0
        \\
        3n+1, &n \mod 2=1
        \end{cases} \\
    \]
    % Path for given N
    \(\textbf{Path for f(15)}\\[3mm]f(n), n=15
 \\
 \Rightarrow 3(15) + 1
 \\
 \Rightarrow n=46
 \\[3mm]
f(n), n=46
 \\
 \Rightarrow \frac{46}{2}
 \\
 \Rightarrow n=23
 \\[3mm]
f(n), n=23
 \\
 \Rightarrow 3(23) + 1
 \\
 \Rightarrow n=70
 \\[3mm]
f(n), n=70
 \\
 \Rightarrow \frac{70}{2}
 \\
 \Rightarrow n=35
 \\[3mm]
f(n), n=35
 \\
 \Rightarrow 3(35) + 1
 \\
 \Rightarrow n=106
 \\[3mm]
f(n), n=106
 \\
 \Rightarrow \frac{106}{2}
 \\
 \Rightarrow n=53
 \\[3mm]
f(n), n=53
 \\
 \Rightarrow 3(53) + 1
 \\
 \Rightarrow n=160
 \\[3mm]
f(n), n=160
 \\
 \Rightarrow \frac{160}{2}
 \\
 \Rightarrow n=80
 \\[3mm]
f(n), n=80
 \\
 \Rightarrow \frac{80}{2}
 \\
 \Rightarrow n=40
 \\[3mm]
f(n), n=40
 \\
 \Rightarrow \frac{40}{2}
 \\
 \Rightarrow n=20
 \\[3mm]
f(n), n=20
 \\
 \Rightarrow \frac{20}{2}
 \\
 \Rightarrow n=10
 \\[3mm]
f(n), n=10
 \\
 \Rightarrow \frac{10}{2}
 \\
 \Rightarrow n=5
 \\[3mm]
f(n), n=5
 \\
 \Rightarrow 3(5) + 1
 \\
 \Rightarrow n=16
 \\[3mm]
f(n), n=16
 \\
 \Rightarrow \frac{16}{2}
 \\
 \Rightarrow n=8
 \\[3mm]
f(n), n=8
 \\
 \Rightarrow \frac{8}{2}
 \\
 \Rightarrow n=4
 \\[3mm]
f(n), n=4
 \\
 \Rightarrow \frac{4}{2}
 \\
 \Rightarrow n=2
 \\[3mm]
f(n), n=2
 \\
 \Rightarrow \frac{2}{2}
 \\
 \Rightarrow n=1
 \\[3mm]

    \section{Credits}
    %Maybe add link to gitrepo and other shit, idk
    This is created using Collatzer (https://github.com/Z1aaan/Collatzer).
    Created By: Z1aaan

    README:
    A program created to visualize and simulate a user-given value for \textit{N}
    and see what happens when it is put under the Collatz function.
    \end{document}


\subsection{Set-builder notation}

\subsubsection{Notation}

We can use short-hand to describe sets.

\(\{x\in S: P(x)\}\)

This defines a set by a restriction. For example we will later be able to define natural numbers above \(5\) as:

\(\{x\in \mathbb{N} : x>5\}\)

\subsubsection{Class builder notation}

Emuneration can be done through set builder notation too

Can define sets formally! defintion doesn't just affect sets

\(\forall x (x\in C \leftrightarrow P(x))\)

NB: We're not saying C exists

Can then use examples of equiv class

\(\forall x (x\in C \leftrightarrow x=x)\)


\section{Some Properties of Determinant}
\begin{lemma}
    Let $A,B$ be sqaure matrices, then $\det(AB)=\det(A)\det(B)$.
\end{lemma}
\begin{proof}
    Direct expansion does the trick, but alternatively we can define $d_A$ via $d_A(B)=\det(AB)$ which is obviously a volume form.
    Therefore $d_A(B)=d_A(I)\det(B)=\det(A)\det(B)$.
\end{proof}
\begin{definition}
    Let $A\in M_n(F)$.
    We say $A$ is singular if $\det A=0$, otherwise we say $A$ is nonsingular.
\end{definition}
\begin{lemma}
    If $A$ is invertible then it is non-singular.
\end{lemma}
\begin{proof}
    Suppose $B$ is an inverse of $A$, then $\det(A)\det(B)=\det(AB)=\det(I)=1\neq 0$ therefore $\det A\neq 0$.
\end{proof}
\begin{remark}
    In particular $\det(A^{-1})=(\det A)^{-1}$ if $A$ is invertible.
\end{remark}
\begin{theorem}
    Let $A\in M_n(F)$, the followings are equivalent:\\
    1. $A$ is invertible.\\
    2. $A$ is non-singular.\\
    3. $r(A)=n$.
\end{theorem}
\begin{proof}
    We just need to show (ii) implies (iii) since we have already done all others.
    Suppose $r(A)<n$, then $\dim\operatorname{span}(A^{(1)},\ldots, A^{(n)})<n$, so there is some $\lambda_1,\ldots,\lambda_n$ not all zero such that $\sum_i\lambda_iA^{(i)}=0$.
    In particular, there is some $j$ such that $\lambda_j\neq 0$ and hence $c_j=-\sum_{i\neq j}(\lambda_i/\lambda_j)A^{(i)}$.
    Expand $\det A$ using multilinearity gives a linear combination of determinants with repeated entries, which is zero as $\det$ is an alternating form.
\end{proof}
\begin{remark}
    By the theorem, it follows easily that the equation $Ax=y$ where $A\in M_n(F),x,y\in F^n$ has a unique solution iff $\det A\neq 0$.
\end{remark}
\begin{lemma}
    Similar matrices have the same determinant.
\end{lemma}
\begin{proof}
    $\det(PAP^{-1})=\det(P)\det(A)\det(P)^{-1}=\det(A)$ for any invertible $P$.
\end{proof}
Therefore the following definition makes sense.
\begin{definition}
    If $\alpha:V\to V$ be an endomorphism, then $\det\alpha=\det[\alpha_{B,B}]$ for any $B$ basis of $V$.
\end{definition}
\begin{theorem}
    $\det:L(V,V)\to F$ satisfies:\\
    1. $\det\operatorname{id}_V=1$.\\
    2. $\det(\alpha\circ\beta)=\det(\beta)\det(\alpha)$.\\
    3. $\det\alpha\neq 0$ iff $\alpha$ is invertible.
    If indeed $\det\alpha\neq 0$, then $\det(\alpha^{-1})=(\det\alpha)^{-1}$.
\end{theorem}
\begin{proof}
    Choose any basis and the rest follows from previous discussions.
\end{proof}
\begin{lemma}
    Let $A\in M_k(F),B\in M_l(F),C\in M_{k,l}(F)$.
    Consider
    $$M_{k+l}(F)\ni M=\left( \begin{array}{c|c}
        A&C\\ \hline
        0&B
    \end{array} \right)$$
    Then $\det M=(\det A)(\det B)$.
\end{lemma}
\begin{proof}
    Write $n=k+l$ and $M=(m_{ij})$.
    Observe that $m_{\sigma(i)i}=0$ if $i\le k$ and $\sigma(i)>k$.
    So for $m_{\sigma(1)1}\cdots m_{\sigma(n)n}\neq 0$, we must have the decomposition $\sigma_1\circ\sigma_2$ such that $\sigma_1$ fixes anything but $1,\ldots, k$ and $\sigma_2$ fixes anything but $k+1,\ldots,n$.
    But then for such $\sigma$, we have $m_{\sigma(j)j}=a_{\sigma_1(j)j}$ for any $j\in\{1,\ldots,k\}$ and $m_{\sigma(j)j}=b_{\sigma_2(s)s}$ where $s=j-k$ for any $j\in\{k+1,\ldots,n\}$.
    Observe also that $\epsilon(\sigma)=\epsilon(\sigma_1)\epsilon(\sigma_2)$, therefore
    \begin{align*}
        \det M&=\sum_{\sigma\in S_n}\epsilon(\sigma)m_{\sigma(1)1}\cdots m_{\sigma(n)n}\\
        &=\sum_{\sigma_1\in S_k,\sigma_2\in S_l}\epsilon(\sigma_1)\epsilon(\sigma_2)a_{\sigma_1(1)1}\cdots a_{\sigma_1(k)k}b_{\sigma_2(1)1}\cdots b_{\sigma_2(l)l}\\
        &=\sum_{\sigma_1\in S_k}\epsilon(\sigma_1)a_{\sigma_1(1)1}\cdots a_{\sigma_1(k)k}\sum_{\sigma_2\in S_l}\epsilon(\sigma_2)b_{\sigma_2(1)1}\cdots b_{\sigma_2(l)l}\\
        &=(\det A)(\det B)
    \end{align*}
    as desired.
\end{proof}
\begin{corollary}
    If $A_1,\ldots,A_k$ are square matrices, then
    $$\det\begin{pmatrix}
        A_1&&&\ast\\
        &A_2&&\\
        &&\ddots&\\
        0&&&A_k
    \end{pmatrix}=(\det A_1)(\det A_2)\cdots (\det A_k)$$
\end{corollary}
\begin{proof}
    Induction on $k$.
\end{proof}
In particular,
$$\det\begin{pmatrix}
    \lambda_1&&&\ast\\
    &\lambda_2&&\\
    &&\ddots&\\
    0&&&\lambda_k
\end{pmatrix}=\lambda_1\cdots\lambda_k$$
\begin{remark}
    Why is it called a volume from?
    Cconsider the volume form $(\mathbb R^3)^3\to F$ via $(a,b,c)\mapsto a\cdot(b\times c)$.
    Then we know that geometrically this gives the (signed) volume of the parallelopiped formed by $a,b,c$.
\end{remark}


\subsection{Overfitting}

Overfitting is a risk. Instead we split to test, train. risk of using too many features. more features always improve training score, not necessarily test score.

As model gets more complex, both test and train do better. however at some point, test stops doing better, overfitting

Structural risk minimisation can address this trade off. use test and training sets. train model on train, rate it on test

Structural minimisation curve has accuracy of boths sets over complexity

To avoid overfitting:

+ reduce number of features
+ do a model selection
+ use regularisation
+ do cross validation

Can choose other model parameters

How to evalute model?

\subsection{K-fold cross-validation}

Can do k-fold cross validation. given algo A and dataset D, divide D into k equal sized subsets

For each subset, train the model on all other subsets and test on the other subset. average error between folds


\newcommand{\fab}{f_{{\bf a},{\bf b}}}

\lab{Algorithms}{Monte Carlo Integration}{Monte Carlo Integration}
\objective{Learn the benefits of using Monte Carlo methods when numerically
integrating in many dimensions.}

Many numerical integration techniques rely on finding deterministic ways to
approximate the integrand, and using these approximations in a Riemann
summation. However, these techniques do not generalize well when the integration
takes place in a high-dimensional space. This lab presents a way to numerically
approximate the value
\[I = \int_{\Omega} f({\bf x}) \;d{\bf x} \]
when $\Omega$ is a high-dimensional space, and we assume the value of the
integral is finite.

\section*{Monte Carlo Integration}
Monte Carlo methods are algorithms that use random numbers to approximate a
desired result. (The name refers to the famous Monte Carlo Casino.) Monte Carlo
integration uses random vectors taken uniformly from the domain to approximate
an intergal.

Suppose we wish to approximate the above integral. We note the average, or
expected, value of the function $f$ on the space $\Omega$ is given by
\[f_{\text{ave}} = \frac{1}{V(\Omega)}\int_{\Omega} f({\bf x}) \;d{\bf x} \]
where $V$ denotes volume. However, the law of large numbers from probability
theory roughly states that
\[f_{\text{ave}} = \lim_{n \to \infty} \frac{1}{n} \sum_{k=1}^{n} f({\bf x}_k) \]
when the ${\bf x}_i$ are uniform random samples of $\Omega$. We see that we can
approximate the desired value as
\[I \approx I_N = V(\Omega)\frac{1}{N}\sum_{k=1}^{N} f({\bf x}_k) \]
for large $N$, and that this approximation gets better as $N$ increases.
Remarkably, for the estimation of the error we have
\[\abs{I - I_N} \approx V(\Omega)\frac{1}{\sqrt{N}} =
O\left(\frac{1}{\sqrt{N}}\right) \]
The decay of the estimation of the error depends only on $N$, and not on the
dimension of the space $\Omega$. This is the key reason why Monte Carlo
integration works well in high dimensions.

\section*{Example}

In order to demonstrate the benefits of Monte Carlo integration, we introduce a
special class of functions. Let $\Omega = [0,2 \pi ]^d$ be the unit cube in
$\mathbb{R}^d$. For ${\bf a} \in \mathbb{N}^d$, ${\bf b} \in \Omega$, define
\[\fab({\bf x}) = \sum_{i=1}^{d} \sin(2\pi a_i x_i + b_i) \]
This class of functions has desirable properties for our example. First, the
functions are defined on a space of arbitrary dimension. Also, it is easy
to analytically calculate
\[\int_{\Omega} \fab({\bf x}) \; d {\bf x} = 0 \]
for any ${\bf a}, {\bf b}, d$. However, the functions are also interestingly
shaped, and are not trivially approximated. We finally note that $V(\Omega) = 1$.

\begin{problem}
\label{prob:fab}
Define a function that accepts an $N \times d$ dimensional array, makes a random
choice for ${\bf a}$ and ${\bf b}$, and returns $\fab({\bf x}_n)$ evaluated for
each of the $N$ vectors. Restrict your choice of ${\bf a}$ to the set
$\{1,2,\ldots,9\}^d$.
\end{problem}

\begin{problem}
\label{prob:grid}
Estimate $\int_{\Omega} \fab({\bf x}) \;d{\bf x}$ using rectangular quadrature
(or another method), for many values of ${\bf a}, {\bf b}, d, N$. How large can
you make $d$?  What problems do you face as $d$ increases? What happens to the
average error as $d$ and $N$ vary?

\nopagebreak[2]

\vspace{5mm}
\noindent
{\it Hint.} First, create an $N \times d$ grid of points in the domain. Using
the result from {\bf Problem 1}, evaluate the function at each point. Since
$V(\Omega) = 1$, our rectangular quadrature estimate is equivalent to finding
the mean of the resultant array. (To see this is true, examine the
one-dimensional case; the generalization is immediate.) Since the true value of
the integral is $0$, the error is simply the absolute value of this mean.

\nopagebreak[2]

\begin{verbatim}
import numpy as np

def grid(d=3, n=100):
    """
    Return a (d**n) by (d) dimensional array, giving a
    uniform, discrete grid of the space [0,1]**d, with
    n points along each dimension.
    """

    linspaces = [np.linspace(0,1,n, endpoint=False)
                 for i in xrange(d)]
    flatmesh = [x.flatten() for x in np.meshgrid(*linspaces)]
    return np.column_stack(flatmesh)

In []: xx = grid()

In []: np.abs(np.mean(problem_two(xx)))
Out[]: error

In []: np.mean([np.abs(np.mean(problem_one(xx)))
                for i in xrange(100)])
Out[]: average_error
\end{verbatim}
\end{problem}

\break
\begin{problem}
\label{prob:mc}
Estimate $\int_{\Omega} \fab({\bf x}) \;d{\bf x}$ using Monte Carlo integration
for many values of ${\bf a}, {\bf b}, d, N$. How large can you make $d$? What
happens to the average error as $d$ and $N$ vary? Compare this method with the
previous.

\nopagebreak[2]

\vspace{5mm}
\noindent
{\it Hint.} First, choose $N$ random points from the domain. Using the result
from {\bf Problem 1}, evaluate the function at each point. Since $V(\Omega) =
1$, Monte Carlo integration is equivalent to finding the mean of the resultant
array. Since the true value of the integral is $0$, the error is simply the
absolute value of this mean.

\nopagebreak[2]

\begin{verbatim}
import numpy as np

In []: xx = np.random.rand(1000000, 3)

In []: np.abs(np.mean(problem_one(xx)))
Out[]: error

In []: np.mean([np.abs(np.mean(problem_one(xx)))
                for i in xrange(100)])
Out[]: average_error
\end{verbatim}
\end{problem}

\let\fab\undefined


\lab{Applications}{SVD-based Image Compression}{SVD}

\objective{Explore the SVD as a method of image compression}

The singular value decomposition is very useful.  In this lab, we are going to explore how the SVD can be used to compress image data.  Recall that the SVD is a decomposition of an $m \times n$ matrix $A$
of rank $r$ into the product $A = U \Sigma V^H$, where $U$ and $V$
are unitary matrices having dimensions $m \times m$ and $n \times n$,
respectively, and $\Sigma$ is an $m \times n$ diagonal matrix
\begin{equation*}
\Sigma = \mbox{diag}(\sigma_1,\sigma_2,\ldots,\sigma_r,0,\ldots,0)
\end{equation*}
where $\sigma_1 \geq \sigma_2 \geq \ldots \geq \sigma_r > 0$ are the
singular values of $A$.  Upon closer inspection, we can write
\begin{equation*}
U = \begin{pmatrix}U_1 & U_2\end{pmatrix}, \quad \Sigma =
\begin{pmatrix}\Sigma_r & 0\\0 & 0\end{pmatrix}, \quad V =
\begin{pmatrix}V_1 & V_2\end{pmatrix},
\end{equation*}
where $U_1$ and $V_1$ have dimensions $m\times r$ and $n\times r$
respectively and $\Sigma_r$ is the $r\times r$ diagonal matrix of
(nonzero) singular values.  Multiplying this out yields the reduced
form of the SVD
\begin{equation*}
A =
\begin{pmatrix}U_1 & U_2\end{pmatrix}
\begin{pmatrix}\Sigma_r & 0\\0 & 0\end{pmatrix}
\begin{pmatrix}V^H_1 \\ V^H_2\end{pmatrix} =
U_1 \Sigma_r V_1^H
\end{equation*}

\subsection*{Low rank data storage}
If the rank of a given matrix is significantly smaller than its
dimensions, the reduced form of the SVD offers a way to store $A$ with less memory.
Without the SVD, an $m\times n$ matrix requires storing $m*n$ values.
By decomposing the original matrix into the SVD reduced form, $U_1$, $\Sigma_r$ and $V_1$ together require $(m*r)+r+(n*r)$ values.
  Thus if $r$ is much smaller than both $m$ and $n$,
we can obtain considerable efficiency.  For example, suppose
$m=100$, $n=200$ and $r=20$.
Then the original matrix would require storing $20,000$ values whereas the reduces form of the SVD only requires storing $6020$ values.

\subsection*{Low rank approximation}
The reduced form of the SVD also provides a way to approximate a
matrix with another one of lower rank. This idea is used in many areas of applied
mathematics including signal processing, statistics, semantic
indexing (search engines), and control theory.  If we are given a matrix $A$ of rank $r$,
we can find an approximate matrix $\widehat A$ of rank $s<r$ by taking
the SVD of $A$ and setting all of its singular values after
$\sigma_s$ to zero, that is,
\begin{equation*}
\Sigma_{\widehat A} = \sigma_1, \sigma_2, \ldots, \sigma_s,\sigma_{s+1}=0,\ldots,\sigma_r=0
\end{equation*}
and then multiplying the matrix back together again.  The more singular values we keep, the closer our approximation is to $A$.
  The number of singular values we decide to preserve depends on how close of an approximation we need and what our size requirements are for $U_1$, $\Sigma_{\widehat A}$, and $V_1$.
Try plotting the the singular values.  We have plotted the singular values to the image below.  Matrix rank is on the x-axis and the eigenvalues are the y-axis.  Note that SVD orders the singluar values from greatest to least.  The greatest eigenvalues contribute most to the image while the smallest eigenvalues hardly contribute anything to the final approximation.  By looking at the graph we can have a rough idea of how many singular values we need to preserve to have a good approximation of $A$.  The matrix rank of the image below is $670$.  However, as the plot shows, we could easily approximate the image using only the first half of the singular values.

\begin{center}
\includegraphics[scale=.2]{hubble_red.png}

\includegraphics[scale=.4]{hubble_svals.pdf}
\end{center}

\begin{lstlisting}[style=python]
: import scipy as sp
: import numpy.linalg as nla
: A = sp.array([[1,1,3,4],[5,4,3,7],[9,10,10,12],[13,14,15,16],[17,18,19,20]])
: nla.matrix_rank(A)
: U,s,Vt = nla.svd(A)
: S = sp.diag(s)
: Ahat = sp.dot(sp.dot(U[:,0:3], S[0:3,0:3]), Vt[0:3,:])
: nla.matrix_rank(Ahat)
: nla.norm(A)-nla.norm(Ahat)
\end{lstlisting}

% We can compute the rank of a matrix by looking at the number of nonzero singular values in the SVD decompositon.  The following function will compute the rank of a matrix.  The \li{sp.finfo(float).eps} gives us the smallest representable postitive number such that $1.0+\mbox{eps} \neq 1.0$  Anything smaller than the eps value is numerically zero to the computer.
% \begin{lstlisting}[style=python]
% : def matrix_rank(X):
% :.... S = nla.svd(X, compute_uv=False)
% :.... tol = S.max()*sp.finfo(S.dtype).eps
% :.... return sum(S>tol)
% : matrix_rank(Ahat)
% \end{lstlisting}

Note that $\widehat A$ is ``close'' to the original matrix $A$, but
that its rank is 3 instead of 4.

\subsection*{Application to Imaging}

Enter the following into IPython (note that any image you might have will work):
\begin{lstlisting}
: import matplotlib.pyplot as plt
: X = sp.misc.imread('fingerprint.png')[:,:,0].astype(float)
: X.nbytes      #number of bytes needed to store X
: sp.misc.imshow(X)
\end{lstlisting}
Computing the SVD of your image is simple.  Remember to make the singluar values a diagonal matrix before multiplying.
\begin{lstlisting}
: U,s,Vt = la.svd(X)
: S = sp.diag(s)
\end{lstlisting}
In the next code block, $n$ repsents the desired rank of the output.
\begin{lstlisting}
: n=50
: u1, s1, vt1 = U[:,0:n], S[0:n,0:n], Vt[0:n,:]
: Xhat = sp.dot(sp.dot(u1, s1), vt1)
: (u1.nbytes+sp.diag(s1).nbytes+vt1.nbytes) - X.nbytes   #should be negative
: sp.misc.imshow(Xhat)
\end{lstlisting}

%write code that will caluculate final image column by column in rank 1 approximations.  It is actually smaller to transmit the matrix col by col than chunks of cols.  We only send 70% of the data if we send col by col than entire matices.  1017856bytes vs 1445888 bytes.
\begin{problem}
A law enforcement agency has been needing to efficiently store over 50,000 fingerprints.  They have decided to use an SVD based compression algorithm.  Your job is to try several parameters for the SVD algorithm and recommend those parameters that retain the highest quality but compress the most.  There should be no smearing or blocking in reconstructed final image and fingerprint detail must be retained (otherwise the fingerprint is worthless).  As part of your recommendation, calculate how much memory would be needed on average to store each compressed fingerprint.  Expand your results to say how much space could be saved if the entire database of fingerprints were compressed using your algorithm.
\end{problem}
% \begin{problem}
% Explore the clown picture for several different values of rank.
% Conduct the experiments described above.  Note that the original
% image takes 64,000 integers to store.  Compare this with the storage
% needs for various lower-rank SVD approximations. What conclusions
% can you draw? Expirement with other images we've used in this book.
% \end{problem}


\lstset{
  mathescape = true,
  basicstyle = \ttfamily}
\newcommand{\dollar}{\mbox{\textdollar}}

\subsection{Matrix -- matrix multiplication}

\begin{frame}[fragile]{Matrix -- matrix multiplication}

$C = A^{m\cdot r}\cdot B^{r\cdot n} $

$$ c_{i j} = \sum_{k=1}^r a_{i k}  x_{k j} $$
\begin{enumerate}[a)]
  \item dot version
\end{enumerate}

\scriptsize{
\begin{lstlisting}[belowskip=-1.4 \baselineskip, language=Matlab, mathescape]
function: C=matmat.ijk(A,B)
  m=rows(A); r=cols(A); n=cols(B); C(1:m,1:n)=0
  for i=1:m
    for j=1:n $\tikzmark{mark4}$
      for k=1:r $\tikzmark{mark1}$
        C(i,j)=C(i,j)+A(i,k) $\cdot$ B(k,j) $\tikzmark{mark3}$
      end $\tikzmark{mark2}$
    end $\tikzmark{mark5}$
  end
end matmat.ijk
\end{lstlisting}
}
\AddNote{mark1}{mark2}{mark3}{1.5cm}{inner loop $\tikzmark{mark6}$}
\AddNote{mark4}{mark5}{mark6}{1.6cm}{middle loop}
\end{frame}

\subsection{Matrix multiplication: loop ordering and properties}

\begin{frame}[fragile]{Matrix multiplication: loop ordering and properties}

$i j k \rightarrow 3! = 6 \text{ sposobów} $ \\

\vspace{5px}
\begin{tabular}{ |p{1cm}||p{1.6cm}||p{3.4cm}||p{3.7cm}|  }
 \hline

 loop order& Inner loop & Middle loop&Inner loop data access\\
 \hline
 ijk   & dot   &vector x matrix&   A by row, B by column\\
\hline
 jik&   dot & matrix x vector  &A by row, B by column\\
\hline
 ikj &saxpy & row gaxpy&  B by row\\
\hline
 jki    &saxpy & column gaxpy&  A by column\\
\hline
 kij &   saxpy  & row outer product& B by row\\
\hline
 kji& saxpy  & column outer product & A by column\\
 \hline
\end{tabular}

\begin{itemize}
\item Operations $\rightarrow$ dot, saxpy,
\item modes of access.
\end{itemize}

Wybór zależy od architektury komputera.
\end{frame}


\subsection{Defining \(e\) as a binomial}

\subsubsection{Lemma}

\(f(n,i)=\dfrac{n!}{n^i(n-i)!}\)

\(f(n,i)=\dfrac{(n-i)!\prod_{j=n-i+1}^nj}{n^i(n-i)!}\)

\(f(n,i)=\dfrac{\prod_{j=n-i+1}^nj}{n^i}\)

\(f(n,i)=\dfrac{\prod_{j=1}^i(j+n-i)}{n^i}\)

\(f(n,i)=\prod_{j=1}^i\dfrac{j+n-i}{n}\)

\(f(n,i)=\prod_{j=1}^i(\dfrac{n}{n}+\dfrac{j-i}{n})\)

\(f(n,i)=\prod_{j=1}^i(1+\dfrac{j-i}{n})\)

\(\lim_{n\rightarrow \infty }f(n,i)=\lim_{n\rightarrow \infty }\prod_{j=1}^i(1+\dfrac{j-i}{n})\)

\(\lim_{n\rightarrow \infty }f(n,i)=\prod_{j=1}^i1\)

\(\lim_{n\rightarrow \infty }f(n,i)=1\)

\subsubsection{Defining \(e\)}

We know that:

\((a+b)^n=\sum^n_{i=0} a^i b^{n-i} \dfrac{n!}{i!(n-i)!}\)

Let's set \(b=1\)

\((a+1)^n=\sum^n_{i=0} a^i \dfrac{n!}{i!(n-i)!}\)

Let's set \(a=\dfrac{1}{n}\)

\((1+\dfrac{1}{n})^{n}=\sum^{n}_{i=0} \dfrac{1}{n^i} \dfrac{n!}{i!(n-i)!}\)

\((1+\dfrac{1}{n})^{n}=\sum^{n}_{i=0} \dfrac{1}{i!} \dfrac{n!}{n^i(n-i)!}\)

\(\lim_{n\rightarrow \infty }(1+\dfrac{1}{n})^{n}=\lim_{n\rightarrow \infty }\sum^{n}_{i=0} \dfrac{1}{i!} \dfrac{n!}{n^i(n-i)!}\)

From the lemma above:

\(\lim_{n\rightarrow \infty }(1+\dfrac{1}{n})^{n}=\sum^{\infty }_{i=0} \dfrac{1}{i!}\)

\(e=\sum^{\infty }_{i=0} \dfrac{1}{i!}\)

\subsubsection{Defining \(e^x\)}

\(e=\lim_{n\rightarrow \infty }(1+\dfrac{1}{n})^{n}\)

\(e^x=\lim_{n\rightarrow \infty }(1+\dfrac{1}{n})^{nx}\)

\(e^x=\lim_{n\rightarrow \infty }\sum^{nx}_{i=0} \dfrac{1}{n^i} \dfrac{(nx)!}{i!(nx-i)!}\)

\(e^x=\lim_{n\rightarrow \infty }\sum^{nx}_{i=0} \dfrac{x^i}{i!} \dfrac{(nx)!}{(nx)^i(nx-i)!}\)

From the lemma:

\(e^x=\sum^{\infty }_{i=0} \dfrac{x^i}{i!}\)


\documentclass[../main.tex]{subfiles}

\begin{document}
    \section{Parametric Surfaces}
    We consider parametric surfaces, which are higher-dimensional analogues of parametric curves. A parametric surface is parameterized by a function \(\vec{r}:D\to\mathbb{R}^3\), where \(\vec{r}(u,v)\) is continuous and \(D\subseteq\mathbb{R}^2\) is a domain. As with curves, a given surface may be parameterized in many ways.

    The graph of a function \(z=f(x,y)\) can be thought of as a parametric surface parameterized by
    \[
    \vec{r}(u,v)=u\i+v\j+f(u,v)\k
    \]

    Spheres are also common parametric surfaces. In spherical coordinates, we may parameterize the sphere in terms of \(\theta\) and \(\phi\) as
    \[
    \vec{r}(\theta,\phi)=a\cos{\theta}\sin{\phi}\i+a\sin{\theta}\cos{\phi}\j+a\cos{\phi}\k
    \]
    where \(\theta\in[0,2\pi]\) and \(\phi\in[0,\pi]\).

    There is also an analogue to the helical curve, which resembles a spiral stairway, and is parameterized by
    \[
    \vec{r}(u,v)=u\cos{v}\i+u\sin{v}\j+v\k
    \]
    where \(u\in[0,1]\) and \(v\in[0,2\pi]\).

    We generally assume, when working with surfaces, that \(\vec{r}\) and its derivatives are continuous.

    \begin{definition}{Surface Closure}{}
        A surface is \emph{closed} if it bounds a closed subset of \(\mathbb{R}^3\).
    \end{definition}

    As with parametric curves, we wish to find equations for the normal vectors of surfaces, as well as their tangent planes. It is also necessary to find area elements of surfaces, enabling us to integrate over them.

    We wish to find the normal vector to a parametric surface parameterized by \(\vec{r}\) at some point \(P=\vec{r}(u_0,v_0)\). We can construct a pair of parametric curves parameterized by \(\vec{r}(u,v_0)\) and \(\vec{r}(u_0,v)\), both of which pass through \(P\). The normal vector of the surface at \(P\) must then be perpendicular to both curves, or equivalently to their tangent vectors at \(P\). This yields the equation
    \[
    \vec{n}=\vec{r}_u\times\vec{r}_v
    \]

    This normal vector is not independent of parameterization, but it does always point in the direction perpendicular to the surface.

    We also wish to find area elements of our surface by continually subdividing the surface along lines with constant \(u\) and constant \(v\). As we continually take smaller and smaller subdivisions, the area element approximates a parallelogram. The vectors spanning this parallelogram are \(\vec{r}_u\,du\) and \(\vec{r}_v\,dv\), so the area element is given by
    \[
    dS=|\vec{r}_u\,du\times\vec{r}_v\,dv|=|\vec{r}_u\times\vec{r}_v|\,du\,dv=|\vec{n}|\,du\,dv
    \]

    Using this, we can integrate functions along surfaces. If we wish to compute the integral of some function \(f\) along some surface \(\mathcal{S}\), we have
    \[
    \iint_\mathcal{S}f(x,y,z)\,dS=\iint_Df(\vec{r}(u,v))|\vec{n}|\,du\,dv
    \]

    One simple application of surface integrals is computing the areas of surfaces. The area of a surface \(\mathcal{S}\) is given by
    \[
    \iint_\mathcal{S}\,dS
    \]

    We can also compute the masses of surfaces with variable densities. The mass of a surface \(\mathcal{S}\) with variable density \(\rho\) is given by
    \[
    \iint_\mathcal{S}\rho\,dS
    \]

    It is also possible to compute the average value of functions along surfaces. The average value of \(f\) on \(\mathcal{S}\) is given by
    \[
    \frac{\iint_\mathcal{S}f\,dS}{\iint_\mathcal{S}\,dS}
    \]

    \begin{example}{}{}
        Find the area of the part of the plane \(2x-y+z=10\) which lies above the disk \((x-3)^2+y^2\geq1\).
        \tcblower
        We begin by parameterizing the plane. The plane may be treated as a graph, and is parameterized by
        \[
        \vec{r}(x,y)=x\i+y\j+(10-2x+y)\k
        \]

        Which has \(\vec{r}_x=\i-2\k\) and \(\vec{r}_y=\j+\k\), so we compute
        \[
        \vec{n}=\vec{r}_x\times\vec{r}_y=2\i-\j+\k
        \]
        and
        \[
        |\vec{n}|=\sqrt{6}
        \]
        so the area of the ellipse is
        \[
        \iint_D\sqrt{6}\,dx\,dy
        \]
        where \(D\) is the disk. Normally, we would have to compute a double integral, but because \(|\vec{n}|\) is constant and \(D\) is simply a circle, we have
        \[
        \iint_D\sqrt{6}\,dx\,dy=\sqrt{6}\pi
        \]
    \end{example}

        The problem of finding the area of a graph may be solved more generally. Let \(\mathcal{S}\) be the graph of a function \(z=f(x,y)\), where \((x,y)\in D\). We wish to find the area of \(\mathcal{S}\). Because \(\mathcal{S}\) may be parameterized by
        \[
        \vec{r}(x,y)=x\i+y\j+f(x,y)\k
        \]
        which immediately yields \(\vec{r}_x=\i+f_x\k\) and \(\vec{r}_y=\j+f_y\k\), and consequently
        \[
        \vec{n}=-f_x\i-f_y\j+\k
        \]
        which yields
        \[
        |\vec{n}|=\sqrt{f_x^2+f_y^2+1}
        \]

        As such, the area of the graph \(z=f(x,y)\) is simply given by
        \[
        \iint_D\sqrt{f_x^2+f_y^2+1}\,dx\,dy
        \]

        \begin{example}{}{}
            Find the area of the part of \(z=x^2+y^2\) below \(z=9\).
            \tcblower
            First, we compute
            \[
            |\vec{n}|=\sqrt{z_x^2+z_y^2+1}
            \]
            meaning that the area is
            \[
            \iint_D\sqrt{4x^2+4y^2+1}\,dx\,dy=\frac{\pi}{6}(37\sqrt{37}-1)
            \]
            where the integral is computed by conversion to polar coordinates, and \(D=\{(x,y):x^2+y^2<9\}\).
        \end{example}
\end{document}


\subsection{Inverse matrices}

An invertible matrix implies that if the matrix is multiplied by another matrix, the original matrix can be recovered.

That is, if we have matrix \(A\), there exists matrix \(A^{-1}\) such that \(AA^{-1}=I\).

Consider a linear map on a vector space.

\(Ax=y\)

If \(A\) is invertible we can have:

\(A^{-1}Ax=A^{-1}y\)

\(x=A^{-1}y\)

If we set \(y=\mathbf 0\) then:

\(x=\mathbf 0\)

So if there is a non-zero vector \(x\) such that:

\(Ax=\mathbf 0\) then \(A\) is not invertible.

\subsection{Left and right inverses}

That is, for all matrices \(A\), the left and right inverses of \(B\), \(B_L^{-1}\) and \(B_R^{-1}\), are defined such that:

\(A(BB_R^{-1})=A\)

\(A(B_L^{-1}B)=A\)

Left and right inversions are equal

Note that if the left inverse exists then:

\(B_L^{-1}B=I\)

And if the right inverse exists:

\(BB_R^{-1}=I\)

Let’s take the first:

\(B_L^{-1}B=I\)

\(B_L^{-1}BB_L^{-1}=B_L^{-1}\)

\(B_L^{-1}BB_L^{-1}-B_L^{-1}=0\)

\(B_L^{-1}(BB_L^{-1}-I)=0\)

\subsection{Inversion of products}

\((AB)(AB)^{-1}=I\)

\(A^{-1}AB(AB)^{-1}=A^{-1}\)

\(B^{-1}B(AB)^{-1}=B^{-1}A^{-1}\)

\((AB)^{-1}=B^{-1}A^{-1}\)

\subsection{Inversion of a diagonal matrix}

\(DD^{-1}=I\)

\(D_{ii}D_{ii}^{-1}=1\)

\(D_{ii}^{-1}=\dfrac{1}{D_{ii}}\)


\subsection{Inverses of elementary matrices}

Suppose we have applied a row operation to a matrix $A$. Consider the
row operation required to return $A$ to its original form, i.e., to
undo the row operation. It turns out that this action is described by
the inverse of an elementary matrix. The following theorem ensures
that the inverse of each elementary matrix is itself an elementary
matrix.

\begin{theorem}{Inverses of elementary matrices}{inverse-elementary-matrix}
  Every elementary matrix is invertible and its inverse is also an
  elementary matrix%
  \index{elementary matrix!inverse}%
  \index{matrix!elementary matrix!inverse}.
\end{theorem}

In fact, the inverse of an elementary matrix is constructed by doing
the {\em reverse} row operation on $I$. $E^{-1}$ is obtained by
performing the row operation which would carry $E$ back to $I$.

\begin{itemize}
\item If $E$ is obtained by switching rows $i$ and $j$, then $E^{-1}$
  is also obtained by switching rows $i$ and $j$.
\item If $E$ is obtained by multiplying row $i$ by the scalar $k$,
  then $E^{-1}$ is obtained by multiplying row $i$ by the scalar
  $\frac{1}{k}$.
\item If $E$ is obtained by adding $k$ times row $i$ to row $j$, then
  $E^{-1}$ is obtained by subtracting $k$ times row $i$ from row $j$.
\end{itemize}

\begin{example}{Inverse of an elementary matrix}{inverse-elementary-matrix}
  Find $E^{-1}$, where $E$ is the elementary matrix
  \begin{equation*}
    E
    =
    \begin{mymatrix}{rr}
      1 & 0 \\
      0 & 2
    \end{mymatrix}
  \end{equation*}
\end{example}

\begin{solution}
  $E$ is obtained from the $2\times 2$ identity matrix by multiplying
  the second row by $2$. In order to carry $E$ back to the identity,
  we need to multiply the second row of $E$ by $\frac{1}{2}$.  Hence,
  $E^{-1}$ is given by
  \begin{equation*}
    E^{-1}
    =
    \begin{mymatrix}{rr}
      1 & 0 \\
      0 & \frac{1}{2}
    \end{mymatrix}.
  \end{equation*}
\end{solution}


\subsection{Stokes' theorem}

Stokes' theorem says that in typical problems a surface integral can be
computed using a line integral.
(There is some fine print regarding continuity and boundary conditions.)
This is a useful theorem because usually the line integral is easier to
compute.
In rectangular coordinates the equivalence between a line integral
on the left and a surface integral on the right is
%
$$\oint P\,dx+Q\,dy+R\,dz
=\int\!\!\!\int_S(\mathop{\rm curl}{\bf F})\cdot{\bf n}\,d\sigma
$$
%
where ${\bf F}=(P,Q,R)$.
For $S$ parametrized by $x$ and $y$ we have
$${\bf n}\,d\sigma=\left(
\frac{\partial S}{\partial x}\times\frac{\partial S}{\partial y}
\right)dx\,dy$$

\noindent
Example:
Let ${\bf F}=(y,z,x)$ and let $S$ be the part of the paraboloid
$z=4-x^2-y^2$
that is above the $xy$ plane.
The perimeter of the paraboloid is the circle $x^2+y^2=2$.
The following script computes both the line and surface integrals.
It turns out that we need to use polar coordinates for the
line integral so that {\it defint} can succeed.

{\color{blue}
\begin{verbatim}
-- www.eigenmath.org/stokes-theorem.txt
"Surface integral"
z = 4 - x^2 - y^2
F = (y,z,x)
S = (x,y,z)
f = dot(curl(F),cross(d(S,x),d(S,y)))
x = r cos(theta)
y = r sin(theta)
defint(f r,r,0,2,theta,0,2pi)
"Line integral"
x = 2 cos(t)
y = 2 sin(t)
z = 4 - x^2 - y^2
P = y
Q = z
R = x
f = P d(x,t) + Q d(y,t) + R d(z,t)
f = circexp(f)
defint(f,t,0,2pi)
\end{verbatim}
}

\noindent
This is the result when the script runs.
Both the surface integral and the line integral
yield the same result.

\bigskip
\noindent
Surface integral\\
$\displaystyle -4\pi$\\
Line integral\\
$\displaystyle -4\pi$


\chapter[Boundary value problems]{Boundary value problems for Ordinary Differential Equations}

\section{Introduction}

A boundary value problem consists in finding a solution of a
differential equation in an interval $[a,b]$ that satisfies
constraints at both ends (boundary conditions).  A typical example of
a boundary value problem is
%
\begin{equation}
  y'' = f(x,y,y'), \quad y(a) = A, \quad y(b) = B, \quad a \le x \le b.
  \label{eq:1}
\end{equation}
%
The question of the existence of solutions of problems
like~(\ref{eq:1}) is non trivial.  However, here we assume that we are
always trying to find a solution to a well posed problem, that is a
problem that has one and only one solution.

Boundary value problems describe many important physical and
engineering problems, from the sagging of a beam to the shape of the
electron orbitals in atoms.

\section{Shooting method}

We describe this method to solve boundary value problems using
equation~(\ref{eq:1}) as an example.  One natural way to attack this
problem is to solve the related initial-value problem, with a guess of
the appropriate initial value $y'(a)$. Then we can integrate the
equation to obtain the approximate solution, hoping that $y(b)=B$. If
not, then the guessed value of $y'(a)$ can be altered and we can try
again. The process is called {\bf shooting}, and there are ways of
doing it systematically.

\medskip

\noindent
Denote the guessed value of $y'(a)$ by $z$, so that the corresponding
initial value problem is
%
\begin{equation}
  y''=f(x,y,y'), \quad y(a)=A, \quad y'(a)=z, \quad a \le x \le b.\label{ivp}
\end{equation}
%
The solution of this problem is $y=y(x,z)$. The objective is to select
$z$ so that $y(b,z)=B$. We call
%
\begin{equation}
  \phi(z) \equiv y(b,z)-B ,
  \label{eq:phiz}
\end{equation}
%
so that our objective is simply to solve for $z$ the equation
%
\begin{equation}
 \phi(z)=0 \,.
 \label{eq:2}
\end{equation}
%

In general equation~(\ref{eq:2}) is nonlinear.  We can use any
numerical root finding method to solve it.  One of the most effective
is the secant method:
%
\begin{equation*}
  z_{n+1} = z_{n} - \phi(z_n) \frac{z_n - z_{n-1}}
  {\phi(z_n)-\phi(z_{n-1})} ,
  \qquad n > 1.
\end{equation*}
%
The advantage of this method is that we need only to know $\phi(z)$ in
order to apply it and that it is fairly accurate.  The disadvantage is
that we need two initial guesses $z_0$ and $z_1$ in order to start the
method.

\medskip

In the case of Newton's method, the iteration formula for $z$ is
%
\begin{equation}
 z_{n+1} = z_n-\frac{\phi(z_n)}{\phi'(z_n)}.
 \label{eq:3}
\end{equation}
%
To determine $\phi'(z)$, we can proceed in two ways.  Firstly, we
could evaluate the derivative numerically by writing
%
\begin{equation*}
  \phi'(z_n) \simeq \frac{\phi(z_n+h)-\phi(z_n)}{h}, \qquad h \ll 1.
\end{equation*}
%
A second option is to introduce
%
\begin{equation*}
  u(x,z)=\pdv{y}{z}
\end{equation*}
%
and we differentiate with respect to $z$ all the equations in
(\ref{ivp}). This becomes
%
\begin{equation*}
  u'' = f_y(x,y,y') u + f_{y'}(x,y,y') u', \quad u(a)=0, \quad u'(a)=1.
\end{equation*}
%
The last differential equation is called the \textit{first variational
  equation}.  It can integrated numerically using for $y$ and $y'$ the
values obtained by the numerical integration of equation~(\ref{ivp}).
Finally, by differentiating~(\ref{eq:2}) we obtain
%
\begin{equation*}
  \phi'(z)=u(b,z)
\end{equation*}
%
that enables us to use the Newton's method to find a root of $\phi$.

\bigskip

To summarise, the steps of the algorithm are as follows:
%
\begin{enumerate}
  %
\item Guess a value for the missing initial condition, $z_0$, and set
  the iteration counter $n$ to zero, $n=0$.
  %
\item Compute $y=y(x,z_n)$ by integrating the initial value problem
  %
  \begin{equation*}
    y''=f(x,y,y'), \quad y(a)=A, \quad y'(a)=z_n, \quad a\le x\le b.
  \end{equation*}
%
\item Compute $\phi(z_n) = y(b,z_n) - B$.  Use a nonlinear solver
  (e.g.\ Newton's or the secant method) to find a new value $z_{n+1}$
  for the missing initial condition.
%
\item If $z_{n+1}$ is not sufficiently accurate increase the iteration
  counter by one, $n \to n + 1$ and repeat steps (2-4) until
  $\phi(z_{n+1})$ is sufficiently small.
%
\end{enumerate}

\section{Finite-difference method}

Another approach to solving boundary value problems is to discretise
the derivatives that appear in the differential equation and transform
the problem in a linear system.  As an example, consider the equation
%
\begin{equation}
  y'' + p(x) y' + q(x) y = f(x), \quad y(a)=A, \quad y(b)=B,
  \quad a \le x \le b.\label{lbvp}
\end{equation}
%
We can represent the derivatives using their finite difference
approximations:
%
\begin{subequations}
  \label{er}
  \begin{align}
    y'(x_i) &= \frac{y_{i+1}-y_{i-1}}{2h} -
    \frac{h^2}{6} y'''(\xi_1), \\
    y''(x_i) &=  \frac{y_{i+1}+y_{i-1}-2 y_i}{h^2} -
    \frac{h^2}{12}y^{(4)}(\xi_2),
  \end{align}
\end{subequations}
%
where $x_i=a+i h$, $i=0,1,2,...,n+1$, $h=(b-a)/(n+1)$. Thus, the
discrete version of~(\ref{lbvp}) is the system of $n+2$ linear
algebraic equations
%
\begin{align*}
  y_0 &=A, \\
  y_{i-1} \left ( 1 - \frac{h}{2} p_i \right ) -
  y_i \left ( 2 - h^2 q_i \right ) +
  y_{i+1} \left (1 + \frac{h}{2} p_i \right )  &= h^2 f_i , \\
  y_{n+1} &=B,
\end{align*}
%
where $p_i=p(x_i)$, $q_i=q(x_i)$, $f_i=f(x_i)$.  This can be solved as
such or reduced to an $n \times n$ system.  For example, if $n=4$, the
corresponding matrix form is given by
%
\begin{equation*}
  \begin{pmatrix}
    -2 + h^2 q_1 & 1 + \frac{h}{2} p_1 & 0 & 0 \\
    1 - \frac{h}{2} p_2 & -2  + h^2 q_2 & 1 + \frac{h}{2} p_2 & 0\\
    0 & 1 - \frac{h}{2} p_3 & -2  + h^2 q_3 & 1 + \frac{h}{2} p_3 \\
    0 & 0 & 1 - \frac{h}{2} p_4 & -2  + h^2 q_4
  \end{pmatrix}
  \begin{pmatrix} y_1\\ y_2\\ y_3\\ y_4 \end{pmatrix} =
  \begin{pmatrix} F_1\\ F_2\\ F_3\\ F_4 \end{pmatrix}
\end{equation*}
%
with
%
\begin{align*}
  & &  F_1 & = h^2 f_1 - A (1 - \frac{h}{2} p_1), &
  F_2 & = h^2 f_2, \\
  & &  F_3 & = h^2 f_3, & F_4 & = h^2 f_4 - B(1 + \frac{h}{2} p_4).
\end{align*}
%
This system is tridiagonal and can be solved by a special form of the
Gaussian elimination algorithm. In the general case, we have an
$n\times n$ system
%
\begin{equation}
  T \boldsymbol{y} = \boldsymbol{F},
  \label{eq:5}
\end{equation}
%
where $T$ is a tridiagonal $n\times n$ matrix,
$\boldsymbol{y}=(y_1,...,y_n)^T$ and $\boldsymbol{F}=(F_1,...,F_n)^T$.

To analyse the error induced by the representation~(\ref{er}) of the
derivatives we introduce an error vector $(e_1,...,e_n)$, where
$e_i=y(x_i)-y_i$, $i=1,\ldots,n$ and $y(x)$ is assumed to be the exact
solution.  Substituting into the finite difference
representation~(\ref{eq:5}) of equation~(\ref{lbvp}), we obtain the
system
%
\begin{equation*}
  T \be = h^4 \boldsymbol{G}
\end{equation*}
%
where $\boldsymbol{G}$ is a constant vector.   If $\det T \ne 0$, we have
%
\begin{equation*}
  \boldsymbol{e}=h^4T^{-1}\boldsymbol{G}.
\end{equation*}
%
We can use this relation to show that the error is $O(h^2)$ [sic] as
$h \to 0$.

\section{The Ritz method}

The Ritz, Galerkin, Square Least methods are used widely on problems
in which it is required to determine an unknown function. Of course,
boundary value problems for differential equations are in this
category. Suppose we are confronted with a problem of the form
%
\begin{equation}
  \mathcal{L} u(x)=f(x)
\end{equation}
%
in which $\mathcal{L}$ is the linear operator
%
\begin{equation}
  \mathcal{L} u=-\dv{}{x} \left[ p(x) \dv{u}{x} \right ] + q(x)u = f(x).
\end{equation}
%
Here $f(x)$ is a given function and $u(x)$ is is the function to be
determined from the equation and boundary conditions
%
\begin{equation}
  u(a)=0, \qquad u(b)=0.
\end{equation}
%
We assume that $p(x)\ge p_0>0$, $q(x)\ge 0$ for $x\in [a,b]$. This
means that the operator $\mathcal{L}$ is symmetric and positive
definite in the real Hilbert space $L_2[a,b]$ since, using the
integration by parts, we have
%
\begin{align*}
  <\mathcal{L}u,u> & = \mint{a}{b}{\mathcal{L} u \cdot u}{x} \\
  & = -\mint{a}{b}{(p u')'u}{x} + \mint{a}{b}{q u^2}{x} \\
  & = \mint{a}{b}{p(u')^2}{x} + \mint{a}{b}{q u^2}{x} \ge 0
\end{align*}
%
for arbitrary $u$. The quantity $<\mathcal{L}u,u>$ is called the
energy of the element $u$ relative the operator $\mathcal{L}$.

\noindent Consider a linear quadratic functional
%
\begin{subequations}
  \label{fun}
  \begin{align}
    J(u) & = <\mathcal{L}u,u>-2<f,u> \\
         & = \mint{a}{b}{p(u')^2}{x} + \mint{a}{b}{q u^2}{x} -2 \mint{a}{b}{f
         u}{x}.
  \end{align}
\end{subequations}
%
A theorem states that if the equation $\mathcal{L}u(x)=f(x)$ has a
solution $u_0$, this solution minimises the functional
$J(u)$. Conversely, if there exists an element $u_0$ that minimises
the functional $J(u)$, this element satisfies the equation
$\mathcal{L}u(x)=f(x)$.  The proof is based on the ability to reduce
the functional $J(u)$ to the form
%
\begin{equation*}
  J(u)=<\mathcal{L}(u-u_0),u-u_0>-<\mathcal{L}u_0,u_0>,
\end{equation*}
%
and then analysing the function
%
\begin{align*}
 g(t) & = J[u_0(x)+t v(x)] \\
      & = <\mathcal{L}u_0,u_0> + 2t<\mathcal{L}u_0,v> +
      t^2<\mathcal{L} v,v> - 2<f,u_0> - 2t<f,v>.
\end{align*}
%
The basic idea of the Ritz method consists in replacing the boundary
value problem for $\mathcal{L}u(x)=f(x)$ by the problem of minimising
the functional $J(u)$. Suppose we select basis functions $u_1$,
$u_2$,...,$u_n$,... in $L_2[a,b]$.  Hence, we seek a solution of the
form
%
\begin{equation}
  u_n(x)=\sum_{m=1}^n c_m u_m(x),\label{rsol}
\end{equation}
%
where $c_1$, $c_2$,...,$c_n$ are unknown constants.  Substituting this
sum into the functional $J(u)=<\mathcal{L}u,u>-2<f,u>$, we obtain
%
\begin{equation}
  J(u_n)=J(c_1,c_2,...,c_n)=\sum_{m,k=1}^n c_m c_k<\mathcal{L}u_m,u_k>-
  2\sum_{m=1}^n c_m<f,u_m>,
\end{equation}
%
which is a quadratic form with respect to the unknown constants
$c_1,c_2,...,c_n$.  Looking for a minimum element of
$J(u_n)=J(c_1,c_2,...,c_n)$, we require that
%
\begin{equation}
  \pdv{}{c_m} J(c_1,c_2,...,c_n)=0, \qquad m=1,2,...,n.
\end{equation}
%
Thus, we come to the linear $n\times n$ system of algebraic equations
for the unknown constants $c_1$, $c_2$,...,$c_n$
%
\begin{equation}
  \sum_{m=1}^nc_m<\mathcal{L}u_m,u_k>=<f,u_k>,~~~~~k=1,2,...,n,\label{sys}
\end{equation}
%
which is called the Ritz system.  After we have found the unknown
constants $c_1$, $c_2$,...,$c_n$, the expression (\ref{rsol}) gives us
an approximate solution.

\section{The collocation method}

\subsection{Introduction}

The method of collocation can be used to tackle many problems in the
numerical analysis of ordinary and partial differential equations.
Here we give a general description of this method that can be adapted
easily to the solution of a boundary value problem for an ordinary
differential equation.

Suppose that we have a linear operator $\mathcal{L}$ (for example, a
linear differential equation) that acts on a space of functions.  We
wish to solve the equation
%
\begin{equation}
  \mathcal{L} u(x) = w(x) ,
  \label{cwk4.eq:1}
\end{equation}
%
where $w(x)$ is a known function and $u(x)$ is the solution we are
looking for.  We indicate with
%
\begin{equation*}
  \{v_1(x), v_2(x), \ldots, v_n(x)\}
\end{equation*}
%
a set of $n$ known functions (\textit{basis functions}) and we write
the (unknown) solution of equation~(\ref{cwk4.eq:1}) as
%
\begin{equation}
  u(x) = \sum_{j=1}^n c_j v_j(x) ,
  \label{cwk4.eq:2}
\end{equation}
%
where the coefficients $c_j$ are (at this stage) unknown.  In general
$u(x)$ written as in equation~(\ref{cwk4.eq:2}) cannot be an exact
solution of equation~(\ref{cwk4.eq:1}).  However, we can find a set of
coefficients $\{c_j\}$ such that~(\ref{cwk4.eq:2}) is a good
approximation of the solution of the equation~(\ref{cwk4.eq:1}).

\medskip

\noindent \textbf{Remark} - Note that the series solution of a
differential equation falls into this class of methods.   In that case
the functions $v_j(x)=x^j$, with $j=0,1,2,\ldots,n$ and the
coefficients $c_j$ of the expansion are determined by requiring that
equation~(\ref{cwk4.eq:1}) is satisfied at each order in $x$.

\subsection{The norm method}

There are various ways of defining ``good approximation''.  As a first
example, consider the boundary value problem
%
\begin{equation}
  u''(x) - u(x) = 0 \, , \qquad
  u(0) = 1, \quad u(1) = e.
  \label{cwk4.eq:4}
\end{equation}
%
We wish to find an approximate solution to this problem using as basis
the functions
%
\begin{equation*}
  v_0(x) = 1 , \quad v_1(x) = x \quad \text{and} \quad v_2(x) = x^2,
\end{equation*}
%
and write the approximate solution of equation~(\ref{cwk4.eq:4}) as a
polynomial of order two:
%
\begin{equation}
  u^{(a)}(x) = c_0 v_0(x) + c_1 v_1(x) + c_2 v_2(x) =
  c_0 + c_1 x + c_2 x^2 .
 \label{cwk4.eq:5}
\end{equation}
%
It is clear that there are no values of the constants $c_j$ that can
make $u^{(a)}(x)$ as defined in equation~(\ref{cwk4.eq:5}) equal to
the exact solution of equation~(\ref{cwk4.eq:4}),
%
\begin{equation}
  u^{(e)}(x) = e^x
  \label{cwk4.eq:3}
\end{equation}
%
for all values of $x$.  However, we can attempt to find some values of
these parameters that minimise the error in approximating $u^{(e)}(x)$
with $u^{(a)}(x)$.  There are many different measures of the error of
the approximation.  For example, we could require that the
coefficients $c_j$ are such that

\begin{enumerate}
  %
\item $u^{(a)}(x)$ satisfies the boundary conditions, i.e.\ $u^{(a)}(0)=1$ and $u^{(a)}(1) = e$.
  %
\item $u^{(a)}(x)$ satisfies as well as possible
  equation~(\ref{cwk4.eq:4}) in the sense that
  %
  \begin{equation}
    F(c_j) \equiv
    \left \| \dv[2]{}{x} u^{(a)}(x) - u^{(a)}(x) \right \|_2^2 =
    \mint{0}{1}{\left [ \dv[2]{}{x} u^{(a)}(x) - u^{(a)}(x)
    \right ]^2}{x}
    \label{eq:10}
  \end{equation}
  %
  is as small as possible.  A criterion (distantly?) related to this
  is used in the finite element method to solve partial differential
  equations.
  %
\end{enumerate}

\begin{figure}
  \centerline{\includegraphics[width=80mm]{figures/ColloEner}}
  \caption{\label{fig:ColloEner} \it Graphs of the exact
    [Eq.~(\ref{cwk4.eq:3})] and approximate [Eq.~(\ref{eq:12})]
    solution of the boundary value problem~(\ref{cwk4.eq:4}).}
\end{figure}

The left boundary condition fixes $c_0$:
%
\begin{equation}
  u^{(a)}(0) = c_0 = 1 .
  \label{eq:9}
\end{equation}
%
The right boundary condition gives a relation between $c_1$ and $c_2$:
%
\begin{equation}
  u^{(a)}(1) = c_0 + c_1 + c_2 = e \implies c_1 = e - 1 - c_2 .
  \label{eq:8}
\end{equation}
%
Substituting~(\ref{eq:9}) and~(\ref{eq:8}) into~(\ref{cwk4.eq:4})
gives that $F(c_j)$ defined by~(\ref{eq:10}) is
%
\begin{equation}
  F(c_2) = \frac{47}{10} c_2^2 - \frac{13}{6}(1+e)c_2 + \frac{1+e+e^2}{3} .
  \label{eq:11}
\end{equation}
%
We solve for $c_2$ by requiring that $F(c_2)$ is as small as possible,
i.e.\ that the derivative of $F(c_2)$ with respect to $c_2$ is zero.
Differentiating~(\ref{eq:11}) we obtain
%
\begin{equation*}
 \dv{F}{c_2} = \frac{47}{5} c_2 - \frac{13}{6}(1+e) = 0 \implies
 c_2 = \frac{65}{282}(1+e) ,
\end{equation*}
%
so that from equation~(\ref{eq:8}) we have
%
\begin{equation*}
  c_1 = \frac{217 e - 347}{282}
\end{equation*}
%
and the approximate solution of the boundary value
problem~(\ref{cwk4.eq:4}) is
%
\begin{equation}
  u^{(a)}(x) = 1 + \frac{217 e - 347}{282} x + \frac{65}{282}(1+e) x^2 .
  \label{eq:12}
\end{equation}
%
The graphs of the approximate and exact solutions are shown in
Figure~\ref{fig:ColloEner}: the match is pretty impressive for a three
node approximation.

\subsection{The collocation method}

In the \textit{collocation} method we substitute the approximate
solution~(\ref{cwk4.eq:2}) in~(\ref{cwk4.eq:1}),
%
\begin{equation*}
  \mathcal{L} \sum_{j=1}^n c_j v_j(x) = w(x) \implies
  \sum_{j=1}^n c_j \mathcal{L} v_j(x) = w(x) ,
\end{equation*}
%
and we require that this equation should be satisfied at a set of $n$
\textit{collocation points} $\{x_j\}$, i.e.\ that the coefficients
$c_j$ are the solution of the system of linear equations
%
\begin{equation*}
  \sum_{j=1}^n c_j \mathcal{L} v_j(x_k) = w(x_k) , \qquad k=1,2,\ldots,n.
\end{equation*}
%
Note that the values of the basis functions $v_j(x)$ at the
collocation points $x_k$ should be such that the matrix of the
coefficients of this system is non-singular.

\begin{figure}
  \centerline{\includegraphics[width=80mm]{figures/Collocation}}
  \caption{\label{fig:Collocation} \it Graphs of the exact
    [Eq.~(\ref{cwk4.eq:3})] and approximate [Eq.~(\ref{eq:13})]
    solution of the boundary value problem~(\ref{cwk4.eq:4}).}
\end{figure}

Once again consider as an example the boundary value
problem~(\ref{cwk4.eq:4}) and write the approximate solution as in
equation~(\ref{cwk4.eq:5}).  We require that it should satisfy the
boundary conditions as in equations~(\ref{eq:9}) and~(\ref{eq:8}), so
that the approximate solution is now given by
%
\begin{equation*}
  u^{(a)}(x) = 1 + (e-1-c_2) x + c_2 x^2 .
\end{equation*}
%
Finally, we require that this function should satisfy the differential
equation~(\ref{cwk4.eq:4}) at $x=1/2$.  Taking into account that
%
\begin{equation*}
 \dv[2]{}{x} u^{(a)}(x) = 2 c_2
\end{equation*}
%
we have that this requirement is equivalent to
%
\begin{equation*}
  2 c_2 - \left [ 1 + (e-1-c_2) \frac{1}{2} +
    c_2 \left ( \frac{1}{2} \right )^2 \right ] = 0 \implies
  c_2 = \frac{2}{9}(1+e) \implies c_1 = \frac{7 e - 11}{9}
\end{equation*}
%
so that the approximate solution is given by
%
\begin{equation}
  u^{(a)}(x) = 1 + \frac{7 e - 11}{9} x + \frac{2}{9}(1+e) x^2 .
  \label{eq:13}
\end{equation}
%
The graphs of the approximate and exact solutions are shown in
Figure~\ref{fig:Collocation}: the match is pretty impressive for a
three node approximation and is comparable, but different, from that
in Figure~\ref{fig:ColloEner}.

\smallskip

\noindent \textbf{Remark} - In general it is not advisable to use as
basis functions the powers of $x$ or a uniformly spaced set of nodes.
Orthogonal polynomials (like the Legendre and the Chebyschev
polynomials) and non-uniform grids (e.g.\ Gauss-Lobatto grids) give
more accurate and numerically stable results.

\section*{Further reading}

Topics covered here are also covered in
\begin{itemize}
\item Chapter 11 of Linz \& Wang, \textit{Exploring Numerical Methods}
  (QA297 LIN),
\item Chapter 8 of Kincaid \& Cheney, \textit{Numerical Analysis}
  (QA297 KIN),
\item Chapter 13 (and to some extent 14) of S{\"u}li \& Mayers,
  \textit{An Introduction to Numerical Analysis} (not in library).
\end{itemize}


% !TEX root = Main.tex
\section{Matrix Approximation \& Reconstruction}
problem of SVD is unobserved entries. Want to only consider I = observed entries. \\
$min_{rank(B)=k}[\sum_{(i,j) \in I}(a_{ij} - b_{ij})^2]$
\subsection*{Convex Relaxation}
\textbullet $rank(B) \geq \|B\|_*$ for $\|B\|_2 \leq 1$. proof: $rank(B)=\#\{\sigma_i >0\}=\sum_{i:\sigma_i>0}1 \geq \sum_{i:\sigma_i>0}\sigma_i = \|B\|_*$ since $\sigma_1 = \|B\|_2  \leq 1$. Thus $Q_k = \{B:rank(B) \leq k\} \subseteq P_k = \{B:\|B\|_* \leq k\}$ a convex relaxation (in fact convex hull). \textbullet
Singular Value Tresholding: if $A=UDV^\top$ then $shrink_\tau(A)=UD_\tau V^\top$ where $D_\tau=diag(max\{0,\sigma_i-\tau\})$. $B_{t+1}=B_t+\eta_t\pi(A-shrink_\tau(B_t))$ where $\pi$ zeros out unobserved entries.

\subsection*{Alternating Least Squares}
reparametrize $B=UV$ where $U \in \mathbb{R}^{mxk}, V \in \mathbb{R}^{kxn}$. If we have product of two matrices, rank cannot be bigger than smallest rank occuring in it. $min[\sum_{(i,j) \in I}(a_{ij} - u_i^\top v_j)^2]$. ALS = optimize over $u_i$'s while keeping $v_j$'s fixed and vice versa.