\textbf{Modify the code diffusion} \verb+eq1_Neumann.m+ \textbf{to solve the diffusion equation with Neumann boundary conditions using finite differences.}

In this problem we use second order centered finite differences to calculate the second derivative in the interior
\begin{align*}
u''_j=\frac{u_{j-1}-2u_{j}+u_{j+1}}{h^2}.
\end{align*}
This discretization, if expressed in matrix form, gives us a tridiagonal matrix with $1,-2,1$ as lower diagonal, diagonal and upper diagonal, respectively. We only need to solve the following matrix equation
\begin{align*}
\frac{d}{dt}\textbf{u}=D_2\textbf{u},
\end{align*}
for the interior since we have the boundary conditions. They have been taking into account using a first order sided finite differences
\begin{align*}
u'_1=\frac{u_2-u_1}{h}=g_1(t)~\Rightarrow~u_1=u_2-hg_1(t),
\end{align*}
\begin{align*}
u'_{N+1}=\frac{u_{N+1}-u_N}{h}=g_2(t)~\Rightarrow~u_{N+1}=hg_2(t)+u_N,
\end{align*}
We will get rid of the first and last row and column the matrix $D_2$, obtaining $\tilde{D}_2$ and add another term to account for the boundary conditions of the Neumann kind in this case. The system obtained is the following
\begin{align*}
\frac{d}{dt}\tilde{\textbf{u}}=\tilde{D}_2\tilde{\textbf{u}}+\frac{1}{h}\begin{pmatrix} -1 & 1 \end{pmatrix}\begin{pmatrix} g_1(t) \\ g_2(t) \end{pmatrix},
\end{align*}
where $g_1(t)$ and $g_2(t)$ are the Neumann boundary condition functions at $x=-1$ and $x=1$, respectively. We defined them generally althought in this problem $g_1(t)=g_2(t)=1$. When solving for the interior, we will need the values at the boundaries that will be substituted by the two equations above. As a result, including the boundary conditions hasn't just altered the system adding that extra term, but it also changes the first and last entries of the matrix $\tilde{D}_2$. Computing the first and last nodes of the interior, and including the boundary conditions, we have
\begin{align*}
u''_2=\frac{u_1-2u_2+u_3}{h^2}&=\frac{u_2-hg_1(t)-2u_2+u_3}{h^2}\\
&=\frac{-u_2+u_3}{h^2}-\frac{1}{h} g_1(t),
\end{align*}
and
\begin{align*}
u''_N=\frac{u_{N-1}-2u_N+u_{N+1}}{h^2}&=\frac{u_{N-1}-2u_N+hg_2(t)+u_N}{h^2}\\
&=\frac{u_{N-1}-u_N}{h^2}+\frac{1}{h} g_2(t).
\end{align*}
This implies that the system is modified as showed above and the first and second entries of the matrix $\tilde{D}_2$ are
\begin{align*}
\tilde{D}_2(1,1)=\tilde{D}_2(N-1,N-1)=-\frac{1}{h^2}.
\end{align*}
Note that the $1/h^2$ factor is already included in $\tilde{D}_2$. The solution for the PDE is shown in the next figure.
\begin{figure}[H]
\centering     %%% not \center
{\includegraphics[scale=0.75]{P3_surf.eps}}
\caption{Solution of the diffusion equation, $N=50$.}
\end{figure}
We can see how at time zero we have the initial condition and in two seconds we reach the steady state where the whole domain has a slope 1, as imposed by the boundary conditions in the first and last nodes.

\subsection*{Matlab code for this problem}
\begin{verbatim}
%% Problem 3
% Simple code to solve the diffusion equation
% u_t = u_xx -1<x<1, with Neumann boundary conditions
clear all; close all; clc
legendfontsize=14;
axisfontsize=14;
labelfontsize=16;
N = 50;
L=2;
h=L/N;
x=-1:h:1;
D2 = gallery('tridiag',N+1,1,-2,1); % In sparse form.
%%

% initial condition
u0 = cos(pi*x)+0.2*cos(5*pi*x)+1;
% boundary conditions
g1 = @(t) 0*t+1;
g2 = @(t) 0*t+1;

A = D2(2:N,2:N)/h^2;
A(1,1)=-1/h^2;
A(end,end)=-1/h^2;
B = zeros(N-1,2);
B(1,1) = -1/h;
B(end,end) = 1/h;

t = 0:0.001:2;
%%
[T,U] = ode45(@(t,u) A*u+B*[g1(t);g2(t)], t, u0(2:end-1));
for k = 2:length(t)
    plot(x(2:end-1),U(k,:)','*-')
    ylim([-1 2])
    shg
    drawnow
end
[T,X]= meshgrid(t,x(2:end-1));
surf(T,X,U','edgecolor','none')
xlabel('$t$','fontsize',labelfontsize,...
            'interpreter','latex')
ylabel('$x$','fontsize',labelfontsize,...
            'interpreter','latex')
zlabel('$u(x,t)$','fontsize',labelfontsize,...
            'interpreter','latex')
set(gca,'fontsize',axisfontsize)
txt='Latex/FIGURES/P3_surf';
saveas(gcf,txt,'epsc')
\end{verbatim}


\subsection{Ridge regression}

Regularisation of LLS. The cost function now includes a norm on \(M\theta \).

\(L_2\) regularisation

This allows us to solve problems where there are too many features. \(L_1\) also allows us to to do this.

\subsubsection{Overspecified}

If \(n>d\) we can minimise weights subject to Xw=y. This is the same as the least norm.

\subsubsection{Maximum A-Priori (MAP) estimator for linear regression}

Maximum a priori estimation. equiv to ridge regression with a priori estimate of \(0\)

\(W_{RR}=(\lambda I+X^TX)^{-1}X^Ty\)

\(E_[w{RR}]=(\lambda I+X^TX)^{-1}X^TXw\)

\(Var[W_{RR}]=a\)


\paragraph{Mobius inversion formula}

$$[x = 1] = \sum_{d|x}\mu(d)$$

\paragraph{Gcd inversion}
\begin{align*}
\sum_{a=1}^n\sum_{b=1}^ngcd^2(a,b)&=\sum_{d=1}^nd^2\sum_{i=1}^{\floor{\frac{n}{d}}}\sum_{j=1}^{\floor{\frac{n}{d}}}[gcd(i,j)=1]\\
&=\sum_{d=1}^nd^2\sum_{i=1}^{\floor{\frac{n}{d}}}\sum_{j=1}^{\floor{\frac{n}{d}}}\sum_{t|gcd(i,j)}\mu(t)\\
&=\sum_{d=1}^nd^2\sum_{t=1}^{\floor{\frac{n}{d}}}\mu(t)\sum_{i=1}^{\floor{\frac{n}{d}}}[t|i]\sum_{j=1}^{\floor{\frac{n}{d}}}[t|j]\\
&=\sum_{d=1}^nd^2\sum_{t=1}^{\floor{\frac{n}{d}}}\mu(t){\floor{\frac{n}{dt}}}^2
\end{align*}

The formula can be computed in $O(nlogn)$ complexity.

Moreover, let $l=dt$, then

\begin{align*}
\sum_{d=1}^nd^2\sum_{t=1}^{\floor{\frac{n}{d}}}\mu(t){\floor{\frac{n}{dt}}}^2&=\sum_{l=1}^n{\floor{\frac{n}{l}}}^2\sum_{d|l}d^2\mu(\frac{l}{d})
\end{align*}

Let $f(l)=\sum_{d|l}d^2\mu(\frac{l}{d})$. It can be proven that $f(l)$ is multiplicative. Besides, $f(p^k)=p^{2k}-p^{2k-2}$.

Therefore, with linear sieve the formula can be computed in $O(n)$ complexity.


\section{Types of Matrices, Matrix Properties}

\subsection{$\mathrm{sgn}\left(\mathbf{x}^{\dagger}\mathbf{H}\mathbf{x}\right)$ Definite}
\label{sec:definite}

A hermitian matrix $\mathbf{H}\in\mathbb{C}^n$ is positive definite, if for any non-zero column vector $\mathbf{x}\in\mathbb{C}^n$, the quadratic form $\mathbf{x}^{\dagger}\mathbf{H}\mathbf{x} > 0$, and negative definite if $\mathbf{x}^{\dagger}\mathbf{H}\mathbf{x} < 0$. The matrix is positive or negative \textit{semidefinite} if  $\mathbf{x}^{\dagger}\mathbf{H}\mathbf{x} \geq 0$ or $\mathbf{x}^{\dagger}\mathbf{H}\mathbf{x} \leq 0$, respectively. Definiteness plays a role in investigating the convexity of a function by looking at the Hessian (section \ref{sec:hessian}). Sometimes notation with curly comparison symbols are used. The relationship between definiteness and eigenvalues is intuitive:

\begin{tabular}{lll}
$\mathbf{H} \succ 0$ & positive definite & all eigenvalues are positive\\
$\mathbf{H} \prec 0$ & negative definite & all eigenvalues are negative\\
$\mathbf{H} \succeq 0$ & positive semidefinite & all eigenvalues are positive or 0\\
$\mathbf{H} \preceq 0$ & negative semidefinite & all eigenvalues are negative or 0
\centering
\end{tabular}

The curly comparison symbols can mean other stuff though, for example in the context of partially ordered sets (order theory) or comparisons between multidimensional arrays.

% unitary
\subsection{Triangular}
\label{sec:triangular}
A lower triangular matrix is a matrix that has all-zero entries above the diagonal.

\begin{equation}
\mathbf{L} = \left[\begin{array}{cccccc} l_{1,1}&&&&&0\\l_{2,1}&l_{2,2}&&&&\\l_{3,1}&l_{3,2}&\ddots&&&\\  \vdots&\vdots&\ddots&\ddots&&\\ \vdots&\vdots&&\ddots&\ddots&\\  l_{n,1}&l_{n,2}&\hdots&\hdots&l_{n,n-1}&l_{n,n}\end{array}\right]
\end{equation}

Upper triangular matrices are matrices that have all-zero entries below the diagonal.

% commuting
\subsection{$\mathbf{AB}-\mathbf{BA}=0$ Commuting}
Two matrices commute if $\mathbf{AB}=\mathbf{BA}$, or, equivalently, their \textit{commutator} $[\mathbf{A,B}] = \mathbf{AB}-\mathbf{BA}$ is zero. This means that $\mathbf{A}$ and $\mathbf{B}$ both have to be square. Matrices commute when they have the same eigenspace, i.e. they have the same eigenvectors. This can be seen by considering the diagonal representations of $\mathbf{A}$ and $\mathbf{B}$.

Let $\mathbf{A}$ and $\mathbf{B}$ be two square matrices with the same eigenvectors $\mathbf{V}$, then they can be diagonalized as:

\begin{equation}
\mathbf{A} = \mathbf{V\Lambda_A V^{-1}}\\
\mathbf{B} = \mathbf{V\Lambda_B V^{-1}}
\end{equation}

They commute, because:

\begin{equation}
\mathbf{AB} = \mathbf{V\Lambda_A V^{-1}V\Lambda_B V^{-1}} = \mathbf{V\Lambda_B \Lambda_A V^{-1}} = \mathbf{V\Lambda_B V^{-1}V\Lambda_A V^{-1}} = \mathbf{BA}
\end{equation}

% anticommuting
\subsection{$\mathbf{AB}+\mathbf{BA}=0$ Anticommuting}
Two matrices anticommute if $\mathbf{AB}=-\mathbf{BA}$, or, equivalently, their \textit{anticommutator} $\{\mathbf{A,B}\} = \mathbf{AB}+\mathbf{BA}$ is zero.

% Hermitian / Symmetric
\subsection{$\mathbf{A}^{\dagger} = \mathbf{A}$ Hermitian, Symmetric}
\label{sec:hermitian}
Hermitian matrices are matrices that are equal to their complex transpose. That is:

\begin{equation}
{\mathbf{A}^{*}}^T = \mathbf{A}^\dagger = \mathbf{A}
\end{equation}

\subsubsection{Properties}
(There are more)
\begin{itemize}
\item By definition: $\mathbf{A} = \mathbf{A}^\dagger$
\item Diagonal Entries are all real, since $a_{i,i} = a_{i,i}^*$, but not necessarily positive (physics will mislead you there...)
\item Inverse is also hermitian:  $\mathbf{A}^{-1} ={ \mathbf{A}^{-1}}^\dagger$
\item Diagonalizable with real eigenvalues and orthogonal eigenvectors $\in \mathbb{C}^n$.
\end{itemize}

Hermitian matrices with only real entries are called symmetric matrices. In that case $\mathbf{A}^T = \mathbf{A}$.

Hermitian matrices can only have real elements along their diagonal.

% Skew Hermitian / Skew Symmetric
\subsection{$\mathbf{A}^{\dagger} = -\mathbf{A}$ Skew Hermitian, Skew Symmetric}
Skew Hermitian matrices that are equal to the negative of their complex transpose. That is:

\begin{equation}
{\mathbf{A}^{*}}^T = \mathbf{A}^\dagger = -\mathbf{A}
\end{equation}

Real matrices that are skew Hermitian are called skew symmetric. In that case:

\begin{equation}
\mathbf{A}^T = -\mathbf{A}
\end{equation}

Skew Hermitian matrices can only have complex values on their diagonal, and skew symmetric matrices can only have zeros as diagonal elements.

% Involutory
\subsection{$\mathbf{A}\mathbf{A}=\mathbf{I}$ Involutory}
Involutory matrixes are matrices that are their own inverse, so that:

\begin{equation}
\mathbf{A}\mathbf{A}=\mathbf{I}
\end{equation}

Involutory matrices are all square roots of the identity matrix. A famous example are the $2\times 2$ Pauli matrices.

% Isometric
\subsection{$||\mathbf{A}\mathbf{x}||_{\alpha} = ||\mathbf{x}||_{\alpha}$ Isometric}
\label{sec:isometric}
An isometric transformation with respect to some norm $||\cdot||_{\alpha}$ preserves that norm (it's in the name: iso-metric). The linear case is represented by isometric matrices, which satisfy:

\begin{equation}
||\mathbf{A}\mathbf{x}||_{\alpha} = ||\mathbf{x}||_{\alpha}
\end{equation}

For some vector $\mathbf{x}$. Isometries are also known as distance-preserving maps. To see this, define the distance between two points $\mathbf{a}$ and $\mathbf{b}$ as $||\mathbf{a} - \mathbf{b}||_{\alpha} = ||\mathbf{x}||_{\alpha}$. Isometries are usually understood be bijective.

In terms of operator norms, isometries must satisfy $||\mathbf{A}||_{(\alpha)} = 1$ where $||\cdot||_{(\alpha)}$ denotes the operator norm (cf. section \ref{sec:operatornorm}). However, operator norms give the upper bound on the distortion of the input, so a unit operator norm is necessary but not sufficient.

\subsubsection{Isometries with Respect to $L^2$}
The $L^2$ norm is unique in that the points $\{\mathbf{x}: ||\mathbf{x}||_{2} = 1\}$ lie on a perfect circle, which remains a circle regardless of the orientation of the underlying coordinate system. (A sphere looks the same regardless of angle.) This symmetry is broken for all other norms. Unitary matrices are isometries with respect to $L^2$, though unitarity is not a necessary condition for an isometry.

\subsubsection{Isometries with Respect to $L^1$ and general $L^q\neq L^2$}
Isometries with respect to $L^1$ are relevant because they are permissible transformations of (classical) probability distributions. For example, the transition matrix that describes the flow of probability between the time steps of a Markov Chain has to ensure that the entries of the state space probability still sum to $1$.

For norms $L^q$ with $q\neq2$, the unit "circle" $\{\mathbf{x}: ||\mathbf{x}||_{q\neq2} = 1\}$ is not perfectly round. Instead the symmetry is broken along the coordinate axes. That means that a rotation of the coordinate axes gives a different unit distance, and so two points $\mathbf{a}$ and $\mathbf{b}$ that are $||\mathbf{a}-\mathbf{b}||_{q\neq2} = 1$ apart in one coordinate system may have a different separation in some other coordinate system.

In particular, for the $L^1$, or \textit{Manhattan} norm (cf. section \ref{sec:l1norm}), the points $\{\mathbf{x}: ||\mathbf{x}||_{1} = 1\}$ lie on a diamond with axes aligned to the axes of the coordinate system. In Manhattan, reaching a point that is $1$ mile away "as the crow flies", depends on the position of that point with respect to the grid of streets and avenues. When that grid is rotated, the point may be quicker or take longer to reach.

Since this rules out rotations, the internet tells me that the only linear maps that are isometries for $L^q$ with $q\neq 2$ are signed permutation matrices. That is, matrices that assign $\hat{x}\rightarrow \hat{y}$ or $\hat{z} \rightarrow -\hat{x}$ and so on.

However, in general, any transformation that maps a point on the $L^q$ unit circle to another point on the $L^q$ unit circle is an isometry with respect to $L^q$, and this class of transformation is more general than signed permutations. The stochastic matrices are an example with respect to $L^1$.

\subsection{Stochastic}

Stochastic matrices are used to describe transitions between states of a Markov chain. The matrices are square and each entry is non-negative and represents a conditional probability of moving from one state to another. The entries along either the row or the column, or both, must sum to 1 according to the requirement of marginalization.

\begin{itemize}
\item Right stochastic has rows that sum to 1, so that $\mathbf{P}\mathbf{1}=\mathbf{1}$. It is applied $\mathbf{\pi}\mathbf{P}$. Column index gives "from", row index gives "to".
\item Left stochastic has columns that sum to 1, so that $\mathbf{1}\mathbf{P}=\mathbf{1}$. It is applied $\mathbf{P}\mathbf{\pi}$. Row index gives "from", column index gives "to".
\item doubly stochastic has rows and columns that sum to 1.
\end{itemize}

Products of stochastic matrices are also stochastic matrices. The spectral radius (largest eigenvalue) of a stochastic matrix is always $1$. Since $\mathbf{1}$ is an eigenvector and the left and right eigenvalues of a square matrix are the same, there is at least one stationary state $\mathbf{\pi}\mathbf{P}=\mathbf{\pi}$ (that is, in this case of a right stochastic matrix, a left eigenvector) with eigenvalue $1$.

% Unitary / Orthogonal
\subsection{$\mathbf{U}^{\dagger} = \mathbf{U}^{-1}$ Unitary, Orthogonal}
\label{sec:unitary}
Unitary matrices satisfy $\mathbf{U}^{\dagger}\mathbf{U} = \mathbf{UU}^{\dagger}=\mathbf{I}$, and they have $det(\mathbf{U}) = 1$. They are diagonalizable and can be expressed as $e^{i\mathbf{H}}$ where $\mathbf{H}$ is a Hermitian matrix.

Unitary matrices that are real are called orthogonal.  Orthogonal matrices satisfy $\mathbf{A}^{-1} = \mathbf{A}^T$. The rows (and columns) of $\mathbf{A}$ are an orthonormal basis in $\mathrm{R}^n$.

Unitary matrices are necessarily invertible, and have determinant $|U|=1$ or $|U|=-1$. They represent \textit{unitary transformations}, which means that they preserve the inner product between two vectors.

The set of $n \times n$ orthogonal matrices is known as the orthogonal group $O(n)$ and the subgroup of orthogonal matrices with determinant $1$ is known as the special orthogonal group $SO(n)$. The elements of $SO(n)$ are rotations, and the elements of $O(n)$ represent translations, reflections or rotations. Similarly, the group of $n \times n$ unitary matrices is the unitary group $U(n)$ and the subgroup of $U(n)$ that has determinant $1$ is the special unitary group $SU(n)$.

Unitary transformations preserve the $L^2$ norm of vectors.

% Similarity
\subsection{$\mathbf{A} = \mathbf{TBT^{-1}}$ Similarity}
\label{sec:similiarity}
Two matrices are said to be similar if they can be related through a similarity transformation  $\mathbf{A} = \mathbf{TBT^{-1}}$ where $\mathbf{T}$ is some nonsingular matrix (cf. section \ref{sec:similaritytrans}). An important example is that square matrices are similar to diagonal matrices, see section \ref{sec:diagonalization}.


\subsubsection{Review of matrix norms}
%
\label{sec:matrix_norms}
%
Given a matrix $A \in \R^{N \times M}$, we define its {\em operator
  norm}
\begin{equation}
  \label{eq:defn_matrix_norm}
  \| A \| := \sup_{\substack{h \in \R^N \\ |h| = 1}} |A h| = \sup_{h
    \in \R^N \setminus \{0\}} \frac{|Ah|}{|h|}.
\end{equation}

\begin{proposition}
  \label{th:matrix_norm}
  (a) $\|\cdot\|$ is a norm on $\R^{N \times M}$:
  \begin{enumerate}\ilist
  \item $\| A \| \geq 0$
  \item $\| A \| = 0$ if and only if $A = 0$
  \item $\| A + \lambda B \| \leq \| A \| + |\lambda| \| B \|$ \\

  (b) $\|\cdot\|$ is an operator norm:
\item $| A h | \leq \| A \| |h| \qquad \forall A \in \R^{N \times M}, h
  \in \R^M, \quad \text{and}$
\item $\| A B \| \leq \| A \| \, \| B \| \qquad \forall A \in \R^{N
    \times M}, B \in \R^{M \times P}.$
  \end{enumerate}
\end{proposition}
\begin{exercise}
  Prove Proposition \ref{th:matrix_norm}.
\end{exercise}

In addition to the canonical operator norm $\|\cdot\|$ we also define
the Frobenius norm
\begin{displaymath}
  \| A \|_F := \bg(\sum_{i = 1}^N \sum_{j = 1}^M |A_{ij}|^2 \bg)^{1/2}
  \qquad \text{ for } A \in \R^{N \times M}.
\end{displaymath}
Then we have the following relationship between $\|\cdot\|$ and
$\|\cdot\|_F$. It is clear from its definition that $\|\cdot\|_F$ is a
norm on $\R^{N \times M}$. In addition it is also an operator norm:

\begin{lemma}
  \label{th:opnrm_vs_frobnrm}
  The Frobenius norm satisfies the following properties:
  \begin{enumerate} \ilist
  \item  $|A h | \leq \|A\|_F |h|$ for all $h \in \R^N$.

  \item $\| A B \|_F \leq \|A \|_F \| B \|_F$ for all $A \in \R^{N
      \times M}, B \in \R^{M \times P}$.

  \item $\|A\| \leq \|A\|_F$ for all $A \in \R^{M \times N}$.
\end{enumerate}
\end{lemma}

\begin{exercise}
  Prove Lemma \ref{th:opnrm_vs_frobnrm}.\footnote{HINT: Show (i),
    using the Cauchy--Schwarz inequality on $\R^N$. (ii), (iii) are
    corollaries of (i).}
\end{exercise}

\begin{exercise}
  (i) Show that $\| A \|_F = \sqrt{{\rm trace}(A^T A)}$.

  (ii) Show that $(A, B) \mapsto {\rm trace}(A^T B)$ is an
  inner product on $\R^{M \times N}$ (in fact, it is the standard
  Euclidean inner product).
\end{exercise}

% If $A \in \R^{M \times N}$ is a matrix, then we define its operator norm
% $\|A\|$ and its Frobenius norm, respectively, as
% \begin{align*}
%   \| A \| &:= \sup_{h \in \R^N \setminus \{h\}} \frac{|A h|}{|h|},
%   \qquad \text{and} \\
%   \| A\|_F &:= \bigg( \sum_{i=1}^M \sum_{j=1}^N |A_{ij}|^2
%   \bigg)^{1/2}.
% \end{align*}

\subsubsection{Spectral decomposition}
%
Let $A \in \R^{N \times N}$ be symmetric, then there exists an
orthonormal basis (ONB) $\{v_1, \dots, v_N\}$ of $\R^N$ and
$\lambda_i \in \R, i = 1, \dots, N$
\begin{displaymath}
  A v_i = \lambda_i v_i.
\end{displaymath}
The $\lambda_i$ are called the eigenvalues of $A$ and $v_i$ the
associated eigenvectors.

\begin{remark}
  In general, a matrix $A \in \mathbb{C}^{N \times N}$ is unitarily
  diagonalisable if and only if it is {\em normal}: $A^* A = A A^*$,
  where $A^* = (\bar{A})^T$.
\end{remark}

\begin{exercise}
  (i) Prove the spectral decomposition theorem for symmetric matrices.

  (ii**) Prove the spectral decoposition theorem for normal matrices.
\end{exercise}

We call a matrix $A \in \R^{N \times N}$ diagonalisable if there
exists a basis $\{v_1, \dots, v_N\}$ of $\mathbb{C}^N$ (not
necessarily ONB) and eigenvalues $\lambda_i \in \C$ such that $A v_i =
\lambda_i v_i$.

We call a matrix $A \in \R^{N \times N}$ positive definite if $x^T A x
> 0$ for all $x \in \R^N \setminus \{0\}$. A symmetric and positive
matrix is simply called spd.

Not all matrices are diagonalisable, but all matrices have a {\em
  Jordan normal form}. That is, there exists a basis $\{v_1, \dots,
v_N\}$ of $\mathbb{C}^N$ such that, writing $V = (v_1, \dots, v_N)$ we
have
\begin{displaymath}
  V A V^{-1} = \left( \begin{matrix}
      J_1 & 0 & \cdots & \\
      0 & J_2 & 0 & \cdots \\
      \vdots & 0 & \ddots &
      \end{matrix} \right) \quad \text{where} \quad
      J_i = \left(\begin{matrix}
      \lambda_i & 1 & & & \\
            & \lambda_i & 1 & &   \\
            &   & \ddots & \ddots &  \\
            & & & \lambda_i & 1 \\
            & & & &  \lambda_i
    \end{matrix} \right).
\end{displaymath}
The $J_i$ are called {\em Jordan blocks}.

\begin{exercise}
  Prove that a matrix $C \in \R^{N \times N}$ is spd if and only if it
  is symmetric and all its eigenvalues are positive.
\end{exercise}

\subsubsection{Convexity}
%
A set $U \subset \R^N$ is convex if, for $x, y \in U, \lambda \in [0,
1]$, the point $(1-\lambda) x + \lambda y \in U$ as well.

If $U$ is convex and $f : U \to \R$, then $f$ is convex if, for $x, y
\in U, \lambda \in [0, 1]$, we have
\begin{equation}
  \label{eq:convex_f}
  f\b( (1-\lambda) x + \lambda y \b) \leq (1-\lambda) f(x) + \lambda f(y).
\end{equation}

\begin{exercise}
  (i) Prove that the functions $f(x) = |x|^n, n \in \N$, $f(x) =
  \exp(x), f(x) = \min(0, x)$ are convex.

  (ii) Prove that $f(x) = -x^2, f(x) = \cos(x), f(x) = \max(0, x)$ are
  non-convex.

  (iii) Prove that, if $C \in \R^{N \times N}$ and $b \in \R^N$, then
  $f(x) = \smfrac12 x^T C x - b^T x$ is convex.
\end{exercise}

\subsection{Crash Course in Differentiation}

\examnote{ The material of this section is understood as background and
  can/should be used in the exam without proofs. }

This section is intended as a rapid introduction into and reference
for differentiation of multi-variate and/or vector valued functions as
required in the remainder of the lecture notes.

\subsubsection{Motivation and definition}
%
\paragraph{Univariate:}
%
Recall first the definition of differentiability in 1D. Let $f : (a,
b) \to \R$ and $t \in (a, b)$. Then we say that $f$ is differentiable
at $t$ if there exists $T \in \R$ such that
\begin{displaymath}
  % \label{eq:diff:diff_1d}
  f(t+h) = f(t) + T h + o(h),
\end{displaymath}
where $o(h) / h \to 0$ as $h \to 0$. We write $T = f'(t)$. In words:
$f$ can be ``well approximated'' in a small neighbourhood of $t$ by an
affine function.

For vector valued functions with scalar argument, nothing changes. $f
: (a, b) \to \R^N$ is differentiable at $t$ if there exists $T \in
\R^N$ such that
\begin{displaymath}
  f(t+h) = f(t) + T h + o(h),
\end{displaymath}
and we call $T = \dot{f}(t)$. Clearly this is equivalent to saying
that $f_i$ is differentiable at $t$ for all $i = 1, \dots, N$.

We define spaces of continuously differentiable functions. For an
interval $A \subset \R$ and $j \geq 1$,
\begin{align*}
  C^j(A;\R^N) = \b\{ f \in C(A;\R^N) \bsep & \text{ $f$ is $j$ times
    differentiable in ${\rm int}(A)$} \\
  & \text{ and $f^{(k)} \in C(A;\R^N)$ for $k = 0, \dots, j$} \b\}.
\end{align*}
(Note that if $A$ contains one of its endpoints, then by $f^{(k)} \in
C(A; \R^N)$ we mean that the derivative $f^{(k)}$ first defined on
${\rm int}(A)$ can be {\em continuously extended} to $A$. This is
subtle point that can in fact be ignored for the purpose of these
lecture notes.)

\paragraph{Multi-variate:} With this idea of differentiation in mind,
the generalisation to functions with vectorial arguments is
immediate. Let $U \subset \R^N$ be open and let $f : U \to \R^M$. We
say that $f$ is (Fr\'echet) differentiable at $x \in U$ if there
exists a matrix $T \in \R^{M \times N}$ (recall that every linear
function from $\R^N$ to $\R^M$ is of the form $h \mapsto T h$ for some
$T \in \R^{M \times N}$) such that
\begin{displaymath}
  f(x+h) = f(x) + T h + o(|h|), \qquad \text{as $h \to 0$,}
\end{displaymath}
and we write $T = \partial f(x)$.

There is also a second notion of differentiability: We say that $f$ is
Gateaux differentiable at $x \in U$ if there exists $T \in \R^{M
  \times N}$ such that, for all $h \in \R^N$,
\begin{displaymath}
  f(x + s h) = f(x) + s T h + o(s) \qquad \text{as $s \to 0$.}
\end{displaymath}
We still write $T = \partial f(x)$. The subtle difference of letting
the perturbation tend to zero in a straight line, or in an arbitrary
fashion makes Gateaux differentiability a strictly weaker notion of
differentiability.

(WARNING: some authors define Gateaux derivative slightly
differently. It is always useful to check what is meant. By contrast
Frechet differentiability seems universally accepted as stated above.)

\begin{exercise}[Frechet versus Gateaux]
  \label{eq:diff:F_vs_G}
  (i) Show that if $f$ is Frechet differentiable in $x$ then it is
  continuous in $x$ as well as Gateaux differentiable in $x$.

  (ii) Conversely, consider $f : \R^2 \to \R$,
  \begin{displaymath}
    f(x, y) = \cases{
      \b(\frac{x^2 y}{x^4+y^2} \b)^2, & (x, y) \neq (0, 0), \\
      0, & (x, y) = (0, 0),
    }
  \end{displaymath}
  and show that $f$ is Gateaux differentiable at $(0, 0)$ but not even
  continuous.

  (iii) In other respects, Gateaux and Fr\'echet differentiability are
  not so far from one another: If $f$ is Gateaux differentiable in $U$
  and $\pp f$ is continuous at $x \in U$, then $f$ is Fr\'echet
  differentiable at $x$. {\it (HINT: If you don't see this
    immediately, do Exercise \ref{ex:diff:pd_frechet} first, from
    which this results follows.)}
\end{exercise}

REMINDER: If we say $f$ is {\em differentiable}, then we mean the
Frechet sense!

\begin{definition}
  If $U \subset \R^N$, with ${\rm int}(U) \neq \emptyset$, then we say
  that $f \in C^1(U; \R^M)$ if $f : U \to \R^{M}$ is differentiable at
  each point $x \in {\rm int}(U)$ and $\pp f \in C(U; \R^{M \times
    N})$.

  REMARK: From Exercise \ref{eq:diff:F_vs_G}(iii) it follows
  immediately that $f \in C^1(U; \R^M)$ if and only if $f$ is Gateaux
  differentiable at each point in $U$ and $\pp U \in C(U; \R^{M\times
    N})$, that is, the definition of $C^1$ is in fact independent of
  which of the two notions of differentiability we employ.
\end{definition}

\subsubsection{Jacobi matrix and gradient}
%
To actually compute the derivative $\pp f(x)$, we simply test the
definition
\begin{displaymath}
  f(x + sh) = f(x) + s\partial f(x) h + o(s)
\end{displaymath}
with $h = e_j$ ($e_j$ is a canonical basis vector). Hence, we obtain
that
\begin{displaymath}
  \frac{\pp f_i(x)}{\pp x_j} = \lim_{s \to 0} \frac{f_i(x+s e_j) -
    f_i(x)}{s} = \b(\partial f(x) e_j\b)_i = \b( \partial f(x) \b)_{ij}.
\end{displaymath}
That is, $\pp f(x)$ is simply the matrix of partial derivatives, or,
{\em Jacobi matrix},
\begin{displaymath}
  \pp f(x) = \left(
    \begin{array}{cccc}
      \frac{\pp f_1}{\pp x_1} & \frac{\pp f_1}{\pp x_2} & \cdots &
      \frac{\pp f_1}{\pp x_N} \\
      \frac{\pp f_2}{\pp x_1} & \frac{\pp f_2}{\pp x_2} & \cdots &
      \frac{\pp f_2}{\pp x_N} \\
      \vdots & \vdots & \ddots &  \vdots\\
      \frac{\pp f_M}{\pp x_1} & \frac{\pp f_M}{\pp x_2} & \cdots &
      \frac{\pp f_M}{\pp x_N}
    \end{array} \right).
\end{displaymath}

If $M = 1$, i.e. $f : U \subset \R^N \to \R$ and $f$ is (Frechet or Gateaux)
differentiable at $x \in U$ then we call
\begin{displaymath}
  \nabla f(x) := \pp f(x)^T
\end{displaymath}
the {\em gradient} of $f$ at $x$. Thus, in this case we can
alternatively write Gateaux-differentiability in the form
\begin{displaymath}
  f(x+sh) = f(x) + s \nabla f(x) \cdot h + o(s),
\end{displaymath}
and similarly for F-differentiability.

\examrem{
\begin{remark}
  \notexam
  This remark is to explain why one makes a distinction between $\pp
  f$ and $\nabla f$. It is mostly semantic, until one starts thinking
  in terms of infinite-dimensional vector spaces, or different inner
  products. (But then it can become very relevant.)

  We can think of the gradient in terms of the Riesz representation
  theorem (RRT). Namely, $\pp f(x)$ represents a linear functional
  acting on $\R^N$, $\ell(h) = \pp f(x) h$, and according to the RRT
  there exists an element of $\R^N$, namely $\nabla f(x)$, that {\em
    represents} $\ell$ in terms of an inner product: $\ell(h) = \nabla
  f(x) \cdot h$.

  Suppose now that we change the inner product to $\< x, y \>_M = x^T
  M y$ for some matrix $M$ (we need to check under which conditions
  this defined an inner product: if and only if $M$ is symmetric and
  positive definite). Then the gradient with respect to $\< \cdot,
  \cdot\>_M$ is $\nabla_M f(x) = M^{-1} \nabla f(x)$, since $\partial
  f(x) h = \< \nabla_M f(x), h \>_M$.
\end{remark}
}

\begin{exercise}[Partial derivatives and differentiability]
  \label{ex:diff:pd_frechet}
  We conclude this discussion with a remark on the relationship
  between partial derivatives and Frechet differentiability, which
  extends Exercise \ref{eq:diff:F_vs_G} (iii):

  Suppose that $f : U \subset \R^N \to \R^M$, $U$ open, has
  well-defined partial derivatives $\pp f(x)$ at each point $x \in
  U$. (I.e., $t \mapsto f(x+t e_i)$ is differentiable at $t = 0$ for
  all $x \in U$.)  that $\partial f : U \to \R^{M \times N}$ is
  continuous at a point $x \in U$. Then $f$ is Frechet differentiable
  at $x$.
\end{exercise}

\subsubsection{Some useful properties}
%
\begin{exercise}
  All the usual linearity properties of differentiation hold for
  multivariate differentiation as well:

  If $f, g$ are differentiable at $x$, then $f + \lambda g$ is
  differentiable at $x$ for any $\lambda \in \R$.
\end{exercise}

The only other property we need is the chain rule. For scalar arguments, this reads
\begin{displaymath}
  \frac{\dd}{\dd t} f(g(t)) = \frac{\dd f}{\dd g} (g(t)) \cdot
  \frac{\dd g}{\dd t}(t) \quad \text{or} \quad
  (f \circ g)' = (f' \circ g) g'.
\end{displaymath}
Note also that the statement for scalar-valued functions immediately
implies the same statement for vector-valued functions.

The multi-variate version is formally identical,
\begin{equation}
  \label{eq:diff:chain_rule}
  \pp \b( f(g(x)) \b) = (\pp f)(g(x)) \, \pp g(x), \quad \text{or}
  \quad
  \pp(f \circ g) = (\pp f) \circ g \, \pp g
\end{equation}
Another, possibly more memorable way to write this is
\begin{align*}
  \frac{\pp (f(g(x)))}{\pp x} &= \frac{\pp f(g)}{\pp g}\Big|_{g = g(x)} \,
  \frac{\pp g(x)}{\pp x}
\end{align*}

\begin{proposition}
  Let $f : U \subset \R^N \to \R^M$ and $g : V \subset \R^K \to
  \R^N$. Suppose that $g$ is differentiable at $x \in V$ and $f$ is
  differentiable at $g(x) \in U$, then $f \circ g$ is differentiable
  at $x$ and \eqref{eq:diff:chain_rule} holds.

  The same result holds if we replace ``differentiable'' with
  ``Gateaux differentiable''.
\end{proposition}
\begin{proof}
  \begin{align*}
    f(g(x+h)) &= f\b( g(x) + \pp g(x) h + o(|h|) \b) \\
    &= f(g(x)) + \pp f(g(x)) \, \b(\pp g(x) h + o(|h|)\b)
    + o\b( |\pp g(x) h + o(|h|)| \b) \\
    &= f(g(x)) + \pp f(g(x)) \, \pp g(x) h  + o(|h|).
  \end{align*}

  The proof for the Gateaux case is analogous.
\end{proof}

An important special case is the following: if $f \in C^1(U;\R^N)$, $U
\subset \R^N$ open, and $g \in C^1(a, b; U)$, then
\begin{displaymath}
  \frac{\dd}{\dd t} f(g(t)) = \pp f(g(t)) \dot{g}(t).
\end{displaymath}
In particular, if $N = 1$, then
\begin{displaymath}
  \label{eq:chain_rule_nabla}
  \frac{\dd}{\dd t} f(g(t)) =  \nabla f(g(t)) \cdot \dot{g}(t).
\end{displaymath}

\subsubsection{Differentiation and integration of $f : \R \to \R^N$}
%
We will not be concerned with integrating over $\R^N$, but we do need
some results on integrating vector-valued functions defined on an
interval.

Let $f : [a, b] \to \R^N$ be continuous, then we define its integral
componentwise:
\begin{displaymath}
  \int_a^b f(t) \dt = \bigg( \int_a^b f_i(t) \dt \bigg)_{i = 1}^N.
\end{displaymath}
Clearly, we get the fundamental theorem of calculus again: if $f \in
C^1([a,b]; \R^N)$, then
\begin{displaymath}
  \int_a^b \dot f(t) \dt = f(b) - f(a).
\end{displaymath}

But other results can fail:

\begin{exercise}
  The integral mean value theorem is an example of a result that does
  {\em not} translate (and hence the differential MVT does not
  either):

  (i) Vector-valued, scalar argument: Show the integral MVT fails for
  $f : (0, 2\pi) \to \R^2$, $f(t) = (\cos t, \sin t)^T$.

  \examrem{
  (ii) \notexam For comparison, the integral MVT, scalar valued,
  vectorial argument: Show that if $U \subset \R^N$ is the closed unit
  cube (but any connected, measurable set with unit volume will do)
  and if $f \in C(U; \R)$, then there exists $x_0 \in U$ such that
  $\int_U f \dx = f(x_0)$. (Of course we haven't defined multi-variate
  integration. Either use your immagination or look up the definition
  in a suitable textbook.)}
\end{exercise}

An important consequence of the fundamental theorem are path
integrals: If $\gamma \in C^1([a, b]; \R^N)$, then the intregal of $f$
along its path is
\begin{displaymath}
  \int_a^b f(\gamma(t)) \dot\gamma(t) \dt.
\end{displaymath}

\begin{exercise}
  (i) Show that, if $E \in C^1(U; \R)$ and $\gamma \in C^1((a, b);
  \R^N)$, then $E \circ \gamma \in C^1((a,b); \R)$ and $\frac{\dd}{\dd
    t} E(\gamma(t)) = \nabla E(\gamma(t)) \cdot \dot\gamma(t)$.

  (ii) Therefore, if $f(x) = \nabla E(x)$, then the integral
  \begin{displaymath}
    \int_a^b f(\gamma(t)) \cdot \dot\gamma(t) \dt = E(\gamma(b)) - E(\gamma(a))
  \end{displaymath}
  depends only on the end-points but not on the entire path.

  \examrem{
  (iii) \notexam {\it (MORE DIFFICULT)} Show, conversely, that if $f \in C(U;
  \R^N)$, where $U$ is open and simply connected, and if $f$ is {\em
    conservative} i.e. integrals along closed paths all vanish,
  \begin{displaymath}
    \int_a^b f(\gamma(t)) \dot\gamma(t) \dt = 0 \qquad \forall \gamma \in
    C^1([a,b]; U), \gamma(a) = \gamma(b),
  \end{displaymath}
  then there exists $E \in C^1(U; \R)$ such that $f = \nabla E$.

  (iv) \notexam  {\it (MORE DIFFICULT)} If $f \in C^1(U; \R^N)$, where $U$ is
  open and simply connected, show that $f$ is conservative if and only
  if $\pp f(x)$ is symmetric for all $x \in U$. For $N = 3$ connect
  this to the curl-free condition.
}
\end{exercise}

\subsubsection{Higher derivatives}
%
The notation for higher derivatives can initially be confusing and we
will therefore avoid their use as much as possible. However, there are
some simple definitions and results that we require.

For $U \subset \R^N$ with ${\rm int}(U) \neq \emptyset$ and for $k
\geq 2$, we define (recursively)
\begin{displaymath}
  C^k(U; \R^M) := \b\{ f \in C^1(U; \R^M) \bsep \partial f \in
  C^{k-1}(U; \R^{M \times N}) \b\}.
\end{displaymath}

A useful result is the following:

\begin{exercise}
  Let $f \in C^2(U; \R^M)$ where $U \subset \R^N$ is convex and
  closed, then $\pp f : U \to \R^{M \times N}$ is Lipschitz
  continuous, in the sense that
  \begin{displaymath}
    \| \pp f(x) - \pp f(y) \|_F \leq L |x - y| \qquad \text{ for } x,
    y \in U. \qedhere
  \end{displaymath}
\end{exercise}

There is a particularly important case: Let $U \subset \R^N$ be open
and let $E \in C^2(U; \R)$, then we define the Hessian matrix
\begin{displaymath}
  \D^2 E(x) := \bg( \frac{\pp^2 E(x)}{\pp x_i \pp x_j}
  \bg)_{i, j = 1, \dots, N}.
\end{displaymath}

\begin{exercise}
  Let $E \in C^2(U)$, where $U \subset \R^N$ is open. Prove the
  following statements:

  (i) $\D^2 E(x) = \pp \D E(x)$.

  (ii) $E(x+h) = E(x) + \D E(x) \cdot h + \smfrac12 h^T \D^2 E(x) h +
  o(|h|^2)$.

  (iii) If $E \in C^3(U)$, then $E(x+h) = E(x) + \D E(x) \cdot h +
  \smfrac12 h^T \D^2 E(x) h + O(|h|^3)$.
\end{exercise}


\section*{Essentials}
\subsection*{Matrix/Vector}
\textbf{Multiplication:} $\mathbf{C}=\mathbf{AB} \Leftrightarrow c_{ik} = \sum_{j=1}^m a_{ij}\cdot b_{jk}$ \\
\textbf{Orthogonal Matrix:} (full rank square matrix with orthonormal columns) $\mathbf{A}^{-1} = \mathbf{A}^\top$, $\mathbf{A} \mathbf{A}^\top = \mathbf{A}^\top \mathbf{A} = \mathbf{I}$, $\operatorname{det}(\mathbf{A}) \in \{+1, -1\}$, $\operatorname{det}(\mathbf{A}^\top \mathbf{A}) = 1$,
	preserves: inner product, norm, distance, angle, rank, mat. orthogon. \\
\textbf{Inner Product:} $\langle \mathbf{x}, \mathbf{y} \rangle = \mathbf{x}^\top \mathbf{y} = \sum_{i=1}^{N} \mathbf{x}_i \mathbf{y}_i$. \\
$\langle \mathbf{x} \pm \mathbf{y}, \mathbf{x} \pm \mathbf{y} \rangle = \langle \mathbf{x}, \mathbf{x} \rangle \pm 2 \langle \mathbf{x}, \mathbf{y} \rangle + \langle \mathbf{y}, \mathbf{y} \rangle$ \\
$\langle \mathbf{x}, \mathbf{y} + \mathbf{z} \rangle = \langle \mathbf{x}, \mathbf{y} \rangle + \langle \mathbf{x}, \mathbf{z} \rangle$\\
$\langle \mathbf{x}, \mathbf{y} \rangle = \|\mathbf{x}\|_2 \cdot \|\mathbf{y}\|_2 \cdot \cos(\theta)$\\
If $\mathbf{y}$ is a unit vector then $\langle \mathbf{x}, \mathbf{y} \rangle$ projects $\mathbf{x}$ onto $\mathbf{y}$ \\
$(\mathbf{u}_i^T\mathbf{v}_j)\mathbf{v}_j = (\mathbf{v}_j\mathbf{v}_j^T)\mathbf{u}_i$ \\
\textbf{Outer Product:} $\mathbf{u} \mathbf{v}^\top$, \quad $(\mathbf{u} \mathbf{v}^\top)_{i, j} = \mathbf{u}_i \mathbf{v}_j$ \\
\textbf{Transpose:} $(\mathbf{A}^\top)^{-1} = (\mathbf{A}^{-1})^\top$ \\
\textbf{Determinant:} $|\mathbf{A}|=\sum_i \lambda_i$, \quad $|\mathbf{A}^{-1}| = 1/|\mathbf{A}|$

\subsection*{Norms}
$\|\mathbf{x}\|_0 = |\{i | x_i \neq 0\}|$ \quad
$\|\mathbf{x}\|_2 = \sqrt{\sum_{i=1}^{N} \mathbf{x}_i^2} = \sqrt{\langle \mathbf{x}, \mathbf{x} \rangle}$ \\
$\|\mathbf{x}\|_p = \left( \sum_{i=1}^{N} |x_i|^p \right)^{\frac{1}{p}}$ \quad
$\|\mathbf{M}\|_F =\allowbreak \sqrt{\sum_{i=1}^{m} \sum_{j=1}^{n}\mathbf{m}_{i,j}^2} \allowbreak = \sqrt{\sum_{i=1}^{\min\{m, n\}} \sigma_i^2}$  $\allowbreak = \|\sigma(\mathbf{A})\|_2  = \sqrt{trace(\mathbf{M}^T\mathbf{M})} $\\
$\|\mathbf{M}\|_G=\sqrt{\sum_{ij}{g_{ij}x^2_{ij}}}$ (weighted Frobenius) \\
$\|\mathbf{M}\|_1 = \sum_{i,j} | m_{i,j}|$ \quad
$\|\mathbf{M}\|_p = \max_{\mathbf{v} \neq 0} \frac{\|\mathbf{M}\mathbf{v}\|_p}{\|\mathbf{v}\|_p}$ \\
$\|\mathbf{M}\|_2 = \sigma_{\text{max}}(\mathbf{M}) = \|\sigma(\mathbf(M))\|_\infty$ \\
$\|\mathbf{M}\|_* = \sum_{i=1}^{\min(m, n)} \sigma_i = \|\sigma(\mathbf{A})\|_1$ (nuclear norm) \\
$\text{rank}(\mathbf{B}) \geq ||\mathbf{B}||_* \ \text{for} \ ||B||_2 \leq 1$

\subsection*{Derivatives}
$\frac{\partial}{\partial \mathbf{x}}(\mathbf{b}^\top \mathbf{x}) = \frac{\partial}{\partial \mathbf{x}}(\mathbf{x}^\top \mathbf{b}) = \mathbf{b}$ \quad
$\frac{\partial}{\partial \mathbf{x}}(\mathbf{x}^\top \mathbf{x}) = 2\mathbf{x}$\\
$\frac{\partial}{\partial \mathbf{x}}(\mathbf{x}^\top \mathbf{A}\mathbf{x}) = (\mathbf{A}^\top + \mathbf{A})\mathbf{x}$ \quad
$\frac{\partial}{\partial \mathbf{x}}(\mathbf{b}^\top \mathbf{A}\mathbf{x}) = \mathbf{A}^\top \mathbf{b}$\\
$\frac{\partial}{\partial \mathbf{X}}(\mathbf{c}^\top \mathbf{X} \mathbf{b}) = \mathbf{c}\mathbf{b}^\top$ \quad
$\frac{\partial}{\partial \mathbf{X}}(\mathbf{c}^\top \mathbf{X}^\top \mathbf{b}) = \mathbf{b}\mathbf{c}^\top$\\
$\frac{\partial}{\partial \mathbf{x}}(\| \mathbf{x}-\mathbf{b} \|_2) = \frac{\mathbf{x}-\mathbf{b}}{\|\mathbf{x}-\mathbf{b}\|_2}$ \quad
$\frac{\partial}{\partial \mathbf{x}}\log(x) = \frac{1}{x}$ \\
$\frac{\partial}{\partial \mathbf{x}}(\|\mathbf{Ax - b}\|_2^2) = \mathbf{2(A^\top Ax-A^\top b)}$ \quad
$\frac{\partial}{\partial \mathbf{x}}\frac{1}{f(x)} = \frac{-f'}{f^2}$\\
$\frac{\partial}{\partial \mathbf{X}}(|\mathbf{X}|) = |\mathbf{X}|\cdot \mathbf{X}^{-1}$ \quad $\frac{\partial}{\partial x}(\mathbf{Y}^{-1}) = -\mathbf{Y}^{-1} \frac{\partial\mathbf{Y}}{\partial x} \mathbf{Y}^{-1}$

\subsection*{Eigenvalues \& Eigenvectors}
Eigenvalue problem: $\mathbf{Ax}=\lambda\mathbf{x}$ \\
1. solve $\det(\mathbf{A} - \lambda \mathbf{I}) = 0$ resulting in $\{\lambda_i\}$ \\
2. $\forall \lambda_i$ solve $(\mathbf{A} - \lambda_i \mathbf{I})x_i = \mathbf{0}$ for $\mathbf{x}_i$

\subsection*{Eigendecomposition}
$\mathbf{A} \in \mathbb{R}^{N \times N}$ then $\mathbf{A} = \mathbf{Q} \boldsymbol{\Lambda} \mathbf{Q}^{-1}$ with $\mathbf{Q} \in \mathbb{R}^{N \times N}$\\
if fullrank: $\mathbf{A}^{-1} = \mathbf{Q} \boldsymbol{\Lambda}^{-1} \mathbf{Q}^{-1}$ and $(\boldsymbol{\Lambda}^{-1})_{i,i} = {1}/{\lambda_i}$\\
if $\mathbf{A}$ symmetric: $A = \mathbf{Q} \boldsymbol{\Lambda} \mathbf{Q^\top}$ ($\mathbf{Q}$ orthogonal)

\subsection*{Probability / Statistics}
$P(x) = \sum_{y \in Y} P(x, y)$ \quad
$P(x, y) = P(x|y) P(y)$ \\
$\forall y \in Y: \sum_{x \in X} P(x|y) = 1$ (property for any fixed $y$) \\
$P(x|y) = \frac{P(x,y)}{P(y)},\quad \text{if } P(y) > 0$ \quad
$P(x|y) = \frac{P(y|x)P(x)}{P(y)}$ (Bayes' rule) \quad
$P(x_1, \ldots, x_n) = \prod_{i=1}^n P(x_i)$ (iff i.i.d) \\
$P(x|y) = P(x) \Leftrightarrow P(y|x) = P(y)$ (iff $X$, $Y$ indep.) \\
$E[X]:=\sum_{x \in X}x\cdot P(x)$ \quad
$Var[X]:= E[(X-\mu_x)^2]:=\sum_{x \in X}(x-\mu_x)^2P(x)= E(X^2) - E(X)^2$ \\
standard deviation $\sigma_x := \sqrt{Var[X]}$

\subsection*{Lagrangian Multipliers}
Problem $\min_Q g(Q)$ with constraint $\forall j \ \sum_i Q_{ij}=1$ turn into $L(Q, \alpha) = g(Q)+\sum_j \alpha_j (1-\sum_i Q_{ij})$ and find $\max_\alpha \min_Q L(Q,\alpha)$ (can use constraint form.).

\subsection*{Convex Function}
$\forall x_1, x_2 \in X, \forall t \in [0,1]: $ \quad (also $\text{iff} \ \forall x: \  f''(x) \geq 0$)\\
$f(tx_1 + (1-t)x_2)\leq t f(x_1) + (1-t)f(x_2)$ \\
\textbf{Sum} of convex functions is convex, \textbf{log} is convex
%\vspace{1mm}

\textbf{Exercise:} Show that if $f$ is convex, any local optimum is global. Assume there is a local optimum $\hat x$ that is not the global optimum $x^*$, then if we choose $t$ to be in the ball of the local optimum, so we know that $f(t\hat x + (1-t)x^*) \geq f(\hat x)$. Since $f(x^*) < f(\hat x)$,  we have $t\cdot f(\hat x) + (1-t) f(x^*) < f(\hat x)$. So we get $f(t\hat x + (1-t)x^*) \geq f(\hat x) > t\cdot f(\hat x) + (1-t)f(x^*)$, which contradicts the convexity of $f$.

%vspace{1mm}

\subsection*{Jensen Inequality:}
for convex $\phi$: $\phi (\sum_{i=1}^n \lambda_i x_i) \leq \sum_{i=1}^n \lambda_i f(x_i)$ if $\sum_{i=1}^n \lambda_i = 1$. Also $\phi(E[X])\leq E[\phi(X)]$.


\subsection{Angles and Sectors of Circles}
Mathematicians tend to deal mostly with \dfont{radians} and we
will see later that some formulas are more elegant when using
radians (rather than degrees). The relationship between degrees
and radians is:
$$\pi~\mbox{rad}=180^\circ.$$
Using this formula, some common angles can be derived:
$$\begin{array}{|c|c|c|c|c|c|c|c|c|c|c|c|}
\hline
~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ \\
\mbox{Degrees} & 0^\circ & 30^\circ & 45^\circ & 60^\circ & 90^\circ & 120^\circ & 135^\circ & 150^\circ & 180^\circ & 270^\circ & 360^\circ \\
~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ \\
\hline
~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ \\
\mbox{Radians} & 0 & \ds{\frac{\pi}{6}} & \ds{\frac{\pi}{4}} & \ds{\frac{\pi}{3}} & \ds{\frac{\pi}{2}} & \ds{\frac{2\pi}{3}} & \ds{\frac{3\pi}{4}} & \ds{\frac{5\pi}{6}} & \ds{\pi} & \ds{\frac{3\pi}{2}} & 2\pi\\
~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ & ~ \\
\hline
\end{array}$$

\begin{example}{Degrees to Radians}{DegreesToRadians}
To convert $45^\circ$ to radians, multiply by $\ds{\frac{\pi}{180^\circ}}$ to get $\ds{\frac{\pi}{4}}$.
\end{example}

\begin{example}{Radians to Degrees}{RadiansToDegrees}
To convert $\ds{\frac{5\pi}{6}}$ radians to degrees, multiply by $\ds{\frac{180^\circ}{\pi}}$ to get $150^\circ$.
\end{example}

From now on, unless otherwise indicated, we will \ifont{always} use radian measure.

In the diagram below is a sector of a circle with \dfont{central angle}
$\theta$ and radius $r$ \dfont{subtending} an arc with length $s$.

$$\includegraphics[width=50mm]{images/trig1}$$

When $\theta$ is measure in radians, we have the following
formula relating $\theta$, $s$ and $r$:
$$\theta=\frac{s}{r}\mbox{\quad~or\quad}s=r\theta.$$

\begin{formulabox}[Sector Area]
The area of the sector is equal to:
$$\mbox{Sector Area}=\frac{1}{2}r^2\theta.$$
\end{formulabox}

\begin{example}{Angle Subtended by Arc}{AngleSubtendedArc}
If a circle has radius $3$ cm, then an angle of $2$ rad is subtended
by an arc of $6$ cm ($s=r\theta=3\cdot 2=6$).
\end{example}

\begin{example}{Area of Circle}{AreaOfCircle}
If we substitute $\theta=2\pi$ (a complete revolution) into the sector
area formula we get the area of a circle:
$$A=\frac{1}{2}r^2(2\pi)=\pi r^2.$$
\end{example}


\section{Parabolas}
Associated external model: \texttt{parabolas.py}

This model is a simple $N$-dimensional parabolic response $u(Y)$,
\begin{equation}
  u(Y) = \sum_{n=1}^N -y_n^2,
\end{equation}
where the uncertain inputs are $Y=(y_1,\cdots,y_N)$ and can be defined arbitrarily.  For optimization
searches, it is possible to obtain a maximum in the interior of the input by assuring the range of each
input variable include 0.  In this case, the maximum point will be ${0}^N$.


\chapter{Interval}

\section{Introduction}
\rih{Two-way range.} The current scanning node as the pivot, need to scan its left neighbors and right neighbors.
$$
|\leftarrow p \rightarrow |
$$

If the relationship between the pivot and its neighbors is symmetric, since scanning range is $[i-k, i+k]$ and iterating from left to right, only consider $[i-k, i]$ to avoid duplication.
$$
|\leftarrow p
$$

\section{Operations}
\runinhead{Merge intervals.} Given a collection of intervals, merge all overlapping intervals.

\textbf{Core clues}:
\begin{enumerate}
\item Sort the intervals
\item When does the overlapping happens?
[0, 5) vs. [2, 6); [0, 5) vs. [2, 4)
\end{enumerate}
\begin{python}
def merge(self, itvls):
    if not itvls:
        return []

    itvls.sort(key=lambda x: x.start)
    ret = [itvls[0]]
    for cur in itvls[1:]:
        pre = ret[-1]
        if cur.start <= pre.end:  # overlap
            pre.end = max(pre.end, cur.end)
        else:
            ret.append(cur)

    return ret
\end{python}

\runinhead{Insert intervals.} Given a set of non-overlapping intervals, insert a new interval into the intervals (merge if necessary). Assume that the intervals were initially sorted according to their start times.

\textbf{Core clues}
\begin{enumerate}
\item Partition the original list of intervals to left-side intervals and right-side intervals according to the new interval.
\item Merge the intermediate intervals with the new interval. Need to mathematically prove it works as expected.
\end{enumerate}

\begin{python}
def insert(self, itvls, newItvl):
    s, e = newItvl.start, newItvl.end
    left = filter(lambda x: x.end < s, itvls)
    right = filter(lambda x: x.start > e, itvls)
    if len(left)+len(right) != len(itvls):
        s = min(s, itvls[len(left)].start)
        e = max(e, itvls[-len(right)-1].end)

    return left + [Interval(s, e)] + right

\end{python}

\section{Event-driven algorithm}
\subsection{Introduction}
The core philosophy of event-driven algorithm:
\begin{enumerate}
\item \textbf{Event}: define \textit{event}; the event are sorted by time of appearance.
\item \textbf{Heap}: define \textit{heap meaning}.
\item \textbf{Transition}: define \textit{transition functions} among events impacting the.
heap.
\end{enumerate}

\subsection{Questions}
\runinhead{Maximal overlaps.} Given a list of number intervals, find max number of overlapping
intervals.
\runinhead{Core clues:}
\begin{enumerate}
\item \textbf{Event}: Every new start of an interval is an event. Scan the sorted intervals (sort the interval by \textit{start}).
\item \textbf{Heap meaning}: Heap stores the \textit{end} of the interval.
\item \textbf{Transition}: Put the ending time into heap, and pop the ending time earlier than the new start time from heap.
\end{enumerate}
\newpage
\begin{python}
def max_overlapping(intervals):
    maxa = 0
    intervals.sort(key=operator.attrgetter("start"))
    h_end = []
    for itvl in intervals:
        heapq.heappush(end_heap, itvl.end)

        while h_end and h_end[0] <= itvl.start:
            heapq.heappop(h_end)

        maxa = max(maxa, len(h_end))

    return maxa
\end{python}


\section{Linear Systems of Equations and Pivoting Strategies}
\subsection{Gaussian Elimination with Backward Substitution}
\begin{align*}
    E1: & a_{11}x_{1}+a{12}x_{2}+\cdots+a_{1n}x_{n} = a_{1,n+1} \\
    E2: & a_{21}x_{1}+a{22}x_{2}+\cdots+a_{2n}x_{n} = a_{2,n+1} \\
    & \vdots \\
    En: & a_{n1}x_{1}+a{n2}x_{2}+\cdots+a_{nn}x_{n} = a_{n,n+1} \\
\end{align*}

\subsection{Operation Counts}
\subsubsection{Multiplications / divisions}
\[
\sum_{i=1}^{n-1}(n-i)+(n-i)(n-i+1)=\frac{2n^3+3n^2-5n}{6}
\]
\subsubsection{Additions / subtractions}
\[
\sum_{i=1}^{n-1}(n-i)(n-i+1)=\frac{n^3-n}{3}
\]
\subsection{Backward substitution *\&/}
\[
1+\sum_{i=1}^{n-1}((n-i)+1)=\frac{n^2+n}{2}
\]
\subsection{Backward substitution +\&-}
\[
\sum_{i=1}^{n-1}((n-i-1)+1)=\frac{n^2-n}{2}
\]

\subsection{Gaussian Elimination with Partial Pivoting}
\begin{python}
def gaussian_elimination_partial_pivoting(A, b):
    np_result = A**-1 * b
    print(np_result.T)
    n = A.shape[0]
    x = np.zeros((n,1))
    tmp = 0
    for k in range(n-1):
        M = k
        for m in range(k+1,n):
            if A[m,k] > A[M,k]: M = m
        A[[k,M]] = A[[M,k]]
        b[[k,M]] = b[[M,k]]
        for i in range(k+1,n):
            m = A[i,k] / A[k,k]
            for j in range(k,n):
                A[i,j] = A[i,j] - m*A[k,j]
            b[i,0] = b[i,0] - m*b[k,0]
    x[-1,0] = b[-1,0] / A[-1,-1]
    for i in range(n-2,-1,-1):
        for j in range(i+1,n):
            tmp += A[i,j] * x[j,0]
        x[i,0] = (b[i,0] - tmp) / A[i,i]
        tmp = 0
    return x
\end{python}

\subsection{Gaussian Elimination with Scaled Partial Pivoting}
\begin{python}
def gaussian_elimination_scaled_partial_pivoting(A, b):
    np_result = A**-1 * b
    print(np_result.T)
    n = A.shape[0]
    x = np.zeros((n,1))
    tmp = 0
    for k in range(n):
        M = np.max(A[k,:])
        A[k,:] /= M
        b[k,0] /= M
    for k in range(n-1):
        for i in range(k+1,n):
            m = A[i,k] / A[k,k]
            for j in range(k,n):
                A[i,j] = A[i,j] - m*A[k,j]
            b[i,0] = b[i,0] - m*b[k,0]
    x[-1,0] = b[-1,0] / A[-1,-1]
    for i in range(n-2,-1,-1):
        for j in range(i+1,n):
            tmp += A[i,j] * x[j,0]
        x[i,0] = (b[i,0] - tmp) / A[i,i]
        tmp = 0
    return x
\end{python}


\section{Naive Bayes}

Naive Bayes methods perform classification based on a direct application of Bayes theorem. Let $y$ denote the sought-after class label and $x_1,x_2...,x_k$ be the features. One is looking for the probability of observing a specific class label conditional on the features that were observed:

\begin{equation}
P(y|x_1,x_2,...,x_k) = \frac{P(x_1,x_2,...,x_k|y)}{P(y)}{P(x_1,x_2,...,x_k)}
\end{equation}

The "naivit\'e" comes in by assuming that the featues are conditionally independent from each other, i.e., given a specific class label $y$, the features are independent random variables. This means that the likelihood factorizes:

\begin{equation}
P(x_1,x_2,...,x_k|y) = P(x_1|y)P(x_2|y)...P(x_k|y)
\end{equation}

Since we are only interested in the relative probabilities of observing a certain class $y$, one need not worry about the evidence $P(x_1,x_2,...,x_k)$ because it does not depend on the class label. It follows the classification rule:

\begin{equation}
\hat{y} = \mathrm{argmax}_{y} P(y)\prod_i P(x_i|y)
\end{equation}

A nice feature is that this requires the estimation of only one dimensional probability distributions, which requires less data. It can also be made very computationally efficient as it naturally lends itself to batch training and parallelizability.

Naive Bayes methods differ among each other principally in the way that the conditional probabilities $P(x_i|y)$ are modeled. For example, Gaussian Naive Bayes assumes that the features have conditional Gaussian Distribution, Bernoulli Naive Bayes assumes conditional binary distribution, and so on. The assumption of conditional independence of the featues tends to hold up reasonably well in practice and amounts to the assumption that the features associated with a particular class label have a diagonal covariance matrix. In the case of Gaussian Naive Bayes, this assumption can be relaxed by instead implementing Quadratic Discriminant Analysis as a classifier, which turns out to be identical to Gaussian Naive Bayes except that it allows for the features to have general covariance matrixes.

In principle, one should be able to mix and match the way that the $P(x_i|y)$ are modeled, which enables naive Bayes to handle virtually any type of feature (though I have not seen implementations of that).


\subsection{Defining groups}

\subsubsection{Magma}

A magma, or groupoid, is a set with a single binary operation.

These can be defined as an ordered pair \((s,\odot )\) where \(s\) is the set, and \(\odot \) is the binary operation.

If \(a\) and \(b\) are in \(s\), then \(a\odot b\) is also in \(s\).

The following are magmas:

\begin{itemize}
\item Natural numbers and addition
\item \(n\times n\) matrices with determinants other than \(0\)
\item Natural numbers above \(0\) and addition
\item Integers and addition
\item Rational numbers and division
\item \(\{-1, 1\}\) and multiplication
\end{itemize}

The following are not magmas:

\begin{itemize}
\item Natural numbers up to \(10\) and addition
\end{itemize}

\subsubsection{Semigroup}

A semigroup is a magma whose binary operation is associative.

The following are semigroups:

\begin{itemize}
\item Natural numbers and addition
\item \(n\times n\) matrices with determinants other than \(0\)
\item Natural numbers above \(0\) and addition
\item Integers and addition
\end{itemize}

The following are not semigroups:

\begin{itemize}
\item \(\{-1, 1\}\) and multiplication
\item Rational numbers and division
\item Natural numbers up to \(10\) and addition
\end{itemize}

\subsubsection{Monoid}

A monoid is a semigroup with an identity element

The following are monoids:

\begin{itemize}
\item Natural numbers and addition
\item \(n\times n\) matrices with determinants other than \(0\)
\item Integers and addition
\item \(\{-1, 1\}\) and multiplication
\end{itemize}

The following are not monoids:

\begin{itemize}
\item Natural numbers above \(0\) and addition
\item Rational numbers and division
\item Natural numbers up to \(10\) and addition
\end{itemize}

\subsubsection{Group}

A group is a monoid where there is an inverse operation for the binary operation.

The following are groups:

\begin{itemize}
\item Integers and addition
\item \(n\times n\) matrices with determinants other than \(0\)
\item \(\{-1, 1\}\) and multiplication
\end{itemize}

The following are not groups:

\begin{itemize}
\item Natural numbers above \(0\) and addition
\item Rational numbers and division
\item Natural numbers and addition
\item Natural numbers up to \(10\) and addition
\end{itemize}


\subsection{Cramér-Rao lower bound}

For an unbiased estimator, the variance cannot be below the Cramer-Rao lower bound.

\(Var (\hat \theta )\ge \dfrac{1}{I(\theta )}\)

Where \(I(\theta )\) is the Fisher information.

We can prove this.

We have the score:

\(V=\dfrac{\delta }{\delta \theta }\ln f(X, \theta )\)

\(V=\dfrac{1}{f(X, \theta )}\dfrac{\delta }{\delta \theta } f(X, \theta )\)

The expectation of the score is \(0\):

\(E[V]=E[\dfrac{1}{f(X, \theta )}\dfrac{\delta }{\delta \theta } f(X, \theta )]\)

\(E[V]=\int \dfrac{1}{f(X, \theta )}\dfrac{\delta }{\delta \theta } f(X, \theta )dx\)


\section{Test statistic}
\label{sec:teststat}

To test the level of agreement between the data and the hypothesized value $\mu$, a test statistic $t_{\mu}$ can be defined as~\cite{Cowan:2010js}:
\begin{equation}
    t_{\mu} = -2 ln \lambda (\mu)
\end{equation}
From the definition of $\lambda(\mu)$ in equation~\ref{eq:lambda}, one can see that $0 \le \lambda \le 1$,
while a $\lambda$ with value close to 1 implies good agreement between data and $\mu$.
Thus, smaller value of $t_{\mu}$ means the increase of compatibility between data and $\mu$.
To quantify the level of disagreement, one can calculate the $p$-value as:
\begin{equation}
    p_{\mu} = \int_{t_{\mu, obs}}^{\infty} f(t_{\mu}|\mu) d t_{\mu}
\end{equation}
in which $t_{\mu, obs}$ is the value of test statistic from observed data,
and $f(t_{\mu}|\mu)$ is the pdf of $t_{\mu}$ under the assumption of hypothesized value $\mu$.
This is a one-side $p$-value with its corresponding observed significance, $Z$, is defined as:
\begin{equation}
    Z = \Phi^{-1}(1-2p_{\mu})
\end{equation}
The relationship between the $t_{\mu}$, $p$-value and significance $Z$ are depicted in figure~\ref{fig:pvalue_Z}.
When searching for a signal process, such as Higgs boson, the particle physics community tends to claim a discovery
when the rejection of background-only hypothesis has a significance of at least $Z = 5$.

\begin{figure}[!htbp]
\begin{center}
\includegraphics[width=1.0\textwidth]{figures/Statistic/test_statistic_pvalue_Z.png}
\end{center}
\caption{(a) Illustration of the relationship between the observed $t_{\mu}$ and its $p$-value.
         (b) The relationship between $p$-value and the observed significance $Z$, where $\phi(x)$ is the standard normal distribution~\cite{Cowan:2010js}.}
\label{fig:pvalue_Z}
\end{figure}

In most cases, one assumes that the presence of a new signal can only increase the event rate comparing to the background only model,
which means the signal strength cannot be a negative value ($\mu \ge 0$).
And for the case of discovery, the hypothesis of a positive signal strength should be tested against to the background-only (null) hypothesis by using the test statistic called $q_0$:
\begin{equation} \label{eq:q0}
    q_0 =
    \begin{cases}
        -2 ln(\lambda(0)) &\hat{\mu} \ge 0 \\
        0                 &\hat{\mu} < 0
    \end{cases}
\end{equation}
which corresponds to the $p$-value called $p_0$:
\begin{equation}
    p_0 = \int_{q_{0, obs}}^{\infty} f(q_0|0) d q_0
\end{equation}
to quantify the level of disagreement between the data and the null hypothesis ($\mu = 0$).


% !TEX root = main.tex

%-------------------------------------------------
\section{Generating functions}\label{sec:gen_funs}

%Generating functions are power series used to represent sequences of real numbers. It is often easier to work with generating functions than with the original sequences.
%
%% definition
%\begin{definition}
%The \emph{generating function} of a sequence $a_0,a_1,a_2,$ of real numbers is the function
%\[
%G(t) = \sum_{k=0}^{\infty} a_k t^k,
%\]
%which is defined for every $t\in\R$ for which the sum converges.
%\end{definition}
%
%The sequence can be reconstructed from $G(t)$ by setting
%\[
%\displaystyle a_n = \frac{1}{n!} G^{(n)}(0),
%\]
%where $G^{(n)}(t)$ is $n$th derivative of $G(t)$. In particular,
%\[
%G(0) = a_0,\quad G'(0) = a_1,\quad G''(0) = a_2,\quad \text{and so on.}
%\]
%
%
A \emph{power series} is an infinite series of the form
\[
G(t) = \sum_{k=0}^{\infty} a_k t^k.
\]
%If $G(t)<\infty$ we say that the power series \emph{converges} at $t$; otherwise we say it \emph{diverges} at $t$. If $G(t)$ converges at some $t\neq 0$ (it clearly converges at $t=0$) then there exists a positive number $R>0$ such that $G(t)$ converges for $|t|<R$ and diverges for $|t|>R$. This number is called the \emph{radius of convergence} of $G$.

Power series can be used to represent sequences of real numbers $a_0,a_1,a_2,\ldots$, in which case they are often called \emph{generating functions}. The terms of the sequence can be recovered by repeatedly differentiating the power series and evaluating these derivatives at $t=0$:
\[
G(0) = a_0,\quad G'(0) = a_1,\quad G''(0) = 2a_2,\quad G'''(0) = 6a_3\quad\text{and so on.}
\]
In this way, the power series \emph{generates} the sequence.

%%%%%%%%%%%%%%%%%%%%%%%%%%%
% PGFS
%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Probability generating functions}

Generating functions allow probability distributions to be represented as functions of a single variable.

% definition
\begin{definition}
Let $X$ be a discrete random variable taking values in the range $\{0,1,2,\ldots\}$ and let $f_X$ denote its PMF. The \emph{probability generating function} (PGF) of $X$ is the generating function of its PMF,
\[
G_X(t) = \expe(t^X) = \sum_{k=0}^{\infty} f_X(k) t^k.
\]
\end{definition}

\begin{remark}
Because $f$ is a PMF, $G(t)$ converges for all $|t|\leq 1$ with $G(0) = 0$ and $G(1)=\sum_{k=0}^{\infty} f(k) = 1$.
\end{remark}

As the following theorem shows, probability generating functions turn \emph{sums} of independent random variables into \emph{products} of independent random variables:
% theorem
\begin{theorem}\label{thm:PGF_sum_of_independent}
If $X$ and $Y$ are independent then $G_{X+Y}(t) = G_X(t)G_Y(t)$.
\end{theorem}
\begin{proof}
If $X$ and $Y$ are independent, then $t^X$ and $t^Y$ are also independent, so
\[
G_{X+Y}(t) = \expe(t^{X+Y}) = \expe(t^X t^Y) = \expe(t^X)\expe(t^Y) = G_X(t)G_Y(t).
\]
\end{proof}

\begin{exercise}
If $Y = a + bX$, show that $G_Y(t) = t^a G_X(t^b)$.
\begin{answer}
\[
G_Y(t) = \expe\big(t^{a+bX}\big) = t^a \expe\big((t^b)^X\big) = t^a G_X(t^b).
\]
\end{answer}
\end{exercise}

% corollary
\begin{corollary}\label{cor:PGF_sum_of_independent}
If $X_1,X_2,\ldots,X_n$ are independent random variables taking values in the non-negative integers, the PGF of their sum is equal to the product of their individual PGFs,
\[
G_{X_1+X_2+\ldots+X_n}(t) = G_{X_1}(t)G_{X_2}(t)\cdots G_{X_n}(t).
\]
\end{corollary}

The PGFs of some fundamental discrete distributions on $\{0,1,2,\ldots\}$ are shown in Table~\ref{tab:pmf_pgf}.
\begin{table}[ht]
\[\begin{array}{lclcll}
								& & \text{PMF} 					& & \text{PGF}					& \\ \hline
X\sim\text{Bernoulli}(p)		& & p^k(1-p)^{1-k} 				& & 1 - p + pt			& \\[1ex]
X\sim\text{Binomial}(n,p)		& & \displaystyle\binom{n}{k}p^k(1-p)^{n-k} 	& & (1 - p + pt)^n		& \\[2ex]
X\sim\text{Geometric}(p)		& & (1-p)^{k}p 					& & \displaystyle\frac{p}{1-(1-p)t}	& \text{for $|t|<(1-p)^{-1}$}	\\[2ex]
X\sim\text{Poisson}(\lambda)	& & \lambda^k e^{-\lambda}/k! 	& & e^{\lambda(t-1)} \\ \hline
\end{array}\]
\caption{The PGFs of some fundamental discrete distributions.\label{tab:pmf_pgf}}
\end{table}

\begin{exercise}
For each of the distributions shown in Table~\ref{tab:pmf_pgf} derive the expressions for their PGFs from the corresponding PMFs.
\begin{answer}
\ben
% bernoulli
\it If $X\sim\text{Bernoulli}(p)$, then $\prob(X=0)=1-p$, $\prob(X=1)=p$ and $\prob(X=k)=0$ for all $k\geq 2$ so
\[
G_X(t)	= \sum_{k=0}^\infty f_X(k) t^k
		= (1-p)t^0 + pt^1
		= 1 - p + pt.
\]
% binomial
\it If $X\sim\text{Binomial}(n,p)$, it can be written as $X=X_1+X_2+\ldots+X_n$ where each $X_i$ is independent with $X_i\sim\text{Bernoulli}(p)$. Thus by Corollary~\ref{cor:PGF_sum_of_independent},
\[
G_X(t)	= G_{X_1+X_2+\ldots+X_n}(t)
		= G_{X_1}(t)G_{X_2}(t)\cdots G_{X_n}(t)
		= (1 - p + pt)^n.
\]
% geometric
\it If $X\sim\text{Geometric}(p)$, then $\prob(X=k) = (1-p)^k p \text{ for } k=0,1,2,\ldots$, so
\[
G_X(t)	= \sum_{k=0}^\infty f_X(k) t^k
		= \sum_{k=0}^\infty (1-p)^k p t^k
		= p\sum_{k=0}^\infty \big[(1-p)t\big]^k
		= \frac{p}{1-(1-p)t} \quad\text{ for all } |t|<\frac{1}{1-p}.
\]
where we have used the fact that $\displaystyle\sum_{k=0}^{\infty} r^k = \frac{1}{1-r}$ for $|r|<1$.
% poisson
\it If $X\sim\text{Poisson}(\lambda)$, then $\prob(X=k) = \lambda^k e^{-\lambda}/x!$ so
\[
G_X(t)	= \sum_{k=0}^\infty f_X(k) t^k
		= \sum_{k=0}^{\infty}\left(\frac{\lambda^k e^{-\lambda}}{k!}\right)t^k
		= e^{-\lambda}\sum_{i=1}^\infty \frac{(\lambda t)^k}{k!}
		= e^{-\lambda}e^{\lambda t}
		= e^{\lambda(t-1)}.
\]
\een
\end{answer}
\end{exercise}

%-----------------------------
\subsection{Moments}

The moments of a distribution can be recoverd by repeatedly computing the derivatives of its PGF and evaluating the resulting expressions at $t=1$.
% theorem (factorial moments)
\begin{theorem}
Let $G_X^{(n)}(t)$ denote the $n$th derivative of $G_X(t)$. Then
\[
G_X^{(n)}(1) = \expe\big[X(X-1)\ldots(X-n+1)\big].
\]
These are called the \emph{factorial moments} of $X$.
\begin{proof}
Take $t < 1$, and compute the $n$th derivative of $G$ to obtain
\begin{align*}
G^{(n)}(t)
	& = \frac{d^n}{dt^n}\left[\sum_{k=0}^{\infty} t^{k} f_X(k)\right] \\
	& = \sum_{k=0}^{\infty} \left[\frac{d^n}{dt^n} t^{k}\right] f_X(k) \\
	& = \sum_{k=0}^{\infty} k(k-1)\cdots(k-n+1)t^{k-n} f_X(k) \\
	& = \expe\big[X(X-1)\cdots(X-n+1)t^{X-n} \big]
\end{align*}
Thus $G^{(n)}(1) = \expe\big[X(X-1)\cdots(X-n+1)\big]$, as required.
\end{proof}
\end{theorem}

% exercise (mean and variance)
\begin{exercise}
Show that $\expe(X) = G'_X(1)$ and $\var(X) = G''_X(1) + G'_X(1) - G'_X(1)^2$.
\begin{answer}
\begin{align*}
\var(X)
	& = \expe(X^2) - \expe(X)^2 \\
	& = \expe\big[X(X-1) + X\big] - \expe(X)^2 \\
	& = \expe\big[X(X-1)\big] + \expe(X) - \expe(X)^2 \\
	& = G''_X(1) + G'_X(1) - G'_X(1)^2.
\end{align*}
\end{answer}
\end{exercise}

%-----------------------------
\subsubsection{Sums with a random number of terms}

% theorem
\begin{theorem}\label{thm:GF-NofX}
Let $X_1,X_2,\ldots$ be independent and identically distributed random variables and let $G_X(t)$ denote their common PGF. Let $N$ be another random variable taking values in the non-negative integers and independent of the $X_i$, and let $G_N(t)$ denote its PGF. Then the PGF of the sum $S_N=X_1+X_2+\ldots+X_N$ is
\[
G_{S_N}(t) = G_N\big[G_X(t)\big].
\]
and its expectation satisfies $\expe(S_N) = \expe(N)\expe(X)$. (This is known as Wald's identity.)
\end{theorem}

\begin{proof}
\begin{align*}
G_{S_N}(t) = \expe(t^{S_N})
	& = \expe\big(\expe(t^{S_N}\,|\,N)\big) \text{\quad(law of total expectation)}\\
	& = \sum_{n=0}^{\infty}\expe(t^{S_N}\,|\,N=n)\prob(N=n) \\
	& = \sum_{n=0}^{\infty}\expe(t^{S_n})\prob(N=n) \\
	& = \sum_{n=0}^{\infty}\expe(t^{X_1+X_2+\ldots+X_n}\,|\,N=n)\prob(N=n) \\
	& = \sum_{n=0}^{\infty}\expe(t^{X_1})\expe(t^{X_2})\cdots\expe(t^{X_n})\prob(N=n) \quad\text{(by independence)}\\
	& = \sum_{n=0}^{\infty}G_X(t)^n\prob(N=n) \\
	& = G_N\big[G_X(t)\big].
\end{align*}
To find $\expe(S_N)$, we find $G_{S_N}(t)$ then evaluate its first derivative at $t=1$.
\[
\frac{d}{dt}\big[G_{S_N}(t)\big]
	= \frac{d}{dt}\big[G_N\big(G_X(t)\big)\big]
	= \frac{dG_N(u)}{du}\times\frac{du}{dt}
	\quad\text{where $u=G_X(t)$.}
\]
Setting $t=1$ so that $u=G_X(1)=1$,
\[
\expe(S_N) = G'_{S_N}(1) = \big[G'_N(1)\big]\big[G'_X(1)\big] = \expe(N)\expe(X).
\]
\end{proof}

% example: compound distribution
\begin{example}
A hen lays $N$ eggs where $N$ has Poisson distribution with parameter $\lambda$. If each egg hatches independently with probability $p$ show that the total number of chicks has Poisson distribution with parameter $\lambda p$.
\end{example}
\begin{solution}
Let $S_N = X_1+X_2+\ldots+X_N$ denote the total number of chicks. Each $X_i$ has $\text{Bernoulli}(p)$ distribution; let $G_X(t)$ denote their common generatig function. Then
\[
G_X(t) = 1 - p + pt \qquad\text{and}\qquad G_N(t) = e^{\lambda(t-1)},
\]
so by Theorem~\ref{thm:GF-NofX},
\[
G_{S_N}(t) = G_N\big[G_X(t)\big] = e^{\lambda(-p+pt)} = e^{\lambda p(t-1)}
\]
which we recognise as the PGF of the $\text{Poisson}(\lambda p)$ distribution. Note that by Corollary~\ref{cor:GF-NofX}, the expected number of chicks is $\expe(S_N) = \expe(N)\expe(X) = \lambda p$, which is the expected value of the $\text{Poisson}(\lambda p)$ distribution.
\end{solution}

\begin{exercise}
\begin{questions}
\question
Let $X\sim\text{Binomial}(m,p)$ and $Y\sim\text{Binomial}(n,p)$ be independent. Show that $X+Y\sim\text{Binomial}(m+n,p)$.
\begin{answer}
The PGFs of $X$ and $Y$ are
\[
G_X(t) = (1-p+pt)^m
\quad\text{and}\quad
G_Y(t) = (1-p+pt)^n
\]
Using the properties of PGFs,
\[
G_{X+Y}(t) = G_X(t)G_Y(t) = (1-p+pt)^m (1-p+pt)^n = (1-p+pt)^{m+n},
\]
which we recognise as the PGF of the $\text{Binomial}(m+n,p)$ distribution.
\end{answer}

\question
Let $X\sim\text{Poisson}(\lambda)$ and $Y\sim\text{Poisson}(\mu)$ be independent. Show that $X+Y\sim\text{Poisson}(\lambda+\mu)$.
\begin{answer}
The PGFs of $X$ and $Y$ are respectively $G_X(t)= e^{\lambda(t-1)}$ and $G_Y(t) = e^{\mu(t-1)}$, so the PGF of $Z=X+Y$ is
\[
G_Z(t) = G_X(t)G_Y(t) = e^{(\lambda+\mu)(t-1)}.
\]
We recognise this as the PGF of a $\text{Poisson}(\lambda+\mu)$ random variable, so by the inversion theorem for PGFs, $Z\sim\text{Poisson}(\lambda+\mu)$.
\end{answer}

\question % GS 5.2.4
Let $X\sim\text{Binomial}(n,p)$. Using the PGF of $X$, show that
\[
\expe\left(\frac{1}{1+X}\right) = \frac{1-(1-p)^{n+1}}{(n+1)p}.
\]

\begin{answer}
Let $G(t)$ be the PGF of $X$. Then $G(t)=\expe(t^X) = (q+pt)^n$ where $q=1-p$.

Now
\[
\int_0^1 t^x\,dt = \left[\frac{t^{1+x}}{1+x}\right]_0^1 = \frac{1}{1+x},
\]
so
\[
\expe\left(\frac{1}{1+X}\right)
	= \expe\left(\int_0^1 t^X \,dt\right)
	= \int_0^1 \expe(t^X)\,dt
	= \int_0^1 (q+pt)^n\,dt
	= \frac{1-q^{n+1}}{(n+1)p}
\]
\end{answer}

\end{questions}
\end{exercise}

%%%%%%%%%%%%%%%%%%%%%%%%%%%
% MGFS
%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Moment generating functions}

% definition
\begin{definition}
The \emph{moment generating function} (MGF) of a random variable $X$ is the function
\[
\begin{array}{rccl}
M_X:	& \R	& \to		& [0,\infty] \\
		& t		& \mapsto	& \expe(e^{tX})
\end{array}
\]
\end{definition}

Becaaus $e^{tX}$ is non-negative, its expectation is well-defined and $\expe(e^{tX})\geq 0$. It may however be that $\expe(e^{tX})$ is infinite, which limits the usefulness of MGFs in some cases.

\begin{theorem}
If $X$ takes only non-negative integer values then $M_X(t) = G_X(e^t)$ where $G_X$ is the PGF of $X$.
\end{theorem}
\begin{proof}
\[
M_X(t) = \expe(e^{tX}) = \expe\big[(e^t)^X\big] = G_X(e^t),
\]
\end{proof}

% example: discrete (easy)
\begin{example}
The MGFs of some fundamental discrete distributions on $\{0,1,2,\ldots\}$ can be computed as follows:
\[
\begin{array}{llll}
X\sim\text{Bernoulli}(p):\quad	& G_X(t) = 1 - p + pt\quad	&\quad\Rightarrow\quad & M_X(t) =  1 - p + pe^t		\\[2ex]
X\sim\text{Binomial}(n,p):		& G_X(t) = (1-p+pt)^n			&\quad\Rightarrow\quad & M_X(t) = (1 - p + pe^t)^n	\\[2ex]
X\sim\text{Poisson}(\lambda):	& G_X(t) = e^{\lambda(t-1)}	&\quad\Rightarrow\quad & M_X(t) = e^{\lambda(e^t-1)}	\\
\end{array}
\]
\end{example}

% theorem
\begin{theorem}[Properties of MGFs]\label{thm:props_mgfs}
\ben
\it If $X$ and $Y$ are independent, then $M_{X+Y}(t) = M_X(t)M_Y(t)$.
\it If $Y = a + bX$, then $M_Y(t) = e^{at} M_X(bt)$
\een
\begin{proof}
\ben
\it By independence,
\[
M_{X+Y}(t) = \expe(e^{t(X+Y)}) = \expe(e^{tX}e^{tY}) = \expe(e^{tX})\expe(e^{tY}) = M_X(t)M_Y(t).
\]
\it For $Y=a+bX$,
\[
M_Y(t) = \expe\big(e^{t(a+bX)}\big) = e^{at}\expe\big(e^{btX}\big) = e^{at}M_X(bt).
\]
\een
\end{proof}
\end{theorem}

\begin{corollary}\label{cor:mgfs_for_sums}
If $X_1,X_2,\ldots,X_n$ are independent random variables,
\[
M_{X_1+X_2+\ldots+X_n}(t) = M_{X_1}(t)M_{X_2}(t)\cdots M_{X_n}(t).
\]
\begin{proof}
If $X_1,X_2,\ldots,X_n$ are independent, $e^{tX_1},e^{tX_2},\ldots,e^{tX_n}$ are also independent, so
\begin{align*}
M_{X_1+X_2+\ldots+X_n}(t)
	& = \expe(e^{t(X_1+x_2+\ldots+X_n)}) \\
	& = \expe(e^{tX_1})\expe(e^{tX_2})\ldots\expe(e^{tX_n}) \quad\text{(by independence)}\\
	& = M_{X_1}(t)M_{X_2}(t)\cdots M_{X_n}(t)
\end{align*}
\end{proof}
\end{corollary}

%
%
%\begin{example}
%Find the MGFs of the $\text{Bernoulli}(p)$ and $\text{Binomial}(n,p)$ distributions.
%\begin{solution}
%\bit
%\it
%For $X\sim\text{Bernoulli}(p)$, $\prob(X=0)=1-p$ and $\prob(X=1)=p$ (and zero otherwise) so
%\[
%M_X(t)	= \expe(e^{tX}) = (1-p)e^0 + pe^t = 1 - p + pe^t.
%\]
%\it
%If $X\sim\text{Binomial}(n,p)$ it can be written as $X=X_1+X_2+\ldots+X_n$, where each $X_i$ is independent with $X_i\sim\text{Bernoulli}(p)$. Thus by Corollary~\ref{cor:mgfs_for_sums},
%\[
%M_X(t)	= M_{X_1+X_2+\ldots+X_n}(t)
%		= M_{X_1}(t)M_{X_2}(t)\cdots M_{X_n}(t)
%		= (1 - p + pe^t)^n.
%\]
%\eit
%\end{solution}
%\end{example}
%
%The MGFs of some discrete distributions are shown in Table~\ref{tab:mgfs}.
%\begin{table}[ht]
%\[\begin{array}{lclcll}
%Distribution					& & \text{PMF} 					& & \text{MGF}					& \\ \hline
%X\sim\text{Bernoulli}(p)		& & p^k(1-p)^{1-k} 				& & 1 - p + pe^t			& \\[1ex]
%%X\sim\text{Binomial}(n,p)		& & \displaystyle\binom{n}{k}p^k(1-p)^{n-k} 	& & (1 - p + pe^t)^n		& \\[2ex]
%X\sim\text{Geometric}(p)		& & (1-p)^{k}p 					& & \displaystyle\frac{p}{1-(1-p)e^t}	& \text{for $|t|<(1-p)^{-1}$}	\\[2ex]
%X\sim\text{Poisson}(\lambda)	& & \lambda^k e^{-\lambda}/k! 	& & e^{\lambda(e^t-1)} \\ \hline
%\end{array}\]
%\caption{MGFs of some discrete distributions\label{tab:mgfs}}
%\end{table}

% example
\begin{example}[Normal distribution]\label{exa:mgf_normal}
By first considering the MGF of the $N(0,1)$ distribution, show that the MGF of the $N(\mu,\sigma^2)$ distribution is given by
\[
M_X(t) = \exp\left(\mu t + \frac{1}{2}\sigma^2t^2\right).
\]
\end{example}

\begin{solution}
Let $Z\sim N(0,1)$ be a standard normal variable; this has PDF
\[
f_Z(z) = \frac{1}{\sqrt{2\pi}}e^{-\frac{1}{2}z^2}\qquad (z\in\R).
\]
Its MGF is therefore given by
\[
M_Z(t) = \expe(e^{tZ})
	 = \int_{-\infty}^{\infty} e^{tz} f_Z(z)\,dz
	 = \frac{1}{\sqrt{2\pi}}\int_{-\infty}^{\infty} e^{tz-\frac{1}{2}z^2}\,dz
\]
The exponent $tz-\frac{1}{2}z^2$ can be written as $\frac{1}{2}t^2 -\frac{1}{2}(z-t)^2$, so
\begin{align*}
M_Z(t)
	& = \frac{1}{\sqrt{2\pi}}\int_{-\infty}^{\infty} e^{\frac{1}{2}t^2} e^{-\frac{1}{2}(z-t)^2}\,dz \\
	& = e^{\frac{1}{2}t^2} \left(\frac{1}{\sqrt{2\pi}}\int_{-\infty}^{\infty} e^{-\frac{1}{2}(z-t)^2}\,dz\right)
	= e^{\frac{1}{2}t^2}.
\end{align*}

Let $X=\mu+\sigma Z$. Then $X\sim N(\mu,\sigma^2)$ and by the properties of MGFs,
\[
M_X(t) = e^{\mu t}M_Z(\sigma t) = e^{\mu t}e^{\frac{1}{2}\sigma^2t^2}= e^{\mu t + \frac{1}{2}\sigma^2t^2},
\]
as required.
\end{solution}

\begin{exercise}
\begin{questions}

\question
Let $X\sim\text{Geometric}(p)$ be the distribution of the number of failures before the first success in a sequence of independent Bernoulli trials in which the probability of success is $p$. Show that $M_X(t) = p/\big[1-(1-p)e^t\big]$ for $|t|<-\log(1-p)$.
\begin{answer}
For $X\sim\text{Geometric}(p)$, $\prob(X=k) = (1-p)^k p \text{ for } k=0,1,2,\ldots$, so
\[
M_X(t)	= \sum_{k=1}^\infty e^{tk}\prob(X=k)
		= \sum_{k=1}^\infty (1-p)^k p e^{tk}
		= p\sum_{k=1}^\infty \big[(1-p)e^t\big]^k
		= \frac{pe^t}{1-(1-p)e^t} \quad\text{ for all } t < -\log(1-p).
\]
where we have used the fact that $\sum_{k=0}^{\infty} r^k = \frac{1}{1-r}$ for $|r|<1$.
\end{answer}

\question
Let $X\sim\text{Poisson}(\lambda)$. Show that $M_X(t) = e^{\lambda(e^t-1)}$.
\begin{answer}
For $X\sim\text{Poisson}(\lambda)$, $\prob(X=k) = \lambda^k e^{-\lambda}/x!$ so
\[
M_X(t)	= \sum_{k=0}^\infty e^{tk}\prob(X=k)
		= \sum_{k=0}^{\infty}\left(\frac{\lambda^k e^{-\lambda}}{k!}\right)e^{tk}
		= e^{-\lambda}\sum_{i=1}^\infty \frac{(\lambda e^t)^k}{k!}
		= e^{-\lambda}e^{\lambda e^t}
		= e^{\lambda(e^t-1)}.
\]
\end{answer}
\end{questions}
\end{exercise}

%-----------------------------
%\subsubsection{Moments}

% theorem
\begin{theorem}\label{thm:taylor_mgf}
If $M_X(t)$ converges in some neighbourhood of $0$ then
\[
M_X(t) = \sum_{k=0}^{\infty} \frac{\expe(X^k)}{k!} t^k.
\]
\begin{proof}
Using the series expansion of $e^{tX}$ and the linearity of expectation,
\[
M_X(t)
	= \expe(e^{tX})
	= \expe\left(\sum_{k=0}^{\infty} \frac{(tX)^k}{k!}\right)
	= \sum_{k=0}^{\infty} \frac{\expe(X^k)}{k!} t^k
\]
\end{proof}
\end{theorem}

The moments of a distribution can be recovered from its MGF by repeated differentiation.

\begin{corollary}
Let $X$ be a random variable and let $M_X(t)$ be its MGF. Then
\[
\expe(X^k) = M_X^{(k)}(0)
\]
where $M_X^{(k)}(0)$ is the $k$th derivative of $M_X(t)$ evaluated at $t=0$.
\end{corollary}

% example: exponential
\begin{example}[Exponential distribution]
Let $X\sim\text{Exponential}(\lambda)$ where $\lambda>0$ is a rate parameter.
\ben
\it Show that $M_X(t)=\displaystyle\frac{\lambda}{\lambda-t}$.
\it Use $M_X(t)$ to find the mean and variance of $X$.
\een
\end{example}

\begin{solution}
\ben
\it % << (i)
The PDF of $X$ is $f(x)=\lambda e^{-\lambda x}$ for $x>0$ (and zero otherwise), so
\begin{align*}
M_X(t)
	= \int_0^\infty e^{tx} f(x)\,dx
	= \lambda \int_0^\infty e^{tx} e^{-\lambda x}\,dx
	& = \lambda \int_0^\infty e^{-(\lambda-t)x}\,dx \\
	& = \frac{-\lambda}{\lambda-t} \left[e^{-(\lambda-t)x}\right]_0^\infty
	= \frac{\lambda}{\lambda-t}.
\end{align*}
\it % << (ii)
\bit
\it $M'(t) = \displaystyle \frac{\lambda}{(\lambda-t)^2}$, which yields $\mu_1 = M'(0)  = \displaystyle\frac{1}{\lambda}$.
\it $M''(t) = \displaystyle \frac{2\lambda}{(\lambda-t)^3}$ which yields $\mu_2 = M''(0)  = \displaystyle\frac{2}{\lambda^2}$.
\it Thus $\sigma^2 = \mu_2 - \mu_1^2 = \displaystyle\frac{1}{\lambda^2}$.
\eit
\een
\end{solution}

%%-----------------------------
%\subsection{Inversion}
%
%If the MGF of a random variable converges in some neighbourhood of $0$, the distribution of the random variable is uniquely specified by its MGF. Moreover if another random variable has the same MGF then both random variables have the same distribution. This is encapsulated in the following theorem (which we shall not prove).
%
%\begin{theorem}[Inversion theorem for MGFs]
%\ben
%\it If $M_X(t)<\infty$ in some neighbourhood of $0$, then $M_X$ uniquely determines the distribution of $X$.
%\it If $M_X(t) = M_Y(t)$ in some neighbourhood of $0$, then $X$ and $Y$ have the same distribution.
%\een
%\end{theorem}
%
%Most distributions we encounter are uniquely determined by their MGFs (a notable exception is the \emph{Cauchy distribution}). It is not always easy however to recover a PMF/PDF by explicitly inverting the associated MGF. Instead they are usually inverted by inspection, where we compare the MGF in question with the known MGFs of various standard distributions.
%
%\begin{example}
%Let $X\sim\text{Binomial}(m,p)$ and $Y\sim\text{Binomial}(n,p)$ be independent. Show that $X+Y\sim\text{Binomial}(m+n,p)$,
%\begin{solution}
%\begin{align*}
%M_{X+Y}(t) 	& = M_X(t)M_Y(t) \\
%			& = (1-p+pe^t)^m (1-p+pe^t)^n \\
%			& = (1-p+pe^t)^{m+n}.
%\end{align*}
%We recognise this as the MGF of the $\text{Binomial}(m+n,p)$ distribution. The result then follows by the inversion theorem for MGFs.
%\end{solution}
%\end{example}

%-----------------------------
\begin{exercise}
\begin{questions}

\question % gamma
Let $X\sim\text{Gamma}(\alpha,\beta)$ where $\beta$ is a rate parameter. Find the MGF of $X$ and use this to show that $\expe(X)=\alpha/\beta$ and $\var(X)=\alpha/\beta^2$.
\begin{answer}
Let $X\sim\text{Gamma}(\alpha,\beta)$. The PDF is
\[
f(x) =  \frac{\beta^{\alpha}}{\Gamma(\alpha)}\, x^{\alpha-1} e^{-\beta x}
\quad\text{where}\quad
\Gamma(\alpha) = \int_0^{\infty} t^{\alpha-1}e^{-t}\,dt
\]
for $x>0$, and zero otherwise. The MGF is
\begin{align*}
M(t) = \expe(e^{tX})
	 = \int_{-\infty}^\infty e^{tx} f(x)\,dx
	& = \frac{\beta^\alpha}{\Gamma(\alpha)}\int_0^\infty e^{tx} x^{\alpha-1}e^{-\beta x}\,dx \\
	& = \frac{\beta^\alpha}{\Gamma(\alpha)}\int_0^\infty x^{\alpha-1}e^{-(\beta-1)x}\,dx \\
	& = \frac{\beta^\alpha}{\Gamma(\alpha)}\frac{1}{(\beta-t)^{\alpha}}\int_0^\infty x^{\alpha-1}e^{-u}\,du \\
	& = \left(\frac{\beta}{\beta-t}\right)^\alpha.
\end{align*}
Elementary calculus yields
\[
M'(t) = \frac{\alpha\beta^\alpha}{(\beta-t)^{\alpha+1}}
\quad\text{and}\quad
M''(t) = \frac{\alpha(\alpha+1)\beta^\alpha}{(\beta-t)^{\alpha+2}}.
\]
Evaluating these at $t=0$, we get $\expe(X)	= \alpha/\beta$ and $\expe(X^2)=\alpha(\alpha+1)/\beta^2$, and hence
$\var(X)=\alpha(\alpha+1)/\beta^2$ as required.
\end{answer}

\question % two poisson
Let $X\sim\text{Poisson}(\lambda)$ and $Y\sim\text{Poisson}(\mu)$ be independent. Show that $X+Y\sim\text{Poisson}(\lambda+\mu)$.
\begin{answer}
The PGFs of $X$ and $Y$ are respectively $G_X(t)= e^{\lambda(t-1)}$ and $G_Y(t) = e^{\mu(t-1)}$, so the PGF of $Z=X+Y$ is
\[
G_{X+Y}(t) = G_X(t)G_Y(t) = e^{\lambda(t-1)}e^{\mu(t-1)} = e^{(\lambda+\mu)(t-1)}.
\]
which we recognise as the MGF of the $\text{Poisson}(\lambda+\mu)$ distribution. (The result follows by the inversion theorem.)
\end{answer}

\question % two normal
Let $X\sim N(\mu_1,\sigma_1^2)$ and $Y\sim N(\mu_2,\sigma_2^2)$ be independent. Show that
$
X+Y \sim N(\mu_1+\mu_2,\sigma_1^2+\sigma_2^2).
$
\begin{answer}
Because $X$ and $Y$ are independent,
\begin{align*}
M_{X+Y}(t)
	& = M_X(t)M_Y(t) \\
	& = \big[e^{\mu_1 t + \frac{1}{2}\sigma_1^2t^2}\big]\big[e^{\mu_2 t + \frac{1}{2}\sigma_2^2t^2}\big] \text{(as calculated in Example~\ref{exa:mgf_normal})} \\
	& = e^{(\mu_1+\mu_2) t + \frac{1}{2}(\sigma_1^2+\sigma_2^2)t^2} \\
\end{align*}
We recognise this as the MGF of the normal distribution with mean $\mu_1+\mu_2$ and variance $\sigma_1^2+\sigma_2^2$. By the inversion theorem for MGFs, we conclude that $X+Y \sim N(\mu_1+\mu_2,\sigma_1^2+\sigma_2^2)$.
\end{answer}

\question % mixture
Let $X$ and $Y$ be independent random variables and let $Z$ be equal to $X$ with probability $p$, and equal to $Y$ with probability $1-p$. Use the law of total expectation to show that
\[
M_Z(t) = pM_X(t) + (1-p)M_Y(t).
\]
\begin{answer}
By the law of total expectation,
\begin{align*}
M_Z(t)
	& = \expe(e^{tZ}) \\
	& = \expe(e^{tZ}|\text{$X$ chosen})\prob(\text{$X$ chosen}) + \expe(e^{tZ}|\text{$Y$ chosen})\prob(\text{$Y$ chosen})  \\
	& = \expe(e^{tX})\prob(\text{$X$ chosen}) + \expe(e^{tY})\prob(\text{$Y$ chosen})  \\
	& = pM_X(t) + (1-p)M_Y(t).
\end{align*}
\end{answer}

\end{questions}
\end{exercise}

%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CFS
%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Characteristic functions}
%----------------------------------------------------------------------

MGFs can be useful but the expectations that define them may be infinite or not even defined (a notable example is the Cauchy distribution). Characteristic functions do not suffer this disadvantage.

% definition
\begin{definition}
The \emph{characteristic function} (CF) of a random variable $X$ is the function
\[
\begin{array}{rccl}
\phi_X:	& \R	& \to		& \C \\
		& t		& \mapsto	& \expe(e^{itX})
\end{array}
\]
\end{definition}

% fourier
\begin{remark}
\bit
\it $\phi(t) = \expe(\cos tX)+ i\expe(\sin tX)$.
\it $\phi:\R\to\C$ exists for all $t\in\R$.
\it $\phi(t) = M(it)$ provided the latter exists.
\eit
\end{remark}

%The proof of the next result follows that of Theorem~\ref{thm:props_mgfs}
%-------------------------
% theorem
\begin{theorem}[Properties of characteristic functions]\label{thm:properties_cf}
\ben
%\it $\phi(0)=1$ and $|\phi(t)|\leq 1$ for all $t$.
\it If $X$ and $Y$ are independent then $\phi_{X+Y}(t) = \phi_X(t)\phi_Y(t)$.
\it If $Y = a + bX$ then $\phi_Y(t) = e^{iat}\phi_X(bt)$
\een
\end{theorem}

% proof
\begin{proof}
\ben
\it % << (i)
$\phi(0)=\expe(e^0)=\expe(1) = 1$, and
$\displaystyle |\phi(t)| = \left|\int e^{itx}\,dF(x)\right| \leq \int |e^{itx}|\,dF(x) = 1$.
\it % << (ii)
By independence, $\phi_{X+Y}(t) = \expe(e^{it(X+Y)}) = \expe(e^{itX}e^{itY})$, and since
\[
e^{itX} = \cos(tX) + i\sin(tX)\quad\text{and}\quad e^{itY} = \cos(tY) + i\sin(tY),
\]
we see that
\[
e^{itX}e^{itY} = \big[\cos(tX)\cos(tY) - \sin(tX)\sin(tY)\big] + i\big[\cos(tX)\sin(tY) + \sin(tX)\cos(tY)\big]
\]
Taking the expectation of both sides, and using the fact that $X$ and $Y$ are independent,
\begin{align*}
\phi_{X+Y}(t)
	& = \expe\big(\cos(tX)+i\sin(tX)\big)\expe\big(\cos(tY)+i\sin(tY)\big) \\
	& = \expe(e^{itX})\expe(e^{itY}) \\
	& = \phi_X(t)\phi_Y(t).
\end{align*}
\it % << (ii)
$\phi_Y(t)
	= \expe\big(e^{it(a+bX)}\big)
	= \expe\big(e^{iat}e^{i(bt)X}\big)
	= e^{iat}\expe(e^{i(bt)X})
	= e^{iat}\phi_X(bt)$.
\een
\end{proof}

%-----------------------------
%\subsubsection{Inversion and continuity}
The next two theorems are needed to prove the central limit theorem.

The \emph{inversion theorem} states that the distribution a random variable is uniquely determined by its CF, and if another random variable has the same CF then they both have the same distribution.

\begin{theorem}[Inversion theorem for CFs]
\ben
\it $\phi_X$ uniquely determines the distribution of $X$.
\it If $\phi_X(t) = \phi_Y(t)$ in some neighbourhood of $0$ then $X$ and $Y$ have the same distribution.
\een
\end{theorem}
%\proofomitted

It is not always easy to recover a distribution by explicitly inverting the associated CF. Instead they are usually inverted by inspection, where we compare the CF in question with the CFs of various standard distributions.

\begin{example}
Let $X\sim\text{Binomial}(m,p)$ and $Y\sim\text{Binomial}(n,p)$ be independent. Show that $X+Y\sim\text{Binomial}(m+n,p)$,
\begin{solution}
\begin{align*}
\phi_{X+Y}(t)
	& = M_X(t)M_Y(t) \\
	& = (1-p+pe^{it})^m (1-p+pe^{it})^n \\
	& = (1-p+pe^{it})^{m+n}.
\end{align*}
We recognise this as the CF of the $\text{Binomial}(m+n,p)$ distribution. The result then follows by the inversion theorem for CFs.
\end{solution}
\end{example}

The \emph{continuity theorem} states that a sequence of distributions $F_1,F_2,\ldots$ converges to a limiting distribution $F$ if and only if the corresponding sequence of characteristic functions $\phi_1,\phi_2,\ldots$ converges to the characteristic function of $F$.

%% convergence in distribution
%\begin{definition}
%A sequence $F_1,F_2,\ldots$ of CDFs is said to \emph{converge} to $F$, denoted by $F_n\to F$, if $F_n(x)\to F(x)$ as $n\to\infty$ for each point $x$ at which $F$ is continuous.
%\end{definition}

% theorem: continuity
\begin{theorem}[Continuity theorem for CFs]
Let $F_1,F_2,\ldots$ and $F$ be CDFs and let $\phi_1,\phi_2,\ldots$ and $\phi$ be the corresponding CFs.
Then $F_n\to F$ if and only if $\phi_n\to\phi$ as $n\to\infty$.
%\ben
%\it If $F_n\to F$ then $\phi_n(t)\to\phi(t)$ for all $t$.
%\it If $\phi_n(t)\to\phi(t)$ for all $t$ then $F_n\to F$.
%\een
\end{theorem}
%\proofomitted


\section{Orbitals}
\subsection{Normalisation of Primitive Contractions}

Consider a generic primitive function,
%
\begin{equation}
	\phi_\mu(\mathbf{r}) = N x^\ell y^m z^n \exp\Big[ -\alpha \mathbf{r}\cdot \mathbf{r} \Big]\,,
\end{equation}
%
where $N$ is an as yet undetermined normalisation constant. The normalisation condition is given by
%
\begin{align}
	\int^\infty_{-\infty} d\mathbf{r} \phi_\mu^*(\mathbf{r})\phi_\mu(\mathbf{r}) &= 1 \\
    N^2 \int_{-\infty}^{\infty} d\mathbf{r} x^{2\ell} y^{2m} z^{2n} \exp\Big[ -2\alpha \mathbf{r}\cdot\mathbf{r} \Big] &= 1 \\
    N^2 \int_{-\infty}^\infty dx x^{2\ell} \int_{-\infty}^\infty dy y^{2m} \int_{-\infty}^\infty dz z^{2n} \exp\Big[ -2\alpha \mathbf{r}\cdot\mathbf{r} \Big] &= 1 \,,
\end{align}
%
where we have split $d\mathbf{r} = dxdydz$. Further, the exponential can be split into a product of
exponentials in the three Cartesian degrees of freedom
%
\begin{equation}\label{eq::Normalisation}
	N^2 \int_{-\infty}^\infty dx x^{2\ell} \exp\Big[ -2\alpha x^2 \Big] \int_{-\infty}^\infty dy y^{2m} \exp\Big[ -2\alpha y^2 \Big] \int_{-\infty}^\infty dz z^{2n} \exp\Big[ -2\alpha z^2 \Big]  = 1\,.
\end{equation}
%
The individual integrals can be evaluated in closed form. The integral with respect to $dx$, for
instance, can we written
%
\begin{equation}
	\int_{-\infty}^\infty dx x^{2\ell} \exp\Big[ -2\alpha x^2 \Big] = \sqrt{\frac{\pi}{2\alpha}} \frac{(2\ell - 1)!!}{(4\alpha)^\ell}\,.
\end{equation}
%
The integrals with respect to $dy$ and $dz$ have similar solutions. Then, we can write
\eqref{eq::Normalisation} as
%
\begin{equation}
	N^2 \Bigg(\frac{\pi}{2\alpha}\Bigg)^{3/2} \frac{(2\ell - 1)!!}{(4\alpha)^\ell} \frac{(2m - 1)!!}{(4\alpha)^m}\frac{(2n - 1)!!}{(4\alpha)^n} = 1\,.
\end{equation}
%
Rearranging, the normalisation constant is given by
%
\begin{align}
	N^2 &= \Bigg( \frac{2\alpha}{\pi} \Bigg)^{3/2} \frac{(4\alpha)^{\ell + m + n}}{(2\ell - 1)!!(2m - 1)!!(2n - 1)!!} \nonumber \,, \\
    N &= \Bigg( \frac{2\alpha}{\pi} \Bigg)^{3/4} \sqrt{\frac{(4\alpha)^{\ell + m + n}}{(2\ell - 1)!!(2m - 1)!!(2n - 1)!!}}\,.
\end{align}
%
Let us take the trivial example of $\ell = m = n = 0$ (note that $(-1)!! = 1$), where $N$
reduces to
%
\begin{equation}
	N = \Bigg( \frac{2\alpha}{\pi} \Bigg)^{3/4} \sqrt{\frac{(4\alpha)^0}{(-1)!!(-1)!!(-1)!!}} = \Bigg( \frac{2\alpha}{\pi} \Bigg)^{3/4}\,.
\end{equation}

\subsection{Laplacian of Contracted Gaussian Atomic Orbitals}
%
Consider a Contracted Gaussian Atomic Orbital (CGO), $\Psi$, which is centred on
$\mathbf{R} = \big[ \begin{array}{ccc} X & Y & Z \end{array}\big]^\top$, and is a function of
the electronic position vector, $\mathbf{r} = \big[ \begin{array}{ccc} x & y & z \end{array}\big]^\top$.
$\Psi(\mathbf{r}, \mathbf{R})$ takes the form of a linear combination of $N$ primitive Gaussian functions
%
\begin{equation}
	\Psi(\mathbf{r,R}) = (x - X)^\ell (y - Y)^m (z - Z)^n \sum_{i=1}^N c_i \exp \Big[ -\zeta_i |\mathbf{r-R}|^2 \Big] \,.
\end{equation}
%
By changing our coordinate system so that $\mathbf{R}$ is the origin, we can omit the difference
coordinates and write
%
\begin{equation}
	\Psi(\mathbf{r}) = x^\ell y^m z^n \sum_{i=1}^N c_i \exp \Big[ -\zeta_i r^2 \Big] \,,
\end{equation}
%
where $r^2 = \mathbf{r}\cdot \mathbf{r}$. Since the Laplacian is the divergence of the gradient,
$\nabla \cdot \nabla$, we can evaluate $\nabla^2 \Psi(\mathbf{r})$ by consecutive operation of $\nabla$.
For the sake of simplicity, we consider the action of $\nabla_x^2$ on $\Psi(\mathbf{r})$ and save the
generalisation for later. A key identity used throughout is
%
\begin{displaymath}
	\frac{d}{dx} \exp\Big[ -\zeta_i r^2 \Big] = -2x \exp\Big[ -\zeta_i r^2 \Big] \,,
\end{displaymath}
%
which can be derived trivially. First, by the product theorem
%
\begin{equation}
	\nabla_x \Psi(\mathbf{r}) = \ell x^{\ell-1} y^m z^n \sum_{i=1}^N c_i \exp \Big[ -\zeta_i r^2 \Big] - 2x^{\ell+1}y^m z^n \sum_{i=1}^N \zeta_i c_i \exp \Big[ -\zeta_i r^2 \Big] \,.
\end{equation}
%
Dealing with the action of $\nabla_x$ on the first term,
%
\begin{align}\label{eq::FirstTerm}
	\nabla_x \Big(  \ell x^{\ell-1} y^m z^n \sum_{i=1}^N c_i \exp \Big[ -\zeta_i r^2 \Big] \Big) &= \ell(\ell-1)x^{\ell-2} y^m z^n \sum_{i=1}^N c_i \exp \Big[ -\zeta_i r^2 \Big] \nonumber \\
    &- 2 \ell x^\ell y^m z^n \sum_{i=1}^N \zeta_i c_i \exp \Big[ -\zeta_i r^2 \Big] \,.
\end{align}
%
Similarly, the action of $\nabla_x$ on the second term
%
\begin{align}\label{eq::SecondTerm}
	\nabla_x \Big( 2x^{\ell+1} y^m z^n \sum_{i=1}^N \zeta_i c_i \exp \Big[ -\zeta_i r^2 \Big] \Big) &= 2(\ell+1)x^\ell y^m z^n \sum_{i=1}^N \zeta_i c_i \exp \Big[ -\zeta_i r^2 \Big] \nonumber \\
    &- 4x^{\ell+2} y^m z^n \sum_{i=1}^N \zeta_i^2 c_i \exp \Big[ -\zeta_i r^2 \Big] \,.
\end{align}
%
Combining \eqref{eq::FirstTerm} and \eqref{eq::SecondTerm}, we are left with $\nabla^2_x \Psi(\mathbf{r})$
%
\begin{align}
	\nabla^2_x \Psi(\mathbf{r}) &= \ell(\ell-1)x^{\ell-2} y^m z^n \sum_{i=1}^N c_i \exp \Big[ -\zeta_i r^2 \Big] +  4x^{\ell+2} y^m z^n \sum_{i=1}^N \zeta_i^2 c_i \exp \Big[ -\zeta_i r^2 \Big] \\
    &- 2(2\ell+1)x^\ell y^m z^n \sum_{i=1}^N \zeta_i c_i \exp \Big[ -\zeta_i r^2 \Big] \,.
\end{align}
%
By replacing $ f_i(r) = c_i \exp\Big[ \zeta_i r^2 \Big]$ for the sake of clarity,
%
\begin{equation}
    \nabla^2_x \Psi(\mathbf{r})= \ell(\ell-1)x^{\ell-2} y^m z^n \sum_{i=1}^n f_i(r) + 4x^{\ell+2} y^m z^n \sum_{i=1}^N \zeta_i^2 f_i(r) - 2(2\ell+1)x^\ell y^m z^n \sum_{i=1}^N \zeta_i f_i(r) \,.
\end{equation}
%
Further, we introduce the notation $F_0(r) = \sum_{i=1}^N f_i(r)$, $F_1(r) = \sum_{i=1}^N \zeta_i f_i(r)$
and $F_2(r) = \sum_{i=1}^N \zeta_i^2 f_i(r)$, so that
%
\begin{equation}
    \nabla^2_x \Psi(\mathbf{r})= \ell(\ell-1)x^{\ell-2} y^m z^n F_0(r) + 4x^{\ell+2} y^m z^n F_2(r) - 2(2\ell+1)x^\ell y^m z^n F_1(r) \,.
\end{equation}
%
It should be obvious to the reader that expressions for $\nabla^2_y \Psi(\mathbf{r})$ and
$\nabla^2_z \Psi(\mathbf{r})$ can be given in entirely equivalent manners, so that
%
\begin{align}
	&\nabla^2_y \Psi(\mathbf{r})= m(m-1)x^\ell y^{m-2} z^n F_0(r) + 4x^\ell y^{m+2} z^n F_2(r) - 2(2m+1) x^\ell y^m z^n F_1(r) \\
    &\nabla^2_z \Psi(\mathbf{r})= n(n-1)x^\ell y^{m} z^{n-2} F_0(r) + 4x^\ell y^{m} z^{n+2} F_2(r) - 2(2n+1) x^\ell y^m z^n F_1(r) \,.
\end{align}
%
By summing $\Big( \nabla^2_x + \nabla^2_y + \nabla^2_z \Big)\Psi(\mathbf{r})$, we then have
the Laplacian of the CGO
%
\begin{align}
	\nabla^2 \Psi(\mathbf{r}) &= \Big[ \ell(\ell-1)x^{\ell-2} y^m z^n + m(m-1)x^\ell y^{m-2} z^n + n(n-1)x^\ell y^{m} z^{n-2} \Big] F_0(r) \nonumber \\
    & + 4\Big[x^{\ell+2} y^m z^n + x^\ell y^{m+2} z^n + x^\ell y^{m} z^{n+2} \Big] F_2(r) \nonumber \\
    &- \Big[ 4(\ell + m + n) + 6 \Big] x^\ell y^m z^n F_1(r) \,.
\end{align}
%
Consider the case where $\ell = m = n = 0$. Only the functions of $F_2(r)$ and $F_1(r)$
are non-zero. The coefficient of $F_1(r)$ is $6$ while that of $F_2(r)$ is $4(x^2 + y^2 + z^2) = 4r^2$.
Then
%
\begin{align}
	\nabla^2 \Psi(\mathbf{r}) &= 4r^2 F_2(r) - 6 F_1(r) \nonumber \\
    &= 4r^2 \sum_{i=1}^N \zeta_i^2 c_i \exp \Big[ -\zeta_i r^2 \Big] - 6 \sum_{i=1}^N \zeta_i c_i \exp \Big[ -\zeta_i r^2 \Big] \nonumber \\
    &= 2 \sum_{i=1}^N c_i \Big( 2\zeta_i^2 r^2 - 3\zeta_i \Big) \exp\Big[ -\zeta_i r^2 \Big] \,.
\end{align}
%
Any atomic orbitals of higher angular momentum necessitate the consideration of a number of
additional terms.


\subsection{Matrix Calculus}

The chain rule for a vectors is similar to the chain rule for scalars. Except the order is important. For $\mathbf{z} = f(\mathbf{y})$ and $\mathbf{y} = g(\mathbf{x}) $ the chain rule is:
\begin{equation}
    \frac{\partial \mathbf{z}}{\partial \mathbf{x}} = \frac{\partial \mathbf{y}}{\partial \mathbf{x}}     \frac{\partial \mathbf{z}}{\partial \mathbf{y}}
\end{equation}

\begin{table}[h]
    \centering
    \begin{tabular}{cc}
        \toprule
            $y$ & $\frac{\partial}{\partial x} y$ \\
        \midrule
            $Ax$     & $A^T$ \\
            $x^T A$  & $A$   \\
            $x^T x$  & $2x$  \\
            $x^T Ax$ & $Ax + A^Tx$  \\
        \bottomrule
    \end{tabular}
    \caption{Useful derivatives equations}
\end{table}

\subsection{Fix-Point Arithmetic}

Multiplication of two fix point values yields
\begin{equation}
	v_1 v_2 = \text{right-shift}\left( Q_1 Q_2 \cdot 2^{-(m+m)} ; m\right)
\end{equation}
Note that for multiplication the exponent $m$ for the values can be different.

Addition of two fix point values
\begin{equation}
	v_1 + v_2 = \left( Q_1 + Q_2 \right) \cdot 2^{-m}
\end{equation}


\section*{Exercises}

\begin{ex}
  For the following pairs of matrices, determine if the sum $A+B$
  and the difference $A-B$ are defined. If so, calculate them.
  \begin{enumerate}
  \item
    $A = \begin{mymatrix}{rr}
      1 & 0 \\
      0 & 1
    \end{mymatrix}$,\quad
    $B = \begin{mymatrix}{rr}
      0 & 1 \\
      1 & 0
    \end{mymatrix}$.

  \item
    $A = \begin{mymatrix}{rrr}
      2 & 1 & 2 \\
      1 & 1 & 0
    \end{mymatrix}$,\quad
    $B = \begin{mymatrix}{rrr}
      -1 & 0 & 3 \\
      0 & 1 & 4
    \end{mymatrix}$.

  \item
    $A = \begin{mymatrix}{rr}
      1 & 0 \\
      -2 & 3 \\
      4 & 2
    \end{mymatrix}$,\quad
    $B = \begin{mymatrix}{rrr}
      2 & 7 & -1 \\
      0 & 3 & 4
    \end{mymatrix}$.
  \end{enumerate}
\end{ex}

\begin{ex}
  For each matrix $A$, find the matrix $-A$.
  \begin{enumerate}
  \item
    $A = \begin{mymatrix}{rr}
      1 & 2 \\
      2 & 1
    \end{mymatrix}$

  \item
    $A = \begin{mymatrix}{rr}
      -2 & 3 \\
      0 & 2
    \end{mymatrix}$

  \item
    $A = \begin{mymatrix}{rrr}
      0 & 1 & 2 \\
      1 & -1 & 3 \\
      4 & 2 & 0
    \end{mymatrix}$
  \end{enumerate}
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Let $A = \begin{mymatrix}{rrr}
    1 & 2 & -1 \\
    -1 & 4 & 0 \\
  \end{mymatrix}$ and
  $B = \begin{mymatrix}{rrr}
    0 & 3 & 0 \\
    1 & -1 & 1 \\
  \end{mymatrix}$.\par\noindent
  Find a matrix $X$ such that $(A+X)-(B+0) = B+A$. Hint: first use
  the properties of matrix addition to simplify the equation and
  solve for $X$.
  \begin{sol}
    The equation simplifies to $X=B+B$, so $X = \begin{mymatrix}{rrr}
    0 & 6 & 0 \\
    2 & -2 & 2 \\
    \end{mymatrix}$.
  \end{sol}
\end{ex}

\begin{ex}\label{add-inv-rst-unique}
  Using only the properties given in
  Proposition~\ref{prop:properties-of-addition}, show that if $A+B=0$, then
  $B=-A$.
  \begin{sol}
    Suppose $A+B=0$. Then we have
    $-A = (-A)+0 = (-A)+(A+B) = ((-A)+A)+B = (A+(-A))+B = 0+B = B+0 =
    B$.  Here, we have used the additive unit law, the assumption
    $A+B=0$, associativity, commutativity, the additive inverse law,
    commutativity, and the additive unit law, in that order.
  \end{sol}
\end{ex}

\begin{ex} Using only the properties given in
  Proposition~\ref{prop:properties-of-addition}, show $A+B=A$ implies $B=0$.
  \begin{sol}
    Suppose $A+B=A$. Then $B=B+0=0+B=(A+(-A))+B = ((-A)+A)+B =
    (-A)+(A+B) = (-A)+A = A+(-A) = 0$. Here, we have used the additive
    unit law, commutativity, the additive inverse law, commutativity,
    associativity, the assumption $A+B=A$, commutativity, and the
    additive inverse law, in that order.
  \end{sol}
\end{ex}