\chapter*{Appendix A: The Lifting Scheme}
\addcontentsline{toc}{chapter}{Appendix A: The Lifting Scheme}

\subsection*{Introduction}

\begin{figure}[htb]
\centerline{
\hbox{
\psfig{figure=fig_lift.ps,bbllx=4.5cm,bblly=13cm,bburx=18.5cm,bbury=19.5cm,width=14cm,height=6.5cm,clip=}}
}
\caption{The lifting scheme -- forward direction.}
\label{fig_lift}
\end{figure}
The lifting scheme \cite{wave:sweldens96} is a flexible technique that has
been used in several different settings, for easy construction and implementation
of traditional wavelets \cite{wave:sweldens96}, and of second generation
wavelets \cite{wave:sweldens95b} such as spherical wavelets \cite{wave:sweldens95a}.

Its principle is to compute the difference between a true coefficient and its
prediction:
\begin{eqnarray}
w_{j+1,l} = c_{j,2l+1} - {\cal P}(c_{j,2l-2L}, ..., c_{j,2l-2},c_{j,2l},c_{j,2l+2}, ..., c_{j,2l+2L})
\end{eqnarray}
A pixel at an odd location 2l+1 is then predicted using pixels at even
locations.

The transformation is done in three steps:
\begin{enumerate}
\item Split the signal into even and odd number samples:
\begin{eqnarray*}
c_{j+1,l} & = & c_{j,2l} \\
w_{j+1,l} & = & c_{j,2l+1}
\end{eqnarray*}
\item Set \[w_{j+1,l} = w_{j+1,l} - {\cal P}(c_{j+1,l})\]
\item Set \[c_{j+1,l} = c_{j+1,l} + {\cal U}(w_{j+1,l})\]
where $\cal U$ is the update operator.
\end{enumerate}
The reconstruction is obtained by:
\begin{eqnarray*}
c_{j,2l} & = & c_{j+1,l} - {\cal U}(w_{j+1,l}) \\
c_{j,2l+1} & = & w_{j+1,l} + {\cal P}(c_{j+1,l}) \\
\end{eqnarray*}

\subsection*{Example of transforms}

\subsubsection*{Haar wavelet via lifting}

The Haar transform can be performed via the lifting scheme by taking
the predict operator equal to the identity, and an update operator which
halves the difference. The transform becomes:
\begin{eqnarray*}
w_{j+1,l} =  w_{j+1,l} - c_{j+1,l} \\
c_{j+1,l} = c_{j+1,l} + {w_{j+1,l} \over 2}
\end{eqnarray*}
All computation can be done in place. Every wavelet transform can be written
via lifting.

\subsubsection*{Linear wavelets via lifting}
The identity predictor used before is correct when the signal is constant.
In the same way, we can use a linear predictor which is correct when
the signal is linear. The predictor and update operators are now:
\begin{eqnarray*}
{\cal P}(c_{j-1,l} & = & {1 \over 2} (c_{j-1,l} + c_{j-1,l+1}) \\
 {\cal U}(w_{j-1,l}) & = & {1 \over 4} (w_{j-1,l-1} + w_{j-1,l})
\end{eqnarray*}
It is easy to verify that:
\begin{eqnarray*}
c_{j-1,l} = -{1 \over 8} c_{j,2l-2} + {1 \over 4} c_{j,2l-1} + {3 \over 4} c_{j,2l}
+ {1 \over 4} c_{j,2l+1} -{1 \over 8} c_{j,2l+2}
\end{eqnarray*}
which is the bi-orthogonal Cohen-Daubechies-Feauveau \cite{wave:cohen92}
 wavelet transform.

\subsection*{Integer wavelet transform}

When the input data are integer values, the wavelet transform no longer consists
of integers. For lossless coding, it is useful to have a wavelet
transform which produces integer values.
We can build an integer version of every wavelet
transform \cite{wave:calderbank96}. For instance, denoting $\lfloor x \rfloor$
as the largest integer
not exceeding x, the integer Haar transform  (also called ``S'' transform)
can be calculated by:
\begin{eqnarray}
w_{j+1,l} & = & c_{j,2l+1} - c_{j,2l} \\ \nonumber
c_{j+1,l} & = & c_{j,2l} + \lfloor {w_{j+1,l} \over 2 } \rfloor = c_{j+1,l} + \lfloor {w_{j+1,l} \over 2 } \rfloor
\end{eqnarray}
while the reconstruction is
\begin{eqnarray}
c_{j,2l} & = &  c_{j+1,l} - \lfloor {w_{j+1,l} \over 2} \rfloor \\ \nonumber
c_{j,2l+1} & = &  w_{j+1,l} +  c_{j,2l}
\end{eqnarray}

More generally, the lifting operators for an integer version of the wavelet transform are:
\begin{eqnarray*}
{\cal P}(c_{j+1,l}) & = & \lfloor \sum_k p_k c_{j+1,l-k} + {1 \over 2 } \rfloor\\
 {\cal U}(w_{j+1,l}) & = &  \lfloor \sum_k u_k w_{j+1,l-k} + {1 \over 2 } \rfloor
\end{eqnarray*}

The linear integer wavelet transform is
\begin{eqnarray*}
w_{j+1,l} & = & w_{j+1,l} - \lfloor {1 \over 2 }(c_{j+1,l}+c_{j+1,l+1})+ {1 \over 2 } \rfloor\\
c_{j+1,l} & = & c_{j+1,l} + \lfloor {1 \over 4 }(w_{j+1,l-1}+w_{j+1,l})+ {1 \over 2 } \rfloor
\end{eqnarray*}

Even if there is no filter that consistently performs better than all
the other filters on all images, the following one performs generally better
than others \cite{wave:calderbank96}:
\begin{eqnarray*}
w_{j+1,l} & = & w_{j+1,l} - \lfloor {1 \over 2 }(c_{j+1,l}+c_{j+1,l+1})+ {1 \over 2 } \rfloor\\
c_{j+1,l} & = & c_{j+1,l} + \lfloor {1 \over 4 }(w_{j+1,l-1}+w_{j+1,l})+ {1 \over 2 } \rfloor
\end{eqnarray*}
More filters can be found in \cite{wave:calderbank96}.
% (the notation (x,y) means that the numbers of wanishing moments
% of the analysing and synthesis high pass filter are respectively x and y):
% \item{integer linear transform: (2,2)}
% More details can be found in \cite{wave:calderbank96}. \\


% I removed the definition of a manifold and moved it in-line for better flow.
% I think it's best to use /emph in definitions to show what's being defined.
% Put periods even in display math!
% For spacing reasons, should use \colon instead of : for functions, e.g. f \colon A \to B.
% Since ker is lowercase, im probably should be also.

\section{Introduction, simplices}\label{905}
%In 18.905, which is the first half of this book, we will cover the following topics:
%\begin{enumerate}
%    \item Singular homology
%    \item CW-complexes
%    \item Basics of category theory
%    \item Homological algebra
%    \item The K\"{u}nneth theorem
%    \item UCT, cohomology
%    \item Cup and cap products, and
%    \item Poincar\'{e} duality.
%\end{enumerate}
We will begin by giving some examples of commonly encountered topological
spaces.
\begin{itemize}
    \item The most basic is \emph{$n$-dimensional Euclidean space}, $\mathbf{R}^n$.
    \item The \emph{$n$-sphere} $S^n=\{x\in \mathbf{R}^{n+1}:|x|=1\}$ is topologized as a subspace of $\mathbf{R}^{n+1}$.
    \item Identifying antipodal points in $S^n$ gives \emph{real projective space} $\mathbf{RP}^n=S^n / (x\sim -x)$, i.e. the space of lines through the origin in $\mathbf{R}^{n+1}$.
    \item Call an ordered collection of $k$ orthonormal vectors an \emph{orthonormal $k$-frame}. The space of orthonormal $k$-frames in $\mathbf{R}^n$ forms the \emph{Stiefel manifold} $V_k(\mathbf{R}^n)$, which is topologized as a subspace of $(S^{n-1})^k$.
    \item Let $x\sim y$ if $x$ and $y$ are $k$-frames with the same span. The \emph{Grassmannian} is the quotient $\mathrm{Gr}_k(\mathbf{R}^n)=V_k(\mathbf{R}^n)/\sim$. In particular, $\mathbf{Gr}_1(\mathbf{R}^n) = \mathbf{RP}^{n-1}$.
\end{itemize}
The above are all \emph{manifolds}, which are Hausdorff spaces locally homeomorphic to Euclidean space. Aside from $\mathbf{R}^n$ itself, the preceding examples are also compact. Such spaces exhibit a hidden symmetry, which is the culmination of 18.905: Poincar\'{e} duality.

As the name suggests, the central aim of algebraic topology is the usage of algebraic tools to study topological spaces. A common technique is to probe topological spaces via maps to them. In different ways, this approach gives rise to singular homology and homotopy groups. We now detail the former; the latter takes stage in 18.906.
\begin{definition}
For $n\geq 0$, the \emph{standard $n$-simplex} $\Delta^n$ is the convex hull of the standard basis $\{e_0,\cdots,e_n\}$ in $\mathbf{R}^{n+1}$. More explicitly,
$$\Delta^n = \left\{\sum t_i e_i : \sum t_i = 1, t_i\geq 0\right\}\subseteq\mathbf{R}^{n+1}.$$
The $t_i$ are called barycentric coordinates.
\end{definition}
\begin{figure}[H]
	\centering
	\includegraphics[width=0.9\linewidth]{assets/L01/01-standard-simplices}
	\caption{The standard 1-simplex and 2-simplex.}
	\label{fig:01-standard-simplices}
\end{figure}
We will write $i$ in lieu of $e_i$ to refer to the vertices of $\Delta^n$. The standard simplices are related by face inclusions $d^i\colon \Delta^{n-1} \to \Delta^{n}$ for $0\leq i \leq n$, where $d^i$ misses the vertex $i$.
\begin{figure}[H]
	\centering
	\includegraphics[width=0.9\linewidth]{assets/L01/01-face-inclusions}
	\caption{The face inclusions for $n=1$ (left) and $n=2$ (right).}
	\label{fig:01-face-inclusions}
\end{figure}
\begin{definition}
Let $X$ be any topological space. A \emph{singular $n$-simplex} in $X$ is a continuous map $\Delta^n\to X$. Denote by $\mathrm{Sin}_n(X)$ the set of all $n$-simplices in $X$. This seems like a rather bold construction to make, as $\mathrm{Sin}_n(X)$ is huge. Nonetheless, we will soon make it even larger.
\end{definition}
\begin{figure}[H]
	\centering
	\includegraphics[width=\linewidth]{assets/L01/01-simplices-on-torus}
	\caption{A 1-simplex and a 2-simplex in the torus.}
	\label{fig:01-simplices-on-torus}
\end{figure}

For $0\leq i \leq n$, precomposition by the face inclusion $d^i$ produces a map $d_i\colon \Sin_n(X)\to\Sin_{n-1}(X)$ sending $\sigma\mapsto\sigma\circ d^i$, which is the $i$th face of $\sigma$. This allows us to make sense of the ``boundary'' of a simplex, and we are particularly interested in simplices for which that boundary vanishes.
\begin{figure}
	\centering
	\includegraphics[width=0.7\linewidth]{assets/L01/01-torus-simplex-around-hole}
	\caption{A 1-simplex around the hole in the torus.}
	\label{fig:01-torus-simplex-around-hole}
\end{figure}
For example, if $\sigma$ is a 1-simplex that goes around the hole in a torus $T$ as in Figure \ref{fig:01-torus-simplex-around-hole}, then $d_1\sigma = d_0\sigma$. To express that the boundary vanishes, we want to write $d_0\sigma - d_1\sigma=0$, but this difference is no longer a simplex. To accommodate such formal sums, we will enlarge $\mathrm{Sin}_n(X)$ further by considering the free abelian group it generates.
\begin{definition}
The abelian group $S_n(X)$ of \emph{singular $n$-chains} in $X$ is the free abelian group generated by $n$-simplices
$$S_n(X) = \mathbf{Z}\Sin_n(X).$$
    Its elements are finite linear combinations, i.e. formal sums
    $$\sum_{i\in\text{finite set}}a_i\sigma_i$$
    where $a_i\in\mathbf{Z}$ and $\sigma_i \in \Sin_n(X)$. If $n<0$, say that $S_n(X)=0$. Now, define
$$\partial\colon \Sin_n(X)\to S_{n-1}(X),$$
$$\partial\sigma = \sum_{i=0}^n(-1)^i d_i\sigma.$$
This extends to a homomorphism $\partial \colon S_n(X) \to S_{n-1}(X)$ by additivity.
\end{definition}
We use this homomorphism to obtain something more tractable than the entirety of $S_n(X)$. First we restrict our attention to chains with vanishing boundary.
\begin{definition}
An \emph{$n$-cycle} in $X$ is an $n$-chain $c$ with $\partial c = 0$. Denote $Z_n(X) = \ker(S_n(X)\xrightarrow{\partial}S_{n-1}(X))$.
\end{definition}
For example, with $\sigma$ on the torus described before, $\sigma\in Z_1(X)$
since $\partial \sigma = d_0\sigma - d_1\sigma = 0$.
\begin{theorem}
    Any boundary is a cycle, i.e., $B_n(X) :=
    \mathrm{Im}(\partial:S_{n+1}(X)\to S_n(X))\subseteq Z_n(X)$.
\end{theorem}
\begin{exercise}\label{exer:simplicialidentities}
    Deduce the above theorem by proving the following statements.
    \begin{enumerate}
	\item An order-preserving map $\phi:[n]\to[m]$ extends to an affine map
	    $\phi^\ast:\Delta^n\to \Delta^m$. Give an explicit formula for
	    $\phi^\ast$.
	\item Prove that any order-preserving map factors uniquely as the
	    composite of an order-preserving surjection followed by an
	    order-preserving injection.
	\item Let $d^j:[n-1]\to [n]$ be the order-preserving injection omitting
	    $j$ as a value. Prove that an order-preserving injection
	    $\phi:[n-k]\to[n]$ is uniquely a composition of the form $d^{j_k}
	    d^{j_{k-1}}\cdots d^{j_1}$ with $0\leq j_1<j_2<\cdots<j_k\leq n$.
	    Define the $j_i$'s in terms of $\phi$. Verify the straightening
	    rule
	    $$d^id^j = d^{j+1} d^i \text{ if }i\leq j.$$
	\item Let $s^i:[n+1]\to [n]$ be the order-preserving surjection
	    repeating the value $i$. Show that any order-preserving surjection
	    $\phi:[m]\to [n]$ is uniquely a composition of the form
	    $(s^n)^{i_n}(s^{n-1})^{i_{n-1}}\cdots(s^0)^{i_0}$ by describing the
	    $i_j$'s in terms of $\phi$. Verify a straightening rule for the
	    composite $s^i s^j$.
	\item Verify a straightening rule for $s^i d^j$.
	\item Write down the straightening rules for the induced maps $d_i =
	    (d^i)^\ast:\Sin_n(X)\to \Sin_{n-1}(X)$ and $s_i =
	    (s^i)^\ast:\Sin_m(X)\to \Sin_{m+1}(X)$. Use these to verify that
	    $\partial^2 = 0:S_n(X)\to S_{n-2}(X)$.
	\item Let $f:X\to Y$ be a continuous map. This induces a map
	    $f_\ast:\Sin_n(X)\to \Sin_n(Y)$. Show that the $f_\ast$ assemble to
	    give a map of simplicial sets, i.e., show that the maps $f_\ast$
	    commute with the maps induced by order-preserving maps $\phi:[m]\to
	    [n]$.
    \end{enumerate}
\end{exercise}

With the preceding, we are prepared to define singular homology.
\begin{definition}
The \emph{$n$th singular homology group} of $X$ is:
    $$ H_n(X) = Z_n(X)/B_n(X) = \frac{\ker(\partial:S_n(X)\to S_{n-1}(X))}{\mathrm{im}(\partial:S_{n+1}(X)\to S_n(X))}.$$
\end{definition}
Both $Z_n(X)$ and $B_n(X)$ are free abelian groups because they are subgroups of the free abelian group $S_n(X)$, but the quotient $H_n(X)$ isn't necessarily free. While $Z_n(X)$ and $B_n(X)$ are uncountably generated, $H_n(X)$ is finitely generated for the spaces we are interested in. If $T$ is the torus for example, then $H_1(T) \cong \mathbf{Z} \oplus \mathbf{Z}$ and $\sigma$ as described previously is one of the two generators.


\section*{Exercises}

\begin{ex}
  Consider the matrix
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      1 & -2 & -2 \\
      2 & -3 & -2 \\
      -2 & 2 &  1 \\
    \end{mymatrix}.
  \end{equation*}
  Which of the following vectors are eigenvectors of $A$? Find the
  corresponding eigenvalues.
  \begin{equation*}
    \vect{v}_1 = \begin{mymatrix}{r} 1 \\ 1 \\ -1 \end{mymatrix},\quad
    \vect{v}_2 = \begin{mymatrix}{r} 1 \\ 1 \\ 0 \end{mymatrix},\quad
    \vect{v}_3 = \begin{mymatrix}{r} 2 \\ 2 \\ -1 \end{mymatrix},\quad
    \vect{v}_4 = \begin{mymatrix}{r} 0 \\ -1 \\ 1 \end{mymatrix}.
  \end{equation*}
\end{ex}

\begin{ex}
  Let
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      1  &  0 & 0 \\
      -5 & -1 & 5 \\
      -3 &  0 & 4 \\
    \end{mymatrix}.
  \end{equation*}
  Find the eigenvectors corresponding to the eigenvalue $\eigenvar=4$.
\end{ex}

\begin{ex}
  Let
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      7 &  -4 &   8 \\
      -1 &  4 &  -2 \\
      -2 &  2 &  -1 \\
    \end{mymatrix}.
  \end{equation*}
  Find the eigenvectors corresponding to the eigenvalue $\eigenvar=3$.
\end{ex}

\begin{ex}
  Let
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      4 &   0 &   3 \\
      -3 &   1 &  -3 \\
      0 &   0 &   1 \\
    \end{mymatrix}.
  \end{equation*}
  This matrix has eigenvalues $\eigenvar=1$ and $\eigenvar=4$. Find a
  basis for each eigenspace.
\end{ex}

\begin{ex}
  Let
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      2 &   4 &  -4 \\
      -1 &  6 &  -9 \\
      0 &   0 &  -3 \\
    \end{mymatrix}.
  \end{equation*}
  This matrix has eigenvalues $\eigenvar=3$ and $\eigenvar=4$. Find a
  basis for each eigenspace.
\end{ex}

\begin{ex}
  Suppose $A$ is a $3\times 3$-matrix with eigenvalues
  $\eigenvar_1=1$, $\eigenvar_2=0$, and $\eigenvar_3=2$ and
  corresponding eigenvectors
  \begin{equation*}
    \vect{v}_1 = \begin{mymatrix}{r}
      -1 \\
      -2 \\
      -2
    \end{mymatrix},
    \quad
    \vect{v}_2 = \begin{mymatrix}{r}
      1 \\
      1 \\
      1
    \end{mymatrix},
    \quad\mbox{and}\quad
    \vect{v}_3 = \begin{mymatrix}{r}
      -1 \\
      -4 \\
      -3
    \end{mymatrix}.
  \end{equation*}
  (By ``corresponding'', we mean that $\vect{v}_1$ corresponds to
  $\eigenvar_1$, $\vect{v}_2$ corresponds to $\eigenvar_2$, and so
  on).  Find
  \begin{equation*}
    A\begin{mymatrix}{r}
      3 \\
      -4 \\
      3
    \end{mymatrix}.
  \end{equation*}
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Let $A$ be an $n\times n$-matrix, and assume $\eigenvar$ is an
  eigenvalue of $A$. Show that $\eigenvar^2$ is an eigenvalue of
  $A^2$.
  \begin{sol}
    If $\vect{v}$ is an eigenvector corresponding to the eigenvalue
    $\eigenvar$, then $A^2\vect{v}=A(A\vect{v}) =
    A(\eigenvar\vect{v}) = \eigenvar(A\vect{v}) =
    \eigenvar^2\vect{v}$. Therefore, $\vect{v}$ is an eigenvector of
    $A^2$ with eigenvalue $\eigenvar^2$.
  \end{sol}
\end{ex}

\begin{ex}
  Let $A$ be an invertible $n\times n$-matrix, and assume $\eigenvar$
  is an eigenvalue of $A$. Show that $\eigenvar\neq 0$ and that
  $\eigenvar^{-1}$ is an eigenvalue of $A^{-1}$.
  \begin{sol}
    We have
    $\eigenvar A^{-1}\vect{v} = A^{-1}\eigenvar\vect{v} =
    A^{-1}A\vect{v} = \vect{v}$. Since $\vect{v}\neq 0$, this implies
    $\eigenvar\neq 0$. Moreover, it implies
    $A^{-1}\vect{v} = \eigenvar^{-1}\vect{v}$. Thus, $\eigenvar^{-1}$
    is an eigenvalue of $A^{-1}$.
  \end{sol}
\end{ex}

\begin{ex}
  If $A$ is an $n\times n$-matrix and $c$ is a non-zero constant,
  compare the eigenvalues of $A$ and $cA$.
  \begin{sol}
    Say $A\vect{v}=\eigenvar \vect{v}$. Then
    $cA\vect{v}=c\eigenvar \vect{v}$ and so the eigenvalues of $cA$ are
    just $c\eigenvar$ where $\eigenvar$ is an eigenvalue of $A$.
  \end{sol}
\end{ex}

\begin{ex}
  Let $A,B$ be invertible $n\times n$-matrices which commute. That is,
  $AB=BA$. Suppose $\vect{v}$ is an eigenvector of $B$. Show that then
  $A\vect{v}$ must also be an eigenvector for $B$.
  \begin{sol}
    Suppose $\vect{v}$ is an eigenvector of $B$, i.e.,
    $B\vect{v}=\eigenvar \vect{v}$. Then
    $BA\vect{v}=AB\vect{v} =A\eigenvar \vect{v}=\eigenvar A\vect{v}$,
    and therefore $A\vect{v}$ is an eigenvector of $B$.
  \end{sol}
\end{ex}

\begin{ex}
  Suppose $A$ is an $n\times n$-matrix and it satisfies $A^m=A$ for
  some $m$ a positive integer larger than 1. Show that if $\eigenvar$
  is an eigenvalue of $A$ then $\eigenvar$ equals either $0$, $1$, or
  $-1$.
  \begin{sol}
    Let $\vect{v}$ be the eigenvector. Then
    $A^m\vect{v}=\eigenvar^m\vect{v}$ and
    $A^m\vect{v}=A\vect{v}=\eigenvar\vect{v}$. Therefore
    $\eigenvar^m=\eigenvar$. Hence if $\eigenvar \neq 0$, we must
    have $\eigenvar^{m-1}=1$, which implies that $\eigenvar=\pm 1$.
  \end{sol}
\end{ex}

\begin{ex}
  Show that if $A\vect{v}=\eigenvar \vect{v}$ and
  $A\vect{w}=\eigenvar \vect{w}$, then whenever $k,p$ are scalars,
  \begin{equation*}
    A(k\vect{v}+p\vect{w}) =\eigenvar (k\vect{v}+p\vect{w})
  \end{equation*}
  Does this imply that $k\vect{v}+p\vect{w}$ is an eigenvector? Explain.
  \begin{sol}
    The formula follows from properties of matrix
    multiplication. However, this vector might not be an eigenvector
    because it might equal $0$ and eigenvectors cannot equal $0$.
  \end{sol}
\end{ex}


% !TEX root = main.tex

%-------------------------------------------------
\section{More transformations}\label{sec:more_transfs}
%----------------------------------------------------------------------
%\subsection{Standard normal CDF $\longrightarrow$ Normal CDF}
%----------------------------------------------------------------------

% example: standard normal
\begin{example}
Let $Z\sim N(0,1)$. Find the CDF of $X=\mu+\sigma Z$ in terms of the CDF of $Z$.
\end{example}
\begin{solution}
Let $F_X$ and $F_Z$ respectively denote the CDFs of $X$ and $Z$.
\[
F_X(x)
	= \prob(X\leq x)
	= \prob(\mu+\sigma Z\leq x)
	= \prob\left(Z\leq\frac{x-\mu}{\sigma}\right)
	= F_Z\left(\frac{x-\mu}{\sigma}\right).
\]
\end{solution}

%%----------------------------------------------------------------------
%\subsection{Standard normal distribution $\longrightarrow$ Chi-squared distribution}
%%----------------------------------------------------------------------
%----------------------------------------------------------------------
%\subsection{Standard normal CDF $\longrightarrow$ Chi-squared CDF}
%----------------------------------------------------------------------
\begin{example}[The chi-squared distribution]
Let $X\sim N(0,1)$. Find the CDF of $Y=X^2$.
\end{example}
\begin{solution}
The PDF of $X\sim N(0,1)$ is given by $f_X(x) = \displaystyle\frac{1}{\sqrt{2\pi}}e^{-z^2/2}$. Hence,
\[
\prob(Y\leq y)
	= \prob(-\sqrt{y}\leq X\leq \sqrt{y})
	= \int_{-\sqrt{y}}^{\sqrt{y}} f_X(x)\,dx
	= \frac{2}{\sqrt{2\pi}}\int_0^{\sqrt{y}} e^{-x^2/2}\,dx.
\]
Using a change-of-variable $t=x^2$, we obtain
\[
\prob(Y\leq y)
	= \int_0^{y} \left(\frac{1}{\sqrt{2\pi t}} e^{-t/2}\right)\,dt.
\]

This is the CDF of the \emph{chi-squared} distribution with one degree of freedom.
\end{solution}

%%----------------------------------------------------------------------
%%\subsection{Standard uniform CDF $\longrightarrow$ Exponential CDF}
%%----------------------------------------------------------------------
%% example: uniform to exponential(1)
%\begin{example}
%Let $X\sim\text{Uniform}[0,1]$, and let $Y = -\theta\log X$ where $\theta>0$. Show that $Y\sim\text{Exponential}(\theta)$, where $\theta$ is a scale parameter.
%\end{example}
%\begin{solution}
%Consider the transformation $g(x) = -\theta\log x$.
%\bit
%\it The CDF of $X$ is $F_X(x) = x$ for $x\in[0,1]$ (with $F_X(x)=0$ for $x<0$, and $F_X(x)=1$ for $x>1$).
%\it The PDF of $X$ is $f_X(x) = 1$ for $x\in[0,1]$ (and zero otherwise).
%\it $g(x)$ is a decreasing function (and thus has a unique inverse) over $\supp(f_X) = [0,1]$.
%\it The support of $f_Y$ is $\supp(f_Y) = \{g(x) : x\in\supp(f_X)\} = [0,\infty)$.
%\it The inverse transformation is $g^{-1}(y) = e^{-y/\theta}$ over $\supp(f_Y)$.
%\eit
%By Theorem~\ref{thm:transf_cdf},
%\[
%F_Y(y)
%	= 1 - F_X\big[g^{-1}(y)\big]
%	= 1 - F_X(e^{-y/\theta})
%	= 1 - e^{-y/\theta} \quad\text{for}\quad y>0.
%\]
%This is the CDF of the $\text{Exponential}(\theta)$ distribution, where $\theta$ is a scale parameter.
%\end{solution}

%----------------------------------------------------------------------
%\subsection{Exponential CDF $\longrightarrow$ Pareto CDF}
%----------------------------------------------------------------------

% example: exponential to pareto
\begin{example}
Let $X\sim\text{Exponential}(\alpha)$ where $\alpha$ is a rate parameter, and let $Y=\theta e^X$, where $\theta>0$ is a constant. Show that $Y$ has the so-called $\text{\emph{Pareto}}(\theta,\alpha)$ distribution, whose CDF is given by
\[
F_Y(y)= \begin{cases}
		1 - \left(\frac{\theta}{y}\right)^{\alpha}	& \text{for } y > \theta \\
		0											& \text{otherwise.}
	\end{cases}
\]
\end{example}

\begin{solution}
Consider the transformation $g(x) = \theta e^x$.
\bit
\it The CDF of $X$ is $F_X(x) = 1 - e^{-\alpha x}$ for $x>0$ (and zero otherwise).
\it $\supp(f_X) = [0,\infty)$.
\it $\supp(f_Y) = \{\theta e^x: x\geq 0\} = [\theta,\infty)$.
\it $g(x)$ is an increasing function over $\supp(f_X)$;
\it the inverse transformation is $g^{-1}(y) = \log(y/\theta)$.
\eit
By Theorem~\ref{thm:transf_cdf},
\[
F_Y(y)
	= F_X\left[\log \left(\frac{y}{\theta}\right)\right]
	= 1 - \exp\left[-\alpha\log \left(\frac{y}{\theta}\right)\right]
	= \begin{cases}
		1 - \displaystyle\left(\frac{\theta}{y}\right)^{\alpha}	& \text{for } y > \theta, \\[2ex]
		0					& \text{otherwise.}
	\end{cases}
\]
as required.

\bigskip\textbf{Remark:}
%\begin{remark}
Compare the upper-tail probabilities of $X\sim\text{Exponential}(\alpha)$ and $Y\sim\text{Pareto}(\theta,\alpha)$:
%\begin{align*}
%\prob(X > x) & = e^{-\alpha x} \\
%\prob(Y > y) & = \theta^{\alpha} y^{\alpha}
%\end{align*}
\[
\prob(X > x) = e^{-\alpha x}
\qquad\text{and}\qquad
\prob(Y > y) = \theta^{\alpha} y^{-\alpha}.
\]
In both cases, the rate at which the tail probabilities converge to zero is controlled by the parameter $\alpha$. We can see however that $\prob(X>x)\to 0$ relatively quickly as $x\to\infty$, the rate of convergence depending ``exponentially'' on $x$, while $\prob(Y>y)\to 0$ more slowly as $y\to\infty$, with the rate of convergence depending ``polynomially'' on $y$.
%\eit
%\bit
%\it $\prob(X>x) \to 0$ at an exponential rate as $x\to\infty$, while
%\it $\prob(Y>y) \to 0$ at only a polynomial rate as $y\to\infty$.
%\eit
Consequently, the Pareto distribution belongs to the class of so-called \emph{heavy-tailed} distributions.
%\end{remark}
\end{solution}

%----------------------------------------------------------------------
%\subsection{Normal PDF $\longrightarrow$ Standard normal PDF}
%----------------------------------------------------------------------

% example: standard normal
\begin{example}[The standard normal distribution]
Let $X\sim N(\mu,\sigma^2)$, and define $Z=(X-\mu)/\sigma$. Find the PDF of $Z$.
\end{example}
\begin{solution}
Let $g(X) = \displaystyle \frac{X-\mu}{\sigma}$.
\bit
\it The PDF of $X$ is $f_X(x) = \displaystyle\frac{1}{\sigma\sqrt{2\pi}}\exp\left[-\frac{1}{2}\left(\frac{x-\mu}{\sigma}\right)^2\right]$.
\it Because $\sigma>0$, we see that $g(x)$ is increasing over $\supp(f_X) = (-\infty,\infty)$.
\it $g(x)$ has inverse function $g^{-1}(z) = \mu + \sigma z$, whose first derivative is $\displaystyle\frac{d}{dz}g^{-1}(z) = \sigma$.
\eit
The PDF of $Z$ is therefore given by
\begin{align*}
f_Z(z)
	& = f_X\big[g^{-1}(z)\big]\left|\frac{d}{dz}g^{-1}(z)\right| \\
	& = \frac{1}{\sigma\sqrt{2\pi}}\exp\left[-\frac{1}{2}\left(\frac{(\mu+\sigma z)-\mu}{\sigma}\right)^2\right]\sigma
	  = \frac{1}{\sqrt{2\pi}}e^{-z^2/2}
\end{align*}
\end{solution}

%----------------------------------------------------------------------
%\subsection{Pareto PDF $\longrightarrow$ Standard uniform PDF}
%----------------------------------------------------------------------

%% example: pareto
%\begin{example}[The Pareto distribution]
%The $\text{Pareto}(\theta,\alpha)$ distribution is a continuous distribution with PDF
%\[
%f_X(x) = \begin{cases}
%	\displaystyle\frac{\alpha}{\theta}\left(\frac{\theta}{x}\right)^{\alpha+1}		& \text{for } x > \theta, \\[2ex]
%	0																			& \text{otherwise.}
%\end{cases}
%\]
%Let $X\sim\text{Pareto}(1,1)$. Find the PDF of $Y=1/X$.%\displaystyle\frac{1}{X}$.
%\end{example}
%
%\begin{solution}
%$X\sim\text{Pareto}(1,1)$ has PDF
%\[
%f_X(x) = \displaystyle\frac{1}{x^2} \quad\text{for $x>1$ (and zero otherwise)}.
%\]
%
%Let $g(x) = 1/x$.
%\bit
%\it $g(x)$ is a monotonic decreasing function over $x>1$.
%\it The inverse function is $g^{-1}(y) = 1/y$.
%\it $\supp(f_Y) = \{g(x): x\in\supp(f_X)\} = \{1/x :x\in(0,\infty)\} = (0,1)$.
%\eit
%%
%%
%%\it The support of $Y$ can be determined as follows:
%%\begin{align*}
%%\supp(f_X) = (0,\infty)
%%	\Rightarrow 1 < X < \infty
%%	\Rightarrow 1 < \frac{1}{Y} < \infty
%%	\Rightarrow 0 < Y < 1
%%	\Rightarrow \supp(f_Y) = (0,1).
%%\end{align*}
%%\eit
%
%Hence the PDF of $Y$ is given by
%\begin{align*}
%f_Y(y)
%	  = f_X\big[g^{-1}(y)\big]\left|\frac{d}{dy}g^{-1}(y)\right|
%	& = f_X\left(\frac{1}{y}\right)\left|\frac{d}{dy}\left(\frac{1}{y}\right)\right| \\
%	& = y^2 \left|-\frac{1}{y^2}\right|
%	= \begin{cases}
%		1	& \text{for } 0<y<1 \\
%		0	& \text{otherwise.}
%	\end{cases}
%\end{align*}
%Thus $Y$ has the standard uniform distribution.
%\end{solution}

%----------------------------------------------------------------------
%\subsection{Normal PDF $\longrightarrow$ Lognormal PDF}
%----------------------------------------------------------------------

% example: lognormal
\begin{example}[The lognormal distribution]
If $X\sim(\mu,\sigma^2)$, then $Y=e^X$ is said to have \emph{lognormal} distribution. Find the PDF of $Y$.
\end{example}
\begin{solution}
Let $g(X) = e^X$.
\bit
\it Recall that $f_X(x) = \displaystyle\frac{1}{\sigma\sqrt{2\pi}}\exp\left[-\frac{1}{2}\left(\frac{x-\mu}{\sigma}\right)^2\right]$.
\it $g(x)$ is an increasing function over $\supp(f_X) = (-\infty,\infty)$.
\it $g(x)\in[0,\infty]$ for $x\in\R$ so $\supp(f_Y) = [0,\infty)$.
\it $g(x)$ has inverse function $g^{-1}(y) = \log y$, for which $\displaystyle \frac{d}{dy}g^{-1}(y) = \frac{1}{y}$.
\eit
Thus
\begin{align*}
f_Y(y)
	& = f_X\big[g^{-1}(y)\big]\left|\frac{d}{dy}g^{-1}(y)\right| \\
	& = f_X(\log y)\left|\frac{1}{y}\right|
	=  \frac{1}{y\sigma\sqrt{2\pi}}\exp\left[-\frac{1}{2}\left(\frac{\log y-\mu}{\sigma}\right)^2\right]
\end{align*}
for $y>0$, and zero otherwise.
\end{solution}

%----------------------------------------------------------------------
%\subsection{Lomax PDF $\longrightarrow$ Logistic CDF}
%----------------------------------------------------------------------

% example: pareto - logistic
\begin{example}[The logistic distribution]
The $\text{Lomax}(\theta,\alpha)$ distribution, also known as the Pareto Type II distribution or the shifted Pareto distribution, is a continuous distribution with PDF
\[
f_X(x) = \frac{\alpha}{\theta}\left(1+\frac{x}{\theta}\right)^{-(\alpha+1)}	\quad\text{for $x > 0$ and zero otherwise.}
\]
Let $X\sim\text{Lomax}(1,1)$. Show that the CDF of $Z = \log X$ is given by
\[
F_Z(z) =  \frac{e^z}{1+e^z}.
\]
This is the CDF of the \emph{standard logistic distribution}.
\end{example}
\begin{solution}
Taking $\alpha=1$ and $\theta=1$, the PDF of $X$ is
\[
f_X(x) = \frac{1}{(1 + x)^2}
\]
Consider the transformation $g(x) = \log x$.
\bit
\it $g(x)$ is an increasing function over $\supp(f_X)=(0,\infty)$.
\it $\supp(f_Z) = \{g(x):x\in\supp(f_X)\} = \{\log x: x\in(0,\infty)\} = (-\infty,\infty)$.
\it The inverse transformation is  $g^{-1}(z) = e^z$, and its first derivative is $\displaystyle\frac{d}{dz}g^{-1}(z) = e^z$.
\eit
The PDF of $Z = \log X$ is therefore given by
\[
f_Z(z) = f_X\big[g^{-1}(z)\big]\left|\frac{d}{dz}g^{-1}(z)\right| = \frac{e^z}{(1+e^z)^2},
\]
from which we the required CDF follows by integration.
%This is the PDF of the \emph{standard logistic distribution}.
%Its CDF if given by $\displaystyle F_Z(z) = \frac{e^z}{(1+e^z)}$.
\end{solution}


\section{Cubic Spline Interpolation}
\begin{defn}
Given a function $f$ defined on $[a,b]$, $a=x_0<x_1<\cdots,<x_n=b$, a cubic spline interpolation $S$ for $f$ is a function that satisfies the following conditions.
\begin{enumerate}
    \item $S_j(x)$ is a cubic polynomial, on the subinterval $[x_j,x_{j+1}]$ for $j=0:n-1$.
    \item $S_j(x_j)=f(x_j)$, $S_j(x_{j+1})=f(x_{j+1})$ for $j=0:n-2$.
    \item $S'_j(x_{j+1})=S'_{j+1}(x_{j+1})$ for $j=0:n-2$.
    \item $S''_j(x_{j+1})=S''_{j+1}(x_{j+1})$ for $j=0:n-2$.
    \item $\begin{cases}
          \text{natural boundary:} & S''(x_0)=S''(x_n)=0. \\
          \text{clamped boundary:} & S'(x_0)=f'(x_0)$, $S'(x_n)=f'(x_n).
          \end{cases}$
\end{enumerate}
\end{defn}
\subsection{Construction of a Cubic Spline}
Let $h_j=x_{j+1}-x_j$ (forward):
\begin{enumerate}[(1)]
    \item \begin{align*}
          S_j(x) &= a_j+b_j(x-x_j)+c_j(x-x_j)^2+d_j(x-x_j)^3\quad\text{for $j=0:n-1$} \\
          &\Rightarrow a_{j+1}=S_{j+1}(x_{j+1})=S_j(x_{j+1}) \\
          &\hspace{1.3cm}=a_j+b_jh_j+c_jh_j^2+d_jh_j^3=f(x_{j+1})\quad\text{for $j=0:n-1$}
          \end{align*}
    \item \begin{align*}
          S'_j(x) &= b_j+2c_j(x-x_j)+3d_j(x-x_j)^2 \\
          &\Rightarrow b_{j+1}=S'_{j+1}(x_{j+1})=S'_j(x_{j+1}) \\
          &\hspace{1.3cm}=b_j+2c_jh_j+3d_jh_j^2\quad\text{for $j=0:n-1$}
          \end{align*}
    \item \begin{align*}
          S''_j(x) &= 2c_j+6d_j(x-x_j) \\
          &\Rightarrow 2c_{j+1}=S''_{j+1}(x_{j+1})=S''_j(x_{j+1}) \\
          &\hspace{1.3cm}=2c_j+6d_jh_j\quad\text{for $j=0:n-1$}
          \end{align*}
\end{enumerate}

Above all, the linear system to be solved is:
\begin{align*}
    &Ax=b \\
    &\begin{cases}
    A=diag([1,2(h_0+h_1),\cdots,2(h_{n-2}+h_{n-1}),1]) \\
    \phantom{A=}+diag([0,h_1,\cdots,h_{n-1}],1)+diag([h_0,\cdots,h_{n-2},0],-1) \\
    x = [c_0;c_1;\cdots;c_n] \\
    b = \left[0;\frac{3}{h_1}(a_2-a_1)-\frac{3}{h_0}(a_1-a_0);\cdots;\frac{3}{h_{n-1}}(a_n-a_{n-1})-\frac{3}{h_{n-2}}(a_{n-1}-a_{n-2});0\right] \\
    \end{cases}
\end{align*}
Then we will get $b_j$, $d_j$ by
\begin{equation*}
    \begin{cases}
    b_j &= \frac{1}{h_j}(a_{j+1}-a_j)-\frac{h_j}{3}(2c_j+c_{j+1}) \\
    d_j &= \frac{1}{3h_j}(c_{j+1}-c_j)
    \end{cases}
\end{equation*}

\subsection{Clamped Splines}
\begin{align*}
    &Ax=b \\
    &\begin{cases}
    A=\begin{pmatrix}
      2h_0 & h_0 & 0 & \cdots & 0 \\
      h_0 & 2(h_0+h_1) & h_1 & \ddots & \vdots \\
      0 & h_1 & 2(h_1+h_2) & \ddots & \vdots \\
      \vdots & \ddots & \ddots & \ddots & h_{n-1} \\
      0 & \cdots & \cdots & h_{n-1} & 2h_{n-1} \\
      \end{pmatrix} \\
    \\
    x = \begin{pmatrix}c_0 & c_1 & \cdots & c_n \end{pmatrix}^T \\
    \\
    b = \begin{pmatrix}
        \frac{3}{h_0}(a_1-a_0)-3f'(a) \\
        \frac{3}{h_1}(a_2-a_1)-\frac{3}{h_0}(a_1-a_0) \\
        \vdots \\
        \frac{3}{h_{n-1}}(a_n-a_{n-1})-\frac{3}{h_{n-2}}(a_{n-1}-a_{n-2}) \\
        3f'(b)-\frac{3}{h_{n-1}}(a_n-a_{n-1})
        \end{pmatrix}
    \end{cases}
\end{align*}


\section{Measurement Uncertainty Analysis}

\subsection{Uncertainty of instruments}
For a single measurement of the mass of the experiment setup, the uncertainty of the measurement instruments are

\begin{table}[H]
  \centering
  \begin{tabularx}{\textwidth}{|X|X|X|X|}
    \hline
     & Calliper & Electronic balance & Timer\\
	 \hline
	 Resolution & 0.002cm & 0.1g & 0.0001s \\
	 \hline
	 Relative uncertainty &\multicolumn{2}{c|}{}& 0.004\% \\
	\hline
  \end{tabularx}
  \caption{Precision of the measurement instruments}
  \end{table}

\subsection{Uncertainty of Calliper Measurements}

In order to estimate type–A uncertainty of the period, the standard deviation of the average value is calculated as
$$  s_d = \sqrt{\frac{1}{n-1}\sum_{i=1}^{n}(d_i - \bar{d})^2 }  $$

Thus, the standard deviation of the average value is calculated and shown below

\begin{table}[H]
  \centering
  \begin{tabularx}{\textwidth}{|p{6cm}|X|}
    \hline
    Object & The standard deviation\\
    \hline
    Disk $[cm]$& 0.0020 \\
    Hoop 1 $[cm]$& 1.1003 \\
    Hoop 2 $[cm]$& 1.1017 \\
    Cylinder A $[cm]$& 0 \\
    Cylinder B $[cm]$& 0 \\
    Cone pulley $[cm]$& 0.0075 \\
    \hline
  \end{tabularx}
  \caption{The standard deviation}
  \end{table}

The uncertainty of Timer is $\pm 0.0001s $ and $\pm 0.004\%$

From the equation to calculate the moment of inertia,
$$ I_1=\frac{mR(g-R\beta_2)}{\beta_2-\beta_1} $$
$$ u_F = \sqrt{(\frac{\partial I}{\partial m})^2(u_m)^2 +   (\frac{\partial I}{\partial R})^2(u_R)^2+(\frac{\partial I}{\partial \beta_1 })^2(u_{\beta_1})^2+ (\frac{\partial I}{\partial \beta_2})^2(u_{\beta_2})^2   }$$

$$\frac{\partial I}{\partial m} = \frac{R\left(g-\beta _2 R\right)}{\beta _2-\beta _1} $$
$$\frac{\partial I}{\partial R} = -\frac{\beta _2 m R'\left(g-\beta _2 R\right)}{\beta _2-\beta _1} $$
$$\frac{\partial I}{\partial \beta_1} = \frac{m R\left(g-\beta _2 R\right)}{\left(\beta _2-\beta _1\right){}^2} $$
$$\frac{\partial I}{\partial \beta_2} = -\frac{m R R'\left(g-\beta _2 R\right)}{\beta _2-\beta _1}-\frac{m R\left(g-\beta _2 R\right)}{\left(\beta _2-\beta _1\right){}^2} $$

$$ u_m = 0.1 g $$
$$ u_R = 0.0075 cm $$
$$ u_{\beta_1} = \sqrt{(0.0001 s)^2+(0.0001 \times 95 \% )^2} =0.000137931 $$
$$ u_{\beta_1} = \sqrt{(0.0001 s)^2+(0.0001 \times 95 \% )^2} =0.000137931 $$

Thus,
$$u_{rF} =  0.4\%$$


\subsection{Partial Derivatives of X, Y, and Z}
\noindent
It's common to look at the derivative when slicing a surface in the yz, xz, and xy planes. These are called partial derivatives.\\

\noindent
To compute $\frac{\partial}{\partial x}{f(x,y)}$, we take the derivative with respect to $x$ as if $y$ is constant. Formally,
\begin{equation*}
	\frac{\partial}{\partial x}f(x,y) = \lim_{h \to 0}{\frac{f(x+h,y)-f(x,y)}{h}}
\end{equation*}
and
\begin{equation*}
	\frac{\partial}{\partial y}f(x,y) = \lim_{h \to 0}{\frac{f(x,y+h)-f(x,y)}{h}}
\end{equation*}

\noindent
We also use the shorthand $\frac{\partial}{\partial x}=f_x$ and $\frac{\partial}{\partial y}=f_y$. This shorthand can be extended to higher-order derivatives so that $\frac{\partial}{\partial y}\left(\frac{\partial}{\partial x}f(x,y)\right)=f_{xy}$.

\noindent
\textbf{Fubini's Theorem} (also called Tonelli's or Clairaut's Theorem) says $f_{xy} = f_{yx}$, $f_{xz} = f_{zx}$, and $f_{yz} = f_{zy}$. It extends into higher-order mixed partial derivatives, saying that two mixed partial derivatives of a function are equal as long as they both differentiate the same number of variables the same number of times. So, $f_{abcdab} = f_{aacdbb}$.


\section{Spectrum of Autoregressive Processes}

\begin{enumerate}[label=\alph*), leftmargin=*]
%% a)
\item
%

A $p$ order Autoregressive process with parameters $\va \in \sR^{p}$ satisfies the Yule-Walker (or normal) equation:

\begin{equation}
    \vr_{xx} = \mathbf{R}_{xx} \va \Rightarrow \va = \mathbf{R}_{xx}^{-1} \vr_{xx}
    \label{eq:yule-walker}
\end{equation}

where $\mathbf{R}_{xx}$ the autocorrelation matrix (ACF) of signal $x(n)$. Equation (\ref{eq:yule-walker}) is meaningful for non-singular and thus invertible $\mathbf{R}_{xx}$.
The biased estimator of ACF guarantees positive semi-definiteness and as a result the $\mathbf{R}_{xx}$ can be inverted, providing solutions for the autoregressive parameters $\va$.
On the other hand, as depicted in figure \ref{fig:2_1_a}, the unbiased ACF estimator leads to indefiniteness and thus $\mathbf{R_{xx}}$ may be singular.

%% b)
\item
%

The power spectral density of an AR process with parameters $\va = [2.76,\ -3.81,\ 2.65,\ -0.92]$ in Guassian white noise ($\sigma^{2} = 1$) is estimated using different AR model order
$p = 2,\ 3,\ \ldots,\ 14$. As illustrated in figure \ref{fig:2_2_b_1}, low order models (i.e $p=2$) fail to capture the behaviour of the process, identifying a single peak in the spectrum,
while two are expected. Higher order models (i.e $p=9$) provide better estimates, able to find the two peaks.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/b/ar_2}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/b/ar_4}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/b/ar_9}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/b/ar_14}
    \end{subfigure}
    \caption{AR: spectrum estimates and model order $p$ with $N=500$ samples.}
    \label{fig:2_2_b_1}
\end{figure}

Intuitively, the larger the model order $p$, the more degrees of freedom available to capture the nature of the process.
In figure \ref{fig:2_2_b_2} the noise power (mean squared prediction error) is illustrated as a function of model order $p$.
Unsurprisingly, the error decreases for increasing model order though it plateaus for $p \leq 9$. As a result, to minimize model complexity and avoid fitting error (overfitting)
model order $p = 9$ is selected.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/b/error}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/b/aic}
    \end{subfigure}
    \caption{AR: noise power and model order $p$ with $N=500$ samples.}
    \label{fig:2_2_b_2}
\end{figure}

We also highlight that despite the fact that $p_{original} = 4$, due to the small number of available sample ($N = 500$),
when $p = 4$ is selected the fitted model performs very poorly, thus either the number of samples should be increased or a higher order model should be used instead.

%% c)
\item
%

The experiment is repeated for the same AR process, but using $N = 10000$ samples and figures \ref{fig:2_2_c_1} and \ref{fig:2_2_c_2} are obtained.
When $p < p_{original} = 4$, the model still does not have the capacity to model the process (under-modelling) while for $p > p_{original} = 4$ the error plateaus.
Despite the fact that AR(4) model identifies the two peaks, we note that AR(5) and all higher order models track spectrum much better. To avoid overfitting the noise
and preserve generalisation of the model, criteria such as the Akaike Information Criterion (AIC) or the Bayesian Information Criterion can be used to penalise higher order models.
According to the AIC, AR(4) model is selected for this experiment, agreeing with the true model order.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/c/ar_3}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/c/ar_4}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/c/ar_5}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/c/ar_9}
    \end{subfigure}
    \caption{AR: spectrum estimates and model order $p$ with $N=10000$ samples.}
    \label{fig:2_2_c_1}
\end{figure}

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/c/error}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/parametric-and-line-spectra/spectrum-of-autoregressive-processes/assets/c/aic}
    \end{subfigure}
    \caption{AR: soise power and model order $p$ with $N=10000$ samples.}
    \label{fig:2_2_c_2}
\end{figure}

%
\end{enumerate}


\section{Eigen analysis and spectral smoothing}

For a smoother with symmetric smoother matrix $\bS$, the
eigendecomposition of $\bS$ can be used to describe its
behavior.

Let $\{ \bu_1,\dots,\bu_n \}$ be an orthonormal basis of eigenvectors
of $\bS$ with eigenvalues $\theta_1\geq\theta_2\dots\geq\theta_n$:
\[
\bS\bu_j = \theta_j \bu_j, j=1,\dots,n
\]
or
\[
\bS=\bU \bD \bU' = \sum_{j=1}^n \theta_j \bu_j \bu_j'.
\]
Here $\bD$ is a diagonal matrix with the eigenvalues as the entries.

For simple linear regression we only have two nonzero
eigenvalues. Their eigenvectors are an orthonormal basis for lines.

\begin{figure}[htb]
  \caption{Eigenvalues and eigenvectors of the hat matrix for linear
    regression.}
\begin{center}
\epsfig{figure=Plots/plot-06-04.ps,angle=270,width=.75\textwidth}
\end{center}
\end{figure}

The cubic spline is an important example of a symmetric smoother, and
its eigenvectors resemble polynomials of increasing degree.

It is easy to show that the first two eigenvalues are unity, with
eigenvectors which correspond to linear functions of the predictor on
which the smoother is based. One can also show that the other
eigenvalues are all strictly between zero and one.

The action of the smoother is now transparent: if presented with a
response $\by = \bu_j$, it shrinks it by an amount $\theta_j$ as
above.

\begin{figure}[htb]
\caption{\label{f6.3.1} Eigenvalues and eigenvectors 1 through 10 of
  $\bS$ for a smoothing spline.}
\begin{tabular}{cc}
\epsfig{figure=Plots/plot-06-05.ps,angle=270,width=.5\textwidth}&
\epsfig{figure=Plots/plot-06-06.ps,angle=270,width=.5\textwidth}
\end{tabular}
\end{figure}

Cubic smoothing splines, regression splines, linear regression,
polynomial regression are all symmetric smoothers. However, loess
and other ``nearest neighbor'' smoothers are not.

\begin{figure}[htb]
\caption{Eigen vectors 11 through 30 for a smoothing spline for
$n=30$.}
\begin{tabular}{cc}
\epsfig{figure=Plots/plot-06-07.ps,angle=270,width=.5\textwidth}&
\epsfig{figure=Plots/plot-06-08.ps,angle=270,width=.5\textwidth}
\end{tabular}
\end{figure}

If $\bS$ is not symmetric we have complex eigenvalues and the above
decomposition is not as easy to interpret. However we can use the
singular value decomposition
\[
\bS = \bU \bD \bV'
\]

On can think of smoothing as performing a basis transformation $\bz =
\bV'\by$, shrinking with $\hat{\bz} = \bD\bz$ the components that are
related to ``unsmooth components'' and then transforming back to the
basis $\hat{\by} = \bU \hat{\bz}$ we
started out with... sort of.

In signal processing signals are ``filtered'' using linear
transformations. The transfer function describes how the power of
certain frequency components are reduced. A low-pass filter will
reduce the power of the higher frequency components.
We can view the eigen values of our
smoother matrices as transfer functions.

Notice that the smoothing spline can be considered a low-pass
filter. If we look at the eigenvectors of the smoothing spline we
notice they are similar to sinusoidal components of increasing
frequency. Figure \ref{f6.3.1} shows the ``transfer function'' defined
by the smoothing splines.

The change of basis idea described above has been explored by Donoho
and Johnston 1994, 1995) and  Beran
(2000). In the following section we give a short introduction to these
ideas.


\subsection{Introduction}

\(m_{11}x+m_{12}y+m_{13}z=v_1\)

\(m_{21}x+m_{22}y+m_{23}z=v_2\)

\(m_{31}x+m_{32}y+m_{33}z=v_3\)

\subsection{Matrix and vector notation}

We can write the above as:

\(\mathbf{M}x=\mathbf{v}\)

What are the properties of \(\mathbf{M}\) and \(\mathbf{v}\)?

They are linear in addition and scalar multiplication.


%!TEX root = ../Thesis.tex
\chapter{Detailed Derivation: Coordinate Velocity Transform} \label{app:coordinate-transform}
Recall $(\mathscr{X},\mathscr{Y})$ is the inertial frame coordinates and $(x,y)$ is the rotating frame coordinates \ref{fig:r3b}. The goal is to go from $\mathscr{X}(x,y)$ and $\mathscr{Y}(x,y)$ to $(\dot{\mathscr{X}}^2 + \dot{\mathscr{Y}}^2)(x,y)$. This is used in \eqref{eq:r3b-T} to obtain the kinetic energy $T(x,y)$ From equation \eqref{eq:coordinate-transform} we had
\begin{align}
\notag
\begin{cases}
\ \mathscr{X} = x\cos{\omega t} - y\sin{\omega t} \\
\ \mathscr{Y} = x\sin{\omega t} + y\cos{\omega t}
\end{cases}
\end{align}
Taking the time derivative we get
\begin{align}
\begin{cases}
\ \dot{\mathscr{X}} = \dot{x}\cos{\omega t} - \dot{y}\sin{\omega t} - \omega x\sin{\omega t} - \omega y\cos{\omega t} \\
\ \dot{\mathscr{Y}} = \dot{x}\sin{\omega t} + \dot{y}\cos{\omega t} + \omega x\cos{\omega t} - \omega y\sin{\omega t}
\end{cases}
\end{align}
We'll take the squares, but first, to get a good overview of the terms, we note that $\dot{\mathscr{X}}^2$ matches
\begin{align}
\begin{split}
\label{eq:algebra}
&(a-b-c-d)^2 \\
&= a^2 + b^2 +c^2 +d^2 \\
&- 2 a b -2 a c - 2 a d \\
&+ 2 b c + 2 b d + 2 c d
\end{split}
\end{align}
Now we take the squares, align the terms as in \eqref{eq:algebra} and note what terms add and cancel in the sum $\mathscr{X}^2 + \mathscr{Y}^2$:
\begin{align}
\begin{split}
\dot{\mathscr{X}}^2 &= (\dot{x}\cos{\omega t} - \dot{y}\sin{\omega t} - \omega x\sin{\omega t} - \omega y\cos{\omega t})(\dot{x}\cos{\omega t} - \dot{y}\sin{\omega t} - \omega x\sin{\omega t} - \omega y\cos{\omega t}) \\
&=
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$\dot{x}^2\cos^2{t}$};
} +
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$\dot{y}^2\sin^2{t}$};
} +
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$\omega^2 x^2\sin^2{t}$};
} +
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$\omega^2 y^2\cos^2{t}$};
} \\
&-
\tikz[baseline]{
    \node[fill=red!20,anchor=base]
    {$2 \dot{x}\dot{y}\cos{\omega t}\sin{\omega t}$};
} -
\tikz[baseline]{
    \node[fill=red!20,anchor=base]
    {$2 \omega x \dot{x} \cos{\omega t}\sin{\omega t}$};
} -
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$2 \omega \dot{x}y\cos^2{t}$};
} \\
&+
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$2 \omega x \dot{y}\sin^2{t}$};
} +
\tikz[baseline]{
    \node[fill=red!20,anchor=base]
    {$2 \omega y \dot{y}\cos{\omega t}\sin{\omega t}$};
} +
\tikz[baseline]{
    \node[fill=red!20,anchor=base]
    {$2 \omega^2 x y \cos{\omega t}\sin{\omega t}$};
}
\end{split}
\end{align}
Likewise $\dot{\mathscr{Y}}^2$ matches
\begin{align}
\begin{split}
&(a + b + c - d)^2 \\
&= a^2 + b^2 +c^2 +d^2 \\
&+ 2 a b + 2 a c - 2 a d \\
&+ 2 b c - 2 b d - 2 c d
\end{split}
\end{align}
so we have
\begin{align}
\begin{split}
\dot{\mathscr{Y}}^2 &= (\dot{x}\sin{\omega t} + \dot{y}\cos{\omega t} + \omega x\cos{\omega t} - \omega y\sin{\omega t})(\dot{x}\sin{\omega t} + \dot{y}\cos{\omega t} + \omega x\cos{\omega t} - \omega y\sin{\omega t}) \\
&=
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$\dot{x}^2\sin^2{t}$};
} +
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$\dot{y}^2\cos^2{t}$};
} +
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$\omega^2 x^2\cos^2{t}$};
} +
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$\omega^2 y^2\sin^2{t}$};
} \\
&+
\tikz[baseline]{
    \node[fill=red!20,anchor=base]
    {$2 \dot{x}\dot{y}\cos{\omega t}\sin{\omega t}$};
} +
\tikz[baseline]{
    \node[fill=red!20,anchor=base]
    {$2 \omega x \dot{x}\cos{\omega t}\sin{\omega t}$};
} -
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$2 \omega \dot{x}y\sin^2{t}$};
} \\
&+
\tikz[baseline]{
    \node[fill=green!20,anchor=base]
    {$2 \omega x \dot{y}\cos^2{t}$};
} -
\tikz[baseline]{
    \node[fill=red!20,anchor=base]
    {$2 \omega y \dot{y}\cos{\omega t}\sin{\omega t}$};
} -
\tikz[baseline]{
    \node[fill=red!20,anchor=base]
    {$2 \omega^2 x y \cos{\omega t}\sin{\omega t}$};
}
\end{split}
\end{align}
\begin{align}
\notag &\tikz\node [fill=green!20,draw,circle]{};\ \text{Terms that add in $\mathscr{X}^2 + \mathscr{Y}^2$} \\
\notag &\tikz\node [fill=red!20,draw,circle]{};\ \text{Terms that cancel in $\mathscr{X}^2 + \mathscr{Y}^2$} \\[1cm]
\end{align}
Cancelling terms and repeated use of $\cos^2{t} + \sin^2{t} = 1$ finally gives us
\begin{align}
\notag \dot{\mathscr{X}}^2 + \dot{\mathscr{Y}}^2 &= \dot{x}^2 + \dot{y}^2 + \omega^2 x^2 + \omega^2 y^2 - 2 \omega \dot{x} y + 2 \omega x \dot{y} \\[0.2cm]
&= (\dot{x}-\omega y)^2 + (\dot{y}+\omega x)^2
\end{align}


\documentclass[11pt, a4paper]{article}

\usepackage{tikz}
\usetikzlibrary{shapes,arrows}
\usepackage{amsmath}
\usepackage{placeins}
\usepackage{amssymb}

\begin{document}

\title{PRINCIPAL COMPONENT ANALYSIS}
\date{}
\maketitle

Principal Component Analysis (PCA) is non-parametric approach to transform a set of observations - of possibly correlated variables - into another set of values defined by a calculated set of new orthogonal uncorrelated variables called principal components.

\section{Example}

Consider the following set of four observations in two dimesions,

\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]
		\tikzstyle {margin} = [dashed]

		\draw[step=1.0, gray, very thin] (-4.5, -4.5) grid (4.5, 4.5);

		\draw[-latex] (-5,0) -- (5,0) node[right]{x1};
		\draw[-latex] (0,-5) -- (0,5) node[left]{x2};

		\draw[point line] (2.7, 2) -- (3.3, 2);
		\draw[point line] (3, 1.7) -- (3, 2.3);
		\draw (3, 2) node [below right] {(3, 2)};

		\draw[point line] (-1.3, 0) -- (-0.7, 0);
		\draw[point line] (-1, -0.3) -- (-1, 0.3);
		\draw (-1, 0) node [below left] {(-1, 0)};

		\draw[point line] (-1.3, 2) -- (-0.7, 2);
		\draw[point line] (-1, 1.7) -- (-1, 2.3);
		\draw (-1, 2) node [above left] {(-1, 2)};

		\draw[point line] (-1.3, -4) -- (-0.7, -4);
		\draw[point line] (-1, -4.3) -- (-1, -3.7);
		\draw (-1, -4) node [above left] {(-1, -4)};

	\end{tikzpicture}
\end{figure}

To see how PCA alters the representation of this tiny dataset, the dataset is first written in matrix form as follows,

\begin{align*}
	X & = \begin{pmatrix} 3 & -1 & -1 & -1 \\
	2 & 0 & 2 & -4
	\end{pmatrix}
\end{align*}

The x-axis intercepts are represented by the first row of this matrix while y-axis intercepts are represented by the second row.
This matrix can be thought of as a representation composed of two random variables x1 and x2. The variances of these variables and their covariance are useful quantities to describe the structure of the dataset.

\begin{align*}
	Var(x1)     & = \frac{1}{N} \sum\limits_{i = 1}^N (x1_i - \bar{x1})^2                \\
	            & = \frac{1}{4} \sum\limits_{i = 1}^4 (x1_i - 0)^2                       \\
	            & = \frac{1}{4} |x1|^2                                                   \\
	            & = \frac{3^2 + (-1)^ 2 + (-1)^2 + (-1)^2}{4}                            \\
	            & = 3                                                                    \\
	Var(x2)     & = \frac{1}{4} \sum\limits_{i = 1}^4 (x2_i - 0)^2                       \\
	            & = \frac{1}{4} |x2|^2                                                   \\
	            & = \frac{2^2 + 0^ 2 + 2^2 + (-4)^2}{4}                                  \\
	            & = 6                                                                    \\
	Cov(x1, x2) & = \frac{1}{N} \sum\limits_{i = 1}^N (x1_i - \bar{x1})(x2_i - \bar{x2}) \\
	            & = \frac{1}{4} \sum\limits_{i = 1}^4 (x1_i - 0)(x2_i - 0)               \\
	            & = \frac{1}{4} \vec{x1}.\vec{x2}                                        \\
	            & = \frac{3*2 + (-1)*0 + (-1)*2 + (-1)*(-4)}{4}                          \\
	            & = 2
\end{align*}

Note that the mean of both the vectors is zero in the dataset. This is a necessary precondition for PCA, the reason for this constraint will become apparent in later sections. Next, these variances and covariance are arranged in a matrix as follows.

\begin{align*}
	\sum & = \begin{pmatrix} Var(x1) & Cov(x1, x2) \\
	Cov(x1, x2) & Var(x2) \\
	\end{pmatrix} \\
	     & = \begin{pmatrix} 3       & 2           \\
	2 & 6 \\
	\end{pmatrix}
\end{align*}

Due to zero mean constraint, the following also holds true,

\begin{align*}
	\sum = \frac{1}{N} XX^T \\
\end{align*}

Now eigenvalues and eigenvectors of $\sum$ are calculated.

\begin{align*}
	\sum \phi              & = \lambda \phi \\
	(\sum - \lambda I)\phi & = 0
\end{align*}

$(\sum - \lambda I)$ must be singular if eigenvectors are non-zero. Hence,

\begin{align*}
	\begin{vmatrix} 3 - \lambda & 2           \\
	2                           & 6 - \lambda \\
	\end{vmatrix}               & = 0         \\
	(3 - \lambda)(6 - \lambda)  & = 4         \\
\end{align*}

This gives eigenvalues $\lambda = 2, 7$ and eignevectors $\phi = (\frac{-2}{\sqrt{5}}, \frac{1}{\sqrt{5}}), (\frac{1}{\sqrt{5}}, \frac{2}{\sqrt{5}}) $ respectively. Now the eigenvectors are arranged as rows in a matrix in order of their corresponding eigenvalue.

\begin{align*}
	P & = \begin{pmatrix} \frac{1}{\sqrt{5}}   & \frac{2}{\sqrt{5}} \\
	\frac{-2}{\sqrt{5}} & \frac{1}{\sqrt{5}} \\
	\end{pmatrix}  \\
	  & = \frac{1}{\sqrt{5}} \begin{pmatrix} 1 & 2                  \\
	-2 & 1\\
	\end{pmatrix}
\end{align*}

See the following visualization of eigenvectors.

\FloatBarrier\clearpage
\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]
		\tikzstyle {margin} = [dashed]

		\draw[step=1.0, gray, very thin] (-4.5, -4.5) grid (4.5, 4.5);

		\draw[-latex] (-5,0) -- (5,0) node[right]{x1};
		\draw[-latex] (0,-5) -- (0,5) node[left]{x2};

		\draw[point line] (2.7, 2) -- (3.3, 2);
		\draw[point line] (3, 1.7) -- (3, 2.3);
		\draw (3, 2) node [below right] {(3, 2)};

		\draw[point line] (-1.3, 0) -- (-0.7, 0);
		\draw[point line] (-1, -0.3) -- (-1, 0.3);
		\draw (-1, 0) node [below left] {(-1, 0)};

		\draw[point line] (-1.3, 2) -- (-0.7, 2);
		\draw[point line] (-1, 1.7) -- (-1, 2.3);
		\draw (-1, 2) node [above left] {(-1, 2)};

		\draw[point line] (-1.3, -4) -- (-0.7, -4);
		\draw[point line] (-1, -4.3) -- (-1, -3.7);
		\draw (-1, -4) node [above left] {(-1, -4)};

		\draw[very thick,->] (0,0) -- (.45,.90);
		\draw[dashed] (-2.25, -4.5) -- (2.25,4.5);
		\draw (.45, .90) node [right] {$(\frac{1}{\sqrt{5}}, \frac{2}{\sqrt{5}})$};

		\draw[very thick,->] (0,0) -- (-0.90,0.45);
		\draw[dashed] (-4.5, 2.25) -- (4.5,-2.25);
		\draw (-0.90,0.45) node [above] {$(\frac{-2}{\sqrt{5}}, \frac{1}{\sqrt{5}})$};

	\end{tikzpicture}
\end{figure}

Now the transformed dataset Y is calculated simply as product of P and X.

\begin{align*}
	P &= \frac{1}{\sqrt{5}} \begin{pmatrix} 1 & 2 \\
	-2 & 1\\
	\end{pmatrix} \begin{pmatrix} 3 & -1 & -1 & -1 \\
	2 & 0 & 2 & -4
	\end{pmatrix} \\
	  & = \frac{1}{\sqrt{5}} \begin{pmatrix} 7 & -1 & 3 & -9 \\
	-4 & 2 & 4 & -2
	\end{pmatrix}
\end{align*}

To visualize the result, Y is plotted on a graph,

\FloatBarrier\clearpage
\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]
		\tikzstyle {margin} = [dashed]

		\draw[step=1.0, gray, very thin] (-4.5, -4.5) grid (4.5, 4.5);

		\draw[-latex] (-5,0) -- (5,0) node[right]{y1};
		\draw[-latex] (0,-5) -- (0,5) node[left]{y2};

		\draw[point line] (2.83, -1.78) -- (3.43, -1.78);
		\draw[point line] (3.13, -2.08) -- (3.13, -1.48);
		\draw (3.13, -1.78) node [below left] {$(\frac{7}{\sqrt{5}}, \frac{-4}{\sqrt{5}})$};

		\draw[point line] (-0.74, 0.89) -- (-0.14, 0.89);
		\draw[point line] (-0.44, 0.59) -- (-0.44, 1.19);
		\draw (-0.44, 0.89) node [below left] {$(\frac{-1}{\sqrt{5}}, \frac{2}{\sqrt{5}})$};

		\draw[point line] (1.04, 1.78) -- (1.64, 1.78);
		\draw[point line] (1.34, 1.48) -- (1.34, 2.08);
		\draw (1.34, 1.78) node [below right] {$(\frac{3}{\sqrt{5}}, \frac{4}{\sqrt{5}})$};

		\draw[point line] (-4.32 , -0.89) -- (-3.72 , -0.89);
		\draw[point line] (-4.02, -1.19) -- (-4.02,-0.59 );
		\draw (-4.02, -0.89) node [below right] {$(\frac{-9}{\sqrt{5}}, \frac{-2}{\sqrt{5}})$};

	\end{tikzpicture}
\end{figure}

Visually, the points on the transformed graph arise due to a change of axes from the original axes to the axes defined by the eigenvectors of $\sum$. To see why this choice of axes was made, the variance covariance matrix of the transformed data is calculated.

\begin{align*}
	\sum_{new} &= \frac{1}{N} YY^T \\
	  & = \frac{1}{4} \frac{1}{\sqrt{5}} \begin{pmatrix} 7 & -1 & 3 & -9 \\
	-4 & 2 & 4 & -2
	\end{pmatrix} \frac{1}{\sqrt{5}}  \begin{pmatrix} 7 & -4 \\
	-1 & 2 \\
	3 & 4 \\
	-9 & -2 \\
	\end{pmatrix} \\
	&=  \frac{1}{20} \begin{pmatrix} 140 & 0 \\
	0 & 40
	\end{pmatrix} \\
	&= \begin{pmatrix} 7 & 0 \\
	0 & 2
	\end{pmatrix}
\end{align*}

This results in the variance covariance of transformed data being diagonalized. Also notice that the variance values i.e. the diagonal terms are equal to the corresponding eigenvalues. PCA yields a set of a new set of variables/axes from the original set of variables/axes which have the following properties.

\begin{itemize}
	\item New variables are mutually uncorrelated.
	\item New variables are orthogonal to each other.
\end{itemize}

You are free to choose a subset of these new variables depending on the tradeoff between smaller number of dimensions vs loss in variance. In practice, a large chunk of calculated variables in the bottom right part of the diagonal are near zero in their variance and thus are ignored resulting in dimensionality reduction.
In this example, if the new variable corresponding to lower variance is dropped, the dataset is transformed from 2-d to 1-d as shown below.

\begin{figure}[htbp]
	\centering
	\begin{tikzpicture}
		\tikzstyle {point line} = [line width=0.15em]

		\draw[-latex] (-5,0) -- (5,0) node[right]{y1};

		\draw[point line] (2.83, 0) -- (3.43, 0);
		\draw[point line] (3.13, -0.3) -- (3.13, 0.3);
		\draw (3.13, -0.2) node [below] {$\frac{7}{\sqrt{5}}$};

		\draw[point line] (-0.74, 0) -- (-0.14, 0);
		\draw[point line] (-0.44, -.3) -- (-0.44, .3);
		\draw (-0.44, -0.2) node [below] {$\frac{-1}{\sqrt{5}}$};

		\draw[point line] (1.04, 0) -- (1.64, 0);
		\draw[point line] (1.34, -.3) -- (1.34, .3);
		\draw (1.34, -0.2) node [below] {$\frac{3}{\sqrt{5}}$};

		\draw[point line] (-4.32 , 0) -- (-3.72 , 0);
		\draw[point line] (-4.02, -0.3) -- (-4.02,0.3 );
		\draw (-4.02, -0.2) node [below] {$\frac{-9}{\sqrt{5}}$};

	\end{tikzpicture}
\end{figure}

\section{Mathematics}

Assume a (MxN) matrix X as a set of N observations of M random variables such that each observation is a column in the matrix X. Furthermore, assume that each variable has zero mean meaning that each row of the matrix sums up to zero. Denote by $\sum_x$ the variance covariance matrix of the variables. Due to the zero mean condition,

\begin{align*}
	\sum_x = XX^T
\end{align*}

Let $\phi_i, \phi_2, ..., \phi_m$ be the m orthonormal eigenvectors of $\sum_x$. Let P be defined as the matrix $[\phi_1^T, \phi_2^T, ..., \phi_m^T]^T$. Consider the following transformation,

\begin{align*}
	Y = PX
\end{align*}

It can be shown that the resulting variables are also zero-mean based. So, their variance covariance matrix can be written as,

\begin{align*}
	\sum_y & = YY^T       \\
	       & = (PX)(PX)^T \\
	       & = PXX^TP^T   \\
	       & = P\sum_xP^T \\
\end{align*}

$\sum_x$ is symmetric and all symmetric matrices are diagonalized by a matrix of their orthonormal eigenvectors. Or more precisely,

\begin{align*}
	\sum_x = P^TDP
\end{align*}

where D is a diagonal matrix. Upon substitution in the preceding equation,

\begin{align*}
	\sum_y = PP^TDPP^T
\end{align*}

Since P is orthonormal, its transpose is its inverse and thus $PP^T = I$ which reduces $\sum_y$ to D.

\section{Kernel PCA}

Let X be $[x_1, x_2, ..., x_N]$ where each $x_i$ is an observation of M variables. Consider mapping each $x_i$ to $f(x_i)$ where $f$ is non-linear and may increase the dimensionality of each $x_i$. So,

\begin{align*}
	f(X) = [f(x_1), f(x_2), ..., f(x_N)]
\end{align*}

Now examine the variance covariance matrix of $f(X)$.

\begin{align*}
	\sum_f & = \frac{1}{N}f(X)f(X)^T                                   \\
	       & = \frac{1}{N}[f(x_1), f(x_2), ..., f(x_N)]\begin{pmatrix}
	f(x_1^T) \\
	f(x_2^T) \\
	... \\
	f(x_N^T)
	\end{pmatrix} \\
	       & = \frac{1}{N} \sum_{i = 1}^{N} f(x_i)f(x_i^T)             \\
\end{align*}

To find principal components, eigenvalues and eigenvectors of the variance covariance matrix must be found out. since there are N eigenvalue, eigenvector pairs, they will be subscripted by j.

\begin{align*}
	\lambda_j\phi_j & = \sum_f\phi_j                                      \\
	\lambda_j\phi_j & = \frac{1}{N} \sum_{i = 1}^{N} f(x_i)f(x_i^T)\phi_j \\
\end{align*}

Since $f(x_i^T)\phi_j = \vec{f(x_i)}.\vec{\phi_j}$ is just a scalar, the equation can be simplified to,

\begin{align*}
	\phi_j & = \sum_{i = 1}^{N} \frac{\vec{f(x_i)}.\vec{\phi_j}}{N\lambda_j}f(x_i) \\
\end{align*}

Substituting $\frac{\vec{f(x_i)}.\vec{\phi_j}}{N\lambda_j}$ = $a_{ij}$, the equation becomes,

\begin{align*}
	\phi_j = \sum_{i = 1}^{N} a_{ij} f(x_i)
\end{align*}

This equation reveas that the principal components are linear combinations of the observations.

\begin{align*}
	\phi_j = a_{1j}f(x_1) + a_{2j}f(x_2) + ... + a_{Nj}f(x_N) \\
\end{align*}

Taking dot products by $f(x_1), f(x_2) ... f(x_N)$ successively,

\begin{align*}
	\vec{f(x_1)}.\vec{\phi_j} & = a_{1j}\vec{f(x_1)}.\vec{f(x_1)} + a_{2j}\vec{f(x_1)}.\vec{f(x_2)} + ... + a_{Nj}\vec{f(x_1)}.\vec{f(x_N)} \\
	\vec{f(x_2)}.\vec{\phi_j} & = a_{1j}\vec{f(x_2)}.\vec{f(x_1)} + a_{2j}\vec{f(x_2)}.\vec{f(x_2)} + ... + a_{Nj}\vec{f(x_2)}.\vec{f(x_N)} \\
	...                       & = ...                                                                                \\
	\vec{f(x_N)}.\vec{\phi_j} & = a_{1j}\vec{f(x_N)}.\vec{f(x_1)} + a_{2j}\vec{f(x_N)}.\vec{f(x_2)} + ... + a_{Nj}\vec{f(x_N)}.\vec{f(x_N)}
\end{align*}

In matrix form, these equations look like,

\begin{align*}
	\begin{pmatrix}
	\vec{f(x_1)}.\vec{\phi_j} \\
	\vec{f(x_2)}.\vec{\phi_j} \\
	... \\
	\vec{f(x_N)}.\vec{\phi_j}
	\end{pmatrix} = \begin{pmatrix}
	\vec{f(x_1)}.\vec{f(x_1)} & \vec{f(x_1)}.\vec{f(x_2)} & ... & \vec{f(x_1)}.\vec{f(x_N)} \\
	\vec{f(x_2)}.\vec{f(x_1)} & \vec{f(x_2)}.\vec{f(x_2)} & ... & \vec{f(x_2)}.\vec{f(x_N)} \\
	... \\
	\vec{f(x_N)}.\vec{f(x_1)} & \vec{f(x_N)}.\vec{f(x_2)} & ... & \vec{f(x_N)}.\vec{f(x_N)} \\
	\end{pmatrix} \begin{pmatrix}
	a_{1j} \\
	a_{2j} \\
	... \\
	a_{Nj} \\
	\end{pmatrix}
\end{align*}

The first matrix on the right is a NxN matrix. Denote it by the kernel matrix: K. For every term on the left hand side matrix use the fact that $\vec{f(x_i)}.\vec{\phi_j} = N\lambda_j a_{ij}$.

\begin{align*}
	N\lambda_j \begin{pmatrix}
	a_{1j}                            \\
	a_{2j}                            \\
	...                               \\
	a_{Nj}                            \\
	\end{pmatrix} = K \begin{pmatrix}
	a_{1j}                            \\
	a_{2j}                            \\
	...                               \\
	a_{Nj}                            \\
	\end{pmatrix}
\end{align*}

It is apparent that $N\lambda_j, [a_{1j}, a_{2j}, ..., a_{Nj}]^T \forall j \in [1, N]$are eigenvalue, eigenvector pairs for the matrix K. The PCA transform for $f(X)$ can be written as,

\begin{align*}
	Y &= \begin{pmatrix}
	\phi_1^T \\
	\phi_2^T \\
	... \\
	\phi_N^T \\
	\end{pmatrix} f(X) \\
	&= \begin{pmatrix}
	\phi_1^T \\
	\phi_2^T \\
	... \\
	\phi_N^T \\
	\end{pmatrix} [f(x_1), f(x_2), ..., f(x_N)] \\
	&= \begin{pmatrix}
	\vec{\phi_1}.\vec{f(x_1)} & \vec{\phi_1}.\vec{f(x_2)} & ... & \vec{\phi_1}.\vec{f(x_N)} \\
	\vec{\phi_2}.\vec{f(x_1)} & \vec{\phi_2}.\vec{f(x_2)} & ... & \vec{\phi_2}.\vec{f(x_N)} \\
	... \\
	\vec{\phi_N}.\vec{f(x_1)} & \vec{\phi_N}.\vec{f(x_2)} & ... & \vec{\phi_N}.\vec{f(x_N)} \\
	\end{pmatrix}
\end{align*}

Using the relation $\vec{f(x_i)}.\vec{\phi_j} = N\lambda_j a_{ij}$,

\begin{align*}
	Y &= \begin{pmatrix}
	N\lambda_1 a_{11} & N\lambda_1 a_{21} & ... & N\lambda_1 a_{N1} \\
	N\lambda_2 a_{12} & N\lambda_2 a_{22} & ... & N\lambda_2 a_{N2} \\
	... \\
	N\lambda_N a_{1N} & N\lambda_N a_{2N} & ... & N\lambda_N a_{NN} \\
	\end{pmatrix} \\
	&=  N\begin{pmatrix}
	\lambda_1 a_{11}  & \lambda_1 a_{21}  & ... & \lambda_1 a_{N1}  \\
	\lambda_2 a_{12}  & \lambda_2 a_{22}  & ... & \lambda_2 a_{N2}  \\
	... \\
	\lambda_N a_{1N}  & \lambda_N a_{2N}  & ... & \lambda_N a_{NN}  \\
	\end{pmatrix}
\end{align*}

This is the kernel trick for PCA where it is not required to explicit map each point to higher dimensions whcich may be computationally expensive. PCA transformation in nonlinear higher dimensions can be calculated by performing the following steps in the order below.

\begin{itemize}
	\item Choose a kernel $ k(x_i, x_j) = \vec{f(x_i)}.\vec{f(x_j)}$.
	\item Calculate kernel matrix K = $\begin{pmatrix}
	      k(x_1, x_1) & k(x_1, x_2) & ... & k(x_1, x_N) \\
	      k(x_2, x_1) & k(x_2, x_2) & ... & k(x_2, x_N) \\
	      ... \\
	      k(x_N, x_1) & k(x_N, x_2) & ... & k(x_N, x_N) \\
	\end{pmatrix}$.
	\item Find the eigenvalue, eigenvalue pairs of K.
	\item Use eigenvalue, eigenvector pairs to calcuate the kernel PCA transform as shown above.
	\item Do dimensinality reduction as appropriate.
\end{itemize}

\section{Assumptions and Limitations}

\begin{itemize}
	\item The new variables are linear combinations of the original ones. Kernel PCA helps alleviate this problem.
	\item PCA assumes that mean and variance are sufficient statistics to describe a random variable. Since only Gaussian distributions are described fully by their variance, the assumption that input dataset is Gaussian is inherent.
	\item The new variables are orthogonal which can be limiting but makes the solution to be readily expressible through linear algebra.
\end{itemize}

\end{document}

Principal Component Analysis/Principal Component Analysis.tex


\subsection{Line Integrals of Vector Fields}
\noindent
One can think of line integral of vector fields as the total work done by the vector field as it moves along some simple path.
\begin{align*}
	W &= \int\limits_{C}{\left(\vec{F}\circ\vec{r}\right) \cdot \hat{T}\mathrm{d}s} \\
	&= \int\limits_{C}{\frac{\vec{r^\prime}(t)}{\norm{\vec{r^\prime}(t)}} \cdot \norm{\vec{r^\prime}(t)}\mathrm{d}t} \\
	&= \int\limits_{C}{\left(\vec{F}\circ\vec{r}\right) \cdot \vec{r^\prime}(t)\mathrm{d}t} = \int\limits_{C}{\vec{F} \cdot \mathrm{d}\vec{r}}
\end{align*}

\noindent
For example, let's find the line integral of $\vec{r}(t) = \langle t, t^2, t \rangle$ for $0 \leq t \leq 1$ in the vector field $\vec{F}(x,y,z) = \langle e^z, \sqrt{1-x^2}, \sin{x} \rangle$.
\begin{equation*}
	\vec{F}\circ\vec{r} = \langle e^t, \sqrt{1-t^2}, \sin{t}\rangle
\end{equation*}
\begin{equation*}
	\vec{r^\prime}(t) = \langle 1, 2t, 1 \rangle
\end{equation*}
\begin{align*}
	\int_{0}^{1}{\langle e^t, \sqrt{1-t^2}, \sin{t} \rangle \cdot \langle 1, 2t, 1 \rangle\mathrm{d}t} &= \int_{0}^{1}{e^t + 2t\sqrt{1 - t^2} + \sin{t}\mathrm{d}t} \\
	&= e - \cos{1} + \frac{2}{3}
\end{align*}

\input{./lineSurfaceIntegrals/directionMatters}
\input{./lineSurfaceIntegrals/circulations}