\section{Math III - Number Theory}
  \subsection{Linear Prime Sieve}
    \code{math/numtheory/prime-sieve.cpp}
\begin{comment}
  \subsection{Divisor Sieve}
    \code{math/numtheory/divisor-sieve.cpp}
\end{comment}
  \subsection{Number/Sum of Divisors}
    If a number $n$ is prime factorized where $n = {p_1}^{e_1} \times {p_2}^{e_2} \times \cdots \times {p_k}^{e_k}$, where $\sigma_0$ is the number of divisors while $\sigma_1$ is the sum of divisors:
    \[
    \sum_{d\mid n} d^k = \sigma_k (n) = \prod \frac{{p_i}^{k(e_i)+1}-1}{p_i -1}
    \]
    \[
    \text{Product: } \prod_{d\mid n} d = n^{\frac{\sigma_1 (n)}{2}}
    \]
  \subsection{M\"{o}bius Sieve}
    The M\"{o}bius function $\mu$ is the M\"{o}bius inverse of $e$ such that $e(n) = \sum_{d\mid n} \mu(d)$.
    \code{math/numtheory/moebius-sieve.cpp}
  \subsection{M\"{o}bius Inversion}
    Given arithmetic functions $f$ and $g$:
    \[
    g(n) = \sum_{d\mid n} f(d) \quad \Leftrightarrow \quad f(n) = \sum_{d\mid n} \mu(d)\; g\left(\frac{n}{d}\right)
    \]
  \subsection{GCD Subset Counting}
    Count number of subsets $S \subseteq A$ such that $\gcd(S) = g$ (modifiable).
    \code{math/numtheory/gcd-subsets.cpp}
  \subsection{Euler Totient}
    Counts all integers from 1 to $n$ that are relatively prime to $n$ in $O(\sqrt{n})$ time.
    \code{math/numtheory/totient.cpp}
\begin{comment}
  \subsection{Euler Phi Sieve}
    Sieve version of Euler totient, runs in $O(N \log N)$ time. Note that $n = \sum_{d\mid n} \varphi(d)$.
    \code{math/numtheory/phi-sieve.cpp}
\end{comment}
  \subsection{Extended Euclidean}
    Assigns $x,y$ such that $ax + by = \gcd(a,b)$ and returns $\gcd(a,b)$.
    \code{math/numtheory/extended-euclidean.cpp}
  \subsection{Modular Exponentiation}
    Find $b^e \pmod m$ in $O(log e)$ time.
    \code{math/numtheory/mod_pow.cpp}
  \subsection{Modular Inverse}
    Find unique $x$ such that $ax \equiv 1 \pmod m$. Returns 0 if no unique solution is found. \underline{Please use modulo solver for the non-unique case.}
    \code{math/numtheory/modinv.cpp}
  \subsection{Modulo Solver}
    Solve for values of $x$ for $ax \equiv b \pmod m$. Returns $(-1,-1)$ if there is no solution. Returns a pair $(x, M)$ where solution is $x \bmod M$.
    \code{math/numtheory/modsolver.cpp}
  \subsection{Linear Diophantine}
    Computes integers $x$ and $y$ such that $ax+by=c$, returns $(-1,-1)$ if no solution. \underline{Tries to return positive integer answers for $x$ and $y$ if possible.}
    \code{math/numtheory/linear-diophantine.cpp}
  \subsection{Chinese Remainder Theorem}
    Solves linear congruence $x \equiv b_i \pmod {m_i}$. Returns $(-1,-1)$ if there is no solution. Returns a pair $(x, M)$ where solution is $x \bmod M$.
    \code{math/numtheory/chinese-remainder.cpp}
    \subsubsection{Super Chinese Remainder}
      Solves linear congruence $a_i x \equiv b_i \pmod {m_i}$. Returns $(-1,-1)$ if there is no solution.
      \code{math/numtheory/super-crt.cpp}
  \subsection{Primitive Root}
    \code{math/numtheory/primitive_root.cpp}
  \subsection{Josephus}
    Last man standing out of $n$ if every $kth$ is killed. Zero-based, and does not kill $0$ on first pass.
    \code{math/numtheory/josephus.cpp}
  \subsection{Number of Integer Points under a Lines}
    Count the number of integer solutions to $Ax+By \leq C$, $0 \leq x \leq n$,
    $0 \leq y$. In other words, evaluate the sum $\sum_{x=0}^n\left\lfloor\dfrac{C-Ax}{B}+1\right\rfloor$.
    To count all solutions, let $n = \left\lfloor\dfrac{c}{a}\right\rfloor$.
    In any case, it must hold that $C-nA \geq 0$. Be very careful about overflows.


\subsection{Multilevel coarse grid corrections and a multigrid method}
\Label{sc:mg-fd}
To describe a multigrid algorithm, we first need to have a multiple
level of grids, say $\mathcal T_\ell$ with $\ell=1:J$ and $\mathcal T_1=\mathcal T_h$ being the
finest mesh.  There are many ways to obtain multiple level of grids
and one simple definition of the grid points in $\mathcal T_\ell$ is as follows:
$$
        x_i^\ell=\frac{i}{2^{J+1-\ell}},\quad i=1,2,\cdots, N_\ell, \ell=1,2,\cdots,J,
$$
where $N_\ell=2^{J+1-\ell}-1$.  Note that $\mathcal T_{\ell-1}$ can be viewed as being obtained
by adding midpoints of the subintervals in ${\mathcal T}_{\ell}$.  For each $\ell$
the set of above nodes will be denoted by $\mathcal N_\ell$.

\begin{figure}[!htb]
\begin{center}
\includegraphics[width=3in]{pictures/manygr.pdf}
\end{center}
\caption{Multiple grids in one dimension
\label{fig:manygrids}}
\end{figure}

With our previous experiences in two-grid method, the description of a
multigrid method is not very difficult.  In fact, the multiple level
grids are treated by treating each two consecutive grids, say $\mathcal T_{\ell-1}$
versus $\mathcal T_{\ell}$.  If we think $\mathcal T_{\ell-1}$ versus $\mathcal T_{\ell}$ like
$\mathcal T_h$ and $\mathcal T_{2h}$, then there is not much new in the multigrid
setting.

Let us give some on details the definition of the restriction and
prolongation matrices.  The restriction matrix $R_{\ell-1}^\ell: R^{N_{\ell-1}}
\mapsto R^{N_{\ell}}$ can be defined by
\begin{equation}\Label{restrictk}
\gamma^{\ell}=R_{\ell-1}^\ell\gamma^{\ell-1}: \;
\gamma^{\ell}_i={1\over2}\gamma^{\ell-1}_{2i-1}+\gamma^{\ell-1}_{2i}+{1\over2}\gamma^{\ell-1}_{2i+1}.
\end{equation}
In matrix form
\begin{equation}
\label{1drestriction}
R_{\ell-1}^\ell=\left(
\begin{array}{ccccccccc}
\frac{1}{2}& 1&\frac{1}{2}&&&&&&\\
&&\frac{1}{2}&1&\frac{1}{2}&&&&\\
&&&&\frac{1}{2}&1&\frac{1}{2}&&\\
&&&&&&\ddots&&\\
&&&&&&\frac{1}{2}&1&\frac{1}{2}\\
\end{array}
\right)
\end{equation}
For a special case, when $N_1=7, N_2=3$, we have
\begin{equation}
\label{1drestriction3}
R_1^2=\left(
\begin{array}{ccccccc}
\frac{1}{2}& 1&\frac{1}{2}&0&0&0&0\\
0&0&\frac{1}{2}&1&\frac{1}{2}&0&0\\
0&0&0&0&\frac{1}{2}&1&\frac{1}{2}\\
\end{array}
\right).
\end{equation}

The prolongation matrix
$P_{\ell}^{\ell-1}: R^{N_{\ell}}
\mapsto R^{N_{\ell-1}}$ can be defined as
\begin{equation}\Label{prolongk}
~~\epsilon^{\ell-1}=P^{\ell-1}_{\ell}\epsilon^{\ell}: \; \epsilon^{\ell-1}_{2i}=\epsilon^{\ell}_i, \epsilon^{\ell-1}_{2i+1}
= {1\over~2}(\epsilon^{\ell}_i+\epsilon^{\ell}_{i+1}), \; i=1:N_{\ell}.
\end{equation}

With the restriction and prolongation matrices in hands, we can now
present a multilevel version of the earlier two-grid algorithm.  As
mentioned before, the idea is to repeat this two grid process for the
coarse grid by using an even coarser grid.  The resulting algorithm is
just a desired multigrid algorithm.

Using the convolution with stride notation, the restriction subprocess can also be written as
$R\ast_2: \mathbb R^{N_\ell}\rightarrow  \mathbb R^{N_{\ell+1}}$, for any $v\in \mathbb R^{N_\ell}, u=(R\ast_2 v)\in R^{N_{\ell+1}}$ with
$$
 (R\ast_2 v)_i=\frac 12 v_{2i-1} +v_{2i}+\frac 12 v_{2i+1},\quad  \mbox{namely}\quad R\ast_2v=R_{\ell-1}^\ell v
$$
where $R=[\frac 12,1,\frac12]$ and $R_{\ell-1}^\ell $ is defined by \eqref{1drestriction}.

Next
let $u^{\ell+1}=\sum\limits_{j=1}^{n_{\ell+1}}\mu_{j}^{\ell+1}\phi^{\ell+1}_{j}
=(  \mu^{\ell+1}, \phi^{\ell+1})_{l^2}$, then we have
\begin{equation}
\begin{split}
u^{\ell+1}&=( \mu^{\ell+1}, \phi^{\ell+1})_{l^2}
=( \mu^{\ell+1}, R\ast_2\phi^{\ell})_{l^2}=(R\ast_2^{\top} \mu^{\ell+1}, \phi^{\ell})_{l^2}\\
&=\sum\limits_{j=1}^{n_{\ell}}\left(R\ast_2^{\top} \mu^{\ell+1}\right)_{j}\phi^{\ell}_{j}.
\end{split}
\end{equation}
The prolongation subprocess can be written as
$R\ast_2^T: \mathbb R^{N_{\ell+1}}\rightarrow \mathbb R^{N_\ell}$, for any $v\in \mathbb R^{N_{\ell+1}}, u=(R\ast_2^T v)\in R^{N_\ell}$ with
$$
(R\ast_2^T v)_{2i}=v_i,\quad (R\ast_2^T v)_{2i+1}=\frac 12 (v_{i+1} +v_i),\quad  \mbox{namely}\quad R\ast_2^Tv=P^{\ell-1}_\ell v
$$
where $R=[\frac 12,1,\frac12]$.

Using the convolution notation, the subprocess to apply $A_\ell$ to
a vector $v\in \mathbb R^{N_\ell}$ can be written as
$A_\ell\ast: \mathbb R^{N_\ell}\rightarrow \mathbb R^{N_\ell}$, for any $v\in \mathbb R^{N_\ell}, r=(A_\ell\ast v)\in R^{N_\ell}$ with
$$
(A_\ell\ast v)_{i}=\frac{1}{h_\ell}( -v_{i-1}+2v_i-v_{i+1})
$$
where $A_\ell=\frac{1}{h_\ell}[-1,2,-1]$.

\newpage

\begin{breakablealgorithm}%[!htb]
	\caption{A multigrid algorithm $\mu = {\text{MG1}}(b; \mu^0; J,\nu_1, \cdots, \nu_J)$}
\label{alg:L-Slash11dm}
\begin{algorithmic}
%	 \State
%		$$
%		u \leftarrow u^0.
%		$$
	\State Set up
		$$
		b^1 = b, \quad \mu^{1}=\mu^0.
		$$
		\State Smoothing and restriction from fine to coarse level (nested)
		\For{$\ell = 1:J$}
		\For{$i = 1:\nu_\ell$}
		\State
		\begin{equation}\label{eq:smoothing}
		\mu^{\ell} \leftarrow \mu^{\ell} + S^\ell \ast (b^\ell - A_\ell \ast \mu^{\ell}).
		\end{equation}
		\EndFor
		\State Form restricted residual and set initial guess:
		$$
		\mu^{\ell+1} \leftarrow 0, \quad b^{\ell+1} \leftarrow R \ast_2 (b^\ell -  A_\ell \ast \mu^{\ell}), A_{\ell+1} = R       \ast_2 A_\ell \ast (R\ast_2^\top).
		$$
		\EndFor
		\State Prolongation and restriction from coarse to fine level
		\For{$\ell = J-1:1$}
		\State
		$$
		\mu^{\ell} \leftarrow \mu^{\ell} + R  \ast_2^{\top} \mu^{\ell+1}.
		$$
%		%		\IF{V-cycle}
%		\For{$i = 1:\nu_\ell$}
%		\State
%		$$
%		u^{\ell,i} \leftarrow u^{\ell,i-1} + [B^{\ell,i}]^T (f^\ell - A^{\ell} u^{\ell,i-1})
%		$$
%		\EndFor
%		%		\ENDIF
		\EndFor
		\State
		$$
		\mu \leftarrow \mu^{1}.
		$$
	\end{algorithmic}
\end{breakablealgorithm}

Application of Multigrid:
		Given $\mu^{(0)}$, for $m=1,2,\cdots$ till convergence
		$$
		\mu^{(m)}= {\text{MG1}}(b; \mu^{{(m-1)}}; J,\nu_1, \cdots, \nu_J).
		$$

\example Let $f(x)=1$. Consider
\begin{equation}\label{1Dposi}
\left\{
\begin{aligned}
-u''&= f, \,\, 0<x<1, \\
 u(0)&=u(1)=0.
\end{aligned}
\right.
\end{equation}
The true solution $u=\frac12 x(1-x)$. Given the partition with the grid points
$x_i=\frac{i}{n+1}, i=0,1,\cdots,n+1$, then by finite element discretization,
we obtain
\begin{equation}\label{matrix}
A\ast \mu =b, A=\frac{1}{h}[-1,2,-1].
\end{equation}
Use gradient descent method and multigrid to solve \eqref{matrix} with random initial guess $\mu^0$.

\begin{figure}[!ht]
\centering
\setlength{\abovecaptionskip}{0pt}
\setlength{\belowcaptionskip}{0pt}
\includegraphics[width=8.3cm]{figures/mgcompare.png}
\caption{Comparison GD with Multigrid}
\label{fig:Hmesh}
\end{figure}

497-6DL/8 Convolutional Multigrid Method/8.4-1DMG.tex


\documentclass{article}
\usepackage{amsmath,amssymb}
\title{Some matrix exercises: corrections and solutions}
\author{Paul Hewson}

\begin{document}

\maketitle

\section{Computer exercises for the lab}

Make sure you are comfortable with the matrix operators in R.   Also, be very very sure you can subscript matrices, i.e. use things such as \text{[,1]} to select column 1, \text{[,-1]} to select everything \emph{except} column 1, and \text{[,2:3]} to select columns 2 and 3.

\begin{enumerate}

\item Find (where possible) the determinants and the inverse of the following matrices:

$C_{1} = \left[ \begin {array}{cc} 4& 4\\\noalign{\medskip} 4& 4\end {array} \right]$, $C_{2} = \left[ \begin {array}{cc} 4& 4.001\\\noalign{\medskip} 4.001& 4.002\end {array} \right]$ and $C_{3} = \left[ \begin {array}{cc} 4& 4.001\\\noalign{\medskip} 4.001& 4.002001\end {array} \right] $

Very briefly comment on the magnitude of the difference between $C_{2}^{-1}$ and  $C_{3}^{-1}$ given the only difference between $C_{2}$ and $C_{2}$ amounts to a difference of $0.000001$ in the bottom right position.

\begin{verbatim}
> A <- matrix(c(4,4,4,4),2,2)
> A
     [,1] [,2]
[1,]    4    4
[2,]    4    4
> det(A)
[1] 0
> solve(A)
Error in solve.default(A) : Lapack routine dgesv: system is exactly singular
> B <- matrix(c(4,4.001,4.001,4.002),2,2)
> B
      [,1]  [,2]
[1,] 4.000 4.001
[2,] 4.001 4.002
> det(B)
[1] -1e-06
> solve(B)
         [,1]     [,2]
[1,] -4002000  4001000
[2,]  4001000 -4000000
> C <- matrix(c(4,4.001,4.001,4.002001),2,2)
> C
      [,1]     [,2]
[1,] 4.000 4.001000
[2,] 4.001 4.002001
> det(C)
[1] 3e-06
> solve(C)
         [,1]     [,2]
[1,]  1334000 -1333667
[2,] -1333667  1333333
\end{verbatim}

\textit{Note that A is singular, the determinant is zero and it can't be inverted.   Also note that the inverses of B and C are very very different - but this is something of a pathological example}

\item Matrix partitioning.   Consider Sterling's financial data held in the R object LifeCycleSavings (see \texttt{?LifeCycleSavings}).   To make life a little easier, reorder the columns using \texttt{X <- LifeCycleSavings[,c(2,3,1,4,5)]}.

\begin{itemize}
\item Find the correlation matrix of \texttt{X} (longhand, using the centering matrix), call this matrix \texttt{R}

\begin{verbatim}
> data(LifeCycleSavings)
> X <- LifeCycleSavings[,c(2,3,1,4,5)]
> R <- cor(X)
> R
            pop15       pop75         sr        dpi        ddpi
pop15  1.00000000 -0.90847871 -0.4555381 -0.7561881 -0.04782569
pop75 -0.90847871  1.00000000  0.3165211  0.7869995  0.02532138
sr    -0.45553809  0.31652112  1.0000000  0.2203589  0.30478716
dpi   -0.75618810  0.78699951  0.2203589  1.0000000 -0.12948552
ddpi  -0.04782569  0.02532138  0.3047872 -0.1294855  1.00000000
\end{verbatim}

\item Partition \textit{R = cov(X)} following the scheme below such that $\boldsymbol{X_{11}}$ is a $2 \times 2$ matrix containing the covariance of \texttt{pop15} and \texttt{pop75}, and $\boldsymbol{X_{22}}$ contains the covariance of \texttt{sr}, \texttt{dpi} and \texttt{ddpi}

\begin{displaymath}
\boldsymbol{R} = \left( \begin{array}{l|l} \boldsymbol{R_{11}} & \boldsymbol{R_{12}} \\ \hline    \boldsymbol{R_{21}} & \boldsymbol{R_{22}} \end{array} \right)
\end{displaymath}

You should find for example that  $\boldsymbol{R_{11}}$ is given by:
 % latex table generated in R 2.3.1 by xtable 1.3-2 package
% Wed Nov 15 22:15:30 2006
\begin{table}[ht]
\begin{center}
\begin{tabular}{rrr}
\hline
 & pop15 & pop75 \\
\hline
pop15 & 83.75 & $-$10.73 \\
pop75 & $-$10.73 & 1.67 \\
\hline
\end{tabular}
\end{center}
\end{table}

\begin{verbatim}
> R11 <- R[1:2,1:2]
> R12 <- R[1:2,3:5]
> R21 <- R[3:5,1:2]
> R22 <- R[3:5,3:5]
> R11
           pop15      pop75
pop15  1.0000000 -0.9084787
pop75 -0.9084787  1.0000000
> R12
              sr        dpi        ddpi
pop15 -0.4555381 -0.7561881 -0.04782569
pop75  0.3165211  0.7869995  0.02532138
\end{verbatim}

\item Find the matrix $\boldsymbol{A}$, where:

\begin{displaymath}
\boldsymbol{A} = \boldsymbol{R_{22}^{-1}R_{21}R_{11}^{-1}R_{12}}
\end{displaymath}

\begin{verbatim}
A <- solve(R22) %*% R21 %*% solve(R11) %*% R12
> A
                sr        dpi         ddpi
sr    0.2082835957 0.13567828  0.028218990
dpi   0.2428243102 0.60736349  0.019316733
ddpi -0.0001528633 0.06307685 -0.001930969
\end{verbatim}

\item Find the matrix $\boldsymbol{B}$ where:

\begin{displaymath}
\boldsymbol{B} = \boldsymbol{R_{11}^{-1}R_{12}R_{22}^{-1}R_{21}}
\end{displaymath}

\begin{verbatim}
> B <- solve(R11) %*% R12 %*% solve(R22) %*% R21
> B
           pop15      pop75
pop15  0.4471770 -0.3095406
pop75 -0.2362827  0.3665391
\end{verbatim}

%\begin{itemize}
\item Are $\boldsymbol{A}$ and $\boldsymbol{B}$ symmetric?   What is the difference between symmetric and asymmetric matrices in terms of their eigenvalues and eigenvectors?\\
\textit{Note that they are both asymmetric matrices, it just so happens for these particular matrices that the eigenvalues are positive the the eigenvectors are real.   This isn't always the case for asymmetric matrices}

\item Find the eigenvalues and eigenvectors of $\boldsymbol{A}$ and $\boldsymbol{B}$ then find the square roots of the eigenvalues.

\begin{verbatim}
> eigen(A)
$values
[1]  6.802894e-01  1.334267e-01 -3.516940e-19

$vectors
            [,1]       [,2]        [,3]
[1,] -0.28005745 -0.8743085 -0.15327638
[2,] -0.95591190  0.4395680  0.02986593
[3,] -0.08831912  0.2058267  0.98773194

> eigen(B)
$values
[1] 0.6802894 0.1334267

$vectors
           [,1]      [,2]
[1,]  0.7988131 0.7023149
[2,] -0.6015793 0.7118664

> sqrt(eigen(A)$values)
[1] 0.8247966 0.3652762       NaN
> sqrt(eigen(B)$values)
[1] 0.8247966 0.3652762
\end{verbatim}

\item Do you notice any similarities between the first two eigenvalues from either matrix?\\
\textit{Note that the square roots of the eigen values are identical.}
\end{itemize}

\item Revisit the \texttt{wines} data in the \texttt{Flury} package.   Consider only Y1, Y5, Y6, Y8 and Y9, use matrix algebra to find the means, correlation and covariance of these data.   Compare the eigenvalues and eigenvectors, and the determinants and inverse you get from the covariance matrix and the correlation matrix.\\
\textit{Part of this is straightforward application of algebra covered in notes.   Remainder follows, eigenvalues and eigenvectors or corrlation matrix and covariance matrix are very very different}

\begin{verbatim}
> eigen(cor(X))
$values
[1] 1.9816214 1.5230931 0.7636275 0.4807668 0.2508912

$vectors
           [,1]       [,2]       [,3]         [,4]        [,5]
[1,] -0.2777688  0.6861076 -0.1803411 -0.008575869  0.64769164
[2,] -0.5561121 -0.1831683  0.2691167  0.763621722  0.04058118
[3,] -0.4812999 -0.3654121  0.4287131 -0.604767532  0.29203728
[4,] -0.5632681  0.3692665 -0.1701835 -0.225349857 -0.68310049
[5,] -0.2542899 -0.4752074 -0.8260121 -0.016792704  0.16412387

> eigen(cov(X))
$values
[1] 2330.125258  814.467485  545.200401    3.584549    1.403159

$vectors
            [,1]         [,2]        [,3]         [,4]         [,5]
[1,] -0.05483952  0.735862357  0.67335086  0.007708220  0.045148845
[2,]  0.50030899  0.604193500 -0.61998120 -0.013591715  0.008928181
[3,]  0.86393142 -0.303609773  0.40160604 -0.008022325  0.009582822
[4,]  0.01284463  0.035080362  0.02616497  0.195179471 -0.979706462
[5,]  0.01187679 -0.006876112 -0.01580852  0.980610251  0.194846828
\end{verbatim}

\item Something to add to what we did last week.   \texttt{library(car)} opens up some additional functionality:
\begin{verbatim}
scatterplot.matrix(iris[,-5], smooth = FALSE,
ellipse = TRUE, by.groups = TRUE, groups = iris$Species)
\end{verbatim}
Look at the associated helpfiles and add this function to your eda repertoire.
\end{enumerate}

\section{Consolidation Exercises}

You should complete these exercises over the next week.   You are guaranteed to meet some simple matrix arithmetic in the exam!   We will briefly go through the answers in class next week.   Don't rely on memorising model solutions.

\begin{enumerate}
\item Which of the following are orthogonal to each other:

\begin{displaymath}
\boldsymbol{x} = \left( \begin{array}{r} 1 \\ -2 \\ 3 \\ -4 \end{array} \right)
\boldsymbol{y} = \left( \begin{array}{r} 6 \\ 7 \\ 1 \\ -2 \end{array} \right)
\boldsymbol{z} = \left( \begin{array}{r} 5 \\ -4 \\ 5 \\ 7 \end{array} \right)
\end{displaymath}

Normalise each of the two orthogonal vectors.

\begin{displaymath}
\boldsymbol{x}^{T}\boldsymbol{y} = \left( \begin{array}{rrrr}1 & -2 & 3 & -4 \end{array} \right) \left( \begin{array}{r} 6 \\ 7 \\ 1 \\ -2 \end{array} \right) = \left( \begin{array}{rrrr} 1 \times 6 + & -2 \times 7 + & 3 \times 1 + & -4 \times -2 \end{array} \right) = 3
\end{displaymath}
i.e. not orthogonal

\begin{displaymath}
\boldsymbol{x}^{T}\boldsymbol{z} = \left( \begin{array}{rrrr}1 & -2 & 3 & -4 \end{array} \right) \left( \begin{array}{r} 5 \\ -4 \\ 5 \\ 7 \end{array} \right) = \left( \begin{array}{rrrr} 1 \times 5 + & -2 \times -4 + & 3 \times 5 + & -4 \times 7 \end{array} \right) = 0
\end{displaymath}
i.e. orthogonal

\begin{displaymath}
\boldsymbol{y}^{T}\boldsymbol{z} = \left( \begin{array}{rrrr}6 & 7 & 1 & -2 \end{array} \right) \left( \begin{array}{r} 5 \\ -4 \\ 5 \\ 7 \end{array} \right) = \left( \begin{array}{rrrr} 6 \times 5 + & 7 \times -4 + & 1 \times 5 + & -2 \times 7 \end{array} \right) = -7
\end{displaymath}
i.e. not orthogonal

\begin{eqnarray*}
\lVert \boldsymbol{x} \rVert &=& \sqrt{1 + 4 + 9 + 16} = \sqrt{30} \\
\lVert \boldsymbol{y} \rVert &=& \sqrt{36 + 49 + 1 + 4} = \sqrt{90}\\
\lVert \boldsymbol{z} \rVert &=& \sqrt{25 + 16 + 25 + 49} = \sqrt{115} \\
\end{eqnarray*}

Normalise the orthogonal vector, divide by the length of the vector:

\begin{displaymath}
\boldsymbol{y}_{norm} = \left( \begin{array}{r} 6/\sqrt{90} \\7/\sqrt{90} \\ 1/\sqrt{90} \\ 2/\sqrt{90} \end{array} \right)
\end{displaymath}

\item Find vectors which are orthogonal to:

\begin{displaymath}
\boldsymbol{u} =  \left( \begin{array}{r} 1 \\ 3 \end{array} \right)
\boldsymbol{v} = \left( \begin{array}{r} 2 \\ 4 \\ -1 \\ 2 \end{array} \right)
\end{displaymath}

Any $w_{1}$ and $w_{2}$ that satisfies $w_{1}+3w_{2}=0$ will be orthogonal to $\boldsymbol{u}$, e.g. $w = \left( \begin{array}{r} 3 \\ -1 \end{array} \right)$.   For $\boldsymbol{v}$ we require $2x_{1} + 4x_{2} - x_{3} + 2x_{4}=0$, which can be solved with $\boldsymbol{x} = \left( \begin{array}{r} 0 \\ 0 \\ 2 \\ 1 \end{array} \right)$

\item Find vectors which are orthonormal to:

\begin{displaymath}
\boldsymbol{x} =  \left( \begin{array}{r} \frac{1}{\sqrt{2}} \\ 0 \\ -\frac{1}{\sqrt{2}} \end{array} \right)
\boldsymbol{y} = \left( \begin{array}{r} \frac{1}{2} \\ \frac{1}{6} \\ \frac{1}{6} \\ \frac{5}{6} \end{array} \right)
\end{displaymath}

If $\boldsymbol{u} = \left( \begin{array}{r}u_{1} \\ u_{2} \\u_{3} \end{array} \right)$, then $\boldsymbol{x}^{T}\boldsymbol{u} = \frac{1}{\sqrt{2}}u_{1}-\frac{1}{\sqrt{2}}u_{3}$, so $\boldsymbol{u} = \left( \begin{array}{r} 1 \\ 0 \\1 \end{array} \right)$ is orthogonal, and we only require to normalise it.   $\lVert \boldsymbol{u} \rVert = \sqrt{1 + 0 + 1} = \sqrt{2}$.   Consequently, $\boldsymbol{u} = \left( \begin{array}{r} \frac{1}{\sqrt{2}} \\ 0 \\ \frac{1}{\sqrt{2}} \end{array} \right)$ has length 1 and is orthogonal.

In a similar way, we find $\left( \begin{array}{r} 0 \\ 0 \\ 5 \\ -1 \end{array} \right)$, a vector which has length $\sqrt{25+1}$, so the orthogonal vector of unit length is given by:  $\left( \begin{array}{r} 0 \\ 0 \\ 5/\sqrt{26} \\ -1/\sqrt{26} \end{array} \right)$

\item What are the determinants of:

\begin{displaymath}
(a) \left( \begin{array}{rr} 1 & 3 \\ 6 & 4 \end{array} \right)
(b) \left( \begin{array}{rrr} 3 & 1 & 6 \\ 7 & 4 & 5 \\ 2 & -7 & 1 \end{array} \right)
\end{displaymath}

Solutions are (a) -14, (b) -222.

\item Invert the following matrices:

\begin{displaymath}
(a) \left( \begin{array}{rrr} 3 & 0 & 0 \\ 0 & 4 & 0 \\ 0 & 0 & 9 \end{array} \right)
(b) \left( \begin{array}{rr} 2 & 3 \\ 1 & 5  \end{array} \right)
(c) \left( \begin{array}{rrr} 3 & 2 & -1 \\ 1 & 4 & 7 \\ 0 & 4 & 2 \end{array} \right)
(d) \left( \begin{array}{rrr} 1 & 1 & 1 \\ 2 & 5 & -1 \\ 3 & 1 & -1 \end{array} \right)
\end{displaymath}

Solutions are (a) $\left( \begin{array}{rrr} 0.333 & 0 & 0 \\ 0 & 0.25 & 0 \\ 0 & 0 & 0.111 \end{array} \right)$, note the determinant is 108 (b) $\left( \begin{array}{rr} 0.714 & -0.429 \\ -0.143 & 0.286  \end{array} \right)$ (with a determinant of 7), (c)  $\left( \begin{array}{rrr} 0.294 & 0.118 & -0.265 \\ 0.029 & -0.088 & 0.325 \\ -0.059 & 0.176 & -0.147 \end{array} \right)$ (with a determinant of -68) and (d) $ \left( \begin{array}{rrr} 0.222 & -0.111 & 0.333 \\ 0.056 & 0.222 & -0.167 \\ 0.722 & -0.111 & -0.l67 \end{array} \right)$ (with a determinant -18)

\item Find eigenvalues and corresponding eigen vectors for the following matrices:

\begin{displaymath}
\boldsymbol{a} = \left( \begin{array}{rr} 1 & 4\\ 2 & 3 \end{array} \right)
\boldsymbol{b} = \left( \begin{array}{rr} 1 & 2 \\ 3 & 2  \end{array} \right)
\boldsymbol{c} = \left( \begin{array}{rr} 2 & -2 \\ -2 & 5 \end{array} \right)
\boldsymbol{d} = \left( \begin{array}{rr} 2 & 2 \\ 2 & 5 \end{array} \right)
\end{displaymath}

\begin{displaymath}
\boldsymbol{e} = \left( \begin{array}{rrr} 1 & 4 & 0\\ 4 & 1 & 0 \\ 0 & 0 & 1 \end{array} \right)
\boldsymbol{f} = \left( \begin{array}{rrr} 4 & 0 & 0 \\ 0  & 9 & 0 \\ 0 & 0 & 1  \end{array} \right)
\boldsymbol{g} = \left( \begin{array}{rrr} 13 & -4 & 2\\ -4 & 13 & -2 \\ 2 & -2 & 10 \end{array} \right)
\end{displaymath}

\textit{We will run through some solutions in class}

\item Convert the following covariance matrix (you've seen it earlier) to a correlation matrix, calculate the eigenvalues and eigenvectors and verify that the eigen vectors are orthogonal.

\begin{displaymath}
\boldsymbol{g} = \left( \begin{array}{rrr} 13 & -4 & 2\\ -4 & 13 & -2 \\ 2 & -2 & 10 \end{array} \right)
\end{displaymath}

\item If $\boldsymbol{X} = \left( \begin{array}{rr} 2 & 6\\ 1 & 3 \\ 4 & 2 \end{array} \right)$, use matrix procedures to find $cov(\boldsymbol{X})$ and $cor(\boldsymbol{X})$.   What is a sum of squares and crossproducts matrix?   What are the eigenvalues and eigenvectors of $cov(\boldsymbol{X})$ and $cor(\boldsymbol{X})$?

\item Find $\frac{\lvert| \begin{array}{rr} 2 & 4\\2 & 7 \end{array} \rvert}{\lvert| \begin{array}{rr} 1 & 9\\4 & 5 \end{array} \rvert}$.

\end{enumerate}

\end{document}


\subsection{Wind Speed and Wind Directions}

As we have mentioned, there are four attributes regarding wind (WSF2, WDF2, WSF5, WDF5) in the data. Among the four, each pair of (WSF, WDF) represents a 2-dim \emph{vector} (wind vector) in polar coordinates. Sometimes, there is a significant variation of directions between different records of the same day, and thus, we shouldn't take their mean directly.

To better summarize the wind vectors, we may consider a natural model about the wind vectors in station $i$:
$$\vec v_i(t)=\vec{v}(t)+\vec u_i+\vec\epsilon_i(t),$$
where $\vec{v_i}(t)$ is the wind vector at station $i$ by time $t$, $\vec u_i$ is a fixed effect for station $i$, and $\vec \epsilon_i(t)$ is the error vector.

With the assumption that the term $\vec\epsilon_i(t)$ has i.i.d. bivariate normal distribution, we may estimate the overall wind vector by estimator
$$\hat{\vec v}_i(t)=\frac1n\sum_{i=1}^n\vec v_i(t)$$
with a constant bias
$$\vec b=\frac1n\sum_{i=1}^n\vec u_i.$$
Such bias here can be viewed as a fixed translation of the attribute space (since it is constant), and can thus be omitted in further analyses.

Since we are only interested in the overall wind condition for Seattle, further analyses would only be concerned with the mean vectors $\hat{\vec v}(t)$. For convenience they are still presented in polar coordinates with the same attribute names. (It might be hard to interpret if they are presented in Cartesian coordinates.)

It is perhaps remarkable that since we have polar coordinates as attributes, the attribute space would not be flat. In fact, the subspace regarding WDF2 (and the one regarding WDF5 as well) is homeomorphic to $\mathbb S^1$ instead of $\mathbb R^1$.


\chapter{Linear Systems of Differential Equations}
Although we have a good idea on how to solve many single linear differential equations, it's often useful to think about linear systems of equations, like
\begin{equation*}
	\begin{cases}
		x_1' = 3tx_1 - x_2 + t^2 \\
		x_2' = -x_1 + e^tx_2 - e^{2t}
	\end{cases}
	\to
	\begin{bmatrix}
		x_1 \\
		x_2
	\end{bmatrix}' = \begin{bmatrix}
		3t & -1 \\
		-1 & e^t
	\end{bmatrix} \begin{bmatrix}
		x_1 \\
		x_2
	\end{bmatrix} + \begin{bmatrix}
		t^2 \\
		-e^{2t}
	\end{bmatrix}.
\end{equation*}
This matrix form, $\vec{x'} = A\vec{x} + \vec{f}$, where $A$ is a square matrix, is called normal form.\\

% Solutions to systems
\input{./linearSystems/solutions/solutions.tex}
% Homogenous
\input{./linearSystems/homogeneousSystems/homogeneousSystems.tex}
% Non-Homogenous
\input{./linearSystems/heterogeneousSystems/heterogeneousSystems.tex}
% Higher-order linear diffeq's as systems of 1st order linear diffeq's
\input{./linearSystems/systemsAndHigherOrder/systemsAndHigherOrder.tex}


%!TEX root = ../Thesis.tex
\chapter{Numerical Analysis} \label{ch:numerical-analysis}
\epigraph{``\itshape{A computer lets you make more mistakes faster than any invention in human history - with the possible exceptions of handguns and tequila.}"}{--- \textup{Mitch Ratliffe}, Technology Review, April 1992}

Some systems can be solved analytically in terms of time-dependent coordinates. Other systems are non-linear and the vast majority cannot be solved analytically. This is the case for the restricted three-body problem, \crefrange{eq:Xdot}{eq:Pdot_Y}. To solve these equations of motion, we must use numerical methods.

The simplest numerical integration is the first order linear approximation known as the Euler method. There are three variants of the Euler method: explicit, implicit and symplectic. As we will see, the explicit Euler will typically increase the energy, implicit Euler will decrease the energy and the symplectic Euler will oscillate the energy slightly but on average conserve pretty well.

We will now solve the restricted three-body problem equations of motion, \crefrange{eq:Xdot}{eq:Pdot_Y}. First we discretize the equations with a linear approximation:

\begin{align}
\Delta X &= +\dfrac{\partial H}{\partial P_X}\Delta t = (P_X + Y)\Delta t, \\[0.2cm]
\Delta Y &= +\dfrac{\partial H}{\partial P_Y}\Delta t = (P_Y - X)\Delta t, \\[0.2cm]
\Delta P_X &= -\dfrac{\partial H}{\partial X}\Delta t = \left(P_Y - \dfrac{(1-k)(k+x)}{((k+x)^2+y^2)^{3/2}} + \dfrac{k(X-1+k)}{((X-1+k)^2+y^2)^{3/2}}\right)\Delta t, \\[0.2cm]
\Delta P_Y &= -\dfrac{\partial H}{\partial Y}\Delta t = \left(-P_X - \dfrac{(1-k)y}{((k+x)^2+y^2)^{3/2}} - \dfrac{k y}{((X-1+k)^2+y^2)^{3/2}}\right)\Delta t.
\end{align}

From now on we will use $h$ instead of $\Delta t$ for the time step size.

\section{Explicit Euler Algorithm}
All new values $(X,Y,P_X,P_Y)_{i+1}$ refer to known values from the previous time step \\ $(X,Y,P_X,P_Y)_i$.
\begin{align}
X_{i+1} &= X_i + (P_{X,i} + Y_i)h, \\[0.2cm]
Y_{i+1} &= Y_i + (P_{Y,i} - X_i)h , \\[0.2cm]
P_{X,i+1} &= P_{X,i} + \left(P_{Y,i} - \dfrac{(1-k)(k+X_i)}{((k+X_i)^2+Y_i^2)^{3/2}} + \dfrac{k(X_i-1+k)}{((X_i-1+k)^2+Y_i^2)^{3/2}}\right)h, \\[0.2cm]
P_{Y,i+1} &= P_{Y,i} + \left(-P_{X,i} - \dfrac{(1-k)Y_i}{((k+X_i)^2+Y_i^2)^{3/2}} - \dfrac{k Y_i}{((X_i-1+k)^2+Y_i^2)^{3/2}}\right)h.
\end{align}

\section{Implicit Euler Algorithm}
Same as explicit Euler, except that all variables on the right-hand side involved in the time step refers to variables in the new time step, $i+1$. This algorithm would involve finding four roots simultaneously and numerically. Since we know the symplectic to be more correct, we skip the implicit Euler (formula included in Appendix \ref{app:implicit_euler}).

\section{Symplectic Euler Algorithm}
In the explicit Euler, all quantities from the Hamiltonian derivative on the right hand refer to the old time step \(i\). In the implicit, they are all referring to the new time step \(i+1\), which results in implicit equations that can typically only be solved numerically.

The symplectic Euler is mixture between explicit and implicit Euler \cite{Hairer}:

\begin{equation}
    \begin{split} \label{eq:symplectic-euler1}
        \vec{q}_{i+1} = \vec{p}_i + h \pd{H}{\vec{p}}(q_i, p_{i+1}) \\
        \vec{p}_{i+1} = \vec{p}_i - h \pd{H}{\vec{q}}(q_i, p_{i+1})
    \end{split}
\end{equation}

or

\begin{equation}
    \begin{split} \label{eq:symplectic-euler2}
        \vec{q}_{i+1} = \vec{p}_i + h \pd{H}{\vec{p}}(q_{i+1}, p_i) \\
        \vec{p}_{i+1} = \vec{p}_i - h \pd{H}{\vec{q}}(q_{i+1}, p_i)
    \end{split}
\end{equation}
where \(h\) is the step size, index \(i\) designates previous time step and \(i+1\) designates new time step.

One can choose whichever is easier to solve for the equations at hand. We choose \cref{eq:symplectic-euler2} so we get
\begin{align}
    X_{i+1} &= X_i + h (P_{X,i} + Y_{i+1}), \label{eq:euler-s-x} \\[0.2cm]
    Y_{i+1} &= Y_i + h (P_{Y,i} - X_{i+1}), \label{eq:euler-s-y} \\[0.2cm]
    P_{X,i+1} &= P_{X,i} + h \left(P_{Y,i} - \dfrac{(1-k)(k+X_{i+1})}{((k+X_{i+1})^2+Y_{i+1}^2)^{3/2}} + \dfrac{k(X_{i+1}-1+k)}{((X_{i+1}-1+k)^2+Y_{i+1}^2)^{3/2}}\right), \label{eq:euler-s-px} \\[0.2cm]
    P_{Y,i+1} &= P_{Y,i} + h \left(-P_{X,i} - \dfrac{(1-k)Y_{i+1}}{((k+X_{i+1})^2+Y_{i+1}^2)^{3/2}} - \dfrac{k Y_{i+1}}{((X_{i+1}-1+k)^2+Y_{i+1}^2)^{3/2}}\right). \label{eq:euler-s-py}
\end{align}
We can first solve \cref{eq:euler-s-x,eq:euler-s-y} for $X_{i+1}$ and $Y_{y+1}$ (see Appendix \ref{app:r3b-symplectic-euler} for Mathematica notebook):
\begin{align}
X_{i+1} &= \dfrac{X_i + h (h P_{Y,i} + P_{X,i} + Y_i)}{1+h^2}, \label{eq:euler_s_x2} , \\[0.5cm]
Y_{i+1} &= \dfrac{Y_i - h (h P_{X,i} - P_{Y,i} + X_i)}{1+h^2}, \label{eq:euler_s_y2}.
\end{align}
We can now start by running \cref{eq:euler_s_x2}. Next we can freely choose between running \cref{eq:euler-s-y} and \cref{eq:euler_s_y2} as both are valid. We will run \cref{eq:euler-s-y} since it is faster computationally. Then we can run the impulse equations as is.

In summary, we run the symplectic Euler algorithm in the order: \cref{eq:euler_s_x2}, \cref{eq:euler-s-y}, \cref{,eq:euler-s-px}, \cref{eq:euler-s-py}.

\section{First Test Runs with First-Order Euler Methods}
The explicit and symplectic Euler algorithm was implemented in Python. The first two things we want to investigate are:
\begin{enumerate}
    \item Does the trajectories look like we expect them to?
    \item Is the Hamiltonian conserved?
\end{enumerate}
For symplectic integrators the trajectories look like we expect them to (see \ref{fig:r3b_symplectic_euler}), but not for the non-symplectic explicit Euler (see \ref{fig:r3b_explicit_euler}). The Hamiltonian is not conserved for either first order (Euler) Methods, but $H$ for the symplectic Euler was conserved on average over an orbit in a closed orbit.
The simulations in Figure \ref{fig:r3b_explicit_euler} \ref{fig:r3b_symplectic_euler} was run with the following parameters:
\begin{adjustwidth*}{0cm}{-0.4cm}
\begin{lstlisting}[language=Python,caption={Initial conditions, step size and duration for initial test run},label={lst:testrun}]
# Duration and step size
DURATION = 1.5*(2*np.pi)
H = 0.0001

# Initial Conditions
X0=0.5
Y0=0.2
PX0=0
PY0=0.5
\end{lstlisting}
\end{adjustwidth*}

\begin{figure}
    \centering
        \subbottom[Explicit Euler $(\mathscr{X},\mathscr{Y})$ trajectory. Earth in origin, moon orbit in greY, first half of spacecraft trajectory is red, last half is blue.]{
            \includegraphics[scale=0.41]{fig/r3b-testing/r3b_y(x)_euler_explicit.pdf}
            \label{fig:r3b_y(x)_euler_explicit}
        }
        \subbottom[Explicit Euler $H(t)$.]{
            \includegraphics[scale=0.411]{fig/r3b-testing/r3b_H(t)_euler_explicit.pdf}
            \label{fig:r3b_H(t)_euler_explicit}
        }
        \caption{Explicit Euler trajectory and Hamiltonian. Ideally the $H(t)$ should be constant, but instead we see temporary dips for every Earth pass, and worse, the spacecraft picks up energy after every pass of Earth. Finally the spacecraft escapes the Earth with no apparent energy input. This is clearly wrong and not a valid trajectory}
        \label{fig:r3b_explicit_euler}
\end{figure}

\begin{figure}
    \centering
        \subbottom[Symplectic Euler $(\mathscr{X},\mathscr{Y})$ trajectory. Earth in origin, moon orbit in greY, first half of spacecraft trajectory is red, last half is blue.]{
            \includegraphics[scale=0.41]{fig/r3b-testing/r3b_y(x)_euler_symplectic.pdf}
            \label{fig:r3b_y(x)_euler_symplectic}
        }
        \subbottom[Symplectic Euler $H(t)$.]{
            \includegraphics[scale=0.411]{fig/r3b-testing/r3b_H(t)_euler_symplectic.pdf}
            \label{fig:r3b_H(t)_euler_symplectic}
        }
        \caption{Symplectic Euler trajectory and Hamiltonian. The first half trajectory is red, the last half is blue. First we note the approximate elliptical orbit as expected. The $H(t)$ shows dips is the spacecraft passes the Earth, but on average over an orbit, $H(t)$ is approximately constant} % FIXME: No it's not constant looking - this needs to be fixed.
        \label{fig:r3b_symplectic_euler}
\end{figure}
Having $H$ conserved on average over an orbit is fine for a closed orbit, but for chaotic orbits around both Earth and Moon, it's not nearly good enough. We want a higher order method to conserve $H$ better, to better gauge the numerical errors, and to allow an adaptive numerical method.

Making the algorithm select a step size adaptively decreases the run time of a given simulation (though not necessarily the correctness of the solution). This is a big advantage because it allows us to search for many more orbits, which statistically will give us better orbits.

\section{Symplectic Störmer-Verlet}
We will now describe the second-order Störmer-Verlet method \cite{Hochbruck2008}:
\begin{align}
q_{i+1/2} &= q_i + \dfrac{h}{2}H_p(q_{i+1/2}, p_i), \\
p_{i+1} &= p_i + \dfrac{h}{2}(H_q(q_{i+1/2}, p_i) + H_q(q_{i+1/2}, p_{i+1})), \\
q_{i+1} &= q_{i+1/2} + \dfrac{h}{2}H_p(q_{n+1/2}, p_{i+1}).
\end{align}
where $H_q$ and $H_p$ denotes the partial derivatives of $H$, which can be expressed in terms of $q_i$ and $p_i$ by Hamilton's equations. Implemented for our equations (using~\cref{eq:Xdot,eq:Ydot,eq:Pdot_X,eq:Pdot_Y} to express $H_q$ and $H_p$) we get:
\begin{align}
X_{i+1/2} &= X_i + \dfrac{h}{2} (P_{X,i} + Y_{i+1/2}) , \label{eq:verlet-implicit-x_1/2} \\[0.3cm]
Y_{i+1/2} &= Y_i + \dfrac{h}{2} (P_{Y,i} - X_{i+1/2}), \label{eq:verlet-implicit-y_1/2} \\[0.8cm]
%
%
P_{X,i+1} &= P_{X,i} + \dfrac{h}{2} \left[-\dot{P}_X(q_{i+1/2},p_i) -\dot{P}_X(q_{i+1/2},p_{i+1}) \right], \label{eq:verlet-implicit-px_1} \\[0.3cm]
%
P_{Y,i+1} &= P_{Y,i} + \dfrac{h}{2} \left[-\dot{P}_Y(q_{i+1/2},p_i) -\dot{P}_Y(q_{i+1/2},p_{i+1}) \right], \label{eq:verlet-implicit-py_1} \\[0.8cm]
%
%
X_{i+1} &= X_{i+1/2} + \dfrac{h}{2} (P_{X,i+1} + Y_{i+1/2}), \label{eq:verlet-explicit-x_1} \\[0.3cm]
Y_{i+1} &= Y_{i+1/2} + \dfrac{h}{2} (P_{Y,i+1} - X_{i+1/2}), \label{eq:verlet-explicit-y_1}
\end{align}
where \(\dot{P}_X,\dot{P}_Y\) are \cref{eq:Pdot_X,eq:Pdot_Y}, \(h\) is the step size, index \(i\) designates previous time step and \(i+1\) designates new time step.
We solve solve the first two implicit equations \cref{eq:verlet-implicit-x_1/2,eq:verlet-implicit-y_1/2} for \(x_{i+1/2},y_{i+1/2}\), for which we get

\begin{equation}
    X_{i+1/2} = \frac{h^2 P_{Y,i} + 2 h \dot{P}_{x,i} + 4 X_i}{4 + h^2} \label{eq:verlet-explicit-x_1/2}
\end{equation}

see \cref{app:r3b-verlet} for derivations. Then we solve \cref{eq:verlet-implicit-px_1,eq:verlet-implicit-py_1} for \(P_{X,i+1},P_{Y,i+1}\) for which we get

\begin{equation}
    P_{X,i+1} = \frac{h^2 (2 \dot{P}_{Y,i} + P_{X,i}) + 4 h \dot{P}_{X,i} + 4 P_{X,i} }{4 + h^2} \label{eq:verlet-explicit-px_1}
\end{equation}

see \cref{app:r3b-verlet} for derivations. Equations \cref{eq:verlet-explicit-x_1,eq:verlet-explicit-y_1} are explicit, so these can be used directly.

We can now run the algorithm in the order: \cref{eq:verlet-explicit-x_1/2}, \cref{eq:verlet-implicit-y_1/2}, \cref{eq:verlet-explicit-px_1}, \cref{eq:verlet-implicit-py_1}, \cref{eq:verlet-explicit-x_1}, \cref{eq:verlet-explicit-y_1}.

\section{Adaptive Symplectic Störmer-Verlet}
Let $z$ denote a vector of the position variables $(X,y)$.
\begin{align}
\text{Euler step result:} \qquad z_1 = z + O(h), \\
\text{Verlet step result:} \qquad z_2 = z + O(h^2), \\
\end{align}
where O(h) denotes an error term of order h. Then we take the difference
\begin{align}
\|z_1 - z_2\| &= O(h) - O(h^2) \\
&\approx O(h)\ ,
\end{align}
since $O(h) \gg O(h^2)$. Thus we approximate the error difference between the Euler and Verlet method as the actual error we make at step size $h$. The idea is to make both an Euler and a Verlet step for every time-step to assess the error and adjust the step size accordingly as implemented in listing \ref{lst:adaptive}
\begin{adjustwidth*}{0cm}{-0.4cm}
\begin{lstlisting}[language=Python,caption=Adaptive method implemented in python. We accept the Verlet step only if the error is no more than a given tolerance \texttt{tol}. Subsequently we estimate a new \texttt{h} that will yield an error of \texttt{tol} in the next step and use \texttt{0.8} of this value to avoid frequent rejects. If the step is rejected we reduce the step size by half,label=lst:adaptive]
if err < tol or h <= h_min:

    # Accept step
    x = x2
    y = y2
    px = px2
    py = py2

    # Forward time by step
    t = t+h
    h = max(h_min, h*max(0.1, 0.8*sqrt(tol/err)))

else:
    # No accept, reduce h to half
    h = max(h_min, 0.5*h)
\end{lstlisting}
\end{adjustwidth*}
For every single step the step size is changed either up or down, depending on the errors and tolerance. As a result we always stay near the same error in every step, only taking as small steps as necessary in each iteration.

The non-adaptive algorithm is fixed in step size but varies in error per step.
The adaptive algorithm varies in step size in an attempt to fix the error per step.

For all simulations we have set $10^{-9}$ as the maximum tolerated error per step in the adaptive algorithm. For the non-adaptive algorithm we select a fixed step size, $10^{-6}$, to ensure that it's reasonably low most of the time. For comparison a simulation was run with same initial conditions using the adaptive and non-adaptive algorithm, see Figure \ref{fig:non-adaptive-steP_error} - \ref{fig:adaptive-steP_error}. Position plot \ref{fig:adaptive-xY_XY} shown for completeness.

\begin{figure}
    \centering
        \subbottom[Non-adaptive symplectic Verlet step size.]{
            \includegraphics[scale=0.41]{fig/adaptive/steP_fixed_h.pdf}
        }
        \subbottom[Non-adaptive symplectic Verlet error per step.]{
            \includegraphics[scale=0.41]{fig/adaptive/err_fixed_h.pdf}
        \label{fig:non-adaptive-error}
        }
        \caption{Non-adaptive method step size and error per step. Step size is fixed and as a result the error varies. All the work spend on calculating the many steps with lower error than $10^{-9}$ in \ref{fig:non-adaptive-error} are wasted since the error spikes to somewhere $10^{-8}$ and $10^{-7}$ during a close fly-bY, which might invalidate all decimals after the 7th decimal on the non-adaptive method. The solutions are practically identical, but the adaptive is much faster}
        \label{fig:non-adaptive-steP_error}
\end{figure}

\begin{figure}
    \centering
        \subbottom[Adaptive symplectic Verlet step size.]{
            \includegraphics[scale=0.41]{fig/adaptive/steP_adaptive.pdf}
        \label{fig:adaptive-step size}
        }
        \subbottom[Adaptive symplectic Verlet error per step.]{
            \includegraphics[scale=0.41]{fig/adaptive/err_adaptive.pdf}
        }
        \caption{The adaptive method constantly varies the step-size to ensure a constant error around $10^{-9}$. As a result it takes longer steps when there is less change in the variables, in the space far from the bodies, and shorter steps when passing close by the Earth or Moon}
        \label{fig:adaptive-steP_error}
\end{figure}

\begin{figure}
    \centering
        \subbottom[Non-adaptive symplectic Verlet $(X,y)$ trajectory.]{
            \includegraphics[scale=0.5]{fig/adaptive/_X-Y_fixed_h.pdf}
        }
        \subbottom[Non-adaptive symplectic Verlet $(\mathscr{X},\mathscr{Y})$ trajectory.]{
            \includegraphics[scale=0.5]{fig/adaptive/X-Y_fixed_h.pdf}
        }
        \caption{The trajectories of same initial conditions for the adaptive and non-adaptive method was visually indistinguishable. Included for context to Figure \ref{fig:non-adaptive-steP_error} - \ref{fig:adaptive-steP_error}}
        \label{fig:adaptive-xY_XY}
\end{figure}

Note on the step size Figure \ref{fig:adaptive-step size} that the adaptive ends up with constant step size at the end at 40 days once captured by the Moon. Thus when we are in circular orbit around a single bodY, we don't gain much by using adaptive methods. However as the spacecraft are in free space far away from both Earth/Moon, we gain a lot by taking longer steps (raising the step size). However as we make close passes to Earth or Moon, then the step size dips sharply to maintain constant error. We see this around day 9, 21 and 33 in \ref{fig:adaptive-step size}. For most simulations, the adaptive method ran 10-100 times faster.

The code for the full Python implementation is in Appendix \ref{app:code-r3b}.


\section{Performance analysis}
\label{sec:performance_analysis}

In this section we analyze how the \emph{execution time} varies with respect to the number of countries $C$ and the number of individuals $N$.
From a theoretical perspective, it is easy to see that the complexity of the algorithm is dominated by the main loop, let us define
\begin{itemize}
    \item $S = \lfloor t_{target} / t_{step} \rfloor$ the number of iterations
    \item $C$ the number of countries
    \item $N$ the number of individuals
\end{itemize}
We further assume that individuals are distributed uniformly, thus having $N/C$ individuals for each country.
Each iteration of the loop is characterized by different phases, detailed in section \ref{sec:alg_desc}, which can be divided in two categories of complexity:
\begin{itemize}
    \item \emph{update exposure} compares each susceptible individuals against, at most, each infected one. The number of checks can be upper-bounded by $\bigO((N/C)^2)$.
    \item all the other steps perform a single scan of the lists of individuals at each country, thus having a complexity $\bigO(N/C)$.
\end{itemize}
We conclude that the execution time $T$, assuming that all processes run in parallel, is upper-bounded by \[T = \bigO \left(S \frac{N^2}{C^2} \right)\].

\subsection{Varying the countries}
If we fix the number of iterations and of individuals, we expect to get a time complexity of $\bigO(1/C^2)$. In other words, the advantage is that $N$ is split between the countries and only individuals in the same country are checked against each other.

We performed several simulations with \num{30000} individuals in the same $4096 \times 4096$ world, which we split horizontally between 1 and 20 countries. Each simulation spanned a single day with a step of \SI{60}{s} (1440 iterations). The measured execution times in figure \ref{fig:profile_countries} show a quadratic decay, matching our expectations.

\begin{figure}[p]
    \begin{subfigure}[c]{.6\textwidth}
        \begin{tikzpicture}
            \begin{axis}[
                xlabel=Number of countries,
                ylabel=Execution time (seconds),
                legend pos=north east,
            ]
                \addplot
                    [blue]
                    table[x=countries, y=elapsed, col sep=comma]
                    {tables/profile_countries_1_20.csv};
                \addplot
                    [black, dashed]
                    table[x=countries, y expr=136.7625 + 2899.99 / \thisrow{countries}^2 , col sep=comma]
                    {tables/profile_countries_1_20.csv};
                \legend{real, $\bigO(1/C^2)$}
            \end{axis}
        \end{tikzpicture}
    \end{subfigure}
    ~
    \begin{subfigure}[c]{.35\textwidth}
        \csvreader[
            tabular=c c,
            table head=\toprule countries & exec. time \\ \midrule,
            table foot=\bottomrule]%
        {tables/profile_countries_1_20.csv}%
        {countries=\countries, elapsed_latex=\elapsed}%
        {\countries & \elapsed}
    \end{subfigure}
    \caption{Execution time when splitting computation between countries}
    \label{fig:profile_countries}
\end{figure}

\subsection{Varying the individuals}
If we fix the number of iterations and of countries, we expect the time complexity to be about $\bigO(N^2)$, or slightly lower since we have some optimizations.

We performed several simulations with 4 countries (the number of threads of our machine), spanning a single day with a \SI{60}{s} step (1440 iterations). We varied the number of individuals from \num{10000} to \num{60000} with a \num{5000} increment. The challenge here was to also vary the world size so that the simulation did not terminate early due to a lack of infected individuals. We found that the heuristic $W = L = 20 \sqrt{N}$ works well in our case. The results are shown in figure \ref{fig:profile_individuals}, and they agree with our expectations.

\begin{figure}[p]
    \begin{subfigure}[c]{.6\textwidth}
        \begin{tikzpicture}
            \begin{axis}[
                xlabel=Number of individuals,
                ylabel=Execution time (seconds),
                legend pos=north west,
            ]
                \addplot
                    [blue]
                    table[x=individuals, y=elapsed, col sep=comma]
                    {tables/profile_individuals_10000_60000.csv};
                \addplot
                    [black, dashed]
                    table[x=individuals, y expr=-60.0211 + 9.479e-7 * \thisrow{individuals}^2 , col sep=comma]
                    {tables/profile_individuals_10000_60000.csv};
                \legend{real, $\bigO(N^2)$}
            \end{axis}
        \end{tikzpicture}
    \end{subfigure}
    ~
    \begin{subfigure}[c]{.35\textwidth}
        \csvreader[
            tabular=c c,
            table head=\toprule individuals & exec. time \\ \midrule,
            table foot=\bottomrule]%
        {tables/profile_individuals_10000_60000.csv}%
        {individuals=\individuals, elapsed_latex=\elapsed}%
        {\num{\individuals} & \elapsed}
    \end{subfigure}
    \caption{Execution time when increasing number of individuals}
    \label{fig:profile_individuals}
\end{figure}


\section*{Integrals and Derivatives}

\subsection*{Table of base integrals}
\bgroup
    \def\arraystretch{1.3}
    \begin{longtable}{ll}
        \toprule
        Ableitung & Integral \\
        \hline
        \midrule

        \endhead
        \endfoot
        \endlastfoot

        % -------------------------------------------------------------------------------------
        % Basic shit
        % -------------------------------------------------------------------------------------
        $\frac{d}{dx} c = 0$ &
        $\int 0\,dx = c$
        \\
        $\frac{d}{dx} x = 1$ &
        $\int 1\,dx = x + c$
        \\
        $\frac{d}{dx} x^n = (n-1)x^{n-1}, n\not=0$ &
        $\int x^n\,dx = \frac{1}{n+1} x^{n+1} + c$
        \\
        $\frac{d}{dx} \ln{x} = \frac{1}{x}, x > 0$ &
        $\int \frac{1}{x}\,dx = \ln|x| + c$
        \\
        $\frac{d}{dx} \ln{-x} = \frac{-1}{-x} = \frac{1}{x}, x < 0$ &
        $\int \frac{1}{x}\,dx = \ln|x| + c$
        \\

        % -------------------------------------------------------------------------------------
        % Exponents
        % -------------------------------------------------------------------------------------
        \midrule
        $\frac{d}{dx} e^x = e^x$ &
        $\int e^x\,dx = e^x + c$
        \\
        $\frac{d}{dx} a^x = a^x \ln{a}$ &
        $\int a^x\,dx = \frac{1}{\ln{a}} a^x + c, a \neq 1$
        \\

        % -------------------------------------------------------------------------------------
        % Trig functions
        % -------------------------------------------------------------------------------------
        \midrule
        $\frac{d}{dx} \sin{x} = \cos{x}$ &
        $\int \cos{x}\,dx = \sin{x} + c$
        \\
        $\frac{d}{dx} \cos{x} = -\sin{x}$ &
        $\int \sin{x}\,dx = -\cos{x} + c$
        \\
        \multirow{2}{*}{$\frac{d}{dx} \tan{x} = \frac{1}{\cos^2{x}} = 1 + \tan^2{x}$} &
        $\int \frac{1}{\cos^2{x}}\,dx = \tan{x} + c$ \\* &
        $\int \tan^2{x}\,dx = \tan{x}-x + c$
        \\
        \multirow{2}{*}{$\frac{d}{dx} \cot{x} = -\frac{1}{\sin^2{x}} = -1 - \cot^2{x}$} &
        $\int \frac{1}{\sin^2{x}}\,dx = -\cot{x} + c$ \\* &
        $\int \cot^2{x}\,dx = -\cot{x}-x + c$
        \\
        \midrule
        &
        $\int \cos^2{x}\,dx = \frac{1}{2}\cos{(x)}\sin{(x)}+\frac{1}{2}x + c$
        \\
        &
        $\int \sin^2{x}\,dx = \frac{1}{2}x-\frac{1}{2}\cos{(x)}\sin{(x)} + c$
        \\

        % -------------------------------------------------------------------------------------
        % Inverse trig functions
        % -------------------------------------------------------------------------------------
        \midrule
        $\frac{d}{dx} \arcsin{x} = \frac{1}{\sqrt{1-x^2}}$ &
        $\int \frac{1}{\sqrt{1-x^2}}\,dx = arcsin{x} + c_1$
        \\
        $\frac{d}{dx} \arccos{x} = -\frac{1}{\sqrt{1-x^2}}$ &
        $\int \frac{1}{\sqrt{1-x^2}}\,dx = -\arccos{x} + c_2$
        \\
        $\frac{d}{dx} \arctan{x} = \frac{1}{1+x^2}$ &
        $\int \frac{1}{1+x^2}\,dx = \arctan{x} + c_1$
        \\
        $\frac{d}{dx} \arccot{x} = -\frac{1}{1+x^2} $ &
        $\int \frac{1}{1+x^2} = -\arccot{x} + c_2$
        \\

        % -------------------------------------------------------------------------------------
        % Hyperbolic trig functions
        % -------------------------------------------------------------------------------------
        \midrule
        $\frac{d}{dx} \sinh{x} = \cosh{x}$ &
        $\int \cosh{x}\,dx = \sinh{x} + c$
        \\
        $\frac{d}{dx} \cosh{x} = \sinh{x}$ &
        $\int \sinh{x}\,dx = \cosh{x} + c$
        \\
        \multirow{2}{*}{$\frac{d}{dx} \tanh{x} = \frac{1}{\cosh^2{x}} = 1 - \tanh^2{x}$} &
        $\int \frac{1}{\cosh^2{x}}\,dx = \tanh{x} + c$ \\* &
        $\int \tanh^2{x}\,dx = x - \tanh{x} + c$
        \\
        \multirow{2}{*}{$\frac{d}{dx} \coth{x} = -\frac{1}{\sinh^2{x}} = 1 - \coth^2{x}$} &
        $\int \frac{1}{\sinh^2{x}}\,dx = -\coth{x} = c$ \\* &
        $\int \coth^2{x}\,dx = x - \coth{x} + c$
        \\
        \midrule
        &
        $\int \cosh^2{x}\,dx = \frac{1}{2}\cosh{(x)}\sinh{(x)}+\frac{1}{2}x + c$
        \\
        &
        $\int \sinh^2{x}\,dx = \frac{1}{4}\sinh{(2x)}-\frac{1}{2}x + c$
        \\

        % -------------------------------------------------------------------------------------
        % Inverse hyperbolic trig functions
        % -------------------------------------------------------------------------------------
        \midrule
        \multirow{2}{*}{$\frac{d}{dx} \arsinh{x} = \frac{1}{\sqrt{x^2+1}}$} &
        $\int \frac{1}{\sqrt{x^2+1}}\,dx = \arsinh{x} + c$ \\* &
        $=\ln{\left(x + \sqrt{x^2+1}\right)} + c$
        \\
        \multirow{2}{*}{$\frac{d}{dx} \arcosh{x} = \frac{1}{\sqrt{x^2-1}}$} &
        $\int \frac{1}{\sqrt{x^2-1}}\,dx = \arcosh{x} + c$ \\* &
        $=\ln{\left(x + \sqrt{x^2-1}\right)} + c, x > 1$
        \\
        \multirow{2}{*}{$\frac{d}{dx} \artanh{x} = \frac{1}{1-x^2}, |x| < 1$} &
        $\int \frac{1}{1-x^2}\,dx = \artanh{x} + c$ \\* &
        $=\frac{1}{2}\ln{\frac{1+x}{1-x}} + c, |x| < 1$
        \\
        \multirow{2}{*}{$\frac{d}{dx} \arcoth{x} = \frac{1}{1-x^2}, |x| > 1$} &
        $\int \frac{1}{1-x^2}\,dx = \arcoth{x} + c$ \\* &
        $=\frac{1}{2}\ln{\frac{x+1}{x-1}} + c, |x| > 1$
        \\

        % -------------------------------------------------------------------------------------
        % Substitution, partial integration
        % -------------------------------------------------------------------------------------

        \bottomrule
    \end{longtable}

\egroup

\subsection*{Important operations}
\bgroup
    \def\arraystretch{1.3}
    \begin{tabular}{ll}

        \multirow{2}{*}{Product Rule of Derivatives} &
        $f(x)' = (g(x)h(x)i(x))'$ \\* &
        $\hspace{5mm} = g'(x)h(x)i(x) + g(x)h'(x)i(x) + g(x)h(x)i'(x)$
        \\

        Quotient Rule of Derivatives &
        $f(x)' = \frac{g(x)}{h(x)} = \frac{g'(x)h(x) - g(x)h'(x)}{h^2(x)}$
        \\

        &\\

        \multirow{2}{*}{Substitution} &
        $\int_a^b f[u(x)]u'(x)\,dx = \int_{u(a)}^{u(b)} f(z)\,dz$ \\* &
        $\hspace{4mm}\textrm{with}\hspace{4mm} z = u(x)$
        \\

        Partial Integration &
        $\int_a^a u(x)v'(x)\,dx = u(x)v(x)\rvert_a^b - \int_a^b u'(x)v(x)\,dx$
        \\
    \end{tabular}
\egroup

\subsection*{Substitutions}

\bgroup
    \def\arraystretch{1.1}
    \begin{longtable}{ll}
        \toprule
        Integral Type & Substitution \\
        \midrule

        \endhead
        \endfoot
        \endlastfoot

        \multirow{2}{*}{$\int f(ax+b)\,dx = \frac{1}{a}\int f(z)\,dz$} &
        $z=ax+b$ \\* &
        $dx = \frac{1}{a}\,dz$
        \\

        & \\

        \multirow{2}{*}{$\int f(\varphi(x))^n \cdot \varphi'(x)\,dx = \frac{(\varphi(x))^{n+1}}{n+1} + c$} &
        $z=\varphi(x)$ \\* &
        $dx = \frac{dz}{\varphi'(x)}$
        \\

        & \\

        \multirow{2}{*}{$\int \frac{\varphi'(x)}{\varphi(x)}\,dx = \ln|\varphi(x)| + c$} &
        $z=\varphi(x)$ \\* &
        $dx = \frac{dz}{\varphi'(x)}$
        \\

        & \\

        \multirow{3}{*}{$\int f\left(x, \sqrt{a^2-x^2}\right)\,dx$} &
        $x = a\sin{z}$ \\* &
        $dx = a\cos{z}\,dz$ \\* &
        $\sqrt{a^-x^2} = a\cos{z}$
        \\

        & \\

        \multirow{3}{*}{$\int f\left(x, \sqrt{a^2+x^2}\right)\,dx$} &
        $x = a\sinh{z}$ \\* &
        $dx = a\cosh{z}\,dz$ \\* &
        $\sqrt{a^2+x^2} = a\cosh{z}$
        \\

        & \\

        \multirow{3}{*}{$\int f\left(x, \sqrt{x^2-a^2}\right)\,dx$} &
        $x = a\cosh{z}$ \\* &
        $dx = a\sinh{z}\,dz$ \\* &
        $\sqrt{x^2-a^2} = a\sinh{z}$
        \\

        \bottomrule
    \end{longtable}
\egroup

\subsection*{Lathed Bodies}

\bgroup
    \def\arraystretch{1.2}
    \begin{tabular}{ll}
        Volume, X axis rotation: &
        $V_x = \pi\int_a^b\left(f(x)\right)^2\,dx$ \\

        Volume, Y axis rotation: &
        $V_y = \pi\int_a^b\left(f^{-1}(y)\right)^2\,dy$ \\
        & ($f(x) = f^{-1}(y)$) (Invert function first) \\

        Arc length: &
        $L = \int_a^b \sqrt{1+\left(f'(x)\right)^2}\,dx$ \\

        Lateral surface: &
        \\
    \end{tabular}
\egroup


\documentclass[a4paper]{article}

\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}

\title{MATH 542 Homework 7}
\author{Saket Choudhary\\skchoudh@usc.edu}

\begin{document}
\maketitle

\begin{align*}
Y_i &= \beta_0+\beta_1X_i+\epsilon_i\\
\epsilon_i&=Y_i - \beta_0-\beta_1X_i\\
\epsilon_i^2&=(Y_i - \beta_0-\beta_1X_i)^2\\
Q&=\sum_i\epsilon_i^2\sum_i(Y_i - \beta_0-\beta_1X_i)^2\\
\end{align*}

We want to minimize the residual $\epsilon_i=Y_i - \beta_0-\beta_1X_i = Y_i-\hat{Y_i}$ so $\frac{\partial \sum_i\epsilon_i^2}{\partial \beta_0}=\frac{\partial \sum_i\epsilon_i^2}{\partial \beta_1}=0$
\begin{align*}
\frac{\partial Q}{\partial \beta_0} &= -2\sum_i(Y_i - \beta_0-\beta_1X_i)=0\\
\hat{\beta_0} &= \bar{Y}-\hat{\beta_1}\bar{X}
\end{align*}

\begin{align*}
\frac{\partial Q}{\partial \beta_1} &= -2\sum_iX_i(Y_i - \beta_0-\beta_1X_i)=0\\
\sum_iX_i(Y_i - \hat{\beta_0}-\hat{\beta_1}X_i)&=0\\
\sum_iX_i(Y_i - \bar{Y}+\hat{\beta_1}\bar{X}-\hat{\beta_1}X_i)&=0\\
\hat{\beta_1} &= \frac{\sum_iX_i(Y_i-\bar{Y})}{\sum_iX_i(X_i-\bar{X})}
\end{align*}

Using $\sum \bar{X}(X_i-\bar{X}) =0$ $\implies$ $\sum X_i(X_i-\bar{X})=\sum (X_i-\bar{X})(X_i-\bar{X})$
and using $\sum \bar{X}(Y_i-\bar{Y}) =0$ we get $\sum_iX_i(Y_i-\bar{Y}) = \sum_i(X_i-\bar{X})(Y_i-\bar{Y})$

and hence $\hat{\beta_1} = \frac{\sum_i(X_i-\bar{X})(Y_i-\bar{Y})}{\sum_i(X_i-\bar{X})^2}$

\begin{align*}
E[\hat{\beta_0}] &= E[\bar{Y}]-\bar{X}E[\hat{\beta_1}]\\
E[\bar{Y}] &= E[\beta_0+\beta_1\bar{X}+1/n\sum_i \epsilon_i] \\
&=\beta_0+\beta_1\bar{X}
\end{align*}

Now consider $E[\hat{\beta_1}] = \frac{\sum_i(X_i-\bar{X})(EY_i-E\bar{Y})}{\sum_i(X_i-\bar{X})^2} = \frac{\sum_i(X_i-\bar{X})(\beta_1 X_i-\beta_1 \bar{X})}{\sum_i(X_i-\bar{X})^2} = \beta_1$

using which we get:
$$
E[\hat{\beta_0}] = E[\bar{Y}]-\bar{X}E[\hat{\beta_1}] = \beta_0 + \beta_1 \bar{X} - \beta_1\bar{X} = \beta_0
$$

\section*{Problem 2}
\begin{align*}
Var(\hat{\beta_1}) &= \frac{1}{(\sum(X_i-\bar{X})^2)^2} Var(\sum_i(X_i-\bar{X})(Y_i-\bar{Y}))\\
&=\frac{1}{(\sum(X_i-\bar{X})^2)^2} Var(\sum_i(X_i-\bar{X})(Y_i))\\
&= \frac{1}{(\sum(X_i-\bar{X})^2)^2} \sum_i(X_i-\bar{X})^2Var(Y_i)\\
&= \frac{1}{(\sum(X_i-\bar{X})^2)^2} \sum_i(X_i-\bar{X})^2\sigma^2\\
&= \frac{\sigma^2}{(\sum(X_i-\bar{X})^2)}\\
\end{align*}

\begin{align*}
\hat{\beta_0} &= \bar{Y}-\hat{\beta_1}\bar{X}\\
Var(\hat{\beta_0}) &= Var(\bar{Y}) + \bar{X}^2Var(\hat{\beta_1})\\
&= \frac{\sigma^2}{n} + \bar{X}^2(\frac{\sigma^2}{(\sum(X_i-\bar{X})^2)})
\end{align*}

\section*{Problem 3}
\begin{align*}
Y_i &= \beta_0 + \beta_1X_i + \epsilon_i\\
\hat{Y_i} &= {\beta_0} + {\beta_1}X_i\\
\frac{\partial Q}{\partial \beta_0} &= -2\sum_i(Y_i - \beta_0-\beta_1X_i)=-2\sum_i(Y_i - \hat{Y_i})=0\\
\frac{\partial Q}{\partial \beta_1} &= -2\sum_iX_i(Y_i - \beta_0-\beta_1X_i)=0\\
&= \sum_i(\hat{Y_i}/\beta_1-\beta_0/\beta_1)(Y_i-\hat{Y_i})=0\\
&\implies \sum_i(\hat{Y_i})(Y_i-\hat{Y_i})=0\\
\end{align*}

Now,
\begin{align*}
\sum_i(Y_i-\bar{Y})^2 &= \sum_(Y_i-\hat{Y_i}+\hat{Y_i}-\bar{Y})^2\\
&= \sum(Y_i-\hat{Y_i})^2+\sum_i(\hat{Y_i}-\bar{Y})^2 + 2\sum_i(Y_i-\hat{Y_i})(\hat{Y_i}-\bar{Y})\\
&= \sum(Y_i-\hat{Y_i})^2+\sum_i(\hat{Y_i}-\bar{Y})^2 + 2\sum_i\hat{Y_i}(Y_i-\hat{Y_i}) - 2\sum_i\bar{Y}(Y_i-\hat{Y_i})\\
&= \sum(Y_i-\hat{Y_i})^2+\sum_i(\hat{Y_i}-\bar{Y})^2
\end{align*}

where the two last terms are zero using the properties derived previous to the last set of equations

\section*{Problem 4}
\begin{align*}
r &= \frac{\sum_i(x_i-\bar{x})(y_i-\bar{y})}{\sqrt{\sum_i(x_i-\bar{x})^2\sum_i(y_i-\bar{y})^2 }}\\
r^2 &= \frac{(\sum_i(x_i-\bar{x})(y_i-\bar{y}))^2}{{\sum_i(x_i-\bar{x})^2\sum_i(y_i-\bar{y})^2 }} = \frac{s_{xy}^2}{s_{xx}s_{yy}}\\
R^2 &= 1- \frac{SSE}{SST} = \frac{SSR}{SST}\\
SSE(rror) &= \sum_i(y_i-\hat{y_i})^2\\
SST(otal) &= \sum_i(y_i-\bar{y})^2\\
SSR(egression) &= \sum_i(\hat{y_i}-\bar{y})^2
\end{align*}

Consider $y=a+bx$ then from the first problem we have:
$b = \frac{s_{xy}}{s_{xx}}$

\begin{align*}
SSR &= \sum_i(\hat{y_i}-\bar{y})^2\\
&= \sum_i(a+bx_i-\bar{y})^2\\
&= \sum(\bar{y}-b\bar{x}+bx_i-\bar{y})^2\\
&= b^2s_{xx}\\
&= \frac{s_{xy}^2}{s_{xx}^2}s_{xx}\\
&= \frac{s_{xy}^2}{s_{xx}}\\
\end{align*}
\begin{align*}
R^2 &= \frac{SSR}{SST}\\
&= \frac{\frac{s_{xy}^2}{s_{xx}}}{s_{yy}}\\
&= \frac{s_{xy}^2}{s_{xx}s_{yy}}\\
&=r^2
\end{align*}
\section*{Problem 5}
\begin{align*}
L(\sigma^2,\beta_0,\beta_1) &= (2\pi\sigma^2)^{-\frac{n}{2}}\prod_iexp(-\frac{1}{2\sigma^2}(y_i-\beta_0-\beta_1x_i)^2)\\
\log(L) & = -\frac{n}{2}\log(2\pi) - n \log\sigma^2 -\frac{1}{2\sigma^2}\sum_i(y_i-\beta_0-\beta_1x_i)^2\\
\frac{\partial \log L}{\partial \beta_0}|^{\beta_{MLE}} &= -\frac{-2}{2\sigma^2}\sum_i(y_i-\beta_0-\beta_1x_i) \\
&\implies \beta_0^{MLE} = \bar{Y}-\beta_1^{MLE}\bar{X}\\
\frac{\partial \log L}{\partial \beta_1}|^{\beta_{MLE}} &= -\frac{-2}{2\sigma^2}\sum_ix_i(y_i-\beta_0-\beta_1x_i) \\
&\implies \beta_1^{MLE} = \frac{\sum_i(x_i-\bar{x})(y_i-\bar{y})}{\sum_i(x_i-\bar{x})^2}\\
\frac{\partial \log L}{\partial \sigma^2}|^{\beta_{MLE}} &= -\frac{n}{2\sigma^2} +\frac{1}{2\sigma^4}\sum_i(y_i-\beta_0-\beta_1x_1)^2 \\
&\implies \sigma^2_{MLE} = \frac{1}{n}\sum_i(y_i-\beta_0^{MLE}-\beta_1^{MLE}x_1)^2
\end{align*}
\end{document}


\chapter{Support Vector Machines} \label{Support_Vector_Machine}
Support Vector Machines (SVMs) method is one of the Artificial Intelligence Outlier Detection methods \citep{Silvia} and it can be applied in classification problems. The main idea of SVMs is to find the optimal hyperplane in feature space that best separates classes \citep{Maimon}. In this section classifiers concepts will be described, then it will extended to One Class Support Vector Machine (OCSVM).
\section{Basics of SVMs}
To describe the method consider the set $\Omega = \lbrace (x_1,y_1),(x_2,y_2),\ldots,(x_n,y_n) \rbrace$; where $i \in \mathbb{R}^{+}$ is i$th$ element in the set, $x_i \in \mathbb{R}^d$ is the input observation number $i$ in $d$ dimensional space and $y_i \in \lbrace -1,1 \rbrace$ is the output number $i$, which indicate the class membership \citep{Roemer}. SVM uses a hyperplane to separate the observations into classes.
SVM aims to maximize the margin $\mathcal{M}$ to get the optimal result. In Figure~\ref{hyperplanes} observations are in 2-dimensional space which belongs to one of two classes. The observations can be separated by multiple hyperplanes, but the hyperplane that has the maximum distance between the observations and the optimal hyperplane is in Figure~\ref{maxHyperplane}. Figures are reproduced from \citep{SVM}.
\begin{figure}[H]
\begin{center}
\minipage{0.4\textwidth}
  \includegraphics[width=\linewidth]{images/hyperplanes}
  \caption{The possible hyperplanes with its different distances from the observations, some hyperplanes are close to one observations of the classes.}
  \label{hyperplanes}
\endminipage
\qquad
\minipage{0.4\textwidth}
  \includegraphics[width=\linewidth]{images/maxHyperplane}
  \caption{The optimal hyperplane the one that has maximum distances between the classes, it shouldn't also biased to one side.}
  \label{maxHyperplane}
\endminipage
\end{center}
\end{figure}
SVM has a low computational cost of evaluating the decision function, which is the function that used to classify the observations. SVM doesn't take all of the training examples into account, but a subset of it which is called support vectors \citep{Maimon}. The support vector is chosen to be the closest observations to the optimal hyperplane. In Figure~\ref{maxHyperplane} the filled shapes are referred to support vectors in that space.
\newpage
\large{\textbf{Hyperplane Classifiers}}

The objective of classification in SVM is to finds a function $f:\mathbb{R}^{N} \rightarrow \lbrace -1,+1 \rbrace$ by using the training observations according to the unknown probability distribution $P(x,y)$ of the observations $\lbrace (x_1,y_1),\ldots,(x_n,y_n) \rbrace \in \mathbb{R}^{N} \times Y, Y=\lbrace -1,+1 \rbrace$ such that $f$ will classify the coordinates of a new test observation $(x,y)$ correctly \citep{Maimon}.
\begin{itemize}
\item \textbf{The Linear Classifier}

Let $\Omega$ be a linearly separable observations, the hyperplane will be defined as:
\begin{align}
\label{equ-3-1}
w.x+b=0, \quad w \in \mathbb{R}^N , b \in \mathbb{R}
\end{align}
Which corresponds to:
\begin{align}
\label{equ-3-2}
f(x) = \text{sign}((w.x)+b)
\end{align}
Where $w$ is the weight vector and $b$ is the bias and sign is either positive or negative.

The margin $\mathcal{M}$ is the largest distance between two parallel hyperplanes that separate the data into two classes. $\mathcal{M}$ is given by:
\begin{align}
\label{equ-3-3}
\mathcal{M} = \frac{2}{||w||}
\end{align}
Note that the margin depends on the weight vector. Maximizing $\mathcal{M}$ is equivalent to minimizing $||w||$ value. A constraint to prevent observations from falling into the margin should be introduced. The problem becomes:
\begin{align}
\label{equ-3-4}
&\min_{w,b}{\frac{1}{2}||w||^2}\\
&\text{s.t. } y_i . ((w.x_i)+b) \geq 1, \quad i=1,\ldots,n
\end{align}
Since this problem is convex optimization, it can be solved using the Lagrange function \citep{Maimon} as follows:
\begin{align}
\label{equ-3-5}
\mathcal{L}(w,b,\alpha) = \frac{1}{2} ||w||^2 - \sum_{i=1}^{n}\bigg( \alpha_i\big[ y_i((w.x_i)+b)-1 \big] \bigg)
\end{align}
Where $\alpha_i$ is the Lagrange multiplier. And this has to be minimized with respect to primal variables $w,b$:

\begin{align}
\label{equ-3-6}
\frac{\partial \mathcal{L}}{\partial w} &= w-\sum_{i=1}^{n}{\alpha_i y_i x_i} = 0 \rightarrow w = \sum_{i=1}^{n}{\alpha_i y_i x_i}\\
\label{equ-3-7}
\frac{\partial \mathcal{L}}{\partial b} &= \sum_{i=1}^{n}{\alpha_i y_i} = 0
\end{align}
The value of $\alpha_i$ = 0 for all the constraints in Equation~\ref{equ-3-5}, which means the values of $\alpha_i$ can be eliminating from the equation without affecting the solution except for support vector (by using Karush-Kuhn-Tucker (KKT) conditions of optimization), thus:
\begin{align}
\label{equ-3-8}
\alpha_i(y_i.((w.x_i)+b)-1) = 0, \quad i=1,\ldots,n
\end{align}
% % % % % % % % % % % % % % % % % % % % % % % % %
The Lagrangian ($\mathcal{L}$) can also be maximized with respect to the dual variables $\alpha_i$. In fact, solving the dual problem may be easier than the primal problem; because they may have the same values under some conditions \citep{Maimon}. The dual problem can be obtained by substituting back~\ref{equ-3-6} and ~\ref{equ-3-7} value in Lagrangian function ~\ref{equ-3-5}:
\begin{align}
\label{equ-3-9}
\mathcal{L}=
\frac{1}{2} \bigg[ \big( \sum_{i=1}^{n}{\alpha_i y_i x_i} \big) . \big( \sum_{j=1}^{n}{\alpha_j y_j x_j} \big) \bigg]
-
\bigg[
\big( \sum_{i=1}^{n}{\alpha_i y_i x_i} \big) . \big( \sum_{j=1}^{n}{\alpha_j y_j x_j} \big) \bigg] + \sum_{i=1}^{n}{y_i \alpha_i b} + \sum_{i=1}^{n}{\alpha_i}
\end{align}
And from ~\ref{equ-3-7} the term $\sum_{i=1}^{n}{y_i \alpha_i b}$ is equal to 0. The first two terms differ by a factor, thus it becomes:
\begin{align}
\label{equ-3-10}
\mathcal{L}=\sum_{i=1}^{n}{\alpha_i} - \frac{1}{2}
\sum_{i=1}^{n}\sum_{j=1}^{n}{\alpha_i \alpha_j y_i y_j x_i x_j} \\
\text{s.t. }
\alpha_i \geq 0, i=1,\ldots,n, \sum_{i=1}^{n}{\alpha_i y_i} = 0 \notag
\end{align}
then the decision function in~\ref{equ-3-2} will be:
\begin{align}
\label{equ-3-11}
f(x) = \text{sign}\bigg(\sum_{i=1}^{n} \alpha_i y_i (x.x_i)+b\bigg)
\end{align}
And by using ~\ref{equ-3-8} and the support vectors $x_i, i \in S=\{i:\alpha_i \neq 0\}$ the value of $b$ becomes:
\begin{align}
\label{equ-3-12}
b=\frac{1}{S} \sum_{i\in S}\bigg(y_i - \sum_{j=1}^{n}\alpha_j y_j(x_i. x_j)\bigg)
\end{align}
% % % % % % % % % % % % % % % % % %
\item \textbf{Kernel Function}

When the observations are not linearly separable in the input space, they should be projected in higher dimension space, this makes the observations separable by using a hyperplane. The kernel function ($K$) is used to project the observations from the input space to higher feature space $F$, where feature space is non-linear mapping $K:\mathbb{R}^N \rightarrow \mathbb{R}^M$ where $M>N$ from the input space ~\citep{Maimon}. In this space a hyperplane can be used to separate the observations, then that hyperplane would be projected back in the input space, and it would have a non-linear curve. Figure~\ref{hyperplane-grayscale} illustrate the idea.
\begin{figure}[H]
\begin{center}
  \includegraphics[scale=0.3]{images/hyperplane-grayscale}
  \caption{Kernel function $K$ projected the observations from input space to feature space \citep{Hyperplane}.}
  \label{hyperplane-grayscale}
\end{center}
\end{figure}
The Kernel computes the inner product of the vectors without knowing the function $K$ on it self \citep{Thuso}. There are different choices of kernel function which gives a different classification accuracy. Some of them listed below:
\begin{itemize}
\renewcommand\labelitemii{$\bullet$}
\item Linear: $K(x,x_i) = x^T x_i$
\item Polynomial: $K(x,x_i) = {(\gamma x.x_i+C)}^d$
\item Sigmoidal: $K(x,x_i) = tanh{(\gamma x^Tx_i+C)}$
\item Gaussian Radial Base function (RBF): $K(x,x_i) = \exp \big( - \gamma||x-x_i||^2 \big)$
where $\sigma \in \mathbb{R}$ is kernel parameter, and $||x-x_i||$ is the dissimilarity measure.
\end{itemize}
Where $C$ is the parameter that controls the influence of individual observations; It is also called smoothness parameter because it controls the trade-off between smooth decision boundary and classifying trading observations correctly \citep{OCSVM}.
% % % % % % % % % % % % % % % % % % %
\item \textbf{Non Separable SVM}

Sometimes the observations may have high noise levels leading the classes to overlap. The previous case was for separable observations, so extending the capabilities of hyperplane classifier so the problem of noise can be solved. \citep{Maimon}.
Equation~\ref{equ-3-4} can be extended by introducing the slack variables $\xi_i$ which allow the training data to violate the constraint in Equation~\ref{equ-3-5}.
A soft classifier is used in the non separable case, and controls the classifier capacity and the sum of the slacks $\sum_{i=1}^{n} \xi_i$, the soft classifier minimizes the following function:
\vspace{-4mm}
\begin{align}
\label{equ-3-13}
&\min_{w,b}\frac{1}{2}||w||^2 + C \sum_{i=1}^{n}{\xi_i} \\
&\text{s.t. } \xi_i \geq 0, y_i(w x_i +b) \geq 1-\xi_i \quad \forall i. \notag
\end{align}
The value of $\xi_i$ can be one of three options, $\xi_i = 0$, then $x_i$ on the right side of the margin. $0< \xi_i <1$, then $x_i$ on the right side of the margin but its distance is less than the margin $\mathcal{M}$. $\xi_i > 1$, then $\xi_i$ is on the wrong side \citep{Thuso}.
The difference between the separable and the non-separable case is the upper bound $C$ on the Lagrange multipliers $\alpha_i$, where $C > 0$ is the regularization constant that determines the balance between the empirical risk and the complexity term \citep{Maimon}. The Lagrange function becomes:
\begin{align}
\label{equ-3-14}
\mathcal{L}(w,b,\xi_i) = \frac{1}{2} ||w||^2
 + C \sum_{i=1}^{n}{\xi_i} - \sum_{i=1}^{n}{\alpha_i \lbrace y_i (w x_i + b) - (1-\xi_i) \rbrace} - \sum_{i=1}^{n}{\beta_i \xi_i}
 \end{align}
And this has to be minimized with respect to variables $w,b,\xi_i$:
\begin{align}
\notag
\mathcal{L} &= \frac{1}{2} w.w + C \sum_{i=1}^{n}{\xi_i} - \sum_{i=1}^{n}(\alpha_i y_i w x_i + \alpha_i y_i b - \alpha_i + \alpha_i \xi_i) - \sum_{i=1}^{n} \beta_i \xi_i \\
\label{equ-3-15}
\frac{\partial \mathcal{L}}{\partial w} &= w - \sum_{i=1}^{n} \alpha_i y_i x_i = 0 \rightarrow w = \sum_{i=1}^{n} \alpha_i y_i x_i \\
\label{equ-3-16}
\frac{\partial \mathcal{L}}{\partial b} &= \sum_{i=1}^{n} \alpha_i y_i = 0\\
\label{equ-3-17}
\frac{\partial \mathcal{L}}{\partial \xi_i} &= C - \alpha_i - \beta_i = 0 \rightarrow \beta_i = C - \alpha_i \quad \forall i.
\end{align}
By using Karush-Kuhn-Tucker (KKT) conditions of optimization to minimize $\mathcal{L}$, which include the Equations ~\ref{equ-3-15}, ~\ref{equ-3-16}, ~\ref{equ-3-17} and:
\begin{align}
\label{equ-3-18}
\alpha_i (y_i (w x_i +b) - (1-\xi_i)) = 0 \\
\label{equ-3-19}
y_i(w x_i +b) - (1-\xi_i) \geq 0 \\
\label{equ-3-20}
\beta_i \xi_i = 0
\end{align}
Now by using Equation ~\ref{equ-3-17}, ~\ref{equ-3-18}, ~\ref{equ-3-19} and ~\ref{equ-3-19} the following values obtained:
\begin{align}
\notag
\alpha_i = 0 \rightarrow y_i(w x_i+b) \geq 1 \\
\notag
\alpha_i = C \rightarrow y_i(w x_i+b) \leq 1 \\
\notag
0<\alpha_i<C \rightarrow y_i(w x_i+b)=1 \notag
\end{align}
Now by substituting back in Equation ~\ref{equ-3-14}:
\begin{align}
\label{equ-3-21}
\mathcal{L} = \frac{1}{2}\sum_{i=1}^{n}\sum_{j=1}^{n} \alpha_i \alpha_j y_i y_j + C \sum_{i=1}^{n}{\xi_i} - \sum_{i=1}^{n}{\alpha_i y_i} \sum_{j=1}^{n}{\lbrace \alpha_j y_j x_j \rbrace}.x_i - b \sum_{i=1}^{n}{\alpha_i y_i} \\ + \sum_{i=1}^{n}{\alpha_i (1-\xi_i)} - \sum_{i=1}^{n}{(C-\alpha_i)\xi_i}\notag
\\
\label{equ-3-22}
= \sum_{i=1}^{n} \alpha_i - \frac{1}{2} \sum_{i=1}^{n}\sum_{j=1}^{n}{\alpha_i \alpha_j y_i y_j x_i. x_j}
\end{align}
Now to extend this classifier from input space $\mathbb{R}^N$ to higher dimensional space (feature space $F$), the polynomial kernel function can be used, thus the Lagrange function becomes:
\begin{align}
\label{equ-3-23}
\mathcal{L} = \sum_{i=1}^{n} \alpha_i - \frac{1}{2}\sum_{i=1}^{n}\sum_{j=1}^{n}{\alpha_i \alpha_j y_i y_j (\phi(x_i)). (\phi(x_j))}
\end{align}
\end{itemize}
\section{One Class Support Vector Machine} \label{OCSVM}
The aim of  One Class Support Vector Machine (OCSVM) method is to test if the new data belongs to a defined class or not, unlike previous classifiers where the methods distinguish between classes \citep{Roemer}. The algorithm represent the observations as points in space, mapped to form separable hyperplane. It uses the origin as the only member of the second class. And it separates the image of the one class from the origin. % \citep{Larry}. %

In Figure~\ref{OSVM1} the normal observations have been labelled by value $+1$, while the outliers are labelled by value $-1$.

\begin{figure}[H]
\begin{center}
\minipage{0.35\textwidth}
  \includegraphics[width=\linewidth]{images/OCSVM}
\endminipage
\end{center}
  \caption{One class SVM, whereby the hyperplane separates the normal observations class 1 from anything else (outliers) class -1.}
  \label{OSVM1}
\end{figure}

\textbf{One-Class SVM algorithm:}
The function in this algorithm returns +1 in the small region which is the training data points and -1 elsewhere. The quadratic minimization function is different to the SVM quadratic minimization problem \citep{Scholkopf}:
\begin{align}
&\min_{w,\xi_i,\rho}{\frac{1}{2}||w||^2 + \frac{1}{\nu n} \sum_{i=1}^{n}\xi_i - \rho} \\
&\notag \text{s.t. } (w.\phi(x_i)) \geq \rho - \xi_i, \forall i=1,\ldots,n \\
&\notag \xi_i \geq 0, \forall i=1,\ldots,n
\end{align}
The distance to the origin in feature space is presented by $\rho$. The variable $w$ is the parametrization of the hyperplane to separate the origin from the observations. In the previous formulation $C$ was the smoothness parameter. In this formula $\nu$ is the smoothness parameter which is the proportion of outliers expected in the data, and it characterizes the solution \citep{OCSVM}:
\begin{itemize}
\item It sets the upper bound on the fractions of outliers;
\item It is the lower bound on the number of training examples used as support-vector \citep{Scholkopf}.
\end{itemize}
So it is still a quadratic problem, and by using Lagrange multipliers, the decision function becomes:
$$
f(x) = \text{sign}((w.\phi(x_i)) - \rho) = \text{sign}(\sum_{i=1}^{n} \alpha_i K(x,x_i) - \rho)
$$

Another method for OCSVM is referred as Tax and Duin method \citep{David}.
% % % % % % % % % % % % % % % % % % % % %
\begin{comment}
\textbf{One-Class SVM according to Tax and Duin:}
This function creates a hypersphere around the observations in feature space instead of planar, minimizing hypersphere is minimized the effect of outliers in the solution.
The equation in this case require all the distances between the observation $x_i$ to the centre is strict less than $R$ where $R$ is the radius of the hypersphere which is the distance from any support vector to the boundary, slack variables $\xi_i$ and parameter $C$ are also used to create a soft margin \citep{David}.
$$
\min_{R,a} R^2 + C \sum_{i=1}^{n}\xi_i
$$
s.t.
$$
||x_i - a||^2 \leq R^2 + \xi_i, \forall i=1,\cdots,n
$$
$$
\xi_i \geq 0, \forall i=1,\cdots,n
$$
The result is characterized by:
$a$ is a linear combination of the support vectors(for the training observations for which \textit{Lagrange multiplier} is non-zero) and $R$ where $R>0$. The \textit{Lagrange multipliers} and \textit{Gaussian kernel} produce:
$$
||z-x||^2 = \sum_{i=1}^{n}\alpha_i \exp(\frac{-||z-x_i||^2}{\sigma^2}) \geq -R^2 \setminus 2 + C_R
$$
A new observation $z$ obtained, and it can be used to test if it is in the class and this happens when the distance to the centre is smaller than or equal to the radius \citep{David}.

There is also Multi-Class SVM which is an extension to two class problem, SVM also can be applied in the regression problems. SVM mainly used in supervised learning, however, it can also be applied in unsupervised learning. To conclude, the performance of SVM basically depends on the kernel function and the kernel parameters, they are the factors that determined the complexity and the accuracy of the model \citep{Maimon}.
\end{comment}


\subsection{Linear Approximations}
\noindent
Since $\partial z = f_x\partial x + f_y\partial y$, we can approximate $\Delta z$ (the change in any function) as $\Delta z \approx f_x\Delta x + f_y\Delta y$ because values of $f$ and the tangent plane are close.
We can rewrite this approximation as a dot product:
\begin{equation*}
	\Delta z \approx \langle f_x, f_y\rangle \cdot \langle \Delta x, \Delta y \rangle.
\end{equation*}

For example, say a cylindrical can has a radius $r=1$ and a height $h=5$. If the radius is increased by 0.1 and the height is increased by 1, what is the approximate $\Delta V$?
\begin{align*}
	V(r,h) &= \pi r^2 h \\
	V_r &= 2\pi rh \text{ and } V_h = \pi r^2 \\
	V_{r}(1,5) &= 10\pi  \text{ and } V_{h}(1,5) = \pi \\
	\Delta V &\approx 10\pi(0.1) + \pi(1) = 2\pi.
\end{align*}
Comparing this to the actual answer of $2.26\pi$, we see our approximation is decent.


\section{Double Integrals in Polar Coordinates}\label{sec:DoubleIntegralsinPolarCoordinates}

Suppose we have a surface given in polar coordinates as
$z=f(r,\theta)$ and we wish to find the integral over some region. We
could attempt to translate into rectangular coordinates and do the
integration there, but it is often easier to stay in polar
coordinates.\index{polar coordinates!double integral}

How might we approximate the volume under such a surface in a way that
uses polar coordinates directly? The basic idea is the same as
before: we divide the region into many small regions, multiply the
area of each small region by the height of the surface somewhere in
that little region, and add them up. What changes is the shape of the
small regions; in order to have a nice representation in terms of $r$
and $\theta$, we use small pieces of ring-shaped areas, as shown in
Figure~\ref{fig:polarcoordinatesregions}. Each small region
is roughly rectangular, except that two sides are segments of a circle
and the other two sides are not quite parallel. Near a point
$(r,\theta)$, the length of either circular arc is about
$r\Delta\theta$ and the length of each straight side is simply $\Delta
r$. When $\Delta r$ and $\Delta \theta$ are very small, the region is
nearly a rectangle with area $r\Delta r\Delta\theta$, and the volume
under the surface is approximately
\[\sum\sum f(r_i,\theta_j)r_i\Delta r\Delta\theta.\]
In the limit, this turns into a double integral
\[\int_{\theta_0}^{\theta_1}\int_{r_0}^{r_1} f(r,\theta)r\,dr\,d\theta.\]

\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <1.5truecm,1.5truecm>
\setplotarea x from 0 to 4.1, y from 0 to 4.1
\axis left  /
\axis bottom  /
\circulararc 70 degrees from 3.5 0.5 center at 0 0
\circulararc 70 degrees from 2.5 0.5 center at 0 0
\circulararc 60 degrees from 1.5 0.5 center at 0 0
\setlinear
\plot 3.3 1.166 0 0 2.8 2.1 /
\plot 2 2.87 0 0 /
\put {$\Delta r$} [tl] <0pt,-3pt> at 2.9 1
\put {$r\Delta \theta$} [bl] <2pt,2pt> at 3.1 1.62
\endpicture}}
\caption{A polar coordinates ``grid''.}
\label{fig:polarcoordinatesregions}
\end{figure}

\begin{example}{Volume of One-Eighth of a Sphere}{VolumeOneEighthSphere}
Find the volume under $z=\sqrt{4-r^2}$
above the quarter circle bounded by
the two axes and the circle $x^2+y^2=4$ in the first quadrant.
\end{example}
\begin{solution}
In terms of $r$ and $\theta$, this region is described by the
restrictions $0\le r\le 2$ and $0\le\theta\le\pi/2$, so we have
\begin{align*}
\int_{0}^{\pi/2}\int_{0}^{2} \sqrt{4-r^2}\;r\,dr\,d\theta
&=\int_{0}^{\pi/2}\left. -{1\over3}(4-r^2)^{3/2}\right|_0^2\,d\theta	\\
&=\int_{0}^{\pi/2} {8\over3}\,d\theta	\\
&={4\pi\over3}.
\end{align*}
The surface is a portion of the sphere of radius 2 centered at the
origin, in fact exactly one-eighth of the sphere. We know the formula
for volume of a sphere is $(4/3)\pi r^3$, so the volume we have
computed is $(1/8)(4/3)\pi 2^3=(4/3)\pi$, in agreement with our
answer.
\end{solution}

This example is much like a simple one in rectangular coordinates: the region
of interest may be described exactly by a constant range for
each of the variables. As with rectangular coordinates, we can adapt
the method to deal with more complicated regions.

\begin{example}{Integration in Polar Coordinates}{integrationinpolarcoordinates}
Find the volume under $z=\sqrt{4-r^2}$ above the region enclosed by the
curve $r=2\cos\theta$, $-\pi/2\le\theta\le\pi/2$; see
Figure~\ref{fig:volumewithvariablelimits}.
\end{example}
\begin{solution}
The region is described in polar coordinates by the inequalities
$-\pi/2\le\theta\le\pi/2$ and $0\le r\le2\cos\theta$, so
the double integral is
\[\int_{-\pi/2}^{\pi/2}\int_{0}^{2\cos\theta} \sqrt{4-r^2}\;r\,dr\,d\theta
=2\int_{0}^{\pi/2}\int_{0}^{2\cos\theta} \sqrt{4-r^2}\;r\,dr\,d\theta.\]
We can rewrite the integral as shown because of the symmetry of the
volume; this avoids a complication during the evaluation.
Proceeding:
\begin{align*}
2\int_{0}^{\pi/2}\int_{0}^{2\cos\theta} \sqrt{4-r^2}\;r\,dr\,d\theta
&=2\int_{0}^{\pi/2}-{1\over3}\left.(4-r^2)^{3/2}\right|_0^{2\cos\theta}\,d\theta	\\
&=2\int_{0}^{\pi/2}-{8\over3}\sin^3\theta+{8\over3}\,d\theta	\\
&=\left.2\left(-{8\over3}{\cos^3\theta\over3}-\cos\theta+{8\over3}\theta\right)\right|_0^{\pi/2}	\\
&={8\over3}\pi-{32\over9}.
\end{align*}
\end{solution}

\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <1.5truecm,1.5truecm>
\setplotarea x from 0 to 2.1, y from -1.1 to 1.1
\axis left  /
\axis bottom shiftedto y=0 /
\circulararc 360 degrees from 2 0 center at 1 0
\put {\hbox{\epsfxsize6cm\epsfbox{images/polar_volume.eps}}} at 5 0
\endpicture}}
\caption{Volume over a region with non-constant limits.}
\label{fig:volumewithvariablelimits}
\end{figure}

You might have learned a formula for computing areas in polar
coordinates. It is possible to
compute areas as volumes, so that you need only remember one
technique. Consider the surface $z=1$, a horizontal plane. The volume
under this surface and above a region in the $x$-$y$ plane is
simply $1\cdot(\hbox{area of the region})$, so computing the volume
really just computes the area of the region.

\begin{example}{}{}
Find the area outside the circle $r=2$ and inside
$r=4\sin\theta$; see Figure~\ref{fig:areabyvolume}.
\end{example}
\begin{solution}
The region is described by $\pi/6\le\theta\le5\pi/6$ and
$2\le r\le4\sin\theta$, so the integral is
\begin{align*}
\int_{\pi/6}^{5\pi/6}\int_2^{4\sin\theta}1\,r\,dr\,d\theta
&=\int_{\pi/6}^{5\pi/6}\left. {1\over2}r^2\right|_2^{4\sin\theta}\,d\theta	\\
&=\int_{\pi/6}^{5\pi/6}8\sin^2\theta-2\,d\theta	\\
&={4\over3}\pi+2\sqrt3.
\end{align*}
\end{solution}

\begin{figure}[H]
\centerline{
\hbox{\hfill\begin{tikzpicture}[domain=-2:2,x=6mm,y=6mm]
\draw (-2.1,0) -- (2.1,0) ;
\draw (0,-2.1) -- (0,4.1) ;
\draw[color=black] (0,0) circle (2);
\draw[color=black] (0,2) circle (2);
\fill[opacity=0.5,fill=red!20]
plot[parametric,id=first,domain=0.5236:2.618]
function{4*sin(t)*cos(t),4*sin(t)*sin(t)} node {}
plot[parametric,id=second,domain=0.5236:2.618]
function{2*cos(3.1416-t),2*sin(3.1416-t)} -- cycle;
\end{tikzpicture}\hfill}}
\caption{Finding area by computing volume.}
\label{fig:areabyvolume}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:DoubleIntegralsinPolarCoordinates}}

\begin{enumialphparenastyle}

\begin{ex}
Find the volume above the $x$-$y$ plane, under the surface
$r^2=2z$, and inside $r=2$.
\begin{sol}
$4\pi$
\end{sol}
\end{ex}

\begin{ex}
Find the volume inside both $r=1$ and $r^2+z^2=4$.
\begin{sol}
$32\pi/3-4\sqrt3\pi$
\end{sol}
\end{ex}

\begin{ex}
Find the volume below $z=\sqrt{1-r^2}$ and above
the top half of the cone $z=r$.
\begin{sol}
$(2-\sqrt2)\pi/3$
\end{sol}
\end{ex}

\begin{ex}
Find the volume below  $z=r$, above the $x$-$y$ plane, and
inside $r=\cos\theta$.
\begin{sol}
$4/9$
\end{sol}
\end{ex}

\begin{ex}
Find the volume below  $z=r$, above the $x$-$y$ plane, and
inside $r=1+\cos\theta$.
\begin{sol}
$5\pi/3$
\end{sol}
\end{ex}

\begin{ex}
Find the volume between $x^2+y^2=z^2$ and $x^2+y^2=z$.
\begin{sol}
$\pi/6$
\end{sol}
\end{ex}

\begin{ex}
Find the area inside $r=1+\sin\theta$ and outside
$r=2\sin\theta$.
\begin{sol}
$\pi/2$
\end{sol}
\end{ex}

\begin{ex}
Find the area inside both
$r=2\sin\theta$ and $r=2\cos\theta$.
\begin{sol}
$\pi/2-1$
\end{sol}
\end{ex}

\begin{ex}
Find the area inside the four-leaf rose $r=\cos(2\theta)$
and outside $r=1/2$.
\begin{sol}
$\sqrt3/4+\pi/6$
\end{sol}
\end{ex}

\begin{ex}
Find the area inside the cardioid $r=2(1+\cos\theta)$
and outside $r=2$.
\begin{sol}
$8+\pi$
\end{sol}
\end{ex}

\begin{ex}\label{ex:areaofthreeleafroseloop}
Find the area of one loop of the three-leaf rose
 $r=\cos(3\theta)$.
\begin{sol}
$\pi/12$\
\end{sol}
\end{ex}

\begin{ex}
Compute $\ds \int_{-3}^3\int_0^{\sqrt{9-x^2}}
\sin(x^2+y^2)\,dy\,dx$ by converting to polar coordinates.
\begin{sol}
$(1-\cos(9))\pi/2$
\end{sol}
\end{ex}

\begin{ex}
Compute $\ds \int_{0}^a\int_{-\sqrt{a^2-x^2}}^0 x^2y\,dy\,dx$
by converting to polar coordinates.
\begin{sol}
$-a^5/15$
\end{sol}
\end{ex}

\begin{ex}
Find the volume under $z=y^2+x+2$ above
the region $x^2+y^2\le 4$
\begin{sol}
$12\pi$
\end{sol}
\end{ex}

\begin{ex}
Find the volume between
$z=x^2y^3$ and $z=1$ above
the region $x^2+y^2\le 1$
\begin{sol}
$\pi$
\end{sol}
\end{ex}

\begin{ex}
Find the volume inside
$x^2+y^2=1$ and $x^2+z^2=1$.
\begin{sol}
$16/3$
\end{sol}
\end{ex}

\begin{ex}
Find the volume under $z=r$ above $r=3+\cos\theta$.
\begin{sol}
$21\pi$
\end{sol}
\end{ex}

\begin{ex}
Figure~\ref{fig:doubleflower} shows the plot of
$r=1+4\sin(5\theta)$.
\begin{figure}[H]
\centerline{
\vbox{\beginpicture
\normalgraphs
\setcoordinatesystem units <2truecm,2truecm>
\setplotarea x from -1 to 1, y from -1 to 1
\put {\hbox{\epsfxsize4cm\epsfbox{images/double_flower.eps}}} at 0 0
\endpicture}}
\caption{$r=1+4\sin(5\theta)$}
\label{fig:doubleflower}
\end{figure}
\begin{enumerate}
	\item Describe the behavior of the graph in terms of the given
	  equation.  Specifically, explain maximum and minimum values, number
	  of leaves, and the 'leaves within leaves'.
	\item Give an integral or integrals to determine the area outside a
	  smaller leaf but inside a larger leaf.
	\item How would changing the value of $a$ in the equation
	  $r=1+a\cos(5\theta)$ change the relative sizes of the inner and
	  outer leaves? Focus on values $a\geq 1$.  (Hint: How would we change
	  the maximum and minimum values?)
\end{enumerate}
\end{ex}

\begin{ex}
Consider the integral $\ds\iint_D {1\over\sqrt{x^2+y^2}} \;
dA$, where $D$ is the unit disk centered at the origin.
\begin{enumerate}
	\item Why might this integral be considered improper?
	\item Calculate the value of the integral of the same function
	  $\ds 1/\sqrt{x^2+y^2}$ over the annulus with outer radius 1 and
	  inner radius $\delta$.
	\item Obtain a value for the integral on the whole disk by letting
	  $\delta$ approach 0.
		\begin{sol}
		$2\pi$
		\end{sol}
	\item For which values $\lambda$ can we replace the denominator with
	  $(x^2+y^2)^\lambda$ in the original integral?
\end{enumerate}
\end{ex}

\end{enumialphparenastyle}


\section{Loss functions:  nonlinear least square and cross-entropy}
Given a collection of data $D=(x_j,y_j)_{j=1}^N$ with $x_j\in A_{i_j}$
and $y_j=e_{i_j}$ We now discuss how to find $\theta =(W,b)$ such that
\begin{equation}
\label{fitting}
f(x_i,\theta)=\sigma(Wx_j+b)=y_j,\quad 1\le j \le N.
\end{equation}

\subsection{Least square}
The data fitting problem \eqref{fitting} can be solved by find a minimizer of the following loss function
\begin{equation}
\label{L0}
L_0(\theta)={1\over N}\sum_{i=1}^N\|f(x_i,\theta)-y_i\|^2
\end{equation}

Obviously,
$$
\theta^*=(W^*,b^*) \mathrm{satisfies}\ \eqref{fitting} \Leftrightarrow L_0(\theta^*)=0\Leftrightarrow \theta^*\in\arg\min_{\theta}L_0(\theta)
$$

We make the  following two observations on the loss function $L_0$ given by \eqref{L0}:
\begin{enumerate}
	\item It has infinitely many global minimizers.
	\item It is a non-convex loss function.
\end{enumerate}

\input{6DL/KL-CR-LR}

\subsection{Convex versus non-convex loss functions}
\subsubsection{Convexity}
The maximum likelihood and maximum posterior look difficult to optimize at first glance. Remarkably though,
the logistic regression has the property that the negative log likelihood is a convex function of the parameters $W,b$.
This means that we can apply a well-developed convergence theory for methods such as gradient descent, accelerated gradient
descent, stochastic gradient descent, etc. Our goal in this section will be to prove this remarkable fact.

The first thing to note is that the negative log-likelihood
$$\displaystyle\sum_{j=1}^n \left(\log(\mathbbm{1}^T\exp(W x_j + b)) -
 (W x_j + b)_{i_j})\right)$$
is a sum of terms over all of the data points $(x_i,y_i)$. Since a sum of convex functions is convex, it suffices to show that
the term corresponding to a single data point is convex (i.e. the negative log-likelihood of a single data point is a convex
function). So we consider the function
\begin{equation}
 f_{x,i}(W,b) = \log(\mathbbm{1}^T\exp(W x + b)) -
 (W x + b)_{i}
\end{equation}
We will show that this function is convex for any choice of data $x$ and label $i$. First note that the second term
$$ - (W x + b)_{i}$$
is a linear function, so it suffices to show that the first term
$$\log(\mathbbm{1}^T\exp(W x + b))$$
is convex. To see this, we note that $W x + b$ is a linear function of the parameters $W,b$. Combining
this with the easy fact that a convex function composed with a linear function is convex, we only need to consider the LogSumExp
function $s:\mathbb{R}^k\rightarrow \mathbb{R}$ defined by
\begin{equation}
 s(x) = \log(\exp(x_1) + \cdots + \exp(x_k))
\end{equation}
We can easily verify that this function is convex by calculating its Hessian and noting that $\nabla^2s$ is diagonally dominant.
%\begin{equation}
% D^2s(x) =   \left(\displaystyle\sum_{i=1}^k \exp(x_i)\right)^{-2}\begin{pmatrix}
%    \left(\displaystyle\sum_{i\neq 1} \exp(x_1 + x_i)\right) & -\exp(x_1 + x_2) & -\exp(x_1 + x_3) & \cdots & -\exp(x_1 + x_k) \\
%    -e^{x_1}e^{x_2} & e^{x_2}(e^{x_1} + e^{x_3} + \cdots + e^{x_k}) & -e^{x_2}e^{x_3} & \cdots & -e^{x_2}e^{x_k}
%  \end{pmatrix}
%\end{equation}


\subsection{Symbols}

As we saw earlier, symbols are defined using an equals sign.

\begin{Verbatim}[formatcom=\color{blue}]
N = 212^17
\end{Verbatim}

\noindent
No result is printed when a symbol is defined.
To see the value of a symbol, just evaluate it.

\begin{Verbatim}[formatcom=\color{blue}]
N
\end{Verbatim}

\noindent
$\displaystyle N=3529471145760275132301897342055866171392$

\bigskip
\noindent
Symbols can have more that one letter.
Everything after the first letter is displayed as a subscript.

\begin{Verbatim}[formatcom=\color{blue}]
NA = 6.02214 10^23
NA
\end{Verbatim}

\noindent
$\displaystyle N_A=6.02214\times10^{23}$

\bigskip
\noindent
A symbol can be the name of a Greek letter.

\begin{Verbatim}[formatcom=\color{blue}]
xi = 1/2
xi
\end{Verbatim}

\noindent
$\displaystyle \xi=\tfrac{1}{2}$

\bigskip
\noindent
Greek letters can appear in subscripts.

\begin{Verbatim}[formatcom=\color{blue}]
Amu = 2.0
Amu
\end{Verbatim}

\noindent
$\displaystyle A_\mu=2.0$

\bigskip
\noindent
The following example shows how
Eigenmath scans the entire symbol to find Greek letters.

\begin{Verbatim}[formatcom=\color{blue}]
alphamunu = 1
alphamunu
\end{Verbatim}

\noindent
$\displaystyle \alpha_{\mu\nu}=1$

\bigskip
\noindent
When a symbolic chain is defined,
Eigenmath follows the chain as far as possible.
The following example sets $A=B$ followed by $B=C$.
Then when $A$ is evaluated, the result is $C$.

\begin{Verbatim}[formatcom=\color{blue}]
A = B
B = C
A
\end{Verbatim}

\noindent
$\displaystyle A=C$

\bigskip
\noindent
Although $A=C$ is printed,
inside the program the binding of $A$ is still $B$, as can be seen with
the $binding$ function.

\begin{Verbatim}[formatcom=\color{blue}]
binding(A)
\end{Verbatim}

\noindent
$\displaystyle B$

\bigskip
\noindent
The {\it quote} function returns its argument unevaluated
and can be used to clear a symbol.
The following example clears $A$ so that its evaluation goes back to
being $A$ instead of $C$.

\begin{Verbatim}[formatcom=\color{blue}]
A = quote(A)
A
\end{Verbatim}

\noindent
$\displaystyle A$

\subsection{User defined functions}

Most of the functions commonly used in math and physics are included in Eigenmath.
See the Reference section at the end of the manual for a complete list.
There is also a facility for the user to define additional functions.

\bigskip
\noindent
User functions are defined using the syntax {\it function-name} ( {\it arg-list} ) = {\it expr}.
The {\it arg-list} is an optional comma separated list of symbols that receive arguments.
Unlike symbolic definitions, {\it expr} is not evaluated when {\it function-name} is defined.
Instead, {\it expr} is evaluated when {\it function-name} is used in a subsequent computation.

\bigskip
\noindent
The following example defines a sinc function and evaluates it at $\pi/2$.

\begin{Verbatim}[formatcom=\color{blue}]
f(x) = sin(x)/x
f(pi/2)
\end{Verbatim}

\noindent
$\displaystyle \frac{2}{\pi}$

\bigskip
\noindent
After a user function is defined, {\it expr} can be recalled using the {\it binding} function.

\begin{Verbatim}[formatcom=\color{blue}]
binding(f)
\end{Verbatim}

\noindent
$\displaystyle \frac{\sin(x)}{x}$

\bigskip
\noindent
The following example uses Rodrigues's formula to
compute an associated Legendre function of $\cos\theta$.
\begin{equation*}
P_l^m(x)=\frac{1}{2^ll!}(1-x^2)^{m/2}\frac{d^{l+m}}{dx^{l+m}}(x^2-1)^l
\end{equation*}
The formula is computed for $x$ and then
{\it eval} replaces $x$ in the result with $\cos\theta$.

\begin{Verbatim}[formatcom=\color{blue}]
P(l,m) = eval(1/(2^l l!) (1 - x^2)^(m/2) d((x^2 - 1)^l,x,l + m),x,cos(theta))
P(2,0)
\end{Verbatim}

\noindent
$\displaystyle \tfrac{3}{2} \cos(\theta)^2-\tfrac{1}{2}$

\bigskip
\noindent
When a user function is evaluated, the function arguments are visible
to any symbol definitions that reference them.
The symbol definitions can appear either before or after the function definition.
In the following example, the $x$ in $A$ and $B$ is replaced with $y$.

\begin{Verbatim}[formatcom=\color{blue}]
f(x) = A + B
A = a x
B = b x
f(y)
\end{Verbatim}

\noindent
$\displaystyle ay+by$