\section{Support Vector Machines II}

\subsection{Hard Margin Problem}

\begin{frame}
  \frametitle{Hard Margin Problem}

  The hard margin SVM optimization problem is formulated as:

  \begin{eqnarray*}
    & \mbox{minimize}   & \frac{1}{2} {\|\vec \alpha\|^2_2} \\[.3cm]
    & \mbox{subject to} & \forall i: \quad y_i\cdot (\vec \alpha^T\vec x_i + \alpha_0) -1 \geq 0
  \end{eqnarray*}
\end{frame}

\subsection{Soft Margin Problem}

\begin{frame}
  \frametitle{Soft Margin Problem}

  The soft margin SVM optimization problem is formulated as:

  \begin{eqnarray*}
    & \mbox{minimize}   & \frac{1}{2} {\|\vec \alpha\|^2_2} + \mu \sum_i \xi_i \\[.5cm]
    & \mbox{subject to} & \forall i: \quad -(y_i\cdot (\vec \alpha^T\vec x_i + \alpha_0) -1+\xi_i) \leq 0~, \\[.3cm]
    &                   & \forall i: \quad -\xi_i \leq 0
  \end{eqnarray*}
\end{frame}

\subsection{Lagrangian}

\begin{frame}
  \frametitle{Lagrangian}

  The solution of the \structure{constrained convex optimization problem} \\
  requires the Lagrangian:

  \begin{eqnarray*}
    L(\vec \alpha, \alpha_0, \vec \xi, \vec \lambda, \vec \mu)
     & = & \frac{1}{2} {\|\vec \alpha\|^2_2} + \mu \sum_i \xi_i  -\sum_i \mu_i \xi_i\\
     &   & \quad - \sum_i \lambda_i (y_i\cdot (\vec \alpha^T\vec x_i + \alpha_0) -1+\xi_i)
  \end{eqnarray*}
\end{frame}

\subsection{Lagrangian}

\begin{frame}
  \frametitle{Lagrangian}

  The solution of the \structure{constrained convex optimization problem} \\
  requires the Lagrangian:

~\\
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\bf{meta-}~~~~~~~~\bf{Lagrangian}\\
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\bf{parameter}~~~~\bf{multiplier}
  \begin{eqnarray*}
    L(\vec \alpha, \alpha_0, \vec \xi, \vec \lambda, \vec \mu)
	  & = & \frac{1}{2} {\|\vec \alpha\|^2_2} + {\color{red} \mu} \sum_i \xi_i  -\sum_i {\color{red} \mu_i} \xi_i\\
     &   & \quad - \sum_i \lambda_i (y_i\cdot (\vec \alpha^T\vec x_i + \alpha_0) -1+\xi_i)
  \end{eqnarray*}
\end{frame}

\subsection{Lagrangian}

\begin{frame}
  \frametitle{Lagrangian}

  The solution of the \structure{constrained convex optimization problem} \\
  requires the Lagrangian:

~\\
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\bf{meta-}~~~~~~~~\bf{Lagrangian}\\
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\bf{parameter}~~~~\bf{multiplier}
  \begin{eqnarray*}
    L(\vec \alpha, \alpha_0, \vec \xi, \vec \lambda, \vec \mu)
	  & = & \frac{1}{2} {\|\vec \alpha\|^2_2} + {\color{red} c} \sum_i \xi_i  -\sum_i {\color{red} \mu_i} \xi_i\\
     &   & \quad - \sum_i \lambda_i (y_i\cdot (\vec \alpha^T\vec x_i + \alpha_0) -1+\xi_i)
  \end{eqnarray*}
\end{frame}

\begin{frame}
  \frametitle{Lagrangian \cont}

  \structure{Partial derivatives  I:}

  \begin{displaymath}
    \frac{\partial  L(\vec \alpha, \alpha_0, \vec \xi, \vec \lambda, \vec \mu)}{\partial \vec \alpha} ~=~
    \vec \alpha - \sum_i \lambda_i y_i \vec x_i ~\stackrel{!}{=}~
    \vec 0.
  \end{displaymath}

  Thus we have:

  \begin{displaymath}
    \vec \alpha = \sum_i \lambda_i y_i \vec x_i~.
  \end{displaymath}
\end{frame}

\begin{frame}

  \frametitle{Lagrangian \cont}

  \structure{Partial derivatives  II:}

  \begin{displaymath}
    \frac{\partial  L(\vec \alpha, \alpha_0, \vec \xi, \vec \lambda, \vec \mu)}{\partial \alpha_0} ~=~
   -\sum_i \lambda_i y_i ~\stackrel{!}{=}~
   0
  \end{displaymath}
  \pspread

  \structure{Partial derivatives  III:}

  \begin{displaymath}
    \frac{\partial  L(\vec \alpha, \alpha_0, \vec \xi, \vec \lambda, \vec \mu)}{\partial \xi_i} ~=~
    c - \mu_i - \lambda_i ~\stackrel{!}{=}~
    0
  \end{displaymath}
\end{frame}

\subsection{Lagrange Dual}

\begin{frame}
  \frametitle{Lagrange Dual}

  Let us consider the \structure{Lagrange function for the dual problem} for the hard margin case:

  \begin{eqnarray*}
    L_\text{D}
    &=& \frac{1}{2} {\vec \alpha^T \vec \alpha} -
        \sum_i \lambda_i (y_i\cdot (\vec \alpha^T\vec x_i + \alpha_0) -1) \\[.3cm] \pause
    &=& \frac{1}{2}{\vec \alpha^T \vec \alpha} -
        (\underbrace{\sum_i \lambda_i y_i\cdot \vec x_i)^T}_{{\vec \alpha^T}} \vec \alpha -
        \underbrace{ \sum_i \lambda_iy_i}_{ = 0}\ \alpha_0 +
        \sum_i \lambda_i \\[.3cm] \pause
    &=& -\frac{1}{2} \sum_i \sum_j \lambda_i \lambda_j y_i y_j \cdot \vec x_i^T\vec x_j +  \sum_i \lambda_i
  \end{eqnarray*}
\end{frame}

\begin{frame}
  \frametitle{The Lagrange Dual Problem}

  The Lagrange dual problem is given by the optimization problem: \\[.25cm]

  \begin{center}
    \tikz[baseline]{
      \node[fill=bl1!100,anchor=base,rounded corners=3pt] (d1) {
        \color{bl3}
        $\begin{aligned}
           \displaystyle
           \mbox{maximize}   & \qquad -\frac{1}{2} \sum_i \sum_j \lambda_i \lambda_j y_i y_j \cdot \vec x_i^T\vec x_j + \sum_i \lambda_i \\[.3cm]
           \mbox{subject to} & \qquad \vec{\lambda}\succeq 0 \\
                             & \qquad \sum_{i} \lambda_i \, y_i = 0
         \end{aligned}$
      };
    }
  \end{center}
  \pause

  \structure{Benefits of the dual representation}
  \begin{itemize}
    \item The model can be reformulated using kernels.
    \item SVMs can be applied efficiently to feature spaces whose dimensionality exceeds the number of samples.
  \end{itemize}
\end{frame}

\note{
  From Bishop: Pattern Recognition and Machine Learning

  The solution to a quadratic programming problem in $d$ variables in general has computational complexity that is $\mathcal{O}(d^3)$.
  In going to the dual formulation we have turned the original optimization problem, which involved minimizing
  \begin{eqnarray*}
    & \mbox{minimize}   & \frac{1}{2} {\|\vec \alpha\|^2_2} \\[.3cm]
    & \mbox{subject to} & \forall i: \quad y_i\cdot (\vec \alpha^T\vec x_i + \alpha_0) -1 \geq 0
  \end{eqnarray*}
  over $d$ variables, into the dual problem, which has $m$ variables.
  For a fixed set of basis functions whose number $d$ is smaller than the number $m$ of data points, the move to the dual problem appears disadvantageous.
  However, it allows the model to be reformulated using kernels, and so the maximum margin classifier can be applied efficiently to feature spaces whose dimensionality exceeds the number of data points, including infinite feature spaces.
  The kernel formulation also makes clear the role of the constraint that the kernel function $k(\vec{x},\vec{x}')$ be positive definite, because this ensures that the Lagrangian function $L_\text{D}$ is bounded below, giving rise to a well-defined optimization problem.
}

\begin{frame}
  \frametitle{Lagrange Dual Problem \cont}

  For convex optimization problems with differentiable objective and constraint functions, the duality gap is zero,
  if the KKT conditions are satisfied. \\[.5cm] \pause

  Especially the \structure{complementary slackness} condition is interesting for us:
  \begin{displaymath}
    \forall i: \quad \lambda_i \, f_i(\vec{x}) = 0
  \end{displaymath}
\end{frame}

\begin{frame}
  \frametitle{Lagrange Dual Problem \cont}

  \structure{Complementary slackness for hard margin SVMs:}

  \begin{displaymath}
    \forall i: \quad \lambda_i \, (y_i\cdot (\vec \alpha^T\vec x_i + \alpha_0) - 1) = 0 \pause
  \end{displaymath}

  \structure{Implications:} \\[.2cm]

  \begin{enumerate}
    \item If $\lambda_i>0$, then $y_i \, (\vec{\alpha}^T \vec{x}_i + \alpha_0) - 1=0$, and thus:
      \begin{displaymath}
        y_i(\vec{\alpha}^T \vec{x}_i + \alpha_0) = 1~.
      \end{displaymath}
      All $\vec x_i$ with $\lambda_i>0$  are elements at the boundary of the slab; \\
      these samples are called \structure{\emph{support vectors}}. \\[.2cm] \pause
    \item We have seen that $  \vec \alpha = \sum_i \lambda_i y_i \vec x_i $, thus the norm vector of the decision boundary is a linear combination of support vectors.
  \end{enumerate}
\end{frame}

\begin{frame}
  \frametitle{Dual Representation}

  The \structure{decision function} can also be rewritten using the duality:

  \begin{center}
    \tikz[baseline]{
      \node[fill=bl1!100,anchor=base,rounded corners=3pt] (d1) {
        \color{bl3}
        $\begin{aligned}
           \displaystyle
           f(\vec x) =
           \vec{\alpha}^T \vec{x} +\alpha_0 =
           \sum_{i=1}^m \lambda_i y_i \vec x_i^T \vec x + \alpha_0
         \end{aligned}$
      };
    }
  \end{center}
  \pause

  \vspace{.5cm}
  \structure{Conclusion:} \\[.2cm]

  Feature vectors only appear in inner products, both in the learning and the classification phase.
\end{frame}

\subsection{Feature Transforms}

\begin{frame}
  \frametitle{Feature Transforms}

  \structure{Linear decision boundaries} in its current form have serious \structure{limitations}: \\[.2cm]

  \begin{itemize}
    \item  Non-linearly separable data cannot be classified. \\[.3cm]
    \item  Noisy data cause problems. \\[.3cm]
    \item  Formulation deals with vectorial data only.
  \end{itemize}
  \pspread

  \structure{Possible solution:} \\[.2cm]

  \begin{itemize}
    \item Map data into richer feature space using non-linear feature transform, \\
      then use a linear classifier.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Feature Transforms \cont}

  We select a feature transform
  \begin{displaymath}
    \phi: \real^d \rightarrow \real^D, \quad D \ge d
  \end{displaymath}
  such that the resulting features
  \begin{displaymath}
    \phi(\vec x_i), \quad i = 1, 2, \dots, m
  \end{displaymath}
  are linearly separable.
\end{frame}

\begin{frame}
  \frametitle{Feature Transforms \cont}

  \begin{ovalblock}{Example}
    Assume the decision boundary is given by the quadratic function
    \begin{displaymath}
      f(\vec x) = a_0 + a_1 x_1^2 + a_2 x_2^2 + a_3 x_1 x_2 + a_4 x_1+a_5 x_2.
    \end{displaymath}

    Obviously this is not a linear decision boundary. \\[.25cm] \pause

    By the following mapping, we get features that have a linear decision boundary:
    \small
    \begin{displaymath}
      \phi(\vec x) = \left(\begin{array}{c}
                             1 \\ x_1^2 \\ x_2^2 \\ x_1x_2 \\ x_1 \\ x_2
                           \end{array}
                     \right)
    \end{displaymath}
  \end{ovalblock}
\end{frame}

\begin{frame}
  \frametitle{Feature Transforms \cont}

  These feature transforms can be easily incorporated into SVMs:\\[.25cm]

  \begin{itemize}
    \item \structure{Decision boundary:}
      {\small
        \begin{displaymath}
          f(\vec x) = \sum_i \lambda_i y_i \cdot  \langle\phi(\vec x_i), \phi(\vec x) \rangle + \alpha_0 \pause
        \end{displaymath}
      }
    \item The Lagrange dual problem is given by the \structure{optimization problem}:
      {\small
        \begin{eqnarray*}
          \mbox{maximize}  & & -\frac{1}{2} \sum_i \sum_j \lambda_i \lambda_j y_i y_j \cdot \langle\phi(\vec x_i),\phi(\vec x_j)\rangle + \sum_i \lambda_i \\[.5cm]
          \mbox{subject to} & &\vec{\lambda}\succeq 0, \quad \sum_{i} \lambda_i \, y_i = 0
        \end{eqnarray*}
      }
  \end{itemize}
\end{frame}

\subsection{Kernel Functions}

\begin{frame}
  \frametitle{Kernel Functions}

  We now define \structure{kernel functions}:
  \begin{displaymath}
    k(\vec x, \vec x') =  \langle \phi(\vec x), \phi(\vec x') \rangle
  \end{displaymath}
  \pspread

  Typical kernel functions are: \\[.25cm]

  \small
  \begin{itemize}
    \item \structure{Linear:}
      \begin{displaymath}
        k(\vec x, \vec x') =~ \langle\vec x, \vec x'\rangle
      \end{displaymath}
    \item \structure{Polynomial:}
      \begin{displaymath}
        k(\vec x, \vec x') = (\langle\vec x, \vec x'\rangle + 1)^k
      \end{displaymath}
    \item \structure{Radial basis function:}
      \begin{displaymath}
        k(\vec x, \vec x') = e^{-\gamma\|\vec x- \vec x'\|_2^2}
      \end{displaymath}
    \item \structure{Sigmoid kernel:}
      \begin{displaymath}
        k(\vec x, \vec x') =\mbox{tanh}({ \alpha \langle\vec x, \vec x'\rangle + \beta})
      \end{displaymath}
  \end{itemize}
\end{frame}

\subsection{Lessons Learned}

\begin{frame}
  \frametitle{Lessons Learned}

  \begin{itemize}
    \item Lagrangian formulation of the hard and soft margin problems \\[.5cm]
    \item Lagrange dual representation \\[.5cm]
    \item Idea of feature transforms
  \end{itemize}
\end{frame}

\input{nextTime.tex}

\subsection{Further Readings}

\begin{frame}
  \frametitle{Further Readings}

  \begin{itemize}
    \item Bernhard Sch{\"o}lkopf, Alexander J. Smola: \\
      \structure{Learning with Kernels}, \\
      The MIT Press, Cambridge, 2003. \\[.15cm]
    \item Vladimir N. Vapnik: \\
      \structure{The Nature of Statistical Learning Theory}, \\
      Information Science and Statistics, Springer, Heidelberg, 2000. \\[.15cm]
    \item S.~Boyd, L.~Vandenberghe: \\
      \structure{Convex Optimization}, \\
      Cambridge University Press, 2004. \\
      \point{\small \url{http://www.stanford.edu/~boyd/cvxbook/}} \\[.15cm]
    \item Christopher M.\ Bishop: \\
      \structure{Pattern Recognition and Machine Learning}, \\
      Springer, New York, 2006
  \end{itemize}
\end{frame}

\subsection{Comprehensive Questions}

\begin{frame}
  \frametitle{Comprehensive Questions}

  \begin{itemize}
    \item What is the Lagrangian of the hard margin SVM? \\[1cm]
    \item What are the KKT optimality conditions for the hard margin SVM? \\[1cm]
    \item How do we apply the KKT conditions to the Lagrange Dual? \\[1cm]
    \item What can we conclude from this reformulated Lagrange Dual?
  \end{itemize}
\end{frame}


% !TEX root = main.tex

%-------------------------------------------------
\section{Maximum likelihood estimators}\label{sec:mle}

% defn: MME
\begin{definition}
An estimator $T = T(\mathbf{X})$ is said to be a \emph{maximum likelihood estimator} (MLE) of $\theta$ if
\[
T = \argmax_{\theta\in\Theta} L(\theta;\mathbf{X}),
\]
which means that $T$ is a value of $\theta$ which maximises the likelihood function $L(\theta,\mathbf{X})$ over the parameter space $\Theta$.
%
%Given $\mathbf{X}=\mathbf{x}$, a \emph{maximum likelihood estimate} of the true parameter value is one that maximises the likelihood function over the parameter space $\Theta$. This can be written as
%\[
%T_{\text{\tiny{MLE}}}(\mathbf{x})
%	= \argmax_{\theta\in\Theta} L(\theta;\mathbf{x})
%%	= \argmax_{\theta\in\Theta} \prod_{i=1}^n f(x_i;\theta).
%\]
%The estimator $T_{\text{\tiny{MLE}}}(\mathbf{X})$ is called a \emph{maximum likelihood estimator} (MLE) of $\theta$.
\end{definition}

%%-----------------------------
%\subsection{Log-likelihood}
%
To find a value of $\theta$ that maximises $L(\theta;\mathbf{X})$ we might differentiate it with respect to $\theta$, then set the resulting expression to zero and solve for $\theta$. Computing derivatives of products such as $\prod_{i=1}^n f(X_i;\theta)$ is not always straightforward however, so we work with the \emph{log-likelihood} function whenever possible.

\begin{definition}
Given $\mathbf{X}=\mathbf{x}$, the \emph{log-likelihood function} $\ell:\Theta\to [0,\infty)$ is
\[
\ell(\theta;\mathbf{x}) = \log L(\theta;\mathbf{x}) = \displaystyle\sum_{i=1}^n \log f(x_i;\theta).% \quad\text{for $\theta\in\Theta$.}
\]
\end{definition}
%Because log is a one-to-one function, there is no loss of information in considering $\ell(\theta)$ instead of $L(\theta)$.
%
%
%\begin{definition}
%$\log L(\theta;\mathbf{x})$ is called the \emph{log-likelihood function} of $\theta$, which we denote by
%\[
%\ell(\theta;\mathbf{x})
%%	= \log L(\theta;\mathbf{x})
%	= \sum_{i=1}^n \log f(x_i;\theta)
%\]
%\end{definition}

Because $\log$ is a strictly increasing function, a value of $\theta$ that maximizes $\ell(\theta;\mathbf{x})$ coincides with a value of $\theta$ that maximizes $L(\theta;\mathbf{x})$. A maximum likelihood estimate of $\theta=(\theta_1,\theta_2,\ldots,\theta_k)$ can therefore be obtained by solving the equations
\[
\frac{\partial\ell}{\partial\theta_1}=0,
\quad
\frac{\partial\ell}{\partial\theta_2}=0,
\quad\ldots\quad,
\frac{\partial\ell}{\partial\theta_k}=0.
\]

% example: MLE for bernoulli
\begin{example}
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $\text{Bernoulli}(\theta)$ distribution where $0<\theta<1$ is unknown. Find the MLE of $\theta$.
\begin{solution}
Let $\boldx = (x_1,x_2,\ldots,x_n)$ be a realisation of the sample.
\bit
\it PMF: $ f(x_i;\theta) = \theta^{x_i} (1-\theta)^{1-x_i}$ for $x_i\in\{0,1\}$.
\it Likelihood function: $ L(\theta) = L(\theta;\mathbf{x}) = \prod_{i=1}^n \theta^{x_i} (1-\theta)^{1-x_i}$
\it Log-likelihood: $ \ell(\theta) = \sum_{i=1}^n\big[ x_i\log\theta + (1-x_i)\log(1-\theta) \big]$
\it First derivative: $ \ell'(\theta) = \frac{1}{\theta(1-\theta)}\sum_{i=1}^n (x_i - \theta)$
\it Setting $\ell'(\theta)=0$, we obtain $ \theta = \frac{1}{n}\sum_{i=1}^n x_i$.
\it Second derivative:
$ \ell''(\theta) = -\sum_{i=1}^n \frac{x_i(1-\theta)^2 + (1-x_i)\theta^2}{\theta^2(1-\theta)^2} < 0$ because $x_i\in\{0,1\}$ and $0<\theta<1$.
\it Since $\ell''(\theta) < 0$ for all $\theta>0$, the turning point is indeed a maximum.
\it The MLE of $\theta$ is therefore the proportion of successes in $n$ trials:
\[
T = \frac{1}{n}\sum_{i=1}^n X_i.
\]
\eit
%\textbf{Note}: in the expression for $\ell''(\theta)$ each summand is either $1/\theta^2$ (when $x_i=0$) or $1/(1-\theta)^2$ (when $x_i=1$), and hence greater than one. Thus $\ell''(\theta) < -n$, so the maximum becomes `sharper' as $n$ increases.
\end{solution}
\end{example}

% example: MLE for uniform
\begin{example}\label{ex:mleuniform}
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $\text{Uniform}[0,\theta]$ distribution where $\theta>0$ is unknown. Find the maximum likelihood estimator of $\theta$.
\end{example}

\begin{solution}
Let $\boldx = (x_1,x_2,\ldots,x_n)$ be a realisation of the sample.

\bigskip
The PDF of the $\text{Uniform}[0,\theta]$ distribution is $f(x) = 1/\theta$ for $0 < x < \theta$ (and zero otherwise)

\bigskip
The parameter space is $\Theta = \{\theta:\theta>0\}$, and the likelihood function is
\[
L(\theta;\mathbf{x}) = \left\{\begin{array}{ll}
	\displaystyle\frac{1}{\theta^n}	& \text{for } \theta > \max\{x_1,x_2,\ldots,x_n\} \\[2ex]
	0				& \text{otherwise}.
\end{array}\right.
\]
\bit
\it $L(\theta;\mathbf{x})$ is a decreasing function of $\theta$ for all $\theta>\max\{x_1,x_2,\ldots,x_n\}$.
\it Thus the MLE of $\theta$ is $\max\{X_1,X_2,\ldots,X_n\}$.
\eit
\end{solution}

% example: mle for normal
\begin{example}
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $N(\mu,\sigma^2)$ distribution, where $\mu$ and $\sigma^2$ are both unknown. Find the MLEs of $\mu$ and $\sigma^2$.
\begin{solution}
Let $\mathbf{x} = (x_1,x_2,\ldots,x_n)$ be a realisation of the sample. The common PDF of the $X_i$ is
\[
f(x;\mu,\sigma) = \frac{1}{\sqrt{2\pi\sigma^2}}\exp\left(-\frac{1}{2}\left(\frac{x-\mu}{\sigma}\right)^2\right).
\]
The likelihood function is
\[
L(\mu,\sigma) = \prod_{i=1}^n \frac{1}{\sqrt{2\pi\sigma^2}}\exp\left(-\frac{(x_i-\mu)^2}{2\sigma^2}\right).
\]
The log-likelihood function is
\begin{align*}
\ell(\mu,\sigma)
	= \sum_{i=1}^n \log f(x_i;\mu,\sigma)
	& = \sum_{i=1}^n \left( -\log(\sqrt{2\pi}) - \frac{1}{2}\log(\sigma^2) - \frac{(x_i-\mu)^2}{2\sigma^2}\right) \\
	& = -n\log(\sqrt{2\pi}) - \frac{n}{2}\log(\sigma^2) - \frac{1}{2\sigma^2}\sum_{i=1}^n (x_i-\mu)^2
\end{align*}
The partial derivatives of $\ell(\mu,\sigma)$ with respect to $\mu$ and $\sigma$ are, respectively,
\[
\frac{\partial\ell}{\partial\mu} = \frac{1}{\sigma^2}\sum_{i=1}^n(x_i-\mu)
\qquad\text{and}\qquad
\frac{\partial\ell}{\partial\sigma} = -\frac{n}{\sigma} + \frac{1}{\sigma^3}\sum_{i=1}^n(x_i-\mu)^2.
\]
Setting these to equal zero then solving for $\mu$ and $\sigma$, we obtain
\[
\hat{\mu}_{\text{\scriptsize{MLE}}} = \frac{1}{n}\sum_{i=1}^n X_i = \Xbar
\qquad\text{and}\qquad
\hat{\sigma}^2_{\text{\scriptsize{MLE}}} = \frac{1}{n}\sum_{i=1}^n (X_i-\Xbar)^2.
\]
\bit
\it The MLE of $\mu$ is the \emph{sample mean}.
\it The MLE of $\sigma^2$ is the \emph{empirical mean squared deviation from the sample mean}.
\eit
\end{solution}
\end{example}

% exercises
\begin{exercise}
\begin{questions}

\question % mle for exponential
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $\text{Exponential}(\lambda)$ distribution, where the rate parameter $\lambda>0$ is unknown. Find the MLE of $\lambda$.
\begin{answer}
Let $\mathbf{x}=(x_1,x_2,\ldots,x_n\}$ be a realization of the sample. The PDF of the $\text{Exponential}(\lambda)$ distribution is
\[
f(x;\lambda) = \left\{\begin{array}{ll}
	\lambda\exp(-\lambda x) 	& \text{for } x>0, \\
	0						& \text{otherwise}.
\end{array}\right.
\]
Hence the likelihood function is
\[
L(\lambda) = L(\lambda;\mathbf{x})
	= \prod_{i=1}^nf(x_i;\lambda)
	= \prod_{i=1}^n\lambda e^{-\lambda x_i}
	= \lambda^n\exp\left(-\lambda\sum_{i=1}^n x_i\right).
\]
and the log-likelihood function is therefore
\[
\ell(\lambda) = n\log\lambda - \lambda\sum_{i=1}^nx_i
\]
The first derivative of the log-likelihood function (with respect to $\lambda$) is
\[
\ell'(\lambda) = \frac{n}{\lambda } -\sum_{i=1}^{n}x_{i},
\]
and setting this equal to zero we obtain
\[
\lambda =\left(\frac{1}{n}\sum_{i=1}^n x_i\right)^{-1}.
\]
The second derivative of $\ell(\lambda)$ is
\[
\ell''(\lambda) = -\frac{n}{\lambda^2} < 0 \text{\quad for all $\lambda > 0$.}
\]
Hence the turning point is a maximum, so the MLE is $\hat{\lambda}_{\text{\scriptsize{MLE}}} = \bar{X}^{-1}$.

\end{answer}

\question % mle for Poisson
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $\text{Poisson}(\lambda)$ distribution, where $\lambda>0$ is unknown. Find the MLE of $\lambda$.
\begin{answer}
Let $\mathbf{x}=(x_1,x_2,\ldots,x_n\}$ be a realization of the sample. The PMF of the $\text{Poisson}(\lambda)$ distribution is
\[
f(x;\lambda) = \left\{\begin{array}{ll}
	\displaystyle\frac{\lambda^x\exp(-\lambda)}{x!} 	& \text{for } x=0,1,2,3,\ldots \\[2ex]
	0									& \text{otherwise}.
\end{array}\right.
\]
The likelihood function is
\[
L(\lambda) = L(\lambda;\mathbf{x})
	= \prod_{i=1}^n f(x_i ;\lambda )
	= \frac{\lambda^{\sum_{i=1}^n x_i}\exp(-n\lambda)}{\prod_{i=1}^n x_i!},
\]
and the log-likelihood function is therefore
\[
\ell(\lambda)= \left(\sum_{i=1}^n x_i\right)\log\lambda - n{\lambda}- \sum_{i=1}^n\log(x_i!)
\]
The first derivative of $\ell(\lambda)$ with respect to $\lambda$ is
\[
\ell'(\lambda) = \frac{\sum_{i=1}^n x_i}{\lambda} - n.
\]
Setting this equal to zero, we obtain
\[
\lambda = \frac{1}{n}\sum_{i=1}^n x_i.
\]
The second derivative is
\[
\ell''(\lambda) = \frac{-\sum_{i=1}^n x_i}{\lambda ^{2}} < 0 \text{\quad for all $\lambda > 0$.}
\]
Hence the turning point is a maximum, so the MLE is $\hat{\lambda}_{\text{\scriptsize{MLE}}} = \bar{X}$.
\end{answer}

\question % mle for binomial
Let $X$ be a single observation from the $\text{Binomial}(n,\theta)$ distribution, where $n$ is known but $\theta$ is unknown. Find the MLE of $\theta$.

\begin{answer}
The likelihood function for the observation $X=k$ is
\[
L(\theta) = L(\theta; k) = \binom{n}{\theta} \theta^{k}(1 - \theta)^{n-k}
\]
and the log-likelihood is
\[
\ell(\theta) = \log \binom{n}{k} + k\log \theta + (n - k)\log(1 - \theta).
\]
Taking the derivative of $\ell(\theta)$ with respect to the $\theta$,
\[
\ell'(\theta) = \frac{k}{\theta} -\frac{(n-k)}{(1-\theta)}
\]
Setting this to zero,
\[
\frac{k(1-\theta)-(n-k)\theta}{\theta(1-\theta)} = 0 \quad\Rightarrow\quad \theta=\frac{k}{n}.
\]
The second derivative of $\ell(\theta)$ is
\[
\ell''(\theta) =  \frac{-k}{\theta^{2}} - \frac{(n-k)}{(1-\theta)^{2}}.
\]
Because $k\leq n$, this is always negative, so the turning point is a maximum. Hence the MLE of $\theta$ is
\[
\hat{\theta}(X) = \frac{X}{n},
\]
which is the observed proportion of successes.
\end{answer}

\question % mle for geometric
Let $X_1,X_2,\ldots,X_n$ be a random sample from the $\text{Geometric}(\theta)$ distribution. Find the MLE of $\theta$.
\begin{answer}
Let $X\sim\text{Geometric}(\theta)$ distribution. The PMF of $X$ is
\[
f(x;\theta) = \theta(1-\theta)^{x-1}\quad x = 1,2,\ldots \text{\quad (and zero otherwise).}
\]
Let $(x_1,x_2,\ldots,x_n)$ be a realisation of the sample. The likelihood function is
\[
L(\theta) = L(\theta;x_1,\ldots,x_n)
	= \prod_{i=1}^n f(x_i;\theta)
	= \prod_{i=1}^n \theta(1-\theta)^{x_i-1}
	= \theta^n(1-\theta)^{\sum_{i=1}^{n}(k_i-1)}.
\]
The log-likelihood function is
\[
\ell(\theta) = n\log\theta +  \log(1-\theta)\sum_{i=1}^{n}(k_i-1).
\]
The first derivative of $\ell(\theta)$ is
\[
\ell'(\theta) = \frac{n}{\theta} - \frac{\sum_{i=1}^{n}(k_i-1)}{1-\theta}.
\]
Setting this equal to zero, we obtain
\[
\theta = \left(\frac{1}{n}\sum_{i=1}^n k_i\right)^{-1}.
\]
Hence the maximum likelihood estimator of $\theta$ is $\bar{X}^{-1}$. This makes sense: the longer we wait until the first success, the lower our estimate of the probability of success.
\end{answer}

\question % ordinary-least-squares method
\textbf{Simple Linear Model}. Let $X$ and $Y$ be two random variables, and consider the simple linear model:
\[
Y = \alpha + \beta X + \epsilon \quad\text{where}\quad \epsilon\sim N(0,\sigma^2)
\]
where $\alpha$, $\beta$ and $\sigma^2>0$ are unknown parameters and the \emph{error variable} $\epsilon$ is independent of $X$. Let $(X_1,Y_1),(X_2,Y_2),\ldots,(X_n,Y_n)$ be a random sample of observations from the joint distribution of $X$ and $Y$.
\begin{parts}
\part
Show that the maximum likelihood estimators of $\alpha$ and $\beta$ are
\[
\hat{\alpha} = \bar{Y}-\hat{\beta}\bar{X}
\quad\text{and}\quad
\hat{\beta} = \frac{\sum_{i=1}^n (X_i-\bar{X})(Y_i-\bar{Y})}{\sum_{i=1}^n (X_i-\bar{X})^2}
\]
respectively.
\begin{answer}
If we observe $X=x$ we have that $Y\sim N(\alpha+\beta x, \sigma^2)$, so
\[
f(y;\alpha,\beta) = \frac{1}{\sqrt{2\pi\sigma^2}}\exp\left[-\frac{1}{2}\left(\frac{y-(\alpha+\beta x)}{\sigma}\right)^2\right]
\]
In particular,
\[
\expe(Y|X=x) = \alpha + \beta x \quad\text{and}\quad \var(Y|X=x) = \sigma^2.
\]
Let $\{(x_1,y_1),(x_2,y_2),\ldots,(x_n,y_n)\}$ be a realisation of the sample. The likelihood function and log-likelihood functions are
\begin{align*}
L(\alpha,\beta,\sigma^2)
	& = \left(\frac{1}{\sqrt{2\pi\sigma^2}}\right)^n \exp\left[-\frac{1}{2\sigma^2}\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big]^2\right].
\ell(\alpha,\beta,\sigma^2) \\
	& = - \frac{n}{2}\log(2\pi\sigma^2) - \frac{1}{2\sigma^2}\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big]^2.
\end{align*}
The MLE estimates of $\alpha$ and $\beta$ are obtained by the \emph{method of least squares}, which is to minimise the sum of squared errors
\[
S(\alpha,\beta) = \sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big]^2
\]
% partial derivatives
The partial derivatives of $S(\alpha,\beta)$ with respect to $\alpha$ and $\beta$ are
\begin{align*}
\frac{\partial S}{\partial\alpha}
	& = 2\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big] (-1) \\
\frac{\partial S}{\partial\beta}
	& = 2\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big] (-x_i) \\
\end{align*}
To find the MLEs of $\alpha$ and $\beta$, we set the partial derivatives to equal zero.
% alpha
\begin{align*}
\frac{\partial H}{\partial\alpha} = 0
	& \ \Rightarrow\  \sum_{i=1}^n y_i- n\alpha -\beta\sum_{i=1}^n x_i = 0 \\
	& \ \Rightarrow\  n\alpha  = \sum_{i=1}^n y_i- \beta\sum_{i=1}^n x_i \\
	& \ \Rightarrow\  \alpha = \bar{y}-\beta\bar{x}.
\end{align*}
% beta
\begin{align*}
\frac{\partial H}{\partial\beta} =0
	& \ \Rightarrow\  \sum_{i=1}^n x_i y_i - \alpha\sum_{i=1}^n x_i -\beta\sum_{i=1}^n x_i^2 = 0 \\
	& \ \Rightarrow\  \sum_{i=1}^n x_i y_i - (\bar{y}-\beta\bar{x})\sum_{i=1}^n x_i -\beta\sum_{i=1}^n x_i^2 = 0 \\
	& \ \Rightarrow\  \sum_{i=1}^n x_i(y_i-\bar{y}) - \beta\sum_{i=1}^n x_i(x_i-\bar{x}) = 0 \\
	& \ \Rightarrow\  \sum_{i=1}^n (x_i-\bar{x})(y_i-\bar{y}) - \beta\sum_{i=1}^n (x_i-\bar{x})^2 = 0 \\
	& \ \Rightarrow\  \beta = \frac{\sum_{i=1}^n (x_i-\bar{x})(y_i-\bar{y})}{\sum_{i=1}^n (x_i-\bar{x})^2}
\end{align*}
The maximum-likelihood estimators of $\alpha$ and $\beta$ are therefore
\[
\hat{\alpha} = \bar{Y}-\hat{\beta}\bar{X}
\text{\quad and\quad}
\hat{\beta} = \frac{\sum_{i=1}^n (X_i-\bar{X})(Y_i-\bar{Y})}{\sum_{i=1}^n (X_i-\bar{X})^2}.
\]
\end{answer}

\part % mle for residual variance
Show that the MLE of the error variance $\sigma^2$ is
\[
\hat{\sigma}^2 = \frac{1}{n}\sum_{i=1}^n \hat{\epsilon}_i^2
\text{\quad where\quad}
\hat{\epsilon}_i = Y_i-(\hat{\alpha}+\hat{\beta} X_i).
\]
\begin{answer}
Recall the log-likelihood function:
\[
\ell(\alpha,\beta,\sigma^2)
	= \frac{n}{2}\log(2\pi\sigma^2) + \frac{1}{2\sigma^2}\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big]^2.
\]
The first partial derivative of $\ell(\alpha,\beta,\sigma^2)$ with respect to $\sigma^2$ is
\[
\frac{\partial\ell}{\partial(\sigma^2)}
	= \frac{n}{2\sigma^2} - \frac{1}{2(\sigma^2)^2}\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big]^2.
\]
Setting this equal to zero,
\[
\sigma^2 = \frac{1}{n}\sum_{i=1}^n \big[y_i-(\alpha+\beta x_i)\big]^2.
\]
Substituting our MLEs for $\alpha$ and $\beta$ we obtain the MLE
\[
\hat{\sigma^2} = \frac{1}{n}\sum_{i=1}^n \hat{\epsilon}_i^2 \text{\quad where\quad} \hat{\epsilon}_i = Y_i-(\hat{\alpha}+\hat{\beta} x_i)
\]
as required.
\end{answer}
\end{parts}

\end{questions}
\end{exercise}


\lab{Applications}{Web Page Experiments}{Web Page Experiments}
\objective{This lab applies multi-armed bandit problems to web page experiments.}

\section*{Web Page Experiments}
One application of the multi-armed bandit problem is in web page design.
Bandit problems provide a way to compare the success of different variations of a web page.
Suppose a business wants to test new versions of a web page.
The goal of the page might be to get the user to click a certain link, make a purchase, etc.
When the user does this, we call it a conversion.  The proportion of web page visits
that results in a conversion is called the conversion rate, or CvR.
The website designer wants to determine which variation of the web page has the best CvR.

We can model this situation as a bandit problem by considering each page as a different arm.
Each page has some unknown probability (the CvR) that a user will perform the desired action.
The company then wants to experiment with giving different users different versions of
the page in order to determine which variation is most successful.

This method is the same that is used by Google Analytics.  Take a moment to skim their description located here:
\url{http://analytics.blogspot.com/2013/01/multi-armed-bandit-experiments.html}.

In this lab we will apply the Thompson Sampling method from the previous lab (the same method used by Google)
to solve this problem.  We will simulate the results and attempt to replicate Google's results found at the website above.

\section*{The Experiment}
Here we describe how we will design our web page experiment using bandits and how we will simulate it.
We will have some number of variations of a web page, $n$.  Each day, the web-page will receive 100 visitors
(for the purposes of our simulation).  Twice each day, at the beginning and after 50 visits,
we will compute the number of each variation to deliver to the next 50 visitors using the weights
method described in the previous lab (you should have written a function that does this in Problem 4
of the previous algorithms lab).

\begin{problem}
Write a function that simulates the experiment for one day.
The function should accept a vector of length $n$ of the true probabilities (CvR)
for different web page variations.  It should also accept the state of the variations
at the beginning of the day; this is an $n \times 2$ array with the number of previous
successes plus one in the first column and the number of previous failures plus one in
the second column (so if it is the first day the state for any arm would be $(1,1)$
corresponding to a $Beta(1,1)$ distribution).

The function could be outlined like this: first compute how many times each variation
should be used in the next 50 visits.  When determining the weights you will need to use
the \li{sim_data} function.  Use 100 as the number of draws here and throughout this lab.
Then ``visit'' each page the number of times given by the weights.
Whether each visit results in a conversion (1) or not (0) can be randomly determined using the following function:
\begin{lstlisting}
import scipy as sp
def pull(p):
    return sp.random.binomial(1,p,size = None).
\end{lstlisting}
This will return a random one or zero based on the probability input \li{p}.

After the first 50 visits, update the state of each arm.
Then recompute the weights and do the same for the next 50 visits, resulting in 100 visits total.

This function should return the resulting states and their corresponding weights.
\end{problem}

In this manner we will continue from day to day, always updating the state of each arm
(the beta distribution for the CvR of each arm).  We will have three criteria to determine when to stop the experiment.
The first, is that the experiment must run at least two weeks to make sure the
results are not overly influenced by a small number of random draws.

The second stopping criteria is that there be a $95\%$ probability that one of the variations is the best variation.
This is the same as saying that, of the weights for each variation, the largest is greater than $.95$.

It may seem that these two criteria should be enough; however, in some cases, the
experiment could last a very long time using just these criteria.
For example, consider the case that two of the web page variations have nearly the same CvR.
In this case it will be very difficult to determine which is best.
It will also not be very important since the results are so similar.
Thus we will use a measure that we will call the potential value remaining
in the experiment as the third criteria.  The value remaining is computed by
simulating many draws for each arm.  Using this data, the potential value remaining
for arm $i$ is obtained by computing the following for each simulated data point:
\begin{equation}\label{valrem}
\frac{\theta_{max} - \theta^*}{\theta^*}
\end{equation}
where $\theta_{max}$ is the largest value for the random draw and $\theta^*$ is the
value of the arm that is currently believed to be the best
(the arm with the highest weighting, or probability of being optimal).
The result is some distribution of numbers between $0$ and $1$ that we can think
of as the distribution of value remaining.  For example, if $50\%$ of the numbers are 0,
then about $50\%$ of the time the arm that is currently believed optimal will perform the best.
The potential value remaining is the $95$th percentile of this distribution.
If the potential value remaining were $.2$, we could interpret it as meaning
that there is about a $5\%$ chance that another arm beats the current best arm by $.2$ or more.
We stop the experiment if this value is less than $1\%$ of the current best arm's CvR.
This way we stop the experiment if there seems to be little chance of improvement over
the current best arm, regardless of whether we've met the $95\%$ tolerance for the weights.

The value remaining can be computed using the following code:
\begin{lstlisting}
import scipy as sp
def val_remaining(data,prob):
    champ_ind = sp.argmax(prob)
    thetaM = sp.amax(data,1)
    valrem = (thetaM - data[:,champ_ind])/data[:,champ_ind]
    pvr = sp.stats.mstats.mquantiles(valrem,.95)
    return valrem,pvr
\end{lstlisting}
where data is simulated using the \li{sim_data} function from the previous algorithms lab,
and prob is a vector containing the probabilities that each arm is optimal
(also computed using a function from the previous lab).

\begin{problem}
Write a function that simulates the problem described above, using the stopping criteria described above.
The function should accept a vector of the true probabilities of the arms.
It should use the function from the previous problem to simulate each day and
continue until the stopping criteria are met.

The function should return the state of each arm (i.e. an $n$ by $2$ matrix with the
successes and failures of each arm), a matrix that contains the weights assigned to
each arm each day, the index of the winning variation, and the number of days it took to converge.

Your code should have a while loop that checks for the stopping criteria after each day.
It might look something like this:
\begin{lstlisting}
while ((delta < p_tol) and (champ_cvr/100. < v_quant)) or days < 14:
\end{lstlisting}
where \li{delta} is the largest weight for the current day, meaning if there were two arms and you determined the weights to be .9 and .1, then delta would be .9.  We stop when the largest weight is greater than .95, so \li{p_tol} is .95.  The variables \li{champ_cvr} and \li{v_quant} describe the stop mechanism that accounts for the potential value remaining.  First, \li{champ_cvr} is the conversion rate of the current best arm over the course of the experiment.  So if it has been used 100 times with 4 successes, then \li{champ_cvr} would be .04.  The variable \li{v_quant} is the potential value remaining, i.e. the $95$th percentile of the value remaining distribution described above.  The variable \li{pvr} can be computed using the \li{val_remaining} function given above.
The \li{days} variable simply keeps track of how many days the experiment has been running.
\end{problem}

Now let's see how our bandit performs with specific examples.

\begin{problem}
Suppose a web page has two variations and the true CvR of the original is
$.04$ and the true CvR of the new variation is $.05$.
Create a plot similar to \ref{fig:weights1} that shows how the weights
assigned to the pages changes from day to day until the optimal page is chosen and the experiment stops.

Next run the same simulation 200 times and keep track of how many days
the experiment took in each case.  Create a histogram that shows the
number of days it takes to complete the experiment.
The following code will create such a histogram:
\begin{lstlisting}
import scipy as sp
from matplotlib import pyplot as plt
hist, bins = sp.histogram(dayvec, bins = 12)
width = (bins[1]-bins[0])
center = (bins[:-1]+bins[1:]) / 2
plt.bar(center, hist, align = 'center', width = width, color = 'g')
plt.show()
\end{lstlisting}
where \li{dayvec} is a vector containing the number of days each simulation took to complete.
Also track which arm is determined to be optimal in each simulation.
What percent of the time did the bandit find the optimal arm?

Create the same two types of plots, this time with six variations having
weights $.04,.02,.03,.035,.045,.05$.  This time only run the simulation 100 times.
What percent of the time did the bandit find the optimal arm in this case?
\end{problem}

\begin{figure}[h]
\centering

\begin{subfigure}[t]{.49\textwidth}
\centering
\includegraphics[width=\textwidth]{weights1.pdf}
\caption{Optimal arm probabilities in the two arm case}
\label{fig:weights1}
\end{subfigure}

\begin{subfigure}[t]{.49\textwidth}
\centering
\includegraphics[width=\textwidth]{weights2.pdf}
\caption{Optimal arm probabilities in the six arm case}
\label{fig:weights2}
\end{subfigure}
\end{figure}

% \begin{figure}[h]
% \centering
% \includegraphics[width=\textwidth]{weights2.pdf}
% \caption{Optimal arm probabilities in the six arm case}
% \label{fig:weights2}
% \end{figure}

\section*{Comparison with Classical Tests}
A more classical approach to this problem would be to split traffic between each
variation for a predetermined amount of time, which should give enough data to
determine the best arm with some level of confidence.
Using the bandit approach described here has significant advantages over a classical test.
There are two main reasons why the bandit approach is more efficient.

The first reason is that the bandit method generally converges more quickly.
A standard test would require splitting the web page views between the different
variations over a long period of time.  According to Google's explanation in the
website mentioned at the beginning of this lab, the two arm case would take 223 days
and the 6 arm case would take 919 days.  The results from the simulations you performed
should show that on average the bandit method finishes much faster.
There are other ways we could choose our stopping criteria that may result in even shorter experiment times.
In general, we can always adjust the tolerance of our stopping criteria to shorten experiment time or increase accuracy.

The second reason the bandit approach is more efficient is that, as we gain more information,
we allocate more visits to the variation that we believe has a better CvR.
In the classical method we would split the visits evenly until the end of the experiment.
This way we gain many more conversions during testing than we would using classical tests.


\section{Transformations and Expectations}

\subsection{Transformations}

If $X$ and $Y$ are discrete random variables, with $Y = g(X)$, then
\begin{equation}
    f_Y(y) = \sum_{\{x: g(x) = y\}} f_X(x).
\end{equation}

For the remainder of this section we will take $X$ and $Y$ to be continuous random variables, with $Y = g(X)$. As such, we will define the following sets $\X{} = \{x: f_X(x) > 0\}$ and $\Y{} = \{y: y =g(x) \quad x \in S \subseteq \X{} \}$.\\

\begin{theorem}
    Let $X$ have cdf $F_X(s)$, let $Y = g(X)$ and let $\X{}$ and $\Y{}$ be defined as above. Then
    \begin{enumerate}[a.]
        \item If g is increasing on $\X{}$, then $F_Y(y) = F_X(g^{-1}(y))$ for $y \in \Y{}$.
        \item If g is decreasing on $\X{}$, then $F_Y(y) = 1 - F_X(g^{-1}(y))$ for $y \in \Y{}$.
    \end{enumerate}
\end{theorem}

\begin{theorem}
    Let $X$ have pdf $f_X(x)$ and $Y = g(X)$, where $g$ is monotone. Define $\X{}$ and $\Y{}$ as above. Suppose $f_X(x)$ is continuous on $\X{}$ and $g^{-1}(y)$ has continuous first derivative on $\Y{}$. Then the pdf of $Y$ is given by:
    \begin{equation*}
        f_Y(y) = \1{y \in \Y{}} f_X(g^{-1}(y)) \abs{ \frac{\d{}}{\d{}y} g^{-1}(y) }
    \end{equation*}
\end{theorem}

If $g$ is not globally monotone, then we just partition $\X{}$ into subsets on which $g$ is continuous and monotone and sum the results. If such a partition doesn't exist, then we have technical problems.\\

\begin{theorem}[Probability integral transform]
    Let $X$ have continuous cdf $F_X(x)$ and define the random variable $Y$ by $Y = F_X(X)$. Then $Y$ is uniformly distributed on $(0, 1)$.
\end{theorem}

If $F_X(x) = y$ is constant on some interval then we define the inverse by
\[
    F_X^{-1}(y) = \inf \{x: F_X(x) = y\}.
\]

\subsection{Expectations}

\begin{definition}[Expected value]
    The \emph{expected value} of a random variable $g(X)$, denoted $\E{}[X]$ is defined by:
    \begin{itemize}[-]
        \item $\E{}[X] = \int_{\R{}}g(x)f_X(x)\d{}x$ if $X$ is continuous,
        \item $\E{}[X] = \sum_{x \in \X{}}g(x)\P{}(X = x)$ if $X$ is discrete.
    \end{itemize}
\end{definition}

\begin{theorem}[Properties of expectation]
    \begin{enumerate}[a.]
        \item Linearity.
        \item $g_1(x) \geq g_2(x) \,\, \forall x \quad \implies \quad \E{}[g_1(X)] \geq \E{}[g_2(X)]$
        \item $a \leq g(x) \leq b \,\, \forall x \quad \implies \quad a \leq \E[g(X)] \leq b $
        \item $\argmin_{c}\E{}[(X - c)^2] = \E{}[X]$
    \end{enumerate}
\end{theorem}

\subsection{Moments}

\begin{definition}[Moment]
    For integer $n$, the $n^{\mathrm{th}}$ \emph{moment} of $X$ is
    \[
        \mu_n' = \E{}[X^n].
    \]
    The $n^{\mathrm{th}}$ \emph{central moment}, $\mu_n$ is
    \[
        \mu_n = \E[(X - \mu)^n].
    \]
    Where $\mu = \mu_1 = \E{}[X]$
\end{definition}

\subsubsection{Variance}
\begin{definition}[Variance]
    The \emph{variance} of a random variable $X$, written $\Var{}[X]$ is the second central moment of $X$,
    \[
        \Var{}[X] = \E{}[(X - \E{}[X])^2].
    \]
    The \emph{standard deviation} of $X$, denoted $\sigma_X$, is given by $\sigma_X = \sqrt{\Var{}[X]}$.
\end{definition}

\begin{theorem}[Properties of variance]
    If $X$ has finite variance then:
    \begin{enumerate}[a.]
        \item $\Var{}[aX + b] = a^2\Var{}[X]$
        \item $\Var{}[X] = \E{}[X^2] - \E{}[X]^2$
    \end{enumerate}
\end{theorem}

\subsubsection{Moment Generating Functions}
\begin{definition}[Moment generating function]
    Let $X$ be a random variable with cdf $F_X$. The \emph{moment generating function (mgf)} of $X$, denoted $M_X(t)$, is given by
    \[
        M_X(t) = \E{}[e^{tX}]
    \]
    provided that the expectation exists for $t$ in some (open) neighbourhood of 0 (otherwise we say the mgf does not exist).
\end{definition}

\begin{remark}
    The mgf is the Laplace transform of the pdf.
\end{remark}

\begin{theorem}
    If $X$ has mgf $M_X(t)$ then
    \[
        \E{}[X^n] = \left. \frac{\d}{\d t}M_X(t)\right|_{t=0}
    \]
\end{theorem}

The mgf can be used to calculate moments, but its principal utility is in characterising a distribution. This relationship can run into some technical difficulties. If the mgf exists, it characterises an infinite set of moments. However, it is possible for two distinct random variables to give rise to the same set of moments.\\

The problem of uniqueness of moments does not occur if the random variables have bounded support (in this case an infinite sequence of moments uniquely determines the distribution). Further, if the mgf exists in a neighbourhood of 0 then it uniquely determines the distribution, no matter the support. Thus, existence of an infinite set of moments is not equivalent to the existence of the mgf. We have the following theorem, describing when the mgf determines the distribution.

\begin{theorem}[When mgf determines distribution]
    Let $F_X(x)$ and $F_Y(y)$ be two cdfs all of whose moments exist.
    \begin{enumerate}[a.]
        \item If $X$ and $Y$ have bounded support then $F_X(u) = F_Y(u) \,\, \forall u$ if and only if $\E{}[X^r] = \E{}[Y^r] \,\, \forall r \in \N$. (So the cdfs are equal if and only if all the moments agree.)
        \item If the mgfs exist and are identical in some neighbourhood of 0 then the cdfs are equal.
    \end{enumerate}
\end{theorem}

\begin{theorem}[Convergence of mgfs near 0 implies convergence of cdfs]
    Suppose $\{X_i, i=1, 2, \dots$\} is a sequence of random variables, each with mgf $M_{X_i}(t)$. Suppose also that for all $t$ in a neighbourhood of 0
    \[
        lim_{i \to  \infty} M_{X_i} = M_X(t)
    \]
    where $M_X(t)$ is an mgf. Then there is a unique cdf $F_X$ whose moments are determined by $M_X(t)$ and, for all $x$ at which $F_X(x)$ is continuous, we have
    \[
        lim_{i\to\infty} F_{X_i}(x) = F_X(x).
    \]
\end{theorem}

\begin{remark}
    The convergence of a sequence of moments is not enough to show the convergence of random variables. We need the moment sequence to be unique too. However, if the mgfs converge in a neighbourhood of 0 as above, then we know that the random variables converge. Convergence of mgfs is therefore a sufficient, but not necessary, condition for convergence of the random variables.
\end{remark}

\begin{theorem}
    For any constants $a$ and $b$
    \[
        M_{aX + b}(t) = e^{bt}M_X(at)\
    \]
\end{theorem}

\subsection{Other Generating Functions}

\begin{definition}[Cumulant generating function]
    The \emph{cumulant generating function} is $\log(M_X(t))$. The \emph{cumulants} of $X$ are defined as the coefficients of the Taylor series of this function.
\end{definition}

\begin{definition}[Factorial moment generating function]
    The \emph{factorial moment generating function} is $\E{}[t^X]$. The name comes from
    \[
        \left. \frac{\d^r}{\d t^r} \E{}[t^X] \right|_{t=1} = \E{}[X(X-1)\cdots(X-r+1).
    \]
    For discrete distributions this is the \emph{probability generating function} and the coefficients of the power series give the probabilities
    \[
        \left. \frac{1}{k!} \frac{\d^k}{\d t^k} \E{}[t^X] \right|_{t=0}  = \P{}(X=k).
    \]
\end{definition}

\begin{definition}[Characteristic function]
    The \emph{characteristic function} of a random variable $X$ is
    \[
        \phi_X(t) = \E{}[e^{itX}]
    \]
\end{definition}

\begin{remark}
    The characteristic function is the most useful of the generating functions. Every cdf has a unique characteristic function. When the moments of the cdf exist, the characteristic function can be used to calculate them.
\end{remark}


%---------------------------Shear-----------------------------
\section{Shear\label{s:quad-shear}}

The shear metric
\[
q = \min \left\{ \frac {\alpha_0} {\normvec{L_0} \normvec{L_3}},
                 \frac {\alpha_1} {\normvec{L_1} \normvec{L_0}},
                 \frac {\alpha_2} {\normvec{L_2} \normvec{L_1}},
                 \frac {\alpha_3} {\normvec{L_3} \normvec{L_2}} \right\}
\]
is the same as the scaled Jacobian, except that it has a truncated range.

Note that if $\alpha_i < DBL\_MIN$ or any edge has length $L < DBL\_MIN$, we set $q = 0$.

\quadmetrictable{shear}%
{$1$}%                                      Dimension
{$[0.3,1]$}%                                Acceptable range
{$[0,1]$}%                                  Normal range
{$[0,1]$}%                                  Full range
{$1$}%                                      Unit square
{\cite{knu:03}}%                            Citation
{v\_quad\_shear}%                           Verdict function name


%% ps1_q3.tex
\section{ Is it an isomorphism? }

Yes. Since $g \circ f : c \rightarrow c$ and $f \circ g : d \rightarrow d$ both must be valid morphisms
in the category, and the only morphisms $c \rightarrow c$ and $d \rightarrow d$ are the identities,
then $g \circ f = id_c$ and $f \circ g = id_d,$ q.e.d.


\clearpage
\subsection{Expression} % (fold)
\label{sub:expression}

Some statements need data, this data can be calculated or provided as a literal value in the code. The term \textbf{expression} is used in programming to describe the places in a statement where data must be supplied. At run time each expression becomes a value that is used by the statement.

\begin{figure}[h]
   \centering
   \includegraphics[width=\textwidth]{./topics/program-creation/diagrams/Expression}
   \caption{An expression provides a \textbf{value} to be used in a Statement.}
   \label{fig:program-creation-expression}
\end{figure}

\mynote{
\begin{itemize}
  \item An expression is a \textbf{term} given to code that calculates a value.
  \item The concepts related to expressions are shown in Figure \ref{fig:program-creation-expression}.
  \item An expression provides a \textbf{value} that is used in a Statement.
  \item The expression's value may be calculated or entered directly into the code.
  \item Calculations can use mathematical operators: + for addition, - for subtraction, * for multiplication, $/$ for division, and parenthesis ( ) for grouping.
  \item Expressions are evaluated using the BODMAS\footnote{BODMAS indicates that expressions are evaluated \textbf{B} brackets first, \textbf{O} orders (which includes powers and square roots), \textbf{DM} for division and multiplication (which are of equal precedence, and are evaluated left-to-right), then \textbf{AS} addition and subtraction (of equal precedence, evaluated left-to-right).} order of operations.
  \item Values entered directly within an expression are \textbf{Literal} values.
\end{itemize}
}

% section program (end)


\documentclass{article}
\usepackage{amsmath}
\usepackage[margin=1.0in]{geometry}
\usepackage{xcolor}

\begin{document}

\noindent
Does $\displaystyle \sum_{n=1}^\infty \frac{(-1)^n}{n+1}$
diverge, converge absolutely, or converge conditionally?

\subsection*{Important Remark}

{\color{red}If we were to use the Ratio Test, on this series, we would consider the limit
\begin{align*}
L=\lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|
= \lim_{n \to \infty} \left| \frac{(-1)^{n+1}}{n+2} \cdot \frac{n+1}{(-1)^n}\right|
= \lim_{n \to \infty} \left| \frac{(-1)(n+1)}{n+2}\right|
= \lim_{n \to \infty} \frac{n+1}{n+2}
= \lim_{n \to \infty} \frac{1}{1}
= 1
\end{align*}
with one use of L'Hopital's above. Since $L=1$, we get NO INFORMATION from the Ratio Test. (Note that later in the solution, we well get a limit of $1$, but since we're using the Limit Comparison Test, which has different requirements, we WILL get information.)}

\subsection*{Solution}

The series $\displaystyle \sum_{n=1}^\infty \frac{(-1)^n}{n+1}$ is alternating, and $b_n = |a_n| = \frac1{n+1}$. The sequence $b_n$ is decreasing and has limit $0$. So by the Alternating Series Test, the series $\displaystyle \sum_{n=1}^\infty \frac{(-1)^n}{n+1}$ converges.

To determine if $\displaystyle \sum_{n=1}^\infty \frac{(-1)^n}{n+1}$ converges absolutely or conditionally, we consider the series $\sum |a_n|$, which in this case is
\[\sum_{n=1}^\infty \frac{1}{n+1}.\]
This series can be shown to diverge using the Integral Test (with a substitution of $u=x+1$) or using the Direct Comparison Test, or using the Limit Comparison Test. We'll use the Limit Comparison Test here. Note that the series $\sum \frac1n$ diverges by the $p$-test. Let $a_n = \frac1{n+1}$ and let $b_n = \frac1n$.
\[ \lim_{n \to \infty} \frac{a_n}{b_n} = \lim_{n \to \infty} \frac{n}{n+1} = \lim_{n \to \infty} \frac11 = 1.\]
Therefore, the Limit Comparison Test applies and the series $\sum_{n=1}^\infty \frac{1}{n+1}$ diverges. Thus, the series  $\displaystyle \sum_{n=1}^\infty \frac{(-1)^n}{n+1}$  converges conditionally.

\subsection*{Epiloque}

{\color{blue} When using the Ratio Test, we got \[\displaystyle \lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|=1\] while with the Limit Comparison Test, we got \[\lim_{n \to \infty} \frac{a_n}{b_n} = 1.\]
When the limit of the sequence $\left|\frac{a_{n+1}}{a_n}\right|$ is $1$ in the Ratio Test, we get NO information. However, as long as the limit of the sequence $\frac{a_n}{b_n}$ is a finite positive number (including $1$) we DO get to conclude something in the Limit Comparison Test, simply because the requirements of the two tests are different. Don't let this freak you out too much: in any case, the two tests made you study DIFFERENT sequences anyway!
}

\end{document}%%%%%%%%%%%%%%%%%

\begin{align*}
L&=\lim_{n \to \infty} \sqrt[n]{|a_n|}\\
&= \lim_{n \to \infty} \sqrt[n]{\left| \right|}\\
\end{align*}

Since $\sum |a_n| = \sum a_n$, the series $\displaystyle \sum_{n=1}^\infty AAAAAAAAAAAAAA$ converges absolutely.

Since $|r| < 1$, the series ...  converges by the Geometric Series Test.

Since $|r| \geq 1$, the series ...  diverges by the Geometric Series Test.

The function $f(x)=\frac{}{}$ is continuous, positive, and decreasing on $[1,\infty)$.

\subsection*{Solution}


\documentclass[main.tex]{subfiles}
\begin{document}

\section*{Fri Nov 29 2019}

The Plummer sphere is defined by
%
\begin{subequations}
\begin{align}
  \rho (r) \dd[3]{r} &= \frac{3M}{4 \pi a^3} \qty(1 + \frac{r^2}{a^2})^{-5/2} \dd[3]{r}  \\
  &= \frac{3M}{4 \pi a^3} \qty(1 + \frac{r^2}{a^2})^{-5/2} r^2 \dd{r} \dd{\Omega }
\,,
\end{align}
\end{subequations}
%
and it models the mass density in a star cluster.
We can marginalize over the angles to find the pdf of the radius alone: this just amounts to multiplying by \(4 \pi \) by isotropy,
%
\begin{align}
  \rho (r) \dd{r} = \frac{3M}{a^3} \qty(1 + \frac{r^2}{a^2})^{-5/2} r^2 \dd{r}
\,,
\end{align}
%
and if we rescale the radius as \(R = r/a\) we find the PDF
%
\begin{align}
  \rho (R) \dd{R} = 3M \qty(1 + R^2)^{-5/2} R^2 \dd{R}
\,,
\end{align}
%
which can be integrated analytically: we get
%
\begin{align}
  \int_{0}^{R_{\text{max}}} \rho(R) \dd{R} = \frac{ M R^3 _{\text{max}}}{\qty(R^2 _{\text{max}} + 1)^{3/2}}
\,.
\end{align}
%

The velocities are isotropically distributed, with the probability of their moduli being described by a Maxwellian density:
%
\begin{align}
  p(v) \dd{v} = \sqrt{\frac{2}{\pi }} \frac{v^2}{\sigma^3}
  \exp( -\frac{v^2}{2 \sigma^2}) \dd{v}
\,.
\end{align}

Do note that this is \emph{not} normalized on \(\mathbb{R}\) as given: its integral is \(2\), it is normalized on \(\mathbb{R}^{+}\).

The parameters are given by: \(M = \num{e4} M_{\odot}\), \(a = \SI{5}{\parsec}\), \(\sigma = \SI{5}{km/s}\).

Here is my approach to the problem without reading the suggestions, it might not be the most efficient way to do it.

We need to be able to draw samples from three distributions: the Plummer sphere, the Maxwell distribution and the angular distribution; since both the star's positions and their velocities are isotropically distributed on the 2-sphere.
The volume element on the sphere \(S^{2}\) is given by \(\dd{A} = \sin \theta \dd{\theta } \dd{\varphi }\); so we can draw our \(\varphi \) from a uniform distribution on \([0, 2 \pi ]\), while our \(\theta \) will need to be distributed according to \(p(\theta ) \dd{\theta }= \sin \theta \dd{\theta }  \).

Since we are computing angles spanning the whole \(2\)-sphere, for both the radius \(r\) and the velocity \(v\) we do not need to simulate negative values.

This was my first approach, but then I noticed that it is really hard to sample from a distribution \(f(x) \sim x^2 \exp(-x^2)\).

For the Plummer distribution it makes sense to sample from the radial and angular distribution separately, while for the Maxwell distribution it is much easier to sample the three components of the velocity vector in cartesian coordinates. Let us see this: first, we rescale \(V = v/\sigma \). We get:
%
\begin{align}
  p(V) \dd{V} = \sqrt{\frac{2}{\pi }} \exp(- V^2/2)V^2 \dd{V}
\,,
\end{align}
%
and we can add in the uniform distribution \(1/ 4\pi \) of the angular part: we find
%
\begin{subequations}
\begin{align}
p(V) \dd{V} \dd{\Omega } &= \frac{1}{4 \pi } \sqrt{\frac{2}{\pi }} \exp(-V^2/2) V^2 \dd{V} \dd{\Omega }  \\
&= \frac{1}{(2 \pi )^{3/2}} \exp(-V^2/2) \dd[3]{V}  \\
&= \prod_{i=1}^{3} \qty(\frac{1}{\sqrt{2 \pi }} \exp(-V_i^2/2) \dd{V_i})
\,,
\end{align}
\end{subequations}
%
so we can see that in cartesian coordinates each component is just distributed according to a Gaussian.

\end{document}


\section{Connectedness}
An interval $I$ in $\mathbb R$ has the defining property that $\forall x,y,z,x<y<z$, then $x,z\in I\implies y\in I$.
We know that a real continuous function maps intervals to intervals due to the intermediate value theorem.
But it may not work if the (restricted) domain is not an interval.
\begin{definition}
    A topological space $X$ is disconnected if there are open $U,V\subset X$ such that $U\neq\varnothing$ and $V\neq\varnothing$ partitions $X$, that is $U\cap V=\varnothing$ and $U\cup V=X$.
    In this case, we say $U,V$ disconnect $x$.\\
    A topological space $X$ is connected if it is not disconnected.
\end{definition}
\begin{lemma}\label{image_connected}
    The image of continuous function on connected space is connected.
\end{lemma}
\begin{proof}
    Suppose $f:X\to Y$ is continuous.
    Note that if we consider $f$ as $f:X\to\operatorname{Im}f$ then it is still continuous.
    Then if $U,V$ disconnect $\operatorname{Im}f$, then $f^{-1}(U),f^{-1}(V)$ disconnect $X$.
\end{proof}
\begin{theorem}\label{connected_eqv}
    For a topological space $X$, the followings are equivalent:\\
    1. $X$ is connected.\\
    2. If $f:X\to\mathbb R$ is continuous, then $f(X)$ is an interval.\\
    3. Every continous function $f:X\to D$, where $D$ is discrete and $|D|\ge 2$, is constant.
    \footnote{Most of the time we take $D=\mathbb Z$}
\end{theorem}
\begin{proof}
    $1\implies 2$: Obvious due to the preceding lemma and the trivial fact that an open set in $\mathbb R$ is connected if and only if it is an interval.\\
    $2\implies 3$: Immediate, also from the preceding lemma.\\
    $3\implies 1$: We shall prove the contrapositive.
    Suppose that $U,V$ disconnects $X$, then choose $d,e\in D$ with $d\neq e$, then the function $f$ defined by
    $$f(x)=
    \begin{cases}
        d\text{, if $x\in U$}\\
        e\text{, otherwise, that is if $x\in V$}
    \end{cases}$$
    is continuous but is not constant, contradiction.
\end{proof}
\begin{example}
    1. $\varnothing$ and singletons are connected.\\
    2. Any indiscrete topological space is connected.\\
    3. The cofinite topology on an infinite set is connected.\\
    4. The discrete topology is disconnected if it is not a singleton.
\end{example}
\begin{lemma}
    A subspace $Y\subset X$ is disconnected if and only if there are open sets $U,V\in X$ such that $U\cap Y\neq\varnothing, V\cap Y\neq\varnothing, U\cap V\cap Y=\varnothing, Y\subset U\cup V$.
\end{lemma}
\begin{proof}
    Trivial.
\end{proof}
\begin{proposition}\label{closure_connected}
    Let $Y$ be a connected subspace of $X$, then $\bar Y$ is connected.
\end{proposition}
\begin{proof}
    Assume not, then by the preceding lemma, there exists open sets $U,V$ in $X$ such that $U\cap\bar Y\neq\varnothing, V\cap\bar Y\neq\varnothing, U\cap V\cap\bar Y=\varnothing, \bar Y\subset U\cup V$.
    It follows that $U\cap V\cap Y=\varnothing, Y\subset U\cup V$, so we must have, WLOG, $U\cap Y=\varnothing$, then $Y\subset X\setminus U\implies\bar Y\subset X\setminus U\implies \bar Y\cap U=\varnothing$, contradiction.
\end{proof}
\begin{remark}
    1. Alternatively, we can use the third part of Theorem \ref{connected_eqv}.
    2. In fact, for any $Z$ with $Y\subset Z\subset\bar Y$ is connected since the closure of $Z$ is $\bar Y$.
\end{remark}
\begin{proof}[Alternative proof of Lemma \ref{image_connected}]
    Let $f:X\to Y$ be continuous, for convenience we can just assume $f$ is surjective using the same argument as the original proof, then consider any continuous $g:Y\to\mathbb Z$, then $g\circ f$ is continuous hence constant since $f$ is connected, but $f$ is surjective, so $g$ is constant, then it is done by Theorem \ref{connected_eqv}.
\end{proof}
\begin{remark}
    1. Connectedness is a topological property.\\
    2. If $f:X\to Y$ is continuous and $A\subset X$ and $A$ is connected, then $f(A)$ is connected.
\end{remark}
\begin{corollary}
    If $X$ is connected and $R$ an equivalence relation on $X$, then $X/R$ is connected.
\end{corollary}
\begin{proof}
    The quotient map is continuous and surjective.
\end{proof}
\begin{example}
    let $Y=\{(x,\sin(1/x)):x>0\}\subset\mathbb R^2$ is connected since it is the image of $f(x)=(x,\sin(1/x))$, which is continuous since its components are connected, over $\mathbb R_{>0}$.\\
    By Proposition \ref{closure_connected}, $\bar Y=Y\cup(\{0\}\times [-1,1])$ is also connected.
    This is called the Topologist's Sine Wave.
\end{example}
\begin{lemma}\label{union_connected}
    Let $\mathscr A$ be a family of connected subset of a topological space $X$ such that $\forall A,B\in\mathscr A,A\cap B=\varnothing$, then $\bigcup_{A\in\mathscr A}A$ is connected.
\end{lemma}
\begin{proof}
    Suppose $f:\bigcup_{A\in\mathscr A}A\to\mathbb Z$ is connected, then $f|_A$ is continuous for any $A\in\mathscr A$, thus it is constant, say it is $n_A$, then $\forall A,B\in\mathscr A$, then $n_A=n_B$ since $A\cap B\neq\varnothing$.
    Thus $f$ is constant, hence $\bigcup_{A\in\mathscr A}A$ is connected.
\end{proof}
\begin{proposition}
    If $X,Y$ are connected, so is $X\times Y$.
\end{proposition}
\begin{proof}
    Observe that $\forall x\in X,\{x\}\times Y\cong Y$ is connected and $\forall y\in Y, X\times \{y\}\cong X$ is connected as well, so since $(x,y)\in (\{x\}\times Y)\cap(X\times \{y\})\neq\varnothing$, by the preceding lemma $A_{x,y}=(\{x\}\times Y)\cup(X\times \{y\})$ is connected.
    Now obviously $(x,y')\in A_{x,y}\cap A_{x',y'}\neq\varnothing$, so $X\times Y=\bigcup_{x\in X,y\in Y}A_{x,y}$ is connected by the preceding lemma.
\end{proof}
\begin{definition}
    Let $X$ be a topological space, we define an equivalence relation $R$ by $xRy$ if and only if there is a connected $U\subset X$ such that $x,y\in U$.
    One can check that this is an equivalence relation by Lemma \ref{union_connected}, and the partition of $X$ by $R$ is called the connected components of $X$.
\end{definition}
Let $C_x$ be the equivalence class containing $x$.
\begin{proposition}
    Connected components are nonempty and are maximal (wrt inclusion) connected subset of $X$, also they are closed.
\end{proposition}
\begin{proof}
    Let $C$ be a connected component, so it is the equivalence class of some $x$, so $C=C_x$, so $C$ is nonempty since it contains $X$.
    So given $y\in C$, $\exists A_y\ni x,y$ such that $U$ is connected.
    $A_y\in C$ by definition of the relation.
    Now $\forall y,z\in C, x\in A_y\cap A_z\neq\varnothing$, therefore by Lemma\ref{union_connected}, hence $C=\bigcup_{y\in C}A_y$ is connected.\\
    If $C\subset D$ and $D$ is connected, then $\forall y\in D$, $x,y\in D$, thus since $D$ is connected $y\in C$, so $D\subset C\implies C=D$.\\
    Hence since $\bar C$ is connected and contains $C$, by maximality $C=\bar C$, therefore $C$ is closed.
\end{proof}
\begin{definition}
    A topological space $X$ is called path-connected if $\forall x,y\in X,\exists\gamma:[0,1]\to X$ continuous, $\gamma(0)=x,\gamma(1)=y$.
\end{definition}
\begin{theorem}
    Any path-connected space is connected.
\end{theorem}
\begin{proof}
    Suppose not, then $X$ is path-connected but not connected, so there are open $U,V$ disconnects $X$.
    Then fixing $x\in U,y\in V$, there exists a continuous $\gamma:[0,1]\to X$ such that $\gamma(0)=x,\gamma(1)=y$.
    Thus $\gamma^{-1}(U),\gamma^{-1}(V)$ are nonempty, open, and partitions $[0,1]$, thus $[0,1]$ is disconnected by them, which is a contradiction.
\end{proof}
The converse, however, is not true.
\begin{example}
    Take the Topologist's Sine Wave, $X=\{(x,\sin(1/x)):x>0\}\cup(\{0\}\times [-1,1])$.
    We have already shown it is connected.
    But it is not path-connected.
    Indeed, pick points $(0,0),(1,\sin(1))\in X$.
    Assume that $\gamma:[0,1]\to X$ is continuous and $\gamma(0)=(0,0)=x,\gamma(1)=(1,\sin(1))=y$.
    Let $\gamma_1,\gamma_2$ be the components of $\gamma$, which are continuous.
    For $\gamma_1(t)>0$, then $[0,\gamma_1(t)]\subset \gamma_1([0,t])$ by IVT, so $\exists n\in\mathbb N,(2\pi n)^{-1},(2\pi n+\pi/2)^{-1}\in (0,\gamma_1(t))\subset \gamma_1([0,t])$.
    So there is some $a,b$ with $\gamma_1(a)=(2\pi n)^{-1},\gamma_1(b)=(2\pi n+\pi/2)^{-1}$, hence $\gamma_2(a)=0,\gamma_2(b)=1$, so we can thus find a sequence $1>t_1>t_2>\cdots>0$ with
    $$\gamma_2(t_n)=
    \begin{cases}
        1\text{, if $n$ is even}\\
        0\text{, otherwise}
    \end{cases}$$
    So $t_n$ converges but $\gamma_2(t_n)$ does not.
    This is a contradiction.
\end{example}
\begin{lemma}[Gluing Lemma]
    Let $f:X\to Y$ be a function between topological spaces.
    If $X=A\cup B$ where $A,B$ are closed and $f|_A,f|_B$ are continuous, then $f$ is continuous.
\end{lemma}
\begin{proof}
    Given closed $V$ in $Y$,
    $$f^{-1}(V)=(f^{-1}(V)\cap A)\cup (f^{-1}(V)\cap B)=(f|_A)^{-1}(V)\cup (f|_B)^{-1}(V)$$
    which is closed since $A,B$ are closed.
    Hence $f$ is continuous.
\end{proof}
\begin{corollary}
    Let $X$ be a topological space.
    Define the relation $R$ by $xRy$ if and only if there is a continuous $\gamma:[0,1]\to X$ such that $\gamma(0)=x,\gamma(1)=y$.
    Then this is an equivalence relation.
\end{corollary}
\begin{proof}
    Trivial.
\end{proof}
\begin{theorem}
    Let $U\subset\mathbb R^n$ be open, then $U$ is connected if and only if $U$ is path-connected.
\end{theorem}
\begin{proof}
    It suffice to show every open connected subset of $\mathbb R^n$ is path-connected.\\
    WLOG $U\neq\varnothing$, fix $x_0\in U$, let $V$ be the path-connected component containing $x_0$.
    We shall show that $V,U\setminus V$ are both open, so by assumption $V=U$, thus the proof will be done.\\
    $V$ open: Since $U$ is open, for any $x\in U$, there is $r>0$ such that $D_r(x)\in U$.
    But any ball is path connected, so $\forall x\in V,\exists r_x>0, D_{r_x}(x)\in V$, so $V$ is open.\\
    $U\setminus V$ open: Fix by the same proof as above, any path-connected components in $V$ is open, so since $U\setminus V$ is the union of some of them (the ones except $V$), it is open.
\end{proof}
\begin{example}
    For $n\ge 2$, $\mathbb R^n$ is not homeomorphic to $\mathbb R$.
    Assume $f:\mathbb R^n\to \mathbb R$ is a homeomorphism.
    Fix $x\in\mathbb R^n$, and let $y=f(x)$, then $f|_{\mathbb R^n\setminus\{x\}}$ is still a homeomorphism to $\mathbb R\setminus\{y\}$.
    But then $\mathbb R^n\setminus\{x\}$ is connected by the preceding theorem, but $\mathbb R\setminus\{y\}$ is not, contradiction.
\end{example}


% This is part of the TFTB Reference Manual.
% Copyright (C) 1996 CNRS (France) and Rice University (US).
% See the file refguide.tex for copying conditions.

\markright{fmpar}
\section*{\hspace*{-1.6cm} fmpar}

\vspace*{-.4cm}
\hspace*{-1.6cm}\rule[0in]{16.5cm}{.02cm}
\vspace*{.2cm}

{\bf \large \sf Purpose}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
Signal with parabolic frequency modulation.
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Synopsis}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
[x,iflaw] = fmpar(N,P1)
[x,iflaw] = fmpar(N,P1,P2,P3)
\end{verbatim}
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Description}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
        {\ty fmpar} generates a signal with parabolic frequency modulation
        law : \[x(t) = \exp(j2\pi(a_0 t + \frac{a_1}{2} t^2 +\frac{a_2}{3} t^3)).\]
\vspace*{.2cm}

\hspace*{-.5cm}\begin{tabular*}{14cm}{p{1.5cm} p{8.5cm} c}
Name & Description & Default value\\
\hline
        {\ty N}  & number of points in time\\
        {\ty P1} & if {\ty nargin=2}, {\ty P1} is a vector containing the three
            coefficients {\ty (a0 a1 a2)} of the polynomial instantaneous phase.
            If {\ty nargin=4}, P1 (as {\ty P2} and {\ty P3}) is a
	    time-frequency point of the form {\ty (ti fi)}.
            The coefficients {\ty (a0,a1,a2)} are then deduced such that
            the frequency modulation law fits these three points\\
        {\ty P2, P3} & same as {\ty P1} if {\ty nargin=4}.       & optional\\
  \hline {\ty x}     & time row vector containing the modulated signal samples \\
        {\ty iflaw} & instantaneous frequency law\\
\hline
\end{tabular*}

\end{minipage}
\vspace*{1cm}

{\bf \large \sf Examples}
\begin{verbatim}
         [x,iflaw]=fmpar(200,[1 0.4],[100 0.05],[200 0.4]);
         subplot(211);plot(real(x));subplot(212);plot(iflaw);
         [x,iflaw]=fmpar(100,[0.4 -0.0112 8.6806e-05]);
         subplot(211);plot(real(x));subplot(212);plot(iflaw);
\end{verbatim}
\vspace*{.5cm}

{\bf \large \sf See Also}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
fmconst, fmhyp, fmlin, fmsin, fmodany, fmpower.
\end{verbatim}
\end{minipage}


\documentclass{article}
\usepackage[utf8]{inputenc}

\title{MAT257 Notes}
\author{Jad Elkhaleq Ghalayini}
\date{October 17 2018}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{mathtools}
\usepackage{enumitem}
\usepackage{graphicx}
\usepackage{cancel}

\usepackage[margin=1in]{geometry}

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}{Lemma}
\newtheorem{definition}{Definition}
\newtheorem*{corollary}{Corollary}
\newtheorem{exercise}{Exercise}

\newcommand{\reals}[0]{\mathbb{R}}
\newcommand{\nats}[0]{\mathbb{N}}
\newcommand{\ints}[0]{\mathbb{Z}}
\newcommand{\rationals}[0]{\mathbb{Q}}
\newcommand{\brac}[1]{\left(#1\right)}
\newcommand{\sbrac}[1]{\left[#1\right]}
\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\eval}[3]{\left.#3\right|_{#1}^{#2}}
\newcommand{\ip}[2]{\left\langle#1,#2\right\rangle}
\newcommand{\prt}[2]{\frac{\partial #1}{\partial #2}}

\begin{document}

\maketitle

\section*{The Inverse Function Theorem Implies the Implicit Function Theorem}

We have to start with the hypotheses of the implicit function theorem: given a \(\mc{C}^r\) (where \(r \geq 1\)) function
\(f: U \to \reals^n\)
where \(U \in \reals^{m + n}\), \(f(a, b) = 0\) and \(\det M \neq 0\), where
\[M = \left(\prt{f_i}{y_j}(a, b)\right)\]
Define
\[F: U \to \reals^m\times\reals^n, (x, y) \mapsto (x, f(x, y))\]
In particular,
\[F(a, b) = (a, 0)\]
This is what we're going to apply the inverse function theorem to. So we've got to show that this function satisfies the hypotheses of the inverse function theorem. So we've got to show that its derivative matrix at the point \((a, b)\) is invertable. So let's compute: the derivative is given by
\[F'(a, b) = \left(\begin{array}{c|c} I & 0 \\ \hline * & M \end{array}\right) \implies \det F'(a, b) = \det I \det M = \det M \neq 0\]
These are the conditions under which we can apply the inverse function theorem. So by the inverse function theorem, there exists an open neighborhood \(V\) of \((a, b)\), and an open neighborhood \(W\) of \((0, 0)\) so that \(F: V \to W\) has a \(\mc{C}^r\) inverse
\(F^{-1}: W \to V\). We can assume \(V = A \times B\), where \(A, B\) are open neighborhoods of \(a, b\). \(F^{-1}(u, v)\) has the form \((u, h(u, v))\). So
\[F(F^{-1}(u, v)) = F(u, h(u, v)) = (u, f(u, h(u, v))) = (u, v) \implies f(u, h(u, v)) = v\]
\[\implies f(u, h(u, 0)) = 0\]
Let \(g(x) = h(x, 0)\), which is \(\mc{C}^r\). Then
\[f(x, g(x)) = 0\]
Remark: we can find \(g'(x)\) by \underline{implicit differentiation}. We have
\[\forall i \in \{1,...,n\}, f_i(x, g(x)) = 0\]
We can write
\[\prt{f_i}{x_j}(x, g(x)) + \sum_{k = 1}^n\prt{f_i}{y_k}(x, g(x))\prt{g_k}{x_j}(x) = 0\]
We can solve for \(\prt{g_k}{x_j}\) because \(\left(\prt{f_i}{y_k}(x, y)\right)\) is invertible near \((a, b)\).

\section*{The Implicit Function Theorem Implies the Inverse Function Theorem}

This time, we start with the hypotheses of the \textit{inverse} function theorem. So here we have a \(\mc{C}^r\) function \(f: U \to \reals^n\) with \(\det f'(a) \neq 0\).

Let \(b = f(a)\), and define
\[F(x, y) = y - f(x)\]
This is a \(\mc{C}^r\) function of \((a, b)\) and \(F(a, b) = 0\). We have
\[\prt{F}{x}(a, b) = \det(-f'(a)) \neq 0\]
By the implicit function theorem there exist open neighborhoods \(A, B\) of \(a, b\) respectively such that for all \(y \in A\), there is a unique \(\mc{C}^r\) \(x = g(y)\) in \(B\) such that
\[F(g(y), y) = 0 \iff y - f(g(y)) = 0\]
Take \(V = f^{-1}(A) \cap B\), \(W = A\). Then
\[x \in V \implies f(x) \in A\]
and \(x\) is the unique element of \(B\) such that \(g(f(x)) = 0\), i.e. \(x = g(f(x))\).

\end{document}


\section{Computing the closest point on a patch  \label{app:closest_point}}
%\subsection{Optimization\label{app:closest_point_opt}}
%\begin{figure}%[!htb]
%  \centering
%    \includegraphics[width=.5\linewidth]{figs/newton-opt.pdf}
%    \mcaption{fig:newton-opt}{Closest point optimization schematic}{}
%\end{figure}
We include our algorithm to find the closest point $\vy$ on a patch $\vP$ to a point $\vx \in \mathbb{R}^3$ in the section for completeness.
For a surface or quadrature patch $\vP$ and point $\vx \in \mathbb{R}^3$,
we need to compute a point $\vy = \vP(s^*, t^*)$ such that
\begin{equation}
  (s^*, t^*) = \argmin_{(s,t) \in [-1,1]^2} \|\vx - \vP(s,t)\|_2^2 =  \argmin_{(s,t) \in [-1,1]^2} \vr(s,t)\cdot \vr(s,t)
\end{equation}
where $ \vr = \vr(s,t) = \vx - \vP(s,t)$; let $g(s,t) = \vr\cdot \vr$.
We first consider the unconstrained problem
\begin{equation}
    (s^*, t^*) = \argmin_{(s,t) \in \mathbb{R}^2} \|\vx - \vP(s,t)\|_2^2  = \argmin_{(s,t) \in \mathbb{R}^2} \psi(s,t)
\end{equation}
We solve this optimization problem with Newton's method.
The first and second derivatives of $\psi$ can be evaluated efficiently, since they are polynomials of fixed order.
The gradient and Hessian of the objective function are:
\begin{equation}
  \nabla \psi  =
  \begin{pmatrix}
    -\vP_s\cdot \vr \\
    -\vP_t\cdot \vr \\
  \end{pmatrix}, \quad
  %\label{eq:grad-newton}
  \nabla^2 \psi =
\begin{pmatrix}
  \vP_s \cdot \vP_s - \vr\cdot \vP_{ss} & \vP_s \cdot \vP_t - \vr\cdot \vP_{st}\\
  \vP_s \cdot \vP_t - \vr\cdot \vP_{st} & \vP_t \cdot \vP_t - \vr\cdot \vP_{tt}  \\
\end{pmatrix}.
  \label{equ:grad-hess-newton}
\end{equation}
The optimality conditions are
\begin{equation}
\vP_s^* \cdot \vr^* = 0, \quad \vP_t^* \cdot \vr^* = 0, \quad (u,v) = (s^*, t^*).
  \label{eq:kkt}
\end{equation}
at a local optimum $(s^*, t^*)$.

Let $\psi_i = \psi(s_i,t_i)$, where $(s_i,t_i)$ is the value of the solution during the $i$th iteration of Newton's method.
To solve for the descent direction in Newton's method, we need to solve
\begin{equation}
  \nabla^2 \psi_i \, \eta_i = -\nabla \psi_i
  \label{eq:newton-system}
\end{equation}
where $\eta_i = (\Delta s_i,\Delta t_i)$ is the $i$th Newton update to $(s_i,t_i)$ such that
\begin{equation}
  s_{i+1} = \alpha_i\Delta s_i + s_i,\quad
  t_{i+1} = \alpha_i\Delta t_i + t_i
  \label{}
\end{equation}

We use four iterations of a backtracking line search with an Armijo condition to compute the step length $\alpha_i$ to ensure an appropriate size step is taken in case the initial guess is outside the region of quadratic convergence.
We compute the solution $(s^*, t^*)$ by iterating
\begin{equation}
  (s_n,t_n) = (s_{n-1}, t_{n-1}) + \alpha_{n-1} \eta_{n-1}, \quad \text{ while } \vP_s \cdot \vr > \err{opt}, \quad \vP_t \cdot \vr > \err{opt},
  \label{eq:descent_iter}
\end{equation}
until convergence, i.e., $\psi_i\approx \err{opt}$, $\vr \approx \vn(\vy)$.

If $(s^*, t^*) \in (-1,1)^2$, then the solution to the unconstrained problem is also the solution to the constrained problem.
However, if the closest point lies in $\mathbb{R}\setminus [-1,1]^2$, we need to ensure the inequality constraints are satisfied.
Additionally, if $(s^*, t^*)$ is on the boundary of $[-1,1]^2$, either $s^*$ or $t^*$ should be exactly zero; with the optimization scheme above, we can only claim that $|s^*| < \err{opt}$ (similarly for $t^*$).
To address both of these troubles, we can solve a one-dimensional projection of \cref{eq:newton-system} on to the boundary of $[-1,1]^2$.
For example, to find the closest point along the edge $v=0$, the Newton iteration becomes
\begin{equation}
  s_n = s_{n-1} + \alpha_{n-1}\frac{-\vP_s \cdot \vr}{\vP_s\cdot \vP_s - \vr\cdot \vP_{ss}},
  \label{eq:geom-newton-1d}
\end{equation}
where $\vP_s$, $\vP_{ss}$ and $\vr$ are evaluated at $s_{n-1}$.
Since the boundary is composed of $[-1,t]$, $[1,t]$, $[s,-1]$, $[s,1]$ for $s,t\in[-1,1]$, we solve \cref{eq:geom-newton-1d} once for each interval.

This final algorithm to compute the closest point is as follows:
\begin{enumerate}
  \item We solve \cref{eq:newton-system} on an extended parameter domain $[-1-c, 1+c]^2$, and terminate the Newton iteration if $(s_i,t_i)$ walks outside this boundary.
    If the Newton iteration terminates inside $[-1,1]^2$, then we've found the closest point.
    We typically choose $c = .2$.
  \item  If the solution is outside $[-1,1]^2$, we solve \cref{eq:newton-system} along each component of the boundary of $[-1,1]^2$, also on an extended parameter domain $[-1-c,1+c]$,
    by choosing an initial guess contained within the interval.
    The solution to these four problems that yields a minimal distance to $\vx$ to used as the closest point, if the solution is inside $[-1,1]$.
  \item If the closest point on the boundary is still outside of $[-1,1]^2$, the
      closest point to $\vx$ is chosen from $\vP(-1,-1), \vP(-1,1), \vP(1,-1),$ and $\vP(1,1)$ closest to $\vx$.
\end{enumerate}
This gives us an algorithm to compute the closest point on a quadrature patch $\vP$ to $\vx$.
The \oned and \twod Newton minimizations converge in ten iterations on average.


\documentclass{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}

\begin{document}

\section*{Introduction}
Summary of the Analysis II course given by Sir Thomas Mountford.

\section{Notes}

\begin{itemize}
	\item A \emph{metric space} is a vector space equipped with a metric, that is a function that measures distance between every two elements of the space, whereas a norm space is simply a vector space equipped with a norm: all metric spaces are norm spaces, but the converse is not true.
	\item BASIC CONCEPT OF TOPOLOGY: a \emph{neighborhood}, or \emph{open ball} of x $B(\vec x, r) = \{\vec y \ : \ d(\vec x, \vec y) < r\}$ is a "zone around a point" (a set of points) such that these points are less than a certain distance around $\vec x$ (actually it's not, it's "any set containing $B(\vec x, \epsilon)$ for some $\epsilon > 0$
	\item OPEN SET: An open set $O \subseteq \mathbb{R}^n$ is a set such that $\forall \vec x \in O \exists \epsilon_x > 0$ such that $B(\vec x, \epsilon_x) \subset O$ (So $O$ is open $\iff$ $O$ is an ... for each of its points)
	\item PROPERTIES: union over any collection of open sets is open; the intersection of two open sets is open
	\item CLOSED SET: a set $F \subset \mathbb{R}^n$ is closed if $F^C = \{\vec x \ : \ \vec x \notin F\}$ is open. Intuitively, think of a disc from which you remove all points on the circle. Ex: $\overline{B} (\vec x, r) = \{\vec y \ : \ d(\vec x, \vec y) \leq r\}$ is closed. Ex: in one dimension, take $F = [0, 1]$. Then compute $F^C = ]-\infty, 0[ \cup ]1, \infty[$ is a union of open sets, and therefore is an open set. Therefore, $F$ is a closed set by definition. ($F^C$ means "$F$ complement")
	\item BOUNDARY: Given $S \subset \mathbb{R}^n$, the boundary of $S$, that is, $\delta s$, is the collection $\{\vec x \ : \ \forall \epsilon > 0 \ B(\vec x, \epsilon) \cap S \neq \emptyset \land B(\vec x, \epsilon) \cap S^C \neq \emptyset\}$
	\item given $S$, the closure (adherence) of $S$, $\overline S$, is $S \cup \delta S$
\end{itemize}

\end{document}


\documentclass[../main.tex]{subfiles}

\begin{document}
\chapter{Higher Derivatives}
Higher derivatives are derivatives of derivatives.
For instance, if $y^\prime$ is the derivative of $y'$,
then $y^{\prime\prime}$ is the derivative of $y^\prime$.
\begin{table}[h]
    \centering
    \caption{All the different notations for higher derivates}
    \label{tab:notations}
    \begin{tabular}{c|c|c|c}
    $y'$      & $\frac{d}{dx}y$                   & $\frac{dy}{dx}$     & $Dy$   \\ \hline
    $y''$     & $\left( \frac{d}{dx} \right)^2 y$ & $\frac{d^2y}{dx^2}$ & $D^2y$ \\ \hline
    $y'''$    & $\left( \frac{d}{dx} \right)^3 y$ & $\frac{d^3y}{dx^3}$ & $D^3y$ \\ \hline
    $y^{(4)}$ & $\left( \frac{d}{dx} \right)^4 y$ & $\frac{d^4y}{dx^4}$ & $D^4y$ \\ \hline
    $y^{(n)}$ & $\left( \frac{d}{dx} \right)^n y$ & $\frac{d^ny}{dx^n}$ & $D^ny$
    \end{tabular}
\end{table}

Higher derivatives are pretty straightforward ---
just keep taking the derivative!
\begin{exmp}
    Let us see what happens if we keep taking the derivative of
    $f(x) = \sin x$:
    \begin{align*}
        f'(x)   &= \cos x   \\
        f''(x)  &= - \sin x \\
        f'''(x) &= - \cos x \\
        f^{(4)} &= \sin x
    \end{align*}
    We have, somehow, arrived back at the original function,
    $f''''(x) = f(x)$.
    The sine and cosine functions, both, have this property.
\end{exmp}
\begin{exmp}
    What is $D^n x^n$?  \\
    We will start small and look for a pattern. We know:
    \begin{align*}
        \frac{d}{dx} x^n        &= nx^{n - 1}               \\
        \frac{d^2}{dx^2} x^n    &= n(n - 1)x^{n - 2}        \\
        \frac{d^3}{dx^3} x^n    &= n(n - 1)(n - 2)x^{n - 3} \\
    \end{align*}
    We can reasonably extend this pattern to deduce:
    \[
        \frac{d^{n - 1}}{dx^{n - 1}} x^n =
        n(n - 1)(n - 2)(n - 3) \cdots 3 \cdot 2 \cdot x
    \]
    Finally, we get:
    \[
        \frac{d^{n}}{dx^{n}} x^n =
        n(n - 1)(n - 2)(n - 3) \cdots 3 \cdot 2 \cdot 1
    \]
    There is a name for this pattern of products; \emph{factorials} ($n!$).
    Therefore:
    \[ \frac{d^{n}}{dx^{n}} x^n = n! \]
    Now, we can also see:
    \[ \frac{d^{n + 1}}{dx^{n + 1}} x^n = 0 \]
    We just (unwittingly) did a proof by \emph{mathematical induction}!
    It is an extremely useful tool in every mathematician's toolbox.
\end{exmp}
\end{document}


\subsection{Gluing Riemann Surfaces}
When we were studying the $k^{th}$ roots, we constructed a Riemann surface $R_k$ equipped with analytic $\pi,g$ such that
\[
    \begin{tikzcd}
        R_k\arrow[swap]{dr}{\pi}\arrow{r}{g}&\mathbb C_\star\arrow{d}{p_k}\\
        &\mathbb C_\star
    \end{tikzcd}
\]
commutes.
We also observed that this diagram can be compactified by resolving the removable singularities
\[
    \begin{tikzcd}
        \hat{R}_k\arrow[swap]{dr}{\hat\pi}\arrow{r}{\hat{g}}&\mathbb C_\infty\arrow{d}{\hat{p}_k}\\
        &\mathbb C_\infty
    \end{tikzcd}
\]
How do we do this in general?
The answer is via gluing.
\begin{definition}
    Let $X,Y$ be topological spaces and with subspaces $X'\subset X,Y'\subset Y$ and a homeomorphism $\Phi:X'\to Y'$.
    The result of gluing $X,Y$ along $\Phi$ is the topological space $Z=(X\sqcup Y)/\sim$ where $\sim$ is the minimal equivalence relation such that $x\sim\Phi(x)$ for all $x\in X'$.
    It is sometimes denoted by $X\cup_{\Phi}Y$ or $X\cup_{X'}Y$ if the homeomorphism $\Phi$ is understood.
\end{definition}
We need to understand how gluing gives rise to a new Riemann surface in the case where $X,Y$ are Riemann surfaces.
\begin{proposition}
    Let $R_1,R_2$ be Riemann urfaces and $S_j\in R_j$ are non empty, connected and open subsets and $\Phi:S_1\to S_2$ is a conformal equvalence of Riemann surfaces, then there is a unique conformal stucture on $R=R_1\cup_\Phi R_2$ such that the inclusions $i_j:R_j\to R$ are analytic.
    In particular, if $R$ is Hausdorff, then it is a Riemann surface.
\end{proposition}
\begin{proof}
    Consider the family of charts $(\phi_j\circ i_j^{-1},i_j(U_j))$ where $(\phi_j,U_j)$ is a chart on $R_j$.
    The transition functions are then either transition functions of $R_j$ or $\phi_2\circ i_2^{-1}\circ i_1\circ\phi_1^{-1}=\phi_2\circ\Phi\circ\phi_1^{-1}$ which is analytic as $\Phi$ is.
    This induces a conformal structure on $R$.\\
    For uniqueness, suppose $(\phi_j,U_j)$ is a chart on $R_j$ and $(\psi,V)$ is a chart in another conformal stucture on $R$ such that the condition holds, then $\psi\circ i_j\circ\phi_j^{-1}$ is analytic since $i_j$ is.
    This means that $\phi_j\circ i_j^{-1}$ has analytic transition function with all charts
    So by maximality the two conformal structures are equal.\\
    It is quite obvious that $R$ is connected.
    So if we assume further that $R$ is Hausdorff, then $R$ is a Riemann surface.
\end{proof}
\begin{example}[Non-example]
    Take $R_1=R_2=\mathbb C,S_1=S_2=\mathbb C_\star$, then $R=\mathbb C\cup_{\operatorname{id}_{\mathbb C_\star}}\mathbb C$ is not Hausdorff.
\end{example}
\begin{example}
    Let $R_1=R_2=\mathbb C$ and $S_1=S_2=\mathbb C_\star$ and let $\Phi:\mathbb C_\star\to\mathbb C_\star$ be the inversion $z\mapsto 1/z$.
    Then $R=\mathbb C\cup_\Phi\mathbb C$ is obviously Hausdorff hence is a Riemann surface by the preceding proposition.
    One can also see easily that $R$ is compact.
    Easily, one can see that $R$ is exactly the Riemann sphere $\mathbb C_\infty$.
\end{example}


\clearpage

\begin{appendix}
\section{}
For every individual i in i = 1,\ldots, N:
\[Y_i \sim \mathcal{N}(\mathbf{0}, \Sigma),\] where
\[\Sigma = \Lambda\Psi\Lambda',\] \[\Lambda =
    \begin{bmatrix}
    0.75 & 0 \\
    0.75 & 0 \\
    0.75 & 0.2/0.5 \\
    0.2/0.5 & 0.75 \\
    0 & 0.75 \\
    0 & 0.75
    \end{bmatrix},\] \[\Psi =
    \begin{bmatrix}
     1 & 0.5 \\
     0.5 & 1
    \end{bmatrix}
,\] and \[\Theta = diag[0.3, 0.3, 0.3, 0.3, 0.3, 0.3].\]
\end{appendix}


%!TEX root = ../Thesis.tex
\chapter{Miscellaneous} \label{app:misc}

\section{Implicit Euler algorithm} \label{app:implicit_euler}
All new values $(x,y,p_x,p_y)_{i+1}$ refer to the same unknown values in the same time step $(x,y,p_x,p_y)_{i+1}$.
\begin{align}
    X_{i+1} &= X_i + (P_{X,i+1} + Y_{i+1})h, \\[0.2cm]
    Y_{i+1} &= Y_i + (P_{Y,i+1} - X_{i+1})h , \\[0.2cm]
    P_{X,i+1} &= P_{X,i} + \left(P_{Y,i+1} - \dfrac{(1-k)(k+X_{i+1})}{((k+X_{i+1})^2+Y_{i+1}^2)^{3/2}} + \dfrac{k(X_{i+1}-1+k)}{((X_{i+1}-1+k)^2+Y_{i+1}^2)^{3/2}}\right)h, \\[0.2cm]
    P_{Y,i+1} &= P_{Y,i} + \left(-P_{X,i+1} - \dfrac{(1-k)Y_{i+1}}{((k+X_{i+1})^2+Y_{i+1}^2)^{3/2}} - \dfrac{k Y_{i+1}}{((X_{i+1}-1+k)^2+Y_{i+1}^2)^{3/2}}\right)h.
\end{align}

\section{Restricted Three-Body Problem Symplectic Euler Derivations (Mathematica)} \label{app:r3b-symplectic-euler}
\begin{figure}[h!]
\centering
\includegraphics[scale=0.8]{appendices/Miscellaneous/symplectic_euler_derivation.pdf}
\end{figure}

\section{Restricted Three-Body Problem Störmer-Verlet Derivations (Mathematica)} \label{app:r3b-verlet}
\begin{figure}[h!]
\centering
\includegraphics[scale=0.60]{appendices/Miscellaneous/R3B_Verlet_derivations.pdf}
\end{figure}

\clearpage

\section{Medium and Short Hohmann orbits} \label{app:more_hohmann}
\begin{adjustwidth*}{0cm}{-0.4cm}
\begin{lstlisting}[language=Python,caption=Medium duration Hohmann]
# --------------------------------------------------------------------------
duration = 3/unit_time
pos      = -2.272183066647597
ang      = -0.075821466029764
burn     = 3.135519748743719/unit_vel
x0       = -0.023110975767437
y0       = -0.012972499765730
px0      = 8.032228991913522
py0      = -7.100537706154897
# --------------------------------------------------------------------------
# dV(earth-escape) = 3.135520 km/s
# dV(moon-capture) = 0.879826 km/s
# dV(total)        = 4.015346 km/s
# Flight-time      = 2.999939 days
# --------------------------------------------------------------------------
\end{lstlisting}
\end{adjustwidth*}

\begin{adjustwidth*}{0cm}{-0.4cm}
\begin{lstlisting}[language=Python,caption=Fast duration Hohmann]
# --------------------------------------------------------------------------
duration = 2/unit_time
pos      = -2.277784119105456
ang      = 0.046759232345463
burn     = 3.809267777777778/unit_vel
x0       = -0.023183463163465
y0       = -0.012910923775798
px0      = 8.760501647975921
py0      = -7.267405934327472
# --------------------------------------------------------------------------
# dV(earth-escape) = 3.809268 km/s
# dV(moon-capture) = 3.014142 km/s
# dV(total)        = 6.823410 km/s
# Flight-time      = 1.000007 days
# --------------------------------------------------------------------------
\end{lstlisting}
\end{adjustwidth*}


\documentclass[12pt]{cdblatex}
\usepackage{examples}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% ============================================================================================
\bgroup
\CdbSetup{action=hide}
\begin{cadabra}
   import cdblib
   checkpoint_file = 'tests/semantic/output/example-15.json'
   cdblib.create (checkpoint_file)
   checkpoint = []
\end{cadabra}
\egroup

\clearpage

% ============================================================================================
\section*{Example 15 Verifying the BSSN equations}

This is short example verifies two of the main equations in the Phys Rev D paper
by Miguel Alcubierre, Bernd Brugmann etal. (Phys.Rev.D. (62) 044034 (2000)).

The code for the full set of BSSN equations can be found at
\url{https://github.com/leo-brewin/adm-bssn-equations}

\begin{cadabra}
   {a,b,c,d,e,f,i,j,k,l,m,n,o,p,q,r,s,u#}::Indices(position=independent,values={t,x,y,z}).
   {t,x,y,z}::Coordinate.

   \partial{#}::PartialDerivative.
   D{#}::Derivative.
   DBar{#}::Derivative.

   N::Depends(t,x,y,z).

   g_{a b}::Symmetric.
   g^{a b}::Symmetric.
   g_{a}^{b}::KroneckerDelta.
   g^{a}_{b}::KroneckerDelta.

   g_{a b}::Depends(t,x,y,z).
   g^{a b}::Depends(t,x,y,z).

   gBar_{a b}::Symmetric.
   gBar^{a b}::Symmetric.
   gBar_{a}^{b}::KroneckerDelta.
   gBar^{a}_{b}::KroneckerDelta.

   gBar_{a b}::Depends(t,x,y,z).
   gBar^{a b}::Depends(t,x,y,z).

   trK::LaTeXForm("K").
   detg::LaTeXForm("g").
   ABar{#}::LaTeXForm("{\bar{A}}").
   DBar{#}::LaTeXForm("{\bar{D}}").
\end{cadabra}

\clearpage

% --------------------------------------------------------------------------------------------
\subsection*{15.1 Evolution equation for $\phi$}

\begin{cadabra}
   phi     := \phi -> (1/12) \log(detg).
   gdotK   := g^{i j} K_{i j} -> trK.
   DdetgDt := \partial_{t}{detg} -> detg g^{i j} \partial_{t}{g_{i j}}.

   DgijDt  := \partial_{t}{g_{i j}} -> -2 N K_{i j}.

   dlog    := \partial_{a?}{\log(A?)} -> (1/A?)\partial_{a?}{A?}.
   dexp    := \partial_{a?}{\exp(A?)} -> \exp(A?)\partial_{a?}{A?}.

   dotphi  := \partial_{t}{\phi}.

   substitute (dotphi, phi)                 # cdb (ex-15-02.101,dotphi)
   substitute (dotphi, dlog)                # cdb (ex-15-02.102,dotphi)
   substitute (dotphi, DdetgDt)             # cdb (ex-15-02.103,dotphi)
   substitute (dotphi, DgijDt)              # cdb (ex-15-02.104,dotphi)
   substitute (dotphi, gdotK)               # cdb (ex-15-02.105,dotphi)
   map_sympy  (dotphi, "simplify")          # cdb (ex-15-02.106,dotphi)

   DphiDt := \partial_{t}{\phi} -> @(dotphi).

   checkpoint.append (dotphi)
\end{cadabra}

\begin{align*}
  \frac{d{\phi}}{dt} &=\Cdb{ex-15-02.101}\\[10pt]
                     &=\Cdb{ex-15-02.102}\\[10pt]
                     &=\Cdb{ex-15-02.103}\\[10pt]
                     &=\Cdb{ex-15-02.104}\\[10pt]
                     &=\Cdb{ex-15-02.105}\\[10pt]
                     &=\Cdb{ex-15-02.106}
\end{align*}

\clearpage

% --------------------------------------------------------------------------------------------
\subsection*{15.2 Evolution equation for ${\bar g}_{ij}$}

\begin{cadabra}
   gBarij := gBar_{i j} -> \exp(-4\phi) g_{i j}.
   Kij    := K_{i j} -> A_{i j} + (1/3) g_{i j} trK.
   A2ABar := \exp(-4\phi) A_{i j} -> ABar_{i j}.
   ABar2A := ABar_{i j} -> \exp(-4\phi) A_{i j}.

   dotgBarij := \partial_{t}{gBar_{i j}}.

   substitute   (dotgBarij, gBarij)         # cdb (ex-15-03.101,dotgBarij)
   product_rule (dotgBarij)                 # cdb (ex-15-03.102,dotgBarij)
   substitute   (dotgBarij, dexp)           # cdb (ex-15-03.103,dotgBarij)
   substitute   (dotgBarij, DgijDt)         # cdb (ex-15-03.104,dotgBarij)
   substitute   (dotgBarij, DphiDt)         # cdb (ex-15-03.105,dotgBarij)
   substitute   (dotgBarij, Kij)            # cdb (ex-15-03.106,dotgBarij)
   distribute   (dotgBarij)                 # cdb (ex-15-03.107,dotgBarij)
   map_sympy    (dotgBarij, "simplify")     # cdb (ex-15-03.108,dotgBarij)
   substitute   (dotgBarij, A2ABar)         # cdb (ex-15-03.109,dotgBarij)

   DgBarijDt := \partial_{t}{gBar_{i j}} -> @(dotgBarij).

   checkpoint.append (dotgBarij)
\end{cadabra}

\begin{align*}
  \frac{d{\bar g}_{ij}}{dt} &=\Cdb{ex-15-03.101}\\[10pt]
                            &=\Cdb{ex-15-03.102}\\[10pt]
                            &=\Cdb{ex-15-03.103}\\[10pt]
                            &=\Cdb{ex-15-03.104}\\[10pt]
                            &=\Cdb{ex-15-03.105}\\[10pt]
                            &=\Cdb{ex-15-03.106}\\[10pt]
                            &=\Cdb{ex-15-03.107}\\[10pt]
                            &=\Cdb{ex-15-03.108}\\[10pt]
                            &=\Cdb{ex-15-03.109}
\end{align*}

\clearpage

% ============================================================================================
% export to json format

\bgroup
\CdbSetup{action=hide}
\begin{cadabra}
   for i in range( len(checkpoint) ):
      cdblib.put ('check{:03d}'.format(i),checkpoint[i],checkpoint_file)
\end{cadabra}
\egroup

\end{document}