\section{Deriving the invariant extended Kalman filter}
The filtering problem is a MAP problem given by
\begin{align}
    \mbfhat{X}_{k}
    &= \argmax_{\mbf{X}_{k}\in G} \pdf{\mbf{X}_{k+1}\mid \mbfhat{X}_{k-1}, \mbf{u}_{k-1}, \mbf{y}_{k}}\\
    \label{eq:InEKF derivation:pdf Xhat given Xhatkm1 ukm1 yk}
    &= \argmax_{\mbf{X}_{k-1}\in G} \pdf{\mbf{y}_{k}\mid \mbf{X}_{k}}\pdf{\mbf{X}_{k}\mid\mbfhat{X}_{k-1}, \mbf{u}_{k-1}},
\end{align}
where $\mbfhat{X}_{k-1}$ is the InEKF estimate of $\mbf{X}_{k-1}$.

As briefly discussed in Section~\ref{sec:Random variables on Lie groups}, random variables on Lie groups can be described by
\begin{align}
    \pdf{\mbf{X}; \mbfbar{X}, \mbs{\Sigma}} &=
    \mc{N}\left( \mbf{X}\ominus\mbfbar{X}, \mbs{\Sigma} \right)\\
    &=
    \label{eq:pdf of X in Lie group}
    \eta\exp\left(-\f{1}{2}\norm{
        \mbf{X}\ominus\mbfbar{X}
    }_{\mbs{\Sigma}\inv}^{2}\right).
\end{align}

Let
\begin{align}
    \mbfcheck{X}_{k} &\coloneqq (\mbfhat{X}_{k-1}\rplus\mbf{u}_{k-1}) \oplus\mbfrv{w}_{k-1},
\end{align}
and let the measurement function be given by
\begin{align}
    \mbf{y}_{k} &= \mbf{X}_{k}\mbf{b} + \mbfrv{n}_{k},
\end{align}
where $\mbf{b}$ is a know column matrix.

Taking the above assumptions into consideration, and plugging \eqref{eq:pdf of X in Lie group} into the MAP objective function \eqref{eq:InEKF derivation:pdf Xhat given Xhatkm1 ukm1 yk} and taking the negative log, results in
\begin{align}
    \mbfhat{X}_{k}
    &= \argmax_{\mbf{X}_{k}\in G} \pdf{\mbf{y}_{k}\mid \mbf{X}_{k}}\pdf{\mbf{X}_{k}\mid\mbfhat{X}_{k-1}, \mbf{u}_{k-1}}\\
    &= \argmin_{\mbf{X}_{k}\in G} \f{1}{2}\norm{\mbf{y}_{k} - \mbf{X}_{k}\mbf{b}}^{2}_{\mbf{R}\inv}
    + \f{1}{2}\norm{
        \mbf{X}_{k}\ominus\mbfcheck{X}_{k}
    }^{2}_{\mbfcheck{P}_{k}\inv},
\end{align}
which is a (nonlinear) least squares problem.

The optimization problem can be rewritten as
\begin{align}
    \label{eq:deriving InEKF: NLS et SigmaInv e}
    \mbfhat{X}_{k} &= \argmin_{\mbf{X}_{k}\in G} \f{1}{2}\mbf{e}(\mbf{X}_{k})^{\trans}\mbs{\Sigma}\inv\mbf{e}(\mbf{X}_{k}),
\end{align}
where
\begin{align}
    \label{eq:deriving InEKF: error function def}
    \mbf{e} &: G \to \rnums^{m},\\
    \mbf{e}(\mbf{X}_{k}) &=
    \bbm
        \mbf{X}_{k} \ominus \mbfcheck{X}_{k}\\
        \mbf{X}_{k}\mbf{b} - \mbf{y}_{k}
    \ebm,\\
    \mbs{\Sigma} &=
    \bbm
        \mbfcheck{P}_{k} & \\ & \mbf{R}_{k}
    \ebm.
\end{align}
\begin{remark}
    Note that the error function $\mbf{e}$ is a mapping from the (Lie) group space $G$ to a the Lie algebra vector space $\rnums^{m}$. That's the nice thing about Lie algebra; we can use the standard optimization tools on the Lie algebra Euclidean space.
\end{remark}

The optimization problem \eqref{eq:deriving InEKF: NLS et SigmaInv e} can be solved using Gauss-Newton algorithm \cite{Nocedal_Numerical_2006, Barfoot_State_2017, Dellaert_Factor_2017}.

First, get an affine approximation of the error function \eqref{eq:deriving InEKF: error function def} linearized around the operating point $\mbfbar{X}_{k}$. To do so, define the perturbation
\begin{align}
    \mbf{d}_{k} &\coloneqq \mbf{X}_{k}\ominus\mbfbar{X}_{k},\\
    \label{eq:deriving InEKF: X_k = Xbar oplus dk}
    \mbf{X}_{k} &= \mbfbar{X}_{k} \oplus\mbf{d}_{k},
\end{align}
\marginnote[-1cm]{The variable name $\mbf{d}_{k}\in\rnums^{n_{x}}$ is chosen because it'll be used as a search direction in the optimization.}
according to the chosen error definition\sidenote{That is, replace `$\ominus$' and `$\oplus$' with the appropriate definition such as `$\liplus$'.}.

Then, the affine approximation can be written as
\begin{align}
    \label{eq:InEKF derivation: error affine approx}
    \mbf{e}(\mbf{X}_{k}) &= \mbf{e}(\mbfbar{X}_{k}\oplus\mbf{d}_{k})\\
    &\approx \mbf{e}\left( \mbfbar{X}_{k}  \right) + \mbf{J}\mbf{d}_{k},
\end{align}
where $\mbf{J}\in\rnums^{m\times n}$ is the Jacobian of the error function with respect to the Lie algebra coordinates.
\sidenote{Basically,
    \begin{align}
        \mbf{J} &= \pd{\mbf{e}(\mbfbar{X}_{k}\oplus \mbs{\xi}_{k})}{\mbs{\xi}_{k}},
    \end{align}
    where $\mbs{\xi}_{k}\in \rnums^{n_{k}}$ is the coordinates of the Lie algebra components.
}

Plugging the error affine approximation \eqref{eq:InEKF derivation: error affine approx} into the objective function gives the quadratic\sidenote{Quadratic in $\mbf{d}_{k}$.} approximation
\begin{align}
    \tilde{J}(\mbf{d}_{k}) &\coloneqq
    J(\mbfbar{X}_{k}\oplus\mbf{d}_{k}) \\
    &= \f{1}{2}\mbf{e}(\mbfbar{X}_{k}\oplus\mbf{d}_{k})^{\trans}\mbs{\Sigma}\inv\mbf{e}(\mbfbar{X}_{k}\oplus\mbf{d}_{k})\\
    &\approx \f{1}{2}\left( \mbf{e}\left( \mbfbar{X}_{k}  \right) + \mbf{J}\mbf{d}_{k} \right)^{\trans}\mbs{\Sigma}\inv\left( \mbf{e}\left( \mbfbar{X}_{k}  \right) + \mbf{J}\mbf{d}_{k} \right)\\
    &= \f{1}{2}\mbf{d}_{k}^{\trans}\mbf{J}^{\trans}\mbs{\Sigma}\inv\mbf{J}\mbf{d}_{k} + \mbf{e}\left( \mbfbar{X}_{k} \right)^{\trans}\mbs{\Sigma}\inv\mbf{J}\mbf{d}_{k} + \f{1}{2}\mbf{e}\left( \mbfbar{X}_{k} \right)^{\trans}\mbs{\Sigma}\inv\mbf{e}\left( \mbfbar{X}_{k} \right).
\end{align}
If $\mbf{J}$ is full column rank, then the quadratic function $\tilde{J}$ is strongly convex and has a unique minimizer. The minimizer is given by solving
\begin{align}
    \label{eq:deriving InEKF: optimal search direction}
    \mbf{d}_{k}^{\star} &= - \left(\mbf{J}^{\trans}\mbs{\Sigma}\inv\mbf{J}\right)\inv\mbf{J}^{\trans}\mbs{\Sigma}\inv\mbf{e}(\mbfbar{X}_{k}),
\end{align}
where
\begin{align}
    \label{eq:deriving InEKF: Phat}
    \mbfhat{P}_{k} &= \left(\mbf{J}^{\trans}\mbs{\Sigma}\inv\mbf{J}\right)\inv
\end{align}
at convergence \cite{Barfoot_State_2017}.

We can try to solve for $\mbf{d}_{k}$ analytically.
Say the error function Jacobian is given by
\begin{align}
    \label{eq:deriving InEKF: Jacobian eye C}
    \mbf{J} &= \bbm \gamma\eye \\ \mbf{C}_{k}\ebm,
\end{align}
where $\gamma = \pm 1$ depends on the choice of the error definition.

\subsection{The Kalman gain}
Plugging \eqref{eq:deriving InEKF: Jacobian eye C} into \eqref{eq:deriving InEKF: Phat} gives
\begin{align}
    \mbfhat{P}_{k}\inv &= \left(\mbf{J}^{\trans}\mbs{\Sigma}\inv\mbf{J}\right)\\
    &=
    \mbfcheck{P}_{k}\inv + \mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv\mbf{C}_{k}.
\end{align}
Pre-multiplying by $\mbfhat{P}_{k}$ gives
\begin{align}
    \eye
    &= \mbfhat{P}_{k}\mbfcheck{P}_{k}\inv + \underbrace{\mbfhat{P}_{k}\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv}_{\eqqcolon \mbf{K}_{k}}\mbf{C}_{k}\\
    \label{eq:deriving InEKF: eye = PhatPcheckinv + KkCk}
    &=\mbfhat{P}_{k}\mbfcheck{P}_{k}\inv + \mbf{K}_{k}\mbf{C}_{k},
\end{align}
where $\mbf{K}_{k}$ is the \emph{Kalman gain}.
Solving for $\mbfhat{P}_{k}$ results in
\begin{align}
    \mbfhat{P}_{k} &= \left( \eye - \mbf{K}_{k}\mbf{C}_{k} \right)\mbfcheck{P}_{k}.
\end{align}
Post-multiply by $\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv$ gives
\begin{align}
    \underbrace{\mbfhat{P}_{k}\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv}_{\mbf{K}_{k}} &= \left( \eye - \mbf{K}_{k}\mbf{C}_{k} \right)\mbfcheck{P}_{k}\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv\\
    \mbf{K}_{k} &=  \mbfcheck{P}_{k}\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv - \mbf{K}_{k}\mbf{C}_{k}\mbfcheck{P}_{k}\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv\\
    \mbf{K}_{k}\left( \eye + \mbf{C}_{k}\mbfcheck{P}_{k}\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv  \right) &= \mbfcheck{P}_{k}\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv
\end{align}
Post-multiply by $\mbf{R}_{k}$
\begin{align}
    \mbf{K}_{k}\left( \mbf{R}_{k} + \mbf{C}_{k}\mbfcheck{P}_{k}\mbf{C}_{k}^{\trans}\right) &= \mbfcheck{P}_{k}\mbf{C}_{k}^{\trans}.
\end{align}
Finally, the Kalman gain is given by
\begin{align}
    \mbf{K}_{k} &=
    \mbfcheck{P}_{k}\mbf{C}_{k}^{\trans} \underbrace{\left( \mbf{R} + \mbf{C}_{k}\mbfcheck{P}_{k}\mbf{C}_{k}^{\trans}\right)\inv}_{\mbf{S}_{k}\inv}.
\end{align}

\subsection{The search direction}
Plugging the posterior covariance \eqref{eq:deriving InEKF: Phat} into the optimal search direction \eqref{eq:deriving InEKF: optimal search direction} results in
\begin{align}
    \mbf{d}_{k}^{\star} &= - \mbfhat{P}_{k}\mbf{J}^{\trans}\mbs{\Sigma}\inv\mbf{e}(\mbfbar{X}_{k})\\
    &=
    -\mbfhat{P}_{k} \bbm \gamma\eye & \mbf{C}_{k}^{\trans} \ebm
    \bbm \mbfcheck{P}_{k}\inv & \\ & \mbf{R}_{k}\inv \ebm
    \bbm \mbfbar{X}_{k}\ominus \mbfcheck{X}_{k} \\\mbfbar{X}_{k}\mbf{b} - \mbf{y}_{k} \ebm
    \\
    &= -\mbfhat{P}_{k}\left( \gamma \mbfcheck{P}_{k}\inv\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) + \mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv(\mbfbar{X}_{k}\mbf{b} - \mbf{y}_{k})\right)\\
    &= -\gamma\mbfhat{P}_{k}\mbfcheck{P}_{k}\inv\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) - \underbrace{\mbfhat{P}_{k}\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv}_{\mbf{K}_{k}}(\mbfbar{X}_{k}\mbf{b} - \mbf{y}_{k})\\
    &\overset{\star}{=}
    -\gamma\left(\eye - \mbf{K}_{k}\mbf{C}_{k}  \right)\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) - \underbrace{\mbfhat{P}_{k}\mbf{C}_{k}^{\trans}\mbf{R}_{k}\inv}_{\mbf{K}_{k}}(\mbfbar{X}_{k}\mbf{b} - \mbf{y}_{k}) \\
    &= -\gamma\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) + \gamma\mbf{K}_{k}\mbf{C}_{k}\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) - \mbf{K}_{k}\left( \mbfbar{X}_{k}\mbf{b} - \mbf{y}_{k} \right)\\
    &= -\gamma\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) + \mbf{K}_{k}\left(\gamma \mbf{C}_{k} \left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) + \left( \mbf{y}_{k} - \mbfbar{X}_{k}\mbf{b}\right)\right)\\
    &= -\gamma\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) + \mbf{K}_{k}\left(\gamma \mbf{C}_{k} \left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) + \left( \mbf{y}_{k} - \mbfbar{X}_{k}\mbf{b}\right)\right)\\
    &= -\gamma\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) + \mbf{K}_{k}\mbf{z}_{k}(\mbfbar{X}_{k}),
\end{align}
\marginnote[-3.5cm]{$\star$ From \eqref{eq:deriving InEKF: eye = PhatPcheckinv + KkCk},
    \begin{align}
        \mbfhat{P}_{k}\mbfcheck{P}_{k}\inv &= \eye - \mbf{K}_{k}\mbf{C}_{k}.
    \end{align}
}
where
\begin{align}
    \label{eq:deriving InEKF: general innovation}
    \mbf{z}_{k}(\mbfbar{X}_{k}) &= \gamma\mbf{C}_{k}\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) + \mbf{y}_{k} - \mbfbar{X}_{k}\mbf{b}
\end{align}
is the innovation.

Finally, using \eqref{eq:deriving InEKF: X_k = Xbar oplus dk}, the posterior estimate is
\begin{align}
    \mbfhat{X}_{k}
    &= \mbfbar{X}\oplus\mbf{d}_{k}^{\star}\\
    \label{eq:deriving InEKF: general update equation}
    &= \mbfbar{X}\oplus\left(-\gamma\left( \mbfbar{X}_{k}\ominus\mbfcheck{X}_{k} \right) + \mbf{K}_{k}\mbf{z}_{k}(\mbfbar{X}_{k})\right).
\end{align}
Note that if the linearization point is defined as
\begin{align}
    \mbfbar{X}_{k} &\coloneqq \mbfcheck{X}_{k},
\end{align}
then
the innovation \eqref{eq:deriving InEKF: general innovation} simplifies to
\begin{align}
    \mbf{z}_{k}(\mbfcheck{X}_{k}) &= \mbf{y}_{k} - \mbfbar{X}_{k}\mbf{b},
\end{align}
and the update equation \eqref{eq:deriving InEKF: general update equation} becomes
\begin{align}
    \mbfbar{X}_{k} &= \mbfcheck{X}_{k}\oplus\left( \mbf{K}_{k}\mbf{z}_{k}(\mbfcheck{X}_{k}) \right)\\
    &= \mbfcheck{X}_{k}\oplus\left( \mbf{K}_{k}(\mbf{y}_{k} - \mbfcheck{X}_{k}\mbf{b}) \right).
\end{align}
This is the invariant extended Kalman filter correction.


\subsection{Law of Exponents}
The Law of Exponents is a set of rules for simplifying expressions that governs
the combination of exponents (powers).
Recall that $\sqrt[n]{~}$ denotes the $n$th root.
For example $\sqrt[3]{8}=2$ represents that the cube root of $8$ is equal to $2$.\\

\begin{definition}{Law of Exponents}{ExponentRules1}
{\bf Definitions}

If $m,n$ are positive integers, then:
\begin{multicols}{2}
\begin{enumerate}
	\item $x^n=x\cdot x\cdot\ldots\cdot x$ ($n$ times).
	\item $x^0=1$, for $x\neq 0$.
	\item $\ds{x^{-n}=\frac{1}{x^n}}$, for $x\neq 0$.
	\item $x^{m/n}=\sqrt[n]{x^m}$ or $\left(\sqrt[n]{x}\right)^m$, for $x\geq 0$.
\end{enumerate}
\end{multicols}

{\bf Combining}\\

\hspace{4mm} 1. \hspace{1mm} $x^ax^b=x^{a+b}$.
\hspace{0.5cm} 2. \hspace{1mm}  $\ds{\frac{x^a}{x^b}=x^{a-b}}$, for $x\neq 0$.
\hspace{0.5cm} 3. \hspace{1mm}  $\left(x^a\right)^b=x^{ab}=x^{ba}=\left(x^b\right)^a$. \\

%\begin{multicols}{3}
%\begin{enumerate}
%	\item $x^ax^b=x^{a+b}$.
%	\item $\ds{\frac{x^a}{x^b}=x^{a-b}}$, for $x\neq 0$.
%	\item $\left(x^a\right)^b=x^{ab}=x^{ba}=\left(x^b\right)^a$.
%\end{enumerate}
%\end{multicols}

{\bf Distributing}
\begin{multicols}{2}
\begin{enumerate}
	\item $(xy)^a=x^ay^a$, for $x\geq 0$, $y\geq 0$.
	\item $\ds{\left(\frac{x}{y}\right)^a=\frac{x^a}{y^a}}$, for $x\geq 0$, $y>0$.
\end{enumerate}
\end{multicols}
\end{definition}

In the next example, the word \ifont{simplify} means \ifont{to make simpler} or to write the expression more compactly.\\

\begin{example}{Laws of Exponents}{LawsExponents}\label{LawsExponents}
Simplify the following expression as much as possible assuming $x,y>0$:
$$\frac{3x^{-2}y^3x}{y^2\sqrt x}.$$
\vspace{-0.5cm}
\end{example}

\begin{solution}
Using the Law of Exponents, we have:
$$\def\arraystretch{2.5}
\begin{array}{>{\displaystyle}r>{\displaystyle}c>{\displaystyle}l>{\displaystyle}l}
\frac{3x^{-2}y^3x}{y^2\sqrt x} & = & \frac{3x^{-2}y^3x}{y^2x^{\frac{1}{2}}},
		&\mbox{since $\ds\sqrt x = x^{\frac{1}{2}}$},\\
~ & = & \frac{3x^{-2}yx}{x^{\frac{1}{2}}},
		&\mbox{since $\ds\frac{y^3}{y^2}=y$},\\
~ & = & \frac{3y}{x^{\frac{3}{2}}},
		&\mbox{since $\ds\frac{x^{-2}x}{x^{\frac{1}{2}}}=\frac{x^{-1}}{x^{\frac{1}{2}}} =x^{-\frac{3}{2}}=\frac{1}{x^{\frac{3}{2}}}$},\\
~ & = & \frac{3y}{\sqrt{x^3}},
		&\mbox{since $\ds x^{\frac{3}{2}}=\sqrt{x^3}$}.
\end{array}$$
An answer of $3yx^{-3/2}$ is equally acceptable, and such an expression may prove to be computationally simpler, although a positive exponent may be preferred.
\end{solution}


\section{Matrix Calculus}

\subsection{Layout}

There are two different layout:

\begin{itemize}
	\item \cindex{numerator layout}:
		\begin{equation}
		\left [
		\begin{matrix}
		\nabla f \\
		\nabla g
		\end{matrix}
		\right ]
		\end{equation}
	\item \cindex{denominator layout}:
	\begin{equation}
	\left [
	\nabla f ,	\nabla g	\right ]
	\end{equation}
\end{itemize}

numerator layout is preferred.

\subsection{Jacobian Matrix}
for $\mathbf{y}_{1 \times m} = \mathbf{f}(\mathbf{x}_{1 \times n})$, its \cindex{Jacobian matrix} is:

\begin{equation}
	\nabla{}_{\mathbf{x}} \mathbf{y} =
\frac{\partial \mathbf{y}}{\partial \mathbf{x}} = \left [ \begin{matrix}
	\nabla f_1 (\mathbf{x}) \\
	\nabla f_1 (\mathbf{x}) \\
	\vdots \\
	\nabla f_m (\mathbf{x})
\end{matrix} \right ] = \begin{pmatrix}
    \pd{f_1}{x} \\
    \\
    \pd{f_2}{x} \\
    \vdots \\
    \pd{f_m}{x}
    \end{pmatrix} = \left [ \begin{matrix}
\frac{\partial f_1({\mathbf{x}})}{x_1} & \frac{\partial f_1({\mathbf{x}})}{x_2} & \dots & \frac{\partial f_1({\mathbf{x}})}{x_n} \\
\frac{\partial f_2({\mathbf{x}})}{x_1} & \frac{\partial f_2({\mathbf{x}})}{x_2} & \dots & \frac{\partial f_2({\mathbf{x}})}{x_n} \\
\vdots & \vdots & \ddots & \vdots  \\
\frac{\partial f_m({\mathbf{x}})}{x_1} & \frac{\partial f_m({\mathbf{x}})}{x_2} & \dots & \frac{\partial f_m({\mathbf{x}})}{x_n}
\end{matrix} \right ]
\end{equation}

\subsection{Element-wise binary operator}

for element-wise binary operator
\begin{equation}
	\mathbf{y}= \mathbf{f}(\mathbf{w}) \bigcirc \mathbf{g}(\mathbf{x})
\end{equation}

$\bigcirc$ could be $+,-,\times\footnote{called \emph{hadamard product}}, \div, max $. The gradient is:

\begin{equation}
	\nabla{}_{\mathbf{x}} \mathbf{y} =
	\left [ \begin{matrix}
		y_1 \\
		y_2 \\
		\vdots \\
		y_n
	\end{matrix} \right ] = \left [ \begin{matrix}
	f_1 (\mathbf{w}) \bigcirc g_1 (\mathbf{x}) \\
	f_2 (\mathbf{w}) \bigcirc g_2 (\mathbf{x}) \\
	\vdots \\
	f_n (\mathbf{w}) \bigcirc g_n (\mathbf{x})
\end{matrix} \right ]
\end{equation}

The expanded matrix could be differentiated using Jacobian matrix.

\subsection{Vector Sum}
Vector sum operation $sum$ could be expressed as
\begin{equation}
	y = \text{sum}\Big (\mathbf{f}(\mathbf{x}) \Big ) = \sum_{i=1}^{n} f_i (\mathbf{x})
\end{equation}

$\nabla \mathbf{y}$ could be calculated as usual.

\subsection{Chain Rules}

In machine learning there are two ways of taking \cindex{chain rules}:
\begin{itemize}
	\item forward differentiation: $\frac{dy}{dx} =\frac{du}{dx} \times \frac{dy}{du}$
	\item backward differentiation: $\frac{dy}{dx}=\frac{dy}{du} \times \frac{du}{dx}$
\end{itemize}

Backward differentiation is preferred for matrix operation.

The full expression of $\mathbf{y}=\mathbf{f}(\mathbf{g}(\mathbf{x}))$ is:

\begin{equation}
\begin{aligned}
	\nabla{}_{\mathbf{x}} f &=  \pd{\mathbf{f}(\mathbf{g}(\mathbf{x}))}{\mathbf{x}} \\
	&= \pd{\mathbf{f}}{\mathbf{g}} \times \pd{\mathbf{g}}{\mathbf{x}} \\
	&= \left [ \begin{matrix}
\frac{\partial f_1({\mathbf{x}})}{g_1} & \frac{\partial f_1({\mathbf{x}})}{g_2} & \dots & \frac{\partial f_1({\mathbf{x}})}{g_n} \\
\frac{\partial f_2({\mathbf{x}})}{g_1} & \frac{\partial f_2({\mathbf{x}})}{g_2} & \dots & \frac{\partial f_2({\mathbf{x}})}{g_n} \\
\vdots & \vdots & \ddots & \vdots  \\
\frac{\partial f_m({\mathbf{x}})}{g_1} & \frac{\partial f_m({\mathbf{x}})}{g_2} & \dots & \frac{\partial f_m({\mathbf{x}})}{g_n}
\end{matrix} \right ]_{m \times n}
\times % multiple another one
\left [ \begin{matrix}
\frac{\partial g_1({\mathbf{x}})}{x_1} & \frac{\partial g_1({\mathbf{x}})}{x_2} & \dots & \frac{\partial g_1({\mathbf{x}})}{x_r} \\
\frac{\partial g_2({\mathbf{x}})}{x_1} & \frac{\partial g_2({\mathbf{x}})}{x_2} & \dots & \frac{\partial g_2({\mathbf{x}})}{x_r} \\
\vdots & \vdots & \ddots & \vdots  \\
\frac{\partial g_n({\mathbf{x}})}{x_1} & \frac{\partial g_n({\mathbf{x}})}{x_2} & \dots & \frac{\partial g_n({\mathbf{x}})}{x_r}
\end{matrix} \right ]_{n \times r}
\end{aligned}
\end{equation}


\section{Level Curves}
\noindent
We can look at different cross sections of a surface $f(x,y)$ by looking at the equation $f(x,y) = c$ where $c \in \mathbb{R}$. This curve lives in the xy-plane and is called the C-level curve. We often visualize these curves in the $z=c$ plane as part of the surface. In higher dimensions, like $f(x,y,z)$, a C-level curve becomes a C-level surface.

[INSERT IMAGES]

\noindent
For example, the C-level surface of $f(x,y,z) = e^{-\left(x^2+y^2+z^2\right)}$ is the sphere centered at the origin with radius $\sqrt{-\ln{c}}$: $x^2 + y^2 + z^ 2 =-\ln{c}$.


\section*{Exercises}

\begin{ex}
  Let
  \begin{equation*}
    A = \begin{mymatrix}{rr}
      1 & 3 \\
      5 & 2 \\
    \end{mymatrix},
    \quad
    B = \begin{mymatrix}{rrr}
      1 & 0 & 2 \\
      0 & 2 & 1 \\
    \end{mymatrix},
    \quad
    C = \begin{mymatrix}{rrr}
      2 & 1 & 0 \\
      1 & 0 & 2 \\
      0 & 3 & 1 \\
    \end{mymatrix}
  \end{equation*}
  and compute the following over $\Z_{11}$:
  \begin{enumerate}
  \item $3A$,
  \item $A^2$,
  \item $AB$,
  \item $BC$,
  \item $C^{-1}$.
  \end{enumerate}
\end{ex}


	\subsubsection{Simulations for Fourier-Galerkin Method}

		Following the ideas in the previous section for the Fourier-Galerkin method, we will assume that the expansion of the solution of (\ref{IVP_Burgers}) is as follows
		\begin{align*}
			u(x, t) = \displaystyle \sum_{|n| \leq \infty} \hat{u}_n (t) \phi_n (x), \hspace{2mm} \hat{u}_n (t) = \frac{1}{2 \pi} \left\langle u (x, t), \phi_n (x) \right\rangle,
		\end{align*}
		and as before $\phi_n (x) = e^{inx}$. \\

		Similarly, we consider the space $V_N = \hat{B}_N \cap H^2_p (\mathcal{D})$ where we will look for its truncated expansion given by
		\begin{align*}
			u_N (x, t) = \displaystyle \sum_{ |n| \leq N } \hat{u}_n (t) \phi_n (x), \hspace{2mm} \hat{u}_n (t) = \frac{1}{2 \pi} \left\langle u (x, t), \phi_n (x) \right\rangle,
		\end{align*}
		which we are going to force to satisfy the following
		\begin{align}
			\left\langle R_N, \phi_n \right\rangle = \left\langle \frac{\partial u_N}{\partial t} + \frac{1}{2} (u_N^2)_x - \alpha \frac{\partial^2 u_N}{\partial x^2}, \phi_n \right\rangle = 0, \hspace{2mm}  \hspace{2mm} \phi_n \in V_N, \hspace{2mm} \forall t > 0
		\end{align}
		or equivalently
		\begin{align*}
			\displaystyle \int_{I} \frac{\partial}{\partial t} u_N (x, t) \overline{\phi_n (x)} dx = \alpha \int_{I} \frac{\partial^2}{\partial x^2} u_N(x, t) \overline{\phi_n (x)} dx - \int_{I} \frac{1}{2} \frac{\partial}{\partial x} \left[ u_N(x, t) \right]^2 \overline{\phi_n (x)} dx
		\end{align*}
		Since $u_N (x, t)$ and its derivatives are periodic in space, integrating by parts we obtain that
		\begin{align*}
			\displaystyle \int_{I} \frac{\partial}{\partial t} u_N (x, t) \overline{\phi_n (x)} dx = - \alpha \int_{I} \frac{\partial}{\partial x} u_N(x, t) \frac{\partial}{\partial x} \overline{\phi_n (x)} dx + \int_{I} \frac{1}{2} \left[ u_N(x, t) \right]^2 \frac{\partial}{\partial x} \overline{\phi_n (x)} dx
		\end{align*}

		Then writing again as an inner product
		\begin{align*}
			\left\langle \frac{\partial u_N}{\partial t}, \phi_n  \right\rangle = \left\langle - \alpha \frac{\partial u_N}{\partial x} + \frac{1}{2} [u_N]^2, \frac{\partial \phi_n}{\partial x}  \right\rangle
		\end{align*}
		and using the orthogonality $\langle \phi_l, \phi_n \rangle = 2 \pi \delta_{ln}$, for each $n$ fixed we have to
		\begin{align*}
			\left\langle \frac{\partial u_N}{\partial t}, \phi_n  \right\rangle = \left\langle \displaystyle \sum_{ |l| \leq N} \frac{d \hat{u}_l (t)}{dt} \phi_l, \phi_n  \right\rangle = 2 \pi \frac{d \hat{u}_n (t)}{dt}
		\end{align*}

		\noindent and for the other terms
		\begin{align*}
			\left\langle \frac{\partial u_N}{\partial x} - \frac{1}{2} [u_N]^2, \frac{\partial \phi_n}{\partial x}  \right\rangle = & \alpha \left\langle - i \displaystyle \sum_{ |l| \leq N} l \hat{u}_l (t) \phi_l, in \phi_n \right\rangle \\
			&+ \left\langle\frac{1}{2} \left( \sum_{ |p| \leq N} \hat{u}_p (t) \phi_p \right) \left( \sum_{ |q| \leq N} \hat{u}_q (t) \phi_q \right), in \phi_n \right\rangle \\
			=&  - \alpha n \left\langle  \displaystyle \sum_{ |l| \leq N} l \hat{u}_l (t) \phi_l, \phi_n \right\rangle - \frac{in}{2} \left\langle \sum_{ |p| \leq N} \sum_{ |q| \leq N} \hat{u}_p (t) \hat{u}_q (t) \phi_{p + q}, \phi_n \right \rangle \\
			=& -2 \pi \alpha n^2 \hat{u}_n (t) - in \pi \sum_{ p + q = n} \hat{u}_p (t) \hat{u}_q (t), \hspace{2mm} |q|, |p| \leq N.
		\end{align*}

		\noindent Therefore, using the linear transformation from $\mathcal{D}_p = [0, 2 \pi]$ to $\mathcal{D} = [x_L, x_R]$ given by $x = P z + x_L$ to escalate the problem, where $P = \frac{x_R - x_L}{2 \pi}$ and $z \in \mathcal{D}_p$, we have the following system of nonlinear ordinary differential equations
		\begin{align}
		\label{Galerkin_Nonlinear}
			\frac{d \hat{u}_n (t)}{dt} = - \alpha P^2 n^2 \hat{u}_n (t) - \frac{in}{2} P \sum_{ p + q = n} \hat{u}_p (t) \hat{u}_q (t), \hspace{2mm} |q|, |p| \leq N,
		\end{align}
		or equivalently
		\begin{align*}
			\frac{d \hat{u}_n (t)}{dt} = - \alpha P^2 n^2 \hat{u}_n (t) - \frac{in}{2} P \sum_{|q| \leq N} \hat{u}_{n - q} (t) \hat{u}_q (t), \hspace{2mm} |n| \leq N,
		\end{align*}
		which is solved using the initial condition of the original projected problem on the space $V_N$, that is,
		\begin{align*}
			\mathcal{P}_N u_0 = \displaystyle \sum_{|n| \leq N} a_n \phi_n (x), \hspace{2mm} a_n = \frac{1}{2 \pi} \langle u_0 (x), \phi_n (x) \rangle.
		\end{align*}

		This problem can be treated in different ways and, in this work, we will develop it with the following approach. First, let's discard the terms $\hat{u}_q$ such that $|q| > N$ on the right side of the equation above, to get the following
		\begin{align*}
			\frac{d \hat{u}_n (t)}{dt} =  -\alpha P^2 n^2 \hat{u}_n (t) - \frac{in}{2} P \sum_{|q| \leq N} \hat{u}_{n - q} (t) \hat{u}_q (t), \hspace{2mm} |n - q| \leq N ,
		\end{align*}
		which is a system of $2N + 1$ equations. \\

		There is a wide variety of numerical methods to solve the above problem, and since this is not linear, it is not easy to justify choosing an appropriate method, since advanced knowledge of numerical analysis is required to investigate its characteristics. The way to choose a candidate method is by investigating its numerical stability, which guarantees that the numerical solution does not explode towards infinity and that it is at least bounded. \\

		In general, explicit methods are not suitable for nonlinear problems and are commonly handled by implicit methods. To show the implementation, and in addition to looking at its most relevant characteristics, we will solve the problem using Euler's implicit method. First, note that when $n = 0$ we have $\hat{u}_0 (t) = \hat{u}_0 (0)$, and that the nonlinear term involves the term $\hat{u}_n$ only when $q = 0$. So, using the integrating factor $e^{\lambda_n t}$ with $\lambda_n = \alpha P^2 n^2 + \frac{in}{2} P \hat{u}_0 (0) $ we can obtain the following formulation
		\begin{align*}
			\frac{d \left[ e^{\lambda_n t} \right]  \hat{u}_n (t)}{dt} = - \frac{in}{2} P e^{ \lambda_n t} \left[ \hat{u}_0 (0) \hat{u}_n (t) + \sum_{\substack{|q|\leq N \\ q \neq 0, n}} \hat{u}_{n - q} (t) \hat{u}_q (t) \right],
		\end{align*}
		and its solution using implicit Euler
		\begin{align*}
			\hat{u}^{j+1}_n = e^{- \lambda_n \Delta t} \left[ \hat{u}^j_n - \Delta t \frac{in}{2} P \sum_{\substack{|q|\leq N \\ q \neq 0, n}} \hat{u}^{j}_{n - q} \hat{u}^{j}_q \right] - \Delta t \frac{in}{2} P \hat{u}_0 (0) \hat{u}^{j+1}_n,
		\end{align*}
		and then solving for the term $\hat{u}^{j + 1}_n$ to obtain
		\begin{align*}
			\hat{u}^{j+1}_n &= \frac{ e^{- \lambda_n \Delta t}}{ 1 + \Delta t \frac{in}{2} P \hat{u}_0 (0) } \left[ \hat{u}^j_n - \Delta t \frac{in}{2} P \sum_{\substack{|q|\leq N \\ q \neq 0, n}} \hat{u}^{j}_{n - q} \hat{u}^{j}_q \right].
		\end{align*}

		Furthermore, using this formulation repeatedly for each $j$ gives us
		\begin{align}
		\label{Galerkin_Euler}
			\hat{u}^{j+1}_n &= \left[ \frac{ e^{- \lambda_n \Delta t}}{ 1 + \Delta t \frac{in}{2} P \hat{u}_0 (0) } \right]^{j+1} \hat{u}^0_n - \Delta t \frac{in}{2} P \sum_{k=1}^{j+1} \left[ \frac{ e^{- \lambda_n \Delta t}}{ 1 + \Delta t \frac{in}{2} P \hat{u}_0 (0) } \right]^{k}  \sum_{\substack{|q|\leq N \\ q \neq 0, n}} \hat{u}^{j+1-k}_{n - q} \hat{u}^{j+1-k}_q
		\end{align}
		Now, we will proceed to describe the steps necessary to implement the method based on the above. The simulations we are going to show were based on the formulation (\ref{Galerkin_Euler}), taking advantage of the fast Fourier transformation to calculate the coefficients $\hat{u}_n$, since it allows us to go from physical space to the space of Fourier and conversely. Also, it is worth mentioning that the codes used in this work can be found at \url{https://github.com/alanmatzumiya/pySpectralPDE.git}. \\

		To carry out these numerical experiments it was necessary to follow the following steps:
		\begin{enumerate}
			\item Set the intervals to consider, $[x_L, x_R]$ for physical space and $[t_0, t_f]$ for time, and give an initial condition function $u_0$. It is also necessary to choose the value of $N$, and another for the parameter $\alpha$.

			\item We proceed to calculate the points in the spatial grid where we want to obtain the solution, and for this, we define the points $z_i \in [0, 2 \pi]$ as
			\begin{align*}
				z_i = \frac{2 \pi i}{N}, \hspace{2mm} i = 0, 1, \dots, N
			\end{align*}
			which are used to obtain the points $x_i \in [x_L, x_R]$
			\begin{align*}
				x_i = P z_i + x_L, \hspace{2mm} P = \frac{x_R - x_L}{2 \pi}
			\end{align*}

			To obtain the points $t_j \in [t_0, t_f]$, we define the sequence $j = 0, 1, \dots, M$, for some positive $M$, and calculate the following
			\begin{align*}
				t_j = t_0 + j \Delta t, \hspace{2mm} \Delta t = \frac{t_f - t_0}{M}
			\end{align*}

			The values $| n | = 0, 1, \dots, N $ must be established to calculate $in, n^2$, which are required to approximate the derivatives.

			\item Calculate the coefficients $\hat{u}_0$ of the initial condition function $u_0$, using the fast Fourier transformation based on the already fixed spatial grid. So, it is possible to start the recursive rule given by (\ref{Galerkin_Euler}) for every $t_j$.

			\item Finally, the approximation obtained is evaluated using the equation given by
			\begin{align*}
				u_N(x, t_j) = \displaystyle \sum_{|n| \leq N} \hat{u}_n (t_j) e^{inx}.
			\end{align*}
		\end{enumerate}

		For the numerical study, we will establish the following initial condition
		\begin{align}
			\label{IC}
			u_0 (x) = e^{0.05 x^2}, \hspace{3mm} x \in [x_L, x_R]
		\end{align}

		To compare results, we will use the approximation given by (\ref{Exact_Solution_Approximation}) as the exact solution. In the figure \ref{Galerkin_alphas} shows the maximum distance over every $t \in [0, 100]$ between the exact solution and its approximations given by (\ref{Galerkin_Euler}) for $N = 2^m$, $m = 4, \dots, 12$, $\Delta t = 1.0 \times 10^{-5}$, and different values of $\alpha$. Furthermore, in Tables \ref{Galerkin_tabla_L2_alpha=1} and \ref{Galerkin_tabla_max_alpha=1}, we can see the numerical values of these distances for different configurations of $N$ and $\Delta t$. Similarly, in Tables \ref{Galerkin_tabla_L2_alpha=005} and \ref{Galerkin_tabla_max_alpha=005} but for $\alpha = 0.005$.

	\begin{figure}[H]
		\centering
		\includegraphics[width=15cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/galerkin/alphas_Error_N.png}
		\caption{(a) $L^2$-norm between the exact solution and its approximations using Galerkin method. (b) Max norm between the exact solution and its approximations.}
		\label{Galerkin_alphas}
	\end{figure}
	\newpage
	\begin{figure}[H]
		\centering
		\caption{Numerical solution for (\ref{IVP_Burgers}) using (\ref{Galerkin_Euler}) with $\alpha = 1.0$, $N=2048$, and $\Delta t = 1.0 \times 10^{-5}$.}
		\includegraphics[width=12cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/galerkin/Numerical_Solution_alpha=1.png}
		\label{Galerkin_alpha=1}
		\caption{Numerical solution for (\ref{IVP_Burgers}) using (\ref{Galerkin_Euler}) at the time $T = 100$ with $\alpha = 1.0$, and $\Delta t = 1.0 \times 10^{-5}$. (b) Point-wise error of approximation}
		\includegraphics[width=12.5cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/galerkin/Numerical_Solution_alpha=1_T=100.png}
		\label{Galerkin_alpha=1_T}
	\end{figure}
	\begin{table}[H]
		\begin{tabular}{lcccc}
			\toprule
			\multicolumn{1}{c}{\textbf{Expansion}} & \multicolumn{4}{c}{\textbf{Error}} \\
			$\hspace{9mm}N$ & $\Delta t=1\times 10^{-2}$ & $\Delta t=1\times 10^{-3}$ & $\Delta t=1\times 10^{-4}$ & $\Delta t=1\times 10^{-5}$ \\
			\midrule
			\hspace{7mm} 16 & 0.72504    & 0.72504    & 0.72504    & 0.72504    \\
			\midrule
			\hspace{7mm} 32 & 6.90249 $\times 10 ^{-2}$   & 6.88052 $\times 10 ^{-2}$   & 6.87838 $\times 10 ^{-2}$   & 6.87816 $\times 10 ^{-2}$   \\
			\midrule
			\hspace{7mm} 64 & 1.23827 $\times 10 ^{-3}$  & 8.85367 $\times 10 ^{-4}$ & 8.80521 $\times 10 ^{-4}$ & 8.80410 $\times 10 ^{-4}$  \\
			\midrule
			\hspace{7mm} 128 & 9.43454 $\times 10 ^{-4}$ & 9.41793 $\times 10 ^{-5}$ & 9.41148 $\times 10 ^{-6}$ & 9.41827 $\times 10 ^{-7}$  \\
			\midrule
			\hspace{7mm} 256 & 9.43454 $\times 10 ^{-4}$ & 9.41793 $\times 10 ^{-5}$ & 9.41109 $\times 10 ^{-6}$ & 9.36411 $\times 10 ^{-7}$ \\
			\midrule
			\hspace{7mm} 512 & 9.43454 $\times 10 ^{-4}$ & 9.41793 $\times 10 ^{-5}$ & 9.41109 $\times 10 ^{-6}$ & 9.36411 $\times 10 ^{-7}$ \\
			\midrule
			\hspace{7mm} 1024 & $\ast$ & 9.41793 $\times 10^{-5}$ & 9.41109 $\times 10^{-6}$ & 9.36411 $\times 10^{-7}$              \\
			\midrule
			\hspace{7mm} 2048 & $\ast$ & $\ast$ & 9.41109 $\times 10^{-6}$ & 9.36411 $\times 10^{-7}$   \\
			\bottomrule
		\end{tabular}
		\caption{Error using $L^2$-norm with $\alpha = 1.0$}
		\label{Galerkin_tabla_L2_alpha=1}
		\vspace{1cm}
		\begin{tabular}{lcccc}
			\toprule
			\multicolumn{1}{c}{\textbf{Expansion}} & \multicolumn{4}{c}{\textbf{Error}} \\
			$\hspace{9mm}N$ & $\Delta t=1\times 10^{-2}$ & $\Delta t=1\times 10^{-3}$ & $\Delta t=1\times 10^{-4}$ & $\Delta t=1\times 10^{-5}$ \\
			\midrule
			\hspace{7mm} 16 & 0.203363    & 0.203333    & 0.203331    & 0.20333     \\
			\midrule
			\hspace{7mm} 32 & 2.64192 $\times 10 ^{-2}$   & 2.6248 $\times 10 ^{-2}$    & 2.62491 $\times 10 ^{-2}$  & 2.62492 $\times 10 ^{-2}$   \\
			\midrule
			\hspace{7mm} 64 & 6.93001 $\times 10 ^{-4}$ & 4.11641 $\times 10 ^{-4}$ & 3.85563 $\times 10 ^{-4}$ & 3.82972 $\times 10 ^{-4}$ \\
			\midrule
			\hspace{7mm} 128 & 4.74934 $\times 10 ^{-4}$ & 4.73649 $\times 10 ^{-5}$ & 4.74295 $\times 10 ^{-6}$ & 5.16105 $\times 10 ^{-7}$ \\
			\midrule
			\hspace{7mm} 256 & 4.74936 $\times 10 ^{-4}$ & 4.7368 $\times 10 ^{-5}$  & 4.72569 $\times 10 ^{-6}$ & 4.64922 $\times 10 ^{-7}$ \\
			\midrule
			\hspace{7mm} 512 & 4.74936 $\times 10 ^{-4}$ & 4.7368 $\times 10 ^{-5}$  & 4.72569 $\times 10 ^{-6}$ & 4.64922 $\times 10 ^{-4}$ \\
			\midrule
			\hspace{7mm} 1024 & * & 4.7368 $\times 10 ^{-5}$  & 4.72569 $\times 10 ^{-6}$ & 4.64922 $\times 10 ^{-7}$ \\
			\midrule
			\hspace{7mm} 2048 & * & * & 4.72569 $\times 10 ^{-6}$ & 4.64922 $\times 10 ^{-7}$ \\
			\bottomrule
		\end{tabular}
		\caption{Error using Max norm with $\alpha = 1.0$}
		\label{Galerkin_tabla_max_alpha=1}
	\end{table}

	\begin{figure}[H]
		\centering
		\caption{Numerical solution for (\ref{IVP_Burgers}) using (\ref{Galerkin_Euler}) with $\alpha = 0.005$, $N=2048$, and $\Delta t = 1.0 \times 10^{-5}$.}
		\includegraphics[width=12cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/galerkin/Numerical_Solution_alpha=0005.png}
		\caption{Numerical solution for (\ref{IVP_Burgers}) using (\ref{Galerkin_Euler}) at the time $T = 100$ with $\alpha = 1.0$, and $\Delta t = 1.0 \times 10^{-5}$. (b) Point-wise error of approximation}
		\includegraphics[width=12.5cm]{burgers_equation/deterministic/numerical_experiments/viscid/figures/galerkin/Numerical_Solution_alpha=0005_T=100.png}
		\label{Galerkin_alpha=005_T}
	\end{figure}

	\begin{table}[H]
	\begin{tabular}{lcccc}
		\toprule
		\multicolumn{1}{c}{\textbf{Expansion}} & \multicolumn{4}{c}{\textbf{Error}} \\
		$\hspace{9mm}N$ & $\Delta t=1\times 10^{-2}$ & $\Delta t=1\times 10^{-3}$ & $\Delta t=1\times 10^{-4}$ & $\Delta t=1\times 10^{-5}$ \\
		\midrule
		\hspace{7mm} 16 & 9.95328   & 9.91901    & 9.91597    & 9.91567    \\
		\midrule
		\hspace{7mm} 32 & 2.72607   & 2.70558    & 2.70347    & 2.70326    \\
		\midrule
		\hspace{7mm} 64 & 2.50343   & 2.45988    & 2.45543    & 2.45497    \\
		\midrule
		\hspace{7mm} 128 & 2.16142   & 2.06992    & 2.05918    & 2.05795    \\
		\midrule
		\hspace{7mm} 256 & 1.3658    & 1.19385    & 1.17602    & 1.17412    \\
		\midrule
		\hspace{7mm} 512 & 0.339826  & 0.265843   & 0.262164   & 0.261805   \\
		\midrule
		\hspace{7mm} 1024 & 0.161405  & 0.133743   & 0.131882   & 0.131699   \\
		\midrule
		\hspace{7mm} 2048 & 6.50292 $\times 10^{-2}$ & 4.70602 $\times 10^{-2}$ & 4.57371 $\times 10^{-2}$  & 4.56090 $\times 10^{-2}$  \\
		\bottomrule
	\end{tabular}
	\caption{Error using $L^2$-norm with $\alpha = 0.005$}
	\label{Galerkin_tabla_L2_alpha=005}
	\vspace{1cm}
	\begin{tabular}{lcccc}
		\toprule
		\multicolumn{1}{c}{\textbf{Expansion}} & \multicolumn{4}{c}{\textbf{Error}} \\
		$\hspace{9mm}N$ & $\Delta t=1\times 10^{-2}$ & $\Delta t=1\times 10^{-3}$ & $\Delta t=1\times 10^{-4}$ & $\Delta t=1\times 10^{-5}$ \\
		\midrule
		\hspace{7mm} 16 & 2.50002  & 2.48992   & 2.48891   & 2.48881   \\
		\midrule
		\hspace{7mm} 32 & 1.21263  & 1.20544   & 1.2047    & 1.20463   \\
		\midrule
		\hspace{7mm} 64 & 1.21269  & 1.17736   & 1.17517   & 1.17495   \\
		\midrule
		\hspace{7mm} 128 & 1.10164  & 1.03493   & 1.03093   & 1.03048   \\
		\midrule
		\hspace{7mm} 256 & 0.954369 & 0.881472  & 0.873392  & 0.87259   \\
		\midrule
		\hspace{7mm} 512 & 0.665071 & 0.418735  & 0.398664  & 0.396931  \\
		\midrule
		\hspace{7mm} 1024 & 0.241841 & 0.244188  & 0.244437  & 0.244461  \\
		\midrule
		\hspace{7mm} 2048 & 0.133067 & 0.104675  & 0.109151  & 0.109596  \\
		\bottomrule
	\end{tabular}
	\caption{Error using Max norm with $\alpha =0.005$}
	\label{Galerkin_tabla_max_alpha=005}
	\end{table}

	\newpage


\lab{Algorithms}{Poisson's equation}{Poisson's equation}
\label{lab:finitedifference2}

 % The basic idea behind finite difference methods is to replace a
 % differential operator, defined on some space of continuous functions, with a
 % difference operator defined on a finite vector space (i.e., a space of grid functions).
 %
 % To do this, we replace derivative terms in the differential equation with
 % appropriate difference expressions.
 %
 % \begin{align*}
 % 	u_{xx}(t,x) &\approx \frac{(u(t,x+h)- 2u(t,x) + u(t,x-h))}{h^2},\\
 % 	u_x(t,x) & \approx \frac{u(t,x+h)-u(t,x-h)}{2h}.
 % \end{align*}
% Recall that the heat equation is given by $u_t = u_{xx} + f(x,t,u)$.
Suppose that we want to describe the distribution of heat throughout a region $\Omega$.
Let $g(x)$ represent the temperature on the boundary of $\Omega$ ($\partial \Omega$), and let $h(x)$ represent the initial heat distribution at time $t = 0$.
If we let $f(x,t)$ represent any heat sources/sinks in $\Omega$, then the flow of heat can be described by the boundary value problem (BVP)
\begin{align}
	\begin{split}
		& { } u_t = \triangle u + f(x,t), \quad x \in \Omega, \quad t >0,\\
		& { }u(x,t) = h(x), \quad x \in \partial \Omega, \\
		& { }u(x,0) = g(x).
	\end{split}
\end{align}
When the source term $f$ does not depend on time, there is often a steady-state heat distribution $u_{\infty}$ that is approached as $t \to \infty$.
This steady state $u_{\infty}$ is a solution of the BVP
\begin{align}
	\begin{split}
		& { }  \triangle u + f(x) = 0, \quad x \in \Omega,\\
		& { }u(x,t) = h(x), \quad x \in \partial \Omega.
	\end{split}
\end{align}

This last partial differential equation, $\triangle u = -f$, is called Poisson's equation.
This equation is satisfied by the steady-state solutions of many other evolutionary processes.
Poisson's equation is often used in electrostatics, image processing, surface reconstruction, computational fluid dynamics, and other areas.

\section*{Poisson's equation in one dimension}
The 1D Poisson equation is given by
\begin{align}
	\begin{split}
u'' &= f(x), \quad a \leq x \leq b,\\
	u(a) &= \alpha,\quad u(b) = \beta,
	\end{split} \label{eqn:1d_poisson}
\end{align}
where $f$ is continuous.
We will develop a finite difference method for solving this BVP.

The basic idea behind most numerical methods for differential equations is to approximate the exact solution $u(x)$ at some finite collection of points in the
domain of the problem.
Instead of analytically solving an infinite-dimensional problem, we look for a simple finite collection of algebraic equations that approximate the original problem.

Note that this equation can easily be solved by integrating twice and using the boundary conditions to determine the constants of integration.
Solving this \eqref{eqn:1d_poisson} is equivalent to solving the equation $\triangle u = f,$ where $\triangle = \frac{d^2}{dx^2}$ is a differential operator defined on the infinite-dimensional space of functions $u:[a,b] \to \mathbb{R}$ that are twice continuously differentiable and satisfy the boundary conditions $u(a) = \alpha$, $u(b) = \beta$.
The finite difference method replaces infinite-dimensional differential operators with finite-dimensional difference operators.

We consider an approximate solution $\{U_i\}_{i=-1}^N$ on an evenly spaced grid of $N$ subintervals, $a = x_{-1}, x_0, \ldots, x_{N-1} = b$ with $h = x_{i+1}-x_i$ for each $i$.
Recalling that $\frac{d^2}{dx^2}u(x) = \frac{1}{h^2}\left(u(x-h) -2u(x) + u(x+h)\right) + \mathcal{O}(h^2),$ we define the finite difference operator $D$ by  $DU_i = \frac{1}{h^2}\left(U_{i-1} -2U_i + U_{i+1}\right)$.
We may discretize \eqref{eqn:1d_poisson} using the equations
\begin{align*}
	\frac{1}{h^2} (U_{i+1}- 2U_i + U_{i-1})  &= f(x_i), \quad i = 0, \ldots, N-2,
\end{align*}
along with boundary conditions $U_{-1} = \alpha,$ $U_{N-1} = \beta$.
This gives $N+1$ equations and $N+1$ unknowns, and can be written in matrix form as
\[ \frac{1}{h^2} \begin{bmatrix}h^2 & 0 &0&\hdots &0 \\ 1 &-2 & 1 &\hdots &0\\ \vdots &  & \ddots & &\vdots \\
0 & \hdots & 1 & -2 & 1 \\ 0 & \hdots & & 0 & h^2
\end{bmatrix} \cdot \begin{bmatrix}U_{-1}\\U_0\\ \vdots \\U_{N-2} \\U_{N-1}\end{bmatrix} = \begin{bmatrix}f(x_{-1})\\f(x_0)\\ \vdots \\ f(x_{N-2}) \\ f(x_{N-1}) \end{bmatrix}\]
We can further modify the system to obtain an $(N-1)\times (N-1)$ tridiagonal matrix on the left:
\[\frac{1}{h^2} \begin{bmatrix}-2 & 1 &0 & \hdots &0\\ 1 &-2 & 1 &\hdots &0\\ \vdots &  & \ddots & &\vdots \\ 0 & \hdots & 1 & -2 & 1 \\
0 & \hdots & 0 & 1 & -2
\end{bmatrix} \cdot \begin{bmatrix}U_0\\U_1\\ \vdots \\U_{N-3}\\U_{N-2}\end{bmatrix} = \begin{bmatrix}f(x_0) -\alpha/h^2 \\f(x_1)\\ \vdots \\ f(x_{N-3})\\ f(x_{N-2})-\beta/h^2 \end{bmatrix}\]

The following code uses this finite difference method to solve the BVP
\begin{align*}
u'' &= -3 \sin{x}, \quad 0 \leq x \leq 2,\\
	u(0) &= -2,\quad u(2) = 1
\end{align*}

\begin{lstlisting}
import numpy as np
from scipy.sparse import spdiags
from scipy.sparse.linalg import spsolve

def bvp(func, a, b, alpha, beta, N):
    """ Solving u'' = f(x), x in [a, b],
    u(a) = alpha, u(b) = beta
    N = number of subintervals """

    # We use the finite difference method to construct a system
    # of algebraic equations described by the matrix equation
    # AU = F

    # The length of each subinterval
    h = (b - a) / N

    # Initialize and define the vector F on the right.
    F = np.empty(N - 1)
    F[0] = func(a + 1. * h) - alpha * h**(-2.)
    F[N-2] = func(a + (N-1) * h) - beta * h**(-2.)
    for j in xrange(1,  N-2):
        F[j] = func(a + (j+1) * h)

    # Here we define the arrays that will go on the diagonals of A.
    data = np.empty((3, N-1))
    # main diagonal
    data[0,:] = -2. * np.ones((1, N-1))
    # off-diagonals
    data[1,:], data[2,:] = np.ones((1, N-1)), np.ones((1, N-1))
    # Next we specify on which diagonals they will be placed
    diags = np.array([0, -1, 1])

    A = h**(-2.) * spdiags(data, diags, N-1, N-1).asformat('csr')

    # We create and return the numerical approximation
    U = np.empty(N + 1)
    U[1:-1] = spsolve(A, F)
    U[0], U[-1] = alpha, beta
    return np.linspace(a, b, N+1), U

x, y = bvp(lambda x:(-3. * np.sin(x)), a=0., b=2., alpha=-2., beta=1, N=30)
\end{lstlisting}

\begin{comment}

How do we know if a numerical approximation is reasonable?
One way to determine this is to compute solutions for various step sizes $h$ and see if the solutions are converging to something.
To be more specific, suppose our finite difference method is $\mathcal{O}(h^p)$ accurate.
This means that the error $E(h) \approx Ch^p$ for some constant $C$ as $h \to 0$ (i.e., for $h>0$ small enough).

So compute the approximation $y_k$ for each stepsize $h_k$, $h_1 > h_2> \ldots>h_m$.
We will think of $y_m$ as the true solution.
Then the error of the approximation for
stepsize $h_k, k < m,$ is
\begin{align*}
	E(h_k) &= \max( \abs{ y_k - y_m}) \approx C h_k^p ,\\
	\log(E(h_k)) &= \log(C) + p \log(h_k).
\end{align*}
Thus on a log-log plot of $E(h)$ vs. $h,$ these values should be on a straight line with slope $p$ when $h$ is small enough to start getting convergence.

The following code generates the log-log plot in \ref{figure1}, and demonstrates second-order convergence for this finite difference approximation.
\begin{lstlisting}
import matplotlib.pyplot as plt
a, b = 0., 1.
num_approx = 10 # Number of Approximations
N = np.array([5*2**j for j in range(num_approx)])
h, max_error = (b-a)/N[:-1], np.ones(num_approx-1)

mesh_best, num_sol_best = bvp(lambda x:-3.*np.sin(x), a, b, alpha=-2., beta=1, N=N[-1])
for j in range(len(N)-1):
    mesh, num_sol = bvp(lambda x:-3.*np.sin(x), a, b, alpha=-2., beta=1, N=N[j])
    max_error[j] = np.max(np.abs( num_sol- num_sol_best[::2**(num_approx-j-1)] ) )
plt.loglog(h,max_error,'.-r',label="$E(h)$")
plt.loglog(h,h**(2.),'-k',label="$h^{\, 2}$")
plt.xlabel("$h$")
plt.legend(loc='best')
plt.show()
print "The order of the finite difference approximation is about ", ( (np.log(max_error[0]) -
    np.log(max_error[-1]) )/( np.log(h[0]) - np.log(h[-1]) ) ), "."
\end{lstlisting}

\begin{figure}[ht]
\centering
\includegraphics[width=12cm]{example_convergence.pdf}
\caption{TODO}
\end{figure}

\end{comment}

\begin{problem}
Extend the given finite difference code to the case of a general second order linear BVP with Dirichlet conditions:
\begin{align*}
	&{ } a_1(x)y'' +a_2(x)y'+ a_3(x) y = f(x), \quad x \in (a,b),\\
	&{ } y(a) = \alpha, \quad y(b) = \beta.
\end{align*}
Use your code to solve the singularly perturbed BVP
\begin{align}
	\begin{split}
	&{ } \epsilon y''(x)-y'= -1, \quad x \in (0,1), \\
	&{ } y(0) = 1, \quad y(1) = 3,
	\end{split} \label{eqn:singular_perturbed_BVP}
\end{align}
with $\epsilon = 1/10$.
How many subintervals are needed to obtain 4 digits of accuracy?

Hint: consider using a centered, forward, or backward differencing scheme to approximate $u'$.

% If $\alpha = 1$,  $\beta = 3$, and $f(x) = -1$, there is an exact solution:
% \[y(x) = \alpha + x+ (\beta - \alpha -1)\frac{e^{x/\epsilon -1}}{e^{1/\epsilon -1}}\]
\end{problem}

\begin{figure}
\centering
\includegraphics[width=12cm]{figure2.pdf}
\caption{The solution to \eqref{eqn:singular_perturbed_BVP}.
The solution gets steeper near $x = 1$ as $\epsilon $ gets small.}
\end{figure}

\section*{Poisson's equation in two dimensions}

 Consider Poisson's equation together with Dirichlet boundary conditions on a rectangular  domain $R = [a,b] x [c,d]$:
 \begin{align}
	\begin{split}
 	u_{xx} + u_{yy} &= f,\quad x \text{ in } R \subset \mathbb{R}^2,\\
 	u &= g, \quad x \text{ on } \partial R.
	\end{split}\label{eqn:2d_poisson}
\end{align}
Let $a = x_{-1}, x_0, \ldots, x_{N-1} = b$ be a partition of $[a,b]$, and let $c = y_{-1}, y_0, \ldots, y_{N-1} = d$ be a partition of $[c,d]$.
Suppose that there are $N+1$ evenly spaced points, so that $N$ is the number of subintervals in each dimension, and $x_i, y_j$ are given by
\begin{align*}
	x_i &= a + (i+1)\triangle x, \\
	y_j &= c + (j+1)\triangle y,
\end{align*}
for $i,j = 0, \ldots, N-2$, where $\triangle x = x_i-x_{i-1}, \triangle y = y_i-y_{i-1}$.
We look for an approximation $U_{i,\,j}$ on the grid $\{(x_i,u_j)\}_{i,j=-1}^{N-1}$.

Recall that
 \begin{align*}
 \triangle u = u_{xx}(x,y) + u_{yy}(x,y) &= \frac{u(x+h,y) - 2u(x,y)+ u(x-h,y)}{h^2} \\
 & \qquad{}+
 \frac{u(x,y+h) - 2u(x,y)+ u(x,y-h)}{h^2} + \mathcal{O}(h^2).
 \end{align*}
 We replace $\triangle $ with the finite difference operator $\triangle_h$, defined by
 \begin{align*}
 \triangle_h U_{ij} &= \frac{U_{i+1,\,j} - 2U_{i,\,j} + U_{i-1,\,j}}{h^2} + \frac{U_{i,\,j+1} - 2U_{i,\,j}+ U_{i,\,j-1}}{h^2},\\
&= \frac{1}{h^2}(U_{i-1,\,j} + U_{i+1,\,j} + U_{i,\,j-1} + U_{i,\,j+1}-4U_{i,\,j}).
 \end{align*}
 Then the set of equations  $\triangle_h U_{ij} = f_{ij}$, $i,j = 0,\ldots,N-2$ % $i,j = 1,\ldots,m$
can be written in matrix form as
 \[AU + p +  q  = f\]

$A$ is a block tridiagonal matrix, given by
\[\frac{1}{h^2}
\begin{bmatrix}
T & I & &  &\\
I &T & I & &\\
&\ddots  & \ddots & \ddots & \\
&  & I & T & I \\
&  &  & I & T\end{bmatrix}\]
where $I$ is the $N-1\times N-1$ % $m\times m$
identity matrix, and $T$ is the tridiagonal matrix
\[\begin{bmatrix}
-4 & 1 & &  &\\
1 &-4 & 1 & &\\
&\ddots  & \ddots & \ddots & \\
&  & 1 & -4 & 1 \\
&  &  & 1 & -4 \end{bmatrix}\]

The vector $U$ is given by
\[U = \begin{bmatrix} U^0 \\ U^1 \\ \\ U^{N-2} \end{bmatrix} \text{ where } U^j =
\begin{bmatrix} U_{0,\,j} \\ U_{1,\,j} \\ \\ U_{N-2,\,j} \end{bmatrix} \text{ for each } j, 0\leq j \leq N-2\]

% \[U = \begin{bmatrix} U^1 \\ U^2 \\ \\ U^m \end{bmatrix} \text{ where } U^j =
% \begin{bmatrix} U_{1,\,j} \\ U_{2,\,j} \\ \\ U_{m,\,j} \end{bmatrix} \text{ for each } j, 1\leq j \leq m\]
% So $U^j$ represents the $j$th row of interior points in our grid, where $y_j = jh$

The vectors $p$ and $q$ come from the boundary conditions of \eqref{eqn:2d_poisson}, and are given by
\[p = \begin{bmatrix} p^0 \\ \ldots \\ \\ p^{N-2} \end{bmatrix}, \quad  q = \begin{bmatrix} q^0 \\ \ldots \\ \\ q^{N-2} \end{bmatrix}\]
where
\[p^j = \frac{1}{h^2} \begin{bmatrix} g_{-1,\,j} \\ 0 \\ \vdots \\0\\ g_{N-1,\,j} \end{bmatrix} ,\,\,\, 0 \leq j \leq N-2\]
and
\[q^0 = \frac{1}{h^2}\begin{bmatrix} g_{0,-1}  \\ g_{1,-1} \\ \vdots \\ g_{N-3,-1}\\ g_{N-2,-1} \end{bmatrix}, \quad q^{N-2} = \frac{1}{h^2}\begin{bmatrix} g_{0,N-1} \\ g_{1,N-1} \\ \vdots \\ g_{N-3,N-1}\\ g_{N-2,N-1} \end{bmatrix}, \quad q^{j} = \begin{bmatrix} 0 \\ 0 \\ \vdots \\ 0 \\ 0 \end{bmatrix} ,\,\,\, 1 \leq j \leq N-3\]

% The vector $q$ is given by $u = [q^0 \ldots q^{N-2}]^T$, %   $u = [q^1 \ldots q^m]^T$,
% where
% \[q^j = \frac{1}{h^2} \begin{bmatrix} g_{0,\,j} \\ 0 \\ \vdots \\0\\ g_{m+1,\,j} \end{bmatrix} , \,\,\, 2 \leq j \leq m-1\]
% and
% \[q^1 = \frac{1}{h^2}\begin{bmatrix} g_{1,0} + g_{0,1} \\ g_{2,0} \\ \vdots \\ g_{m-1,0}\\ g_{m,0} + g_{m+1,1}\end{bmatrix}, \quad q^m = \frac{1}{h^2}\begin{bmatrix} g_{1,m+1} + g_{0,m}\\ g_{2,m+1} \\ \vdots \\ g_{m-1,m+1}\\ g_{m,m+1} + g_{m+1,m}\end{bmatrix}\]

% \begin{problem}
% Find the solution $u$ of the 2D Laplace equation $\Delta u = 0$ on the unit
% square $[0,1]\times [0,1] \subset \mathbb{R}^2,$ subject to the (Dirichlet) condition that
% $u(x,y) = x^3$ on the boundary.
%
% Graph your solution, and demonstrate convergence of the numerical approximation by
% creating a log-log plotof the error $E(h).$
% \end{problem}

% \begin{problem}
% Find the solution $u$ of the 2D Poisson equation with the given Dirichlet boundary conditions:
% \begin{align*}
% 	\Delta u &= -\pi^2 \sin(\pi x)\sin(\pi y), \quad (x,y) \in [0,1]\times [0,1], \\
% 	u(x,0) &= 1-x, \\
% 	u(x,1) &= 1-2x, \\
% 	u(0,y) &= 1, \\
% 	u(1,y) &= -y.
% \end{align*}
%
% Graph your solution, and demonstrate convergence of the numerical approximation by
% creating a log-log plot of the error $E(h).$
% \end{problem}

% The matrix $A$ is sparse, and so we can use several functions from the package \texttt{scipy.sparse.linalg}.
% In particular, we use the functions \texttt{spdiags} and \texttt{spsolve}.
%
%  \begin{verbatim}
% D1,D2,D3 = -4*np.ones((1,m**2)), np.ones((1,m**2)), np.ones((1,m**2))
% Dm1, Dm2 = np.ones((1,m**2)), np.ones((1,m**2))
% for j in range(0,D2.shape[1]):
% 	if (j%m)==m-1:
% 		D2[0,j]=0
% 	if (j%m)==0:
% 		D3[0,j]=0
% diags = np.array([0,-1,1,-m,m])
% data = np.concatenate((D1,D2,D3,Dm1,Dm2),axis=0) # This stacks up rows
% A = 1./h**2.*spdiags(data, diags, m**2,m**2).asformat('csr') # This ap
%  \end{verbatim}

\begin{comment}
\subsection{2D Heat Equation}
Recall that the collection of finite difference equations
\[\nabla^2_h U_{ij} = 0, \quad 1 \leq i,j\leq m\]
can be written in matrix form as
\[AU + q  = 0\]

The Crank-Nicolson method for the 2D heat equation is given by
\[U_{i,\,j}^{n+1}- U_{i,\,j}^{n} = \frac{\Delta t}{2}(\nabla_h^2 U_{i,\,j}^{n} + \nabla_h^2 U_{i,\,j}^{n+1}) \text{ for each } 1 \leq i,j \leq m\]
is a second order accurate in both space and time. Basically we're using a midpoint scheme in time,
and a trapezoidal scheme in space. The resulting method is implicit, and can be written in matrix form as
\begin{align*}
	IU^{n+1} &= IU^n + \frac{\Delta t}{2}(AU^n + q + AU^{n+1} + q)\\
	(I - \frac{\Delta t}{2}A)U^{n+1}&= (I + \frac{\Delta t}{2}A)U^n + \Delta t q
\end{align*}

% TODO: What size must the time step be to ensure stability?

We will need to take many time steps, where many equations must be solved with the matrix $(I - \frac{\Delta t}{2}A)$.
The function \texttt{factorized} from \texttt{scipy.sparse.linalg} computes the LU decomposition of the matrix.
This decomposition reduces the time required for solving consecutive time steps.
\end{comment}

\section*{Poisson's Equation and Conservative Forces}
<<<<<<< HEAD
In physics Poisson's equation is used to describe the scalar potential of a conservative force (we'll explain what each of these terms mean).
A conservative force $\bold{F}(\bold{r})$ is a vector function that obeys any of three equivalent conditions
\begin{enumerate}
	\item The curl is identically zero
		\[\nabla \times \bold{F}(\bold{r}) = 0\]
	\item It can be written as the negative of a gradient, called the scalar potential
		\[\bold{F}(\bold{r}) = - \nabla U(\bold{r})\]
	\item The line integral over a closed path is zero
		\[\oint \bold{F}(\bold{r}) \cdot d\bold{r} = 0\]

\end{enumerate}
\textit{Note: Letters in bold are vector quantities}

The scalar potential defined over the space is a measure of the potential energy a particle would have if placed at that point.
It could be electric potential energy, gravitational potential energy, or anything else governed by a conservative force.

Whenever we have a conservative force we can use a few identities from vector calculus to obtain Poisson's equation for it's scalar potential.
Consider for example the electrostatic force, the force that two electric charges exert on each other.
It can be summarized in two of Maxwell's four equations.
\[\nabla \times \bold{E} = -\frac{\delta \bold{B} } {\delta t} \qquad
\nabla \cdot \bold{E} = \frac{\rho}{\epsilon_0}\]
Where $\bold{E}$ is the electric field, $\bold{B}$ is the magnetic field, $\rho$ the charge density, and $\epsilon_0$ the permisivity of free space, a constant. $\nabla \times$ and $\nabla \cdot$ represent respectively the curl and the divergence operators. In the absence of a changing magnetic field the curl is zero and so we have a conservative force (although $\bold{E}$ is not actually a force but a force per charge we can still treat it as a conservative field, proportionaly to a related conservative force). Because the curl is zero we can immediately assume the other properties of a conservative force and write $\bold{E}$ as a gradient
\[\bold{E}  = - \nabla V = 0\]
$V$ is called the electrostatic scalar potential, more commonly known as the voltage.
If we insert this into the equation for the divergence of $E$ we find after the application of the identity $\nabla \cdot (\nabla\psi) = \Delta \psi$
\[\nabla \cdot ( - \nabla V) = - \Delta V = \frac {\rho}{\epsilon_0}\]
With this we have Poisson's equation for electrostatics.
If there are no charges ( or that the divergence of $E$ is zero), then this reduces to Laplace's equation.
A similar analysis can be applied using Gauss's law of gravity (a reformulation of Newton's law of gravity in terms of a divergence) for a description of the gravitational scalar potential in terms of mass density.

Unfortunately, because in most physical systems charges are free to roam, this electric potential feeds back into the system and changes the charge distribution.
This means that in reality $V$ is present on both sides of the equation.
In fact it is usually non-linear.
However, in the following analysis, we'll assume that the charges are fixed in space, or "frozen in".
We'll also ignore all constants (specifically $\epsilon_0$) and units so that we are really dealing with the relative charge.

One simple application of the electric potential is to calculate basic properties of simple molecules, starting from a charge distibution and from that calculating the electric potential field.
We'll assume that the electrons produce a clouds of charge centered around the atoms, appoximated by a Boltzmann Distribution $\rho = q e^{-r}$, where $q$ is the relative charge of the atom and $r$ is the distance from the atom.
To make this easier, we'll make a function (\li{rho1}) to calculate the $\rho$ value at a certain point in space based the position of an atom and it's relative charge.
We'll make another function (\li{rhoSum}) to give the total charge density at a point based on a list of atoms.

\begin{lstlisting}
#definitions for atoms position and charges
#the angle the hydrogen atoms make
theta = 106.0/180.0*np.pi
#Length of the two branches
A = 1.0
# Hydrogen 1 (x0,y0,q)
# Hydrogen 2
# Oxygen
water = ((-np.sin(theta / 2) * A, 0, 1),
         (np.sin(theta / 2) * A, 0, 1),
         (0, -np.cos(theta / 2) * A, -2))

def rho1(x, y, atom):
    return atom[2] * np.exp(-np.sqrt((x - atom[0])**2 + (y - atom[1])**2))

def rhoSum(x,y,atoms):
    return np.sum([rho1(x,y,atom) for atom in atoms],axis=0)

# Generate a color dictionary for use with LinearSegmentedColormap.
# It places red and blue at the min and max values of data
# and white when data is zero.
def genDict(data):
    zero = 1 / (1 - np.max(data) / np.min(data))
    cdict = {'red':[(0.0,  1.0, 1.0),
                   	(zero,  1.0, 1.0),
                   	(1.0,  0.0, 0.0)],
         'green':  [(0.0,  0.0, 0.0),
                   	(zero,  1.0, 1.0),
                   	(1.0,  0.0, 0.0)],
         'blue':   [(0.0,  0.0, 0.0),
                   	(zero,  1.0, 1.0),
                   	(1.0,  1.0, 1.0)]}
    return cdict
X = np.linspace(-5, 5, 100)
X, Y = np.meshgrid(X, X)
# Generate the grid of rho values.
Rho = rhoSum(X, Y, water)
plt.imshow(Rho, cmap =  mcolors.LinearSegmentedColormap('cmap', genDict(Rho)))
plt.colorbar()
plt.show()
\end{lstlisting}
The function \li{genDict} scales the color values to be white when the charge density is zero.
This is mostly to help visualize where there are neutrally charged zones by forcing them to be white.
You may find it useful to also apply it when you solve for the electric  potential.

\begin{figure}
\centering
\includegraphics[width=12cm]{waterRho.png}
\caption{Relative charge density of an H$_2$O molecule}
\end{figure}

With a function for $\rho$, we can solve Poisson's equation for the electric potential field.
\begin{problem}
Solve for the electric potential $V$
\[\Delta V = -\rho(x,y)\]
The electric potential is usually defined to go to zero at inifinity.
We can approximate this using $V=0$ at the boundry conditions on $[-5,5]\times [-5,5]$.
Solve and plot the electric potential of water, using the definition for the atoms above.

\end{problem}

\begin{problem}
Solve for the electric potential of a CO$_2$ molecule.

CO$_2$ can be modeled as two atoms of relative charge $-1$ placed at $x=-1$ and $x=1$ on the $x$ axis and a third atom with relative charge $2$ at the origin.
Simply reuse your code from the last problem with new definitions for the atoms using the template for a water molecule.

If the molecules are moving slowly (ie. when the temperature is low) electrostatic forces will dominate molecular interactions.
Molecules will want to align themselves in the lowest energy configuration, or the lowest electric potential.
Positive potentials will overlap themselves with negative potentials.
From the electric potential plots you obtain, how do you think the molecules will arrange themselves when the temperature is cold, when they form an ice?
\end{problem}

\begin{figure}
\centering
\includegraphics[width=12cm]{co2V.png}
\caption{Relative electric potential field of a CO$_2$ molecule}
\end{figure}


\section{Probabilistic Graphical Models}
Probabilistic Graphical Models are models for which a graph can express the
conditional dependence structure between random variables. We will define
the basics used to understand PGMs here. \\
For starters, a model is a set of probability distributions that may
have generated data. For example, a model could be the set of normal distributions
\begin{equation}
  \left\{ p(x) : p(x) = \frac{1}{\sqrt{2\pi \sigma^2}} \exp \left( -\frac{(x-\mu)^2}{2\sigma^2} \right), \mu \in \mathbb{R}, \sigma^2 \in \mathbb{R}^+\right\}
\end{equation}
A graphical model is defined to be a pair $(\mathcal{G}, \mathcal{P})$ where
$\mathcal{G}$ is a graph and $\mathcal{P}$ is a set of distributions which
factorizes according to $\mathcal{G}$. A graph $\mathcal{G} = (\mathbf{V}, \mathbf{E})$
consists of a set of vertices $\mathbf{V}$ and a set of edges $\mathbf{E}$.
Typically the vertices are random variables and the edges relate the random variables.\\
We say that random variable $X_i$ is conditionally independent of $X_j$ given a set of
variables $\mathbf{Z}$, and write this as: $X_i \indep X_j | \mathbf{Z}$.\\
\subsection{Graphoid Axioms}
Some properties of conditonal independence are as follows:
\begin{enumerate}
  \item $X_1 \indep X_2 | \mathbf{Z} \implies X_2 \indep X_1|\mathbf{Z}$ (symmetry)
  \item $X_1 \indep \left( X_2, X_3 \right) | \mathbf{Z} \implies X_1 \indep X_2|\mathbf{Z}$ (decomposition)
  \item $X_1 \indep \left( X_2, X_3 \right) | \mathbf{Z} \implies X_1 \indep X_2|\left(\mathbf{Z}, X_3\right)$ (weak union)
  \item $X_1 \indep X_2|\left(\mathbf{Z}, X_3\right) \,\, \& \,\, X_1 \indep X_3 | \mathbf{Z} \implies X_1 \indep \left(X_2, X_3 \right) | \mathbf{Z}$ (contraction)
  \item $X_1 \indep X_2 | \left(\mathbf{Z}, X_3\right) \,\, \& \,\, X_1 \indep X_3 | \left(\mathbf{Z}, X_2\right) \implies X_1 \indep \left(X_2, X_3\right)|\mathbf{Z}$ (intersection, for positive distributions)
\end{enumerate}
\subsection{Bayesian Networks}
Bayesian Networks are a type of PGM constructed from directed acyclic graphs.
A directed acyclic graph (DAG) is a graph $\mathcal{G} = (\mathbf{V}, \mathbf{E})$
such that $\mathbf{E}$ contains only directed edges ($\rightarrow)$ and there
does not exist a sequence of directed edges from $X_i$ to $X_i$ for all nodes
in the graph.\\
Due to the structure of the DAG, we can factorize the joint distribution
$p(\mathbf{x})$ with respect to the graph as:
\begin{equation}
  p(\mathbf{x}) = \prod_{i=1}^n p(x_i \, | \, pa(x_i, \mathcal{G}))
\end{equation}
Here, $pa(x_i, \mathcal{G})$ are the parents of $x_i$ with respect to graph $\mathcal{G}$.\\
Using this factorization, we can specify conditional independencies from the graph's structure.
We define three useful set construction operations for these networks:
\begin{equation}
  \begin{matrix}
    an(x_i, \mathcal{G}) \equiv \left\{ x_j \, | \, x_j \rightarrow \cdots \rightarrow x_i \in \mathcal{G} \right\} & (\textrm{ancestors of } x_i)\\
    de(x_i, \mathcal{G}) \equiv \left\{ x_j \, | \, x_j \leftarrow \cdots \leftarrow x_i \in \mathcal{G} \right\} & (\textrm{descendants of } x_i)\\
    nd(x_i, \mathcal{G}) \equiv \left\{ x_j \, | \, x_j \notin de(x_i, \mathcal{G}) \right\} & (\textrm{non-descendants of } x_i)\\
  \end{matrix}
\end{equation}
A distribution $p(\mathbf{x})$ satisfies the local Markov property wrt DAG
$\mathcal{G}$ if:
\begin{equation}
  x_i \indep nd^*(x_i, \mathcal{G}) \, | \, pa(x_i, \mathcal{G})
\end{equation}
where $nd^*(x_i, \mathcal{G}) \equiv nd(x_i, \mathcal{G}) \setminus pa(x_i, \mathcal{G})$\\

\subsection{Markov Random Fields}
\subsection{Exact Inference}
\subsection{Variational Inference}


% Section 3: Problem Background

\section{Problem Background}

For the present context it is sufficient to restrict ourselves to optimal transport on $\mathbb {R}^d$. Let $X$,  $Y$ be subsets of $\mathbb{R}^d$ and let $\mu$ and $\nu$ be probability measures on $X$ and $Y$, respectively. In this paper we will always have X = Y , but using different notation for domain and target space makes definitions easier to grasp.

Transport map $T$ is any (measurable) map $X  \rightarrow Y$ that transforms the measure $\mu$ into the measure $\nu$. More precisely it satisfies $\mu(T^{-1}(B)) = \nu(B)$ for every measurable $B \subset Y$ . A transference plan is a measure $\pi$ on $X \times Y$ with marginals $\pi(\cdot \times Y) = \mu$ and $\pi(X \times \cdot) = \nu$. The set of transference plans from $\mu$ to $\nu$ is denoted by $\Pi(\mu, \nu)$. Any transport map $T$ from $\mu$ to $\nu$ defines a transference plan $\pi_T$ from $\mu$ to $\nu$ as the unique measure satisfying $\pi_{T}(A \times B)=\mu\left(A \cap T^{-1}(B)\right)$ for all measurable $A \subset X$ and $B \subset Y$. Not every transference plan $\pi$ can be represented in this way, because transference plans allow mass from one site $x \in X$ to be split between multiple destinations, which is not possible under a transport map. Figure 1 shows such an example.

\begin{figure}[htbp]
  \centering
  \includegraphics[width=0.8\linewidth]{img/1d_ot}
  \label{fig:ot}
  \caption{Two one-dimensional distributions $\mu$  and $\nu$ , plotted on the x and y axes, and one possible joint distribution that defines a transport plan between them.}
\end{figure}

We assume that the cost of transporting a unit mass from $x \in X$ to $y \in Y$ is $c_p(x,y) = \|x-y\|^p$ for some $p \geq 1$. The minimum cost for transferring $\mu$ to $\nu$ is then given by

\begin{equation}
  C_p (\mu, \nu) = \min_{x \in \Pi(\mu,\nu)} \|x-y\|^p d\pi(x,y)
\end{equation}

Taking the $p^{th}$ root, we obtain the Wasserstein metric $W_p$. More precisely we have

\begin{equation}
  W_p(\mu, \nu) = C_p(\mu, \nu)^{1/p}
\end{equation}

for and measures $\mu$ and $\nu$ that satisfy $\int_{X}\|x\|^{p} d \mu(x)<\infty$ adn $\int_{Y}\|y\|^{p} d \nu(y)<\infty$. In order to evaluate the Wasserstein metric, we need to find an optimal solution to $C_p(\mu,\nu)$, i.e., a minimizing transference plan $\pi$. This problem is often referred to as the Kantorovich formulation of optimal transport. Note that by Theorem 4.1 in [29] a minimizing always exists. However, it neither has to be unique nor representable in terms of an optimal transport map.

Often one would like to compare data sets that are available as images from a certain source, e.g. real photography, astronomical imagery, or microscopy data. We may think of such images as discrete measures on a grid. For example, tiny clippings from STED microscopy images of mitochondrial networks. A question of interest might be whether both images stem from the same part of the network, which can in principle be answered by finding an optimal transference plan (third panel in Figure 1) and computing the Wasserstein distance. Note that this coarse resolution is not representative for a serious analysis, but was only chosen for illustrative purposes.

Assume now that we have discrete measures of the form $\mu = \sum_{i=1}^m \mu_i \delta_{x_i}$ and $\nu = \sum_{j = 1}^n \nu_i \delta_{y_j}$ and write $c_{ij} = ||x_i - y_j||^p$. In what follows, we always have $m = n$, and $(x_i)_{1 \leq i \leq m} = (y_j)_{i\leq j \leq n}$ form a regular square grid in $R^2$, but since it is more intuitive, we keep different notation for source locations and target locations. Let $\pi_{ij}$ be the amount of mass transported from $x_i$ to $y_j$. Then, the problem can be rewritten as a linear program:

\begin{equation}
  \begin{array}{rl}{\mathrm{(primal)}} & {\min_\pi \sum_{i=1}^{m} \sum_{j=1}^{n} c_{i j} \pi_{i j}} \\{\text { subject to }} & {\sum_{j=1}^{n} \pi_{i j}=\mu_{i},\quad \forall i=1, \ldots, m} \\{} & {\sum_{i=1}^{m} \pi_{i j}=\nu_{j},\quad \forall j=1, \ldots, n} \\{} & {\pi_{i j} \geq 0}\end{array}
\end{equation}

Note that this primal form has $mn$ variables which leads to computational difficulties, we can deduct the dual form of this problem as:

\begin{equation}
  \begin{array}{rl}{\mathrm{(dual)}} & {\max _{u, v} \sum_{i=1}^{m} \mu_{i} u_{i}+\sum_{j=1}^{n} \nu_{j} v_{j}} \\{\text { subject to }} & {c_{ij} - u_i - v_j \geq 0\qquad\forall i=1, \ldots, m \quad j = 1,\cdots,n}\end{array}
\end{equation}

Though only $m + n$ variables left, there still exist $mn$ constraints.


\subsection{Damped Forced Vibrations ($b \neq 0$)}
\noindent
Our equation is
\begin{equation*}
	my' + by' + ky = F_0\cos{(\gamma t)}.
\end{equation*}
Depending on if $\Delta = b^2 - 4mk$ is positive, zero, or negative, we'll get different results and thus different guesses for $y_p$ and thus different solutions.
There is, however, one long, complicated and not very useful for $y$.
\begin{equation*}
	y = C_1e^{\frac{-b+\sqrt{b^2-4mk}}{2m}t} + C_2e^{\frac{-b-\sqrt{b^2-4mk}}{2m}t} +
	 \frac{F_0\left(b\gamma\sin{(\gamma t)} + \left(k-\gamma^2m\right)\cos{(\gamma t)}\right)}{b^2\gamma^2 + (k-\gamma^2m)^2}.
\end{equation*}
The one useful thing this formula does tell us is that assuming $m$, $b$, $k$, $F_0$, and $\gamma$ are all positive and non-zero, the exponential terms quickly decrease to 0, so in the limit the function looks like the particular solution part.

\begin{center}
	\includegraphics[width=0.75\textwidth]{./higherOrder/forcedVibrs/damped_forced.png}
\end{center}

\begin{example}
	Solve the following IVP.
	\begin{equation*}
		\begin{cases}
			y'' + 2y' + 10y = 5\cos{(4t)} \\
			y(0) = 0 \\
			y'(0) = -2.6
		\end{cases}
	\end{equation*}
\end{example}
\noindent
Some of the work has been omitted for brevity.

\noindent
Solving the auxiliary equation and finding $y_h$,
\begin{equation*}
	r^2 + 2r + 10 = 0 \implies r = -1 \pm 3i \implies y_h = e^{-t}\left(C_1\cos{(3t) + C_2\sin{(3t)}}\right).
\end{equation*}
Guessing the form of $y_p$, noting that $\omega \neq \gamma$,
\begin{equation*}
	y_p = A\cos{(4t)} + B\sin{(4t)}.
\end{equation*}
Solving for $A$ and $B$,
\begin{equation*}
	y_p'' + 2y_p' + 10y_p = 5\cos{(4t)} \implies A = -3/10 \text{ and } B = 4/10.
\end{equation*}
Solving for $C_1$ and $C_2$,
\begin{equation*}
	\begin{cases}
		y(0) = 0 \\
		y'(0) = -2.6
	\end{cases} \implies C_1 = 3/10 \text{ and } C_2 = -13/10.
\end{equation*}
So, we have a solution for $y$,
\begin{equation*}
	y = e^{-t}\left(\frac{3}{10}\cos{(3t)} - \frac{13}{10}\sin{(3t)}\right) + \left(\frac{-3}{10}\cos{(4t)} + \frac{4}{10}\sin{(4t)}\right).
\end{equation*}
The graph of this solution was shown above before the example.


\chapter{Introduction}

\section{Types of machine learning}
\begin{equation}\nonumber
\begin{cases}
\text{Supervised learning} \begin{cases} \text{Classification} \\ \text{Regression} \end{cases}\\
\text{Unsupervised learning} \begin{cases} \text{Discovering clusters} \\ \text{Discovering latent factors} \\ \text{Discovering graph structure} \\ \text{Matrix completion} \end{cases}\\
\end{cases}
\end{equation}

\section{Three elements of a machine learning model}

\textbf{Model = Representation + Evaluation + Optimization}\footnote{Domingos, P. A few useful things to know about machine learning. Commun. ACM. 55(10):78–87 (2012).}

\subsection{Representation}
In supervised learning, a model must be represented as a conditional probability distribution $P(y|\vec{x})$(usually we call it classifier) or a decision function $f(x)$. The set of classifiers(or decision functions) is called the hypothesis space of the model. Choosing a representation for a model is tantamount to choosing the hypothesis space that it can possibly learn.

\subsection{Evaluation}
In the hypothesis space, an evaluation function (also called objective function or risk function) is needed to distinguish good classifiers(or decision functions) from bad ones.

\subsubsection{Loss function and risk function}
\label{sec:Loss-function-and-risk-function}

\begin{definition}
In order to measure how well a function fits the training data, a \textbf{loss function} $L:Y \times Y \rightarrow R \geq 0$ is defined. For training example $(x_i,y_i)$, the loss of predicting the value $\widehat{y}$ is $L(y_i,\widehat{y})$.
\end{definition}

The following is some common loss functions:
\begin{enumerate}
\item 0-1 loss function \\ $L(Y,f(X))=\mathbb{I}(Y,f(X))=\begin{cases} 1, & Y=f(X) \\ 0, & Y \neq f(X) \end{cases}$
\item Quadratic loss function $L(Y,f(X))=\left(Y-f(X)\right)^2$
\item Absolute loss function $L(Y,f(X))=\abs{Y-f(X)}$
\item Logarithmic loss function \\ $L(Y,P(Y|X))=-\log{P(Y|X)}$
\end{enumerate}

\begin{definition}
The risk of function $f$ is defined as the expected loss of $f$:
\begin{equation}\label{eqn:expected-loss}
R_{\mathrm{exp}}(f)=E\left[L\left(Y,f(X)\right)\right]=\int L\left(y,f(x)\right)P(x,y)\mathrm{d}x\mathrm{d}y
\end{equation}
which is also called expected loss or \textbf{risk function}.
\end{definition}

\begin{definition}
The risk function $R_{\mathrm{exp}}(f)$ can be estimated from the training data as
\begin{equation}
R_{\mathrm{emp}}(f)=\dfrac{1}{N}\sum\limits_{i=1}^{N} L\left(y_i,f(x_i)\right)
\end{equation}
which is also called empirical loss or \textbf{empirical risk}.
\end{definition}

You can define your own loss function, but if you're a novice, you're probably better off using one from the literature. There are conditions that loss functions should meet\footnote{\url{http://t.cn/zTrDxLO}}:
\begin{enumerate}
\item They should approximate the actual loss you're trying to minimize. As was said in the other answer, the standard loss functions for classification is zero-one-loss (misclassification rate) and the ones used for training classifiers are approximations of that loss.
\item The loss function should work with your intended optimization algorithm. That's why zero-one-loss is not used directly: it doesn't work with gradient-based optimization methods since it doesn't have a well-defined gradient (or even a subgradient, like the hinge loss for SVMs has).

The main algorithm that optimizes the zero-one-loss directly is the old perceptron algorithm(chapter \S \ref{chap:Perceptron}).
\end{enumerate}

\subsubsection{ERM and SRM}
\begin{definition}
ERM(Empirical risk minimization)
\begin{equation}
\min\limits _{f \in \mathcal{F}} R_{\mathrm{emp}}(f)=\min\limits _{f \in \mathcal{F}} \dfrac{1}{N}\sum\limits_{i=1}^{N} L\left(y_i,f(x_i)\right)
\end{equation}
\end{definition}

\begin{definition}
Structural risk
\begin{equation}
R_{\mathrm{smp}}(f)=\dfrac{1}{N}\sum\limits_{i=1}^{N} L\left(y_i,f(x_i)\right) +\lambda J(f)
\end{equation}
\end{definition}

\begin{definition}
SRM(Structural risk minimization)
\begin{equation}
\min\limits _{f \in \mathcal{F}} R_{\mathrm{srm}}(f)=\min\limits _{f \in \mathcal{F}} \dfrac{1}{N}\sum\limits_{i=1}^{N} L\left(y_i,f(x_i)\right) +\lambda J(f)
\end{equation}
\end{definition}

\subsection{Optimization}
Finally, we need a \textbf{training algorithm}(also called \textbf{learning algorithm}) to search among the classifiers in the the hypothesis space for the highest-scoring one. The choice of optimization technique is key to the \textbf{efficiency} of the model.

\section{Some basic concepts}

\subsection{Parametric vs non-parametric models}

\subsection{A simple non-parametric classifier: K-nearest neighbours}

\subsubsection{Representation}
\begin{equation}
y=f(\vec{x})=\arg\min_{c}{\sum\limits_{\vec{x}_i \in N_k(\vec{x})} \mathbb{I}(y_i=c)}
\end{equation}
where $N_k(\vec{x})$ is the set of k points that are closest to point $\vec{x}$.

Usually use \textbf{k-d tree} to accelerate the process of finding k nearest points.

\subsubsection{Evaluation}
No training is needed.

\subsubsection{Optimization}
No training is needed.

\subsection{Overfitting}

\subsection{Cross validation}
\label{sec:Cross-validation}
\begin{definition}
\textbf{Cross validation}, sometimes called \emph{rotation estimation}, is a \emph{model validation} technique for assessing how the results of a statistical analysis will generalize to an independent data set\footnote{\url{http://en.wikipedia.org/wiki/Cross-validation_(statistics)}}.
\end{definition}

Common types of cross-validation:
\begin{enumerate}
\item K-fold cross-validation. In k-fold cross-validation, the original sample is randomly partitioned into k equal size subsamples. Of the k subsamples, a single subsample is retained as the validation data for testing the model, and the remaining k − 1 subsamples are used as training data.
\item 2-fold cross-validation. Also, called simple cross-validation or holdout method. This is the simplest variation of k-fold cross-validation, k=2.
\item Leave-one-out cross-validation(\emph{LOOCV}). k=M, the number of original samples.
\end{enumerate}

\subsection{Model selection}
When we have a variety of models of different complexity (e.g., linear or logistic regression models with different degree polynomials, or KNN classifiers with different values ofK), how should we pick the right one? A natural approach is to compute the \textbf{misclassification rate} on the training set for each method.


\documentclass{article}

\usepackage{amsmath}

\begin{document}

\section{Reparametrizing the Two Body Problem}

When fitting radial velocity (RV) data for exoplanet or binary star systems, the
RV model is typically expressed as:
\begin{equation}
    v_r(t) = v_0 + K\,\left[\cos(\omega + f(t)) + e\,\cos(\omega)\right]
\end{equation}
where $v_0$ is the systemic (barycentric) velocity of the system relative to the
Sun, $K$ is the velocity semi-amplitude, $\omega$ is the argument of pericenter,
and $f(t)$ is the True Anomaly. The True Anomaly is computed from the Mean
Anomaly $M(t)$
\begin{equation}
    M = \frac{2\pi}{P} \, (t - t_{\mathrm{peri}})
\end{equation}
through a transcendental equation that relates the Mean Anomaly to the Eccentric
Anomaly $E(t)$:
\begin{equation}
    M = E - e\,\sin E
\end{equation}
The Eccentric Anomaly is related to the True Anomaly by:
\begin{align}
    \tan\frac{f}{2} &= \sqrt{\frac{1+e}{1-e}} \, \tan\frac{E}{2} \\
    \tan\frac{E}{2} &= \sqrt{\frac{1-e}{1+e}} \, \tan\frac{f}{2}
\end{align}

In the above parametrization, the full set of parameters is $(v_0, K, P, e,
\omega, t_{\mathrm{peri}})$.

Another common representation of the Mean Anomaly is in terms of the phase of
pericenter $M_0$, which specifies the phase at which pericenter occurs in the
angle defined by time relative to a reference time $t_{\mathrm{ref}}$, which is
often taken to be the minimum or mean observation time. So, an alternate
definition of Mean Anomaly is:
$$
    M = \frac{2\pi}{P} \, (t - t_{\mathrm{ref}}) - M_0
$$
With this convention, $t_{\mathrm{peri}}$ is replaced by $M_0$ as a parameter so
the list of parameters is $(v_0, K, P, e, \omega, M_0)$, but it is
straightforward to transform between $M_0$ and $t_{\mathrm{peri}}$. In either
case, the Mean Anomaly is defined such at $M=0$ (mod $2\pi$) occurs at
pericenter.

TODO: discussion about how this is not a good parametrization for MCMC sampling,
and history (Ford 2005).

\subsection{Replacing $\omega, M_0$ with phase of max/min velocity}

Observed phase of max/min radial velocity $M^*_{\mathrm{max}},
M^*_{\mathrm{min}}$.
\begin{align}
M^*_{\mathrm{max}} &= \frac{2\pi}{P} \, (t_{\mathrm{max}} - t_{\mathrm{ref}}) \\
M^*_{\mathrm{min}} &= \frac{2\pi}{P} \, (t_{\mathrm{min}} - t_{\mathrm{ref}})
\end{align}
Note: $M^*_{\mathrm{max}}, M^*_{\mathrm{min}}$ are not actually mean anomalies.
They are mean anomaly $+ M_0$, because what we observe are phases relative to
the reference time $t_{\mathrm{ref}}$.

So, how do we relate these quantities to $(\omega, M_0)$?

As far as I can figure, there is no direct, closed-form transformation, but I
have figured out a procedure to do the transformation in terms of a different
transcendental equation. This method is built on the intuition (guess) that the
quantity
\begin{equation}
    \Delta M = M^*_{\mathrm{min}} - M^*_{\mathrm{max}}
\end{equation}
will be related to the argument of pericenter. Defining also
\begin{align}
    \Delta E &= E_{\mathrm{min}} - E_{\mathrm{max}} \\
    \Sigma E &= E_{\mathrm{min}} + E_{\mathrm{max}}
\end{align}
we can re-express the classic Kepler transcendental equation as
\begin{align}
    \Delta M &= \Delta E - e\, (\sin E_{\mathrm{min}} - \sin E_{\mathrm{max}})\\
    \frac{\Delta M}{2} &= \frac{\Delta E}{2} - e\, \sin\frac{\Delta E}{2} \, \cos\frac{\Sigma E}{2}
\end{align}
where the last line makes use of the `difference of sines' trig identity.

Using the relationship between $E$ and $f$,
\begin{align}
    (E_{\mathrm{min}} - E_{\mathrm{max}})/2 =
        \arctan\left[ \sqrt{\frac{1-e}{1+e}} \,
            \tan \frac{f_{\mathrm{min}}}{2} \right] -
        \arctan\left[ \sqrt{\frac{1-e}{1+e}} \,
            \tan \frac{f_{\mathrm{max}}}{2} \right]
\end{align}
However, we know the values of $f_{\mathrm{min}}$ and $f_{\mathrm{max}}$: From
Equation (1), we know that the RV is maximum when $f=f_{\mathrm{max}} \equiv
-\omega$, and minimum when $f=f_{\mathrm{min}} \equiv \pi-\omega$.
Plugging in these values and doing some trig algebra (using the difference of
arctan's trig identity), this simplifies to
\begin{align}
    \Delta E / 2 = \arctan\left(\frac{\sqrt{1 - e^2}}{e\,\sin\omega}\right)
\end{align}
and similarly, for the sum of $E_{\mathrm{min}} + E_{\mathrm{max}}$,
\begin{align}
    \Sigma E / 2 = \arctan\left(\frac{\sqrt{1 - e^2}}{\tan\omega}\right)
        \quad .
\end{align}
So, the relationship between $\omega$, $e$, and $\Delta M$ is defined by the
expression
\begin{align}
    \frac{\Delta M}{2} &= \frac{\Delta E}{2} -
        e\, \sin\frac{\Delta E}{2} \, \cos\frac{\Sigma E}{2} \\
    % \frac{\Delta M}{2} &=
    %     \arctan\left(\frac{\sqrt{1 - e^2}}{e\,\sin\omega}\right) -
    %     \,e \,
    %         \sin\left(\arctan\left(\frac{\sqrt{1 - e^2}}{e\,\sin\omega}\right)\right) \,
    %         \cos\left(\arctan\left(\frac{\sqrt{1 - e^2}}{\tan\omega}\right)\right)
    \frac{M_{\rm min} - M_{\rm max}}{2} &=
        \arctan\left( \frac{\sqrt{1 - e^2}}{e \, \sin\omega} \right) +
        \frac{2\, e \, \sqrt{1 - e^2} \, \sin\omega}{e^2 - 2 + e^2\,\cos(2\omega)}
\end{align}
and the phase $M_0$ can be retrieved with
\begin{align}
    M_0 &= M^*_{\mathrm{max}} - M_{\mathrm{max}} \\
    M_{\mathrm{max}} &= E_{\mathrm{max}} - e \, \sin E_{\mathrm{max}} \\
    E_{\mathrm{max}} &= 2\,\arctan\left(\sqrt{\frac{1-e}{1+e}} \,
        \tan\frac{f_{\mathrm{max}}}{2}\right) \\
    &= -2\,\arctan\left(\sqrt{\frac{1-e}{1+e}} \,
        \tan\frac{\omega}{2}\right) \\
\end{align}

\end{document}


\documentclass {article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{soul}
\begin{document}

\section{Solve graphically}
Find two positive numbers that minimize the sum of twice the first number plus second if product of two is 288.\\
\\
- Assume that the first and second numbers are $a$ and $b$, respectively.\\
- Using the given information, we already know:\\

\begin{align*}
	a > 0 \\
	b > 0 \\
	ab = 288 \\
	b=\frac{288}{a} \\
	f(x)= 2a + b \\
	f(x)= 2a + \frac{288}{a} \\
\end{align*}

and we need to find out the number $a$ that minimizes the function:\\

\begin{align*}
	f(x)= 2a + \frac{288}{a} \\
\end{align*}

\begin{figure}
  \includegraphics[width=\linewidth]{graph1.png}
  \caption{Graph of $f(x)= 2a + \frac{288}{a}$}
\end{figure}

We can tell from the below graph that f(x) has a local minimum of $(12,48)$ and since $a > 0$ in this case, it is also the global minimum of $f(x)$.

\begin{align*}
a = 12 \\
\\
b = \frac{288}{a} \\
\\
b = \frac{288}{12} \\
\\
b = 24 \\
\end{align*}

The two positive numbers the minimize the sum of twice the first number plus the second number if the product of the two is 288 is: $a = 12, b = 24$\\
ANSWER: $a = 12, b = 24$

\section{Solve algebraically}
Find two positive numbers whose product is 750 and for which the sum of one and 10 times the other is a minimum.\\
\\
- Assume that the first and second numbers are $a$ and $b$, respectively.\\
- Using the given information, we already know:\\

\begin{align*}
	a > 0 \\
	b > 0 \\
	ab = 750 \\
	b=\frac{750}{a} \\
	f(x)= a+10b \\
	f(x)= a+\frac{7500}{a}\\
\end{align*}
\\
To algebraically determine the minimum in this case, we have to find which point on the function does its tangent line have a slope of zero. Although this sounds fairly complicated, this point is found by finding the derivative of the function and setting it equal to zero since the derivative $\frac{d}{dx}$ represents rate of change at any given point and we're looking for a slope of zero at a particular point.\\
\\

\begin{align*}
	f(a)=a + 7500a^{-1}\\
	\frac{d}{da}f(a)=\frac{d}{da}(a) + 7500*\frac{d}{da}(a^{-1})\\
	\frac{d}{da}f(a)=1+7500*(-a^{-2})\\
	\frac{d}{da}f(a)=1+\frac{-7500}{a^2}\\
	\frac{d}{da}f(a)=0\\
	1+\frac{-7500}{a^2}=0\\
	\frac{-7500}{a^2}=-1\\
	-7500=(-a)^2\\
	a=50\sqrt{3}\\
	a=-50\sqrt{3} [rejected]\\
	\\
	b=\frac{750}{50\sqrt{3}}\\
	b=5\sqrt{3}
\end{align*}

The answer to the question, "What two positive numbers whose product is 750 and for which the sum of one and 10 times the other is a minimum?" is: $a=50\sqrt{3}, b=5\sqrt{3}$.

\end{document}