% Template:     Informe/Reporte LaTeX
% Documento:    Archivo de ejemplo
% Versión:      5.1.5 (15/05/2018)
% Codificación: UTF-8
%
% Autor: Francisco Muñoz P.
%        Facultad de Ciencias Físicas y Matemáticas
%        Universidad de Chile
%        fjmunoz95@gmail.com, frmunoz@dcc.uchile.cl
%
% Derechos del template corresponden a Pablo Pizarro. pablo.pizarro@ing.uchile.cl
% Manual template: [http://latex.ppizarror.com/Template-Informe/]
% Licencia MIT:    [https://opensource.org/licenses/MIT/]

% define some usefull commands
\newcommand\FF{\mathcal{F}}
\newcommand\prd{Phys. Rev. D}
\newcommand\MF{\text{MF}}
\newcommand\IL{\int_{-\infty}^{\infty}}
\newcommand\WM{\text{WMF}}
\newcommand\wmf[2]{(#1|#2_{\MF})^{\WM}_{t_0}}
% NUEVA SECCIÓN
\section{Introduction}{\label{sec:intro}}
\newp por definir.

\section{Fourier Transform}{\label{sec:ft}}
\newp The Fourier Transform it's a Mathematical tool that allow us to decompone singals by a given dictionary or in other words, to apply a change of basis to a given signal. Thinking on a continuous space for a signal $s(t)$, it's Fourier Transform is given by the following integral, where $i =  \sqrt{-1}$ denotes the imaginary unit:
\insertequation[\label{ft:ft}]{\tilde{s}(f) = \int_{-\infty}^{\infty} s(t)e^{-2i\pi f t} dt = \FF[s(t)]}

\newp And the inverse transform is:

\insertequation[\label{ft:ift}]{s(t) = \int_{-\infty}^{\infty} \tilde{s}(f)e^{2i\pi f t } df = \FF^{-1}[\tilde{s}(f)]}

\subsection{Properties of the Fourier Transform}{\label{sec:ft::subsec:propft}}
\newp Some usefull properties of the Fourier Transform are:
	\begin{description}

	\item \textbf{Fourier Transform is a linear operator}. This means that, given a constant $a$ and 		functions $s(t)$, $h(t)$, we can write:
	\insertequation[\label{propFT:linear1}]{\FF[s(t) + h(t)] =  \FF[s(t)] + \FF[h(t)]}
	\insertequation[\label{propFT:linear2}]{\FF[as(t)] = a\FF[s(t)] }

	\item \textbf{A time-shift impart a phase in the fourier Transform.} For a given function $s(t)$, it's satisfied:
	\insertequation[\label{propFT:time-shift}]{\FF[s(t - t_0)] = \FF[s(t)]e^{-2i\pi f t_0}}

	\newp also can be obtained a conjugate of the transform by:

	\insertequation[\label{propFT:time_shift2}]{\FF[s(t_0 - t)] = \FF[s(t)]^{*}e^{-2\pi f t_0}}

	\item \textbf{The Power Spectrum equal to square of absolute value of the Fourier Transform.} This computation allow us to remove the phase and the complex part which is changed to real by the absolute function.

	\insertequation[\label{psd}]{\mathcal{P}_s = |\FF[s(t)]|^{2}}

	\item \textbf{Convolution Theorem.} The definition of convolution for two functions $s(t)$ and $h(t)$, is given by:

	\insertequation[\label{convolution1}]{[s(t) * h(t)] = [h(t) * s(t)] = \int_{-\infty}^{\infty} s(t-\tau) h(\tau) d\tau}

	Calculating the Fourier Transform of this convolution leads to:

	\insertequation[\label{convolution2}]{\FF[s * h] = \FF[s] \cdotp \FF[h]}

	Also, Fourier Transform of a point-wise multiplication of two functions leads to a convolution of Fourier Transforms:

	\insertequation[\label{convolution3}]{\FF[s(t) \cdotp h(t)](f) = \FF[s](f) * \FF[h](f)}

	In practice using Fourier to calculate a convolution is faster than just calculate the convolution, that's why this is a very important concept.

	\item \textbf{Parseval's Formula}, it says that for $s(t)$ and $h(t)$ square-integrable functions with fourier transforms given by (\ref{ft:ift}) and (\ref{ft:ft}), it's satisfied:

	\insertequation[\label{parseval}]{\int_{-\infty}^{\infty} s(t)h^{*}(t)dt = \int_{-\infty}^{\infty}\tilde{s}(f)\tilde{h}^{*}(f) df}

	 \item \textbf{Plancherel Theorem}, it's a direct consecuence of the Parseval Formula but in the sence of energy:

	 \insertequation[\label{plancherel}]{\int_{-\infty}^{\infty} \mid x(t) \mid^{2} (t)dt = \int_{-\infty}^{\infty}\mid \tilde{s}(f)\mid^{2} df}

	\item \textbf{Correlation.} The cross-correlation of two real signals $s(t)$ and $h(t)$ can be defined in the sense of a convolution by:

	\insertequation[\label{corr_as_conv}]{R_{hs}(t) = h \star s = h^{*}(-t) \ast s(t) = s(t) \ast h^{*}(-t)}

	\newp We are more interested in real signals when $s^{*}(-t) = s(-t)$ but the calculations will be in the general case. Using (\ref{convolution1}) we get:

	\insertequation[\label{corr1}]{R_{hs}(t) = \int_{-\infty}^{\infty} s(\tau - t) h^{*}(-t) dt}

	\newp Applying a simple change of variables($t \longrightarrow -t$) we get:

	\insertequation[\label{corr2}]{R_{hs}(t) = \int_{-\infty}^{\infty} s(\tau + t)h^{*}(t) dt}

	\newp Using the Time shift property (\ref{propFT:time-shift}) with the Parseval's Formula (\ref{parseval}):

	\insertequation[\label{corr3}]{R_{hs}(t) = \int_{-\infty}^{\infty} \tilde{s}(f)\tilde{h}^{*}(f)e^{2i\pi f \tau} df}

	\newp And this can be resumed as Fouriers Transforms:

	\insertequation[\label{corr4}]{R_{hs}(t) = \FF^{-1}[\FF[s] \FF[h]^{*}] = \FF^{-1}[\FF[s(t) \star h^{*}(-t)]]}

	\newp A particular behavior is when calculating the auto-correlation of a signal, which correspond to the Inverse Fourier Transform of the Power Spectral Density:

	 \insertequation[\label{autocorr}]{R_{hh} = \FF^{-1}[\mid \tilde{h}(f)\mid^{2}] = \FF^{-1}[\mathcal{P}_h(f)]}

\end{description}

\subsection{Some useful Fourier Transforms}{\label{subsec_ftusefull}}
\begin{description}
	\item \textbf{Fourier Transform of a Sinusoid is a Delta Function.} \texttt{need to insert a figure in this section}
	\insertequation[\label{sindelta}]{\FF[Cos(2\pi f_0t)] = \frac{1}{2}[\delta(f - f_0) + \delta(f + f_0)]}

	\item \textbf{Fourier Transform of a Dicar Comb is a Dirac Comb.} The Dirac Comb is an infinite sequence of Dirac delta function placed at even intervals of size $T$:

	\insertequation[\label{diraccomb}]{III_T(t) = \sum_{n=-\infty}^{\infty} \delta(t-nT)}

	thus, the fourier Transform correspond to a Dirac Comb in the frequency domain:

	\insertequation[\label{diracft}]{\FF[III_T(t)] = \frac{1}{T} III_{1/T}(f)}

\end{description}

\subsection{Window Function}{\label{subsec_window}}
\newp The Continuous Fourier Transform use the particularity that continuous functions are well defined for all times ($-\infty < t < \infty$). This is just an idealization that doesn't work on real signals because this only involve a finite span of time with some finite rate of sampling. The Fourier Transform of a discrete measured data will no longer be just the transform of the continuous underlying function, but rather the transform of the point-wise product of the signal($s(t)$) and the observing window($W(t)$), which will be a rectangular window of dirac comb.

\insertequation[\label{window1}]{s_{obs}(t) = s(t)W(t)}

\newp Using the convolution theorem (ec. \ref{convolution3}) the transform of eq. \ref{window1} will be:

\insertequation[\label{window2}]{\FF[s_{obs}(t)] = \FF[s(t)] * \FF[W(t)]}

\subsubsection{Fixing periodicity}{\label{subsubsec_fixperiod}}

\newp Fourier Transform works only for periodic signals, this means, for signals where its first point has exactly the same value as the last point (continuity in edges). But in real world, a measured signal has always some grade of noise which lead to a discontinuity in the edges. In order to fix this discontinuity we use a window function.

\newp For these cases, the window function will not be a rectangular, or at least, usualy we don't want to use this kind of window. The most useful windows are kind-of curve like \textit{Hann window}, \textit{Blackmann window} or \textit{Tukey window}. We are going to use mostly the \textit{Tukey window} which is: \texttt{should use a example plot here for all windows.}

\insertequation[\label{tukey}]{w(n) = \begin{cases}
\frac{1}{2} \left[1 + cos\left(\pi \left(\frac{2n}{\alpha (N-1} - 1  \right) \right) \right] & 0 \leq n < \frac{\alpha(N-1)}{2} \\
1 & \frac{\alpha(N-1)}{2} \leq n \leq (N-1)(1-\frac{\alpha}{2})\\
\frac{1}{2} \left[1 + cos\left(\pi \left(\frac{2n}{\alpha (N-1} - \frac{2}{\alpha} +  1  \right) \right) \right] & (N-1)(1 - \frac{\alpha}{2} < n \leq (N-1))
\end{cases}}

\newp Here, $\alpha$ define the form of the window, when $\alpha=0$ we get a rectangular window and for $\alpha=1$ we get a \textit{Hann window}.
\subsection{Nyquist-Shannon Sampling Theorem}{\label{subsec_NSST}}
\newp sampling is a process of converting a signal (function-like of continuous time and/or space) into a numeric sequence, thus the sampling theorem states that, if a function $f(x)$ contains no frequencies higher than B hertz, it is completely determined by givin its ordinates at a series of points spaced $1/(2B)$ seconds apart. This means that a suficient sample-rate for uniform sampling is therefore anything larget than $2B$ samples per second, or from other perspective, for a given sample rate $f_s$, \textbf{perfect reconstruction is guaranteed possible} for a bandlimit $B < f_s/2$.

\newp In the most general case of non-uniform sampling, this theorem states that a band-limited signal can be perfectly reconstructed from its samples if the \textbf{average sampling rate} satisfies the Nyquist condition. this means that even if uniformly spaced samples can be easier reconstruced (easier algorithms), it is not a necessary condition for perfect reconstruction.

\subsection{Nyquist frequency}{\label{subsec_nyqfreq}}
\newp The nyquist frequency is defined as half of the sampling rate  of a discrete signal when we have evenly-sampled data

\insertequation[\label{nyq1}]{\nu_N = 0.5 f_s}

\newp And correspond to the highest frequency limit of one interval over which a periodogram is uniquely defined. If we go far away from this nyquist limit, the periodogram (and the simple Fourier Transform) will have repeated information.

\newp For unevenly-sampled data, the symmetry that allow to define this frequency as eq. \ref{nyq1} is broken and using an average sampling rate as in the Sampling Theorem doesn't work. In order to find a best approximation to the real nyquist frequency (called \textit{pseudo-nyquist frequency}) there are several methods in literature. The one implemented here correspond to the method delevoped by Koen \cite{koen-nyqFreq} who give a calculation formula to be solved for the nyquist frequency for arbitrary time spacing of measurements, this is the smallest positive root of

\insertequation[\label{koen1}]{SS(\nu) = \sum_{l=1}^{N-1}\sum_{k=l+1}^{N} [sin 2 \pi \nu (t_k - t_l)]^2 = 0}

\newp This calculation will take $O(N_{\nu}N^2)$ where $N_{\nu}$ correspond to the number of frequencies to try in the formula. Since is a very expensive calculation, generaly it is more useful to know what is the maximum frequency that we are interested in.

\subsection{Discrete Fourier Transform}{\label{subsec_dft}}
\newp We can pass from cotinuous space to discrete space by applying a Rectangular Window with a dirac comb (eq. \ref{window2}) This is because the delta serve to collapse the Fourier Integral into a Fourier Sumatory. Considering the most general case, where a continuous signal $s(t)$ is observed at an \textbf{irregular sampling} of size $N_t$, leads to dirac comb:

\insertequation[\label{irr-dirac-comb}]{III_{\{t_j\}}(t) = \sum_{j = 1}^{N_t} \delta(t - t_j)}

\newp Using this, the direct transform will be:

\insertequation[\label{to-discrete1}]{
\begin{aligned}[b]
\FF[s_{obs}(t)] = \tilde{s}_{k} & = \FF[s(t){III}_{\{t_j\}}(t)]  \\
\tilde{s}_{k} & = \int_{-\infty}^{\infty}s(t)\sum_{j = 2}^{N_t} \delta(t - t_j) e^{-2\pi i f t} dt \\
\tilde{s}_{k} & = \sum_{j = 1}^{N_t}\int_{-\infty}^{\infty}s(t)\delta(t - t_j) e^{-2\pi i f t} dt  \\
\tilde{s}_{k} & = \sum_{j = 1}^{N_t} s_je^{-2\pi i t_j k \Delta f}
\end{aligned}
}

\newp Following the same idea we can get the inverse transform, considering that the \textbf{frequencies will always be in a regular grid} so $f_k = k \Delta f$:

\insertequation[\label{to_discrete2}]{s_j = \Delta f \sum_{k = 1}^{N_f} \tilde{s}_k e^{2 \pi i t_j k \Delta f}}

\newp the total number of frequencies ($N_f$) and how we define the $\Delta f$ will be explained more clearly in the next sections.

\subsection{Frequency Sampling}{\label{subsec_freqsampling}}
\newp Since we are considering the most general case of Fourier Transform for non-uniform sampling, it's neccessary to define clearly the frequency grid, more precisely we need to define the frequency limit on the low end and the high end, in addition to define the grid spacing.
\begin{description}

\item \textbf{low-end frequency limit.} For a set of observations in a time interval of lenght $T$, a signal with frequency $1/T$ will complete exactly one oscillation in the observing window. Chosing a low-end frequency limit could be this frequency or just set the minimum frequency to zero for simplicity. Also is posible to give a custom low-end limit based on what frequencies we want to explore. If we have complex signals we will ned to define a negative low-end frequency limit.

\item \textbf{high-end frequency limit.} In order to not miss information that could be relevant, it's neccessary to look for the maximum frequency that could give us information, this is given by the nyquist frequency defined in section \ref{subsec_nyqfreq}. Also is posible to give a custom high-end limit based on what frequencies we want to explore.

\item \textbf{grid spacing.} Choose the correct grid spacing it's also an important point, if we use a too fine grid will leads to unnecessarily long computational times, while, use a grid too coarse leads to missing narrow peaks that fall between grid points. A razonable idea is to choose grid spacing smaller than the spected width of the periodogram peaks, these peaks has a widht of $\sim 1/T$ where $T$ is the duration of the window function. To ensure that our grid sufficiently samples each peak, we can over-sample by some factor ($n_0$) using a grid spacing of

\insertequation[\label{grid-spacing}]{\Delta f = \frac{1}{n_0 T}}

\newp And with this we can know the total number of frequencies to evaluate

\insertequation[\label{n-frequencies}]{N_f = n_0 T (f_{max} - f_{min})}

\newp choose the value of $n_0$ is kind-of arbitrary and we can find uses in literature in the range of $n_0 = 5$ to $n_0 = 10$ \cite{vanderplas-lombScargle}.

\end{description}

\section{Fast Fourier Transform}
\newp The normal Discrete Fourier Transform showed in section \ref{subsec_dft} takes times $O(N^2)$ to compute wheter is the direct or the inverse transform, this is a very expensive computation and can be improved to $O(N log(N))$ usign the algorithm of the Fast Fourier Transform(FFT). \texttt{NEED TO READ MORE ABOUT THE ALGORITHM}

\subsection{Non-uniform Fast Fourier Transform}
\newp The same idea used in the optimization of the Discrete Fourier Tranform for evenly-sampled data called FFT can be used for unevnely-sampled data and we call this Non-unifor Fast Fourier Transform, using the same considerations. \texttt{NEED TO EXPLORE MORE THE ALGORITHM}

\section{Fourier Transform as Signal Decomposition}
\newp An alternative approach to realize the Fourier Transfrom is to do a Signal Decomposition by regression, here we just take our data and find what coeficients give the best fit to the original data given a Dictionary which can be interpreted as a change of basis, if we use for the Dictionary the fourier Matrix, then the coeficients of the signal decomposition will be the values of the Fourier Transform.

\subsection{What is a Signal Decomposition?}
\newp given a signal $s \in \mathbb{R}^{N}$ we want to find a representative set of coeficients $\beta \in\mathbb{R}^{M}$ such that:

\insertequation[\label{signaldecom1}]{\hat{s} = \Phi \hat{\beta}}

\newp Where $\Phi \in \mathbb{R}^{N \times M}$ is known as the dictionary matrix. Every column of this dictionary matrix correspond to an \textit{atom}, i.e. the constituents of signal $s$. A dictionary is called \textit{complete} when $M = N$. $\Phi$ can be either pre-defined or learned from data. We will select the atoms previously, for example, in a Fourier Dictionary we choose the atoms by knowing what frequencies we want.
\newp Solve eq. \ref{signaldecom1} for a complete dictionary that is invertible means find $\hat{\beta}^{*}$ such that:

\insertequation[\label{signaldecom2}]{\hat{\beta}^{*} = \Phi^{-1} \hat{s}}

\newp When the matrix is not invertible, the solution to eq. \ref{signaldecom1} is obtained by solving the  Least Square problem:

\insertequation[\label{leastsquare}]{\underset{\hat{\beta}}{\text{min}} \frac{1}{2} \| \hat{s} - \Phi\hat{\beta} \|^{2}_{2}}

\newp Where $\| \cdotp \|_{2}^{2}$ means the square of the L2-norm, the solution to this problem is:

\insertequation[\label{pseudoinv1}]{\hat{\beta}^{*} = (\Phi^{T}\Phi)^{-1}\Phi^{T} \hat{s}}

\newp Which requieres $(\Phi^{T}\Phi)$ to be invertible, means, having linear independent columns. Te matrix $(\Phi^{T}\Phi)^{-1}\Phi^{T}$ correspond to the left Moore-Penrose pseudo-inverse of $\Phi$.

\newp Then complete dictionary is not often the better dictionary, here we are more interested in the overcomplete dictionary, this is a dictionary with more atoms than data samples ($M > N$). For overcomplete dictionaries eq. (\ref{signaldecom1}) is under-determined and many solutions may exists. When $\Phi$ is overcomplete, the product $(\Phi^{T}\Phi)$ is always a singular matrix and then (\ref{pseudoinv1}) cannot be compute. In order to get an optimal its neccessary to include a regularization coeficient which impose conditions to help choose one of the many solutions of the underdetermined system. There are two kind of regularization usualy used, one is smooth-ness and the other is spartisy, the first refers to the least-rough or least-complex solution, this condition usualy leads to dense solutions with almost all points non-zeros, and the second one refeers to solution with less non-zero points and thus a more sparse solution. here we will show many method that try to find a good $\beta$.

\subsubsection{Ridge Regression}
\newp this is an example of a regression that use as regularization smooth-ness, the ridge regression is defined as:

\insertequation[\label{ridge1}]{\underset{\hat{\beta}}{\text{min}}\alpha \| \beta \|^{2}_{2} + \| s - \Phi \beta \|_{2}^{2}}

\newp Where $\alpha > 0$ is the trade-off between reconstruction error and smooth-ness of the solution. Te solution to \ref{ridge1} is:

\insertequation[\label{ridge2}]{\beta^{*} = (\Phi^{T}\Phi + \alpha I)^{-1}\Phi^{T}s}

\textbf{\texttt{UNCOMPLETE, NEED TO EXPLAIN MORE METHODS}}

\subsection{Fourier Dictionary}

\newp The discrete Fourier Transform from section \ref{subsec_dft} can be interpreted as a change of basis of the signals which means change the time representation of the signal $\vec{s}$ to a frequency representation $vec{\beta}$ using a sine/cose basis. This will means that we can write:

\insertequation[\label{signaldecomp1}]{\vec{s} = \Phi \cdotp \vec{\beta}}

\newp Where $\Phi$ is the Dictionary/Basis to use in the change of representation. This Dictionary is the Fourier Matrix where every colummn (atom $\phi$) correspond to a sine/cosine wave of a given frequency $k\Delta f$:

\insertequation[\label{signaldecomp2}]{\phi_k = e^{2 \pi i t_j k \Delta f} \forall t_j}

 \newp and in order to represent Direct and Inverse Fourier Transform Correctly it's neccessary to use an orthogonal Basis which means an square matrix Dictionary with a defined Inverse, if this matrix is defined correctly (ofter in the evenly-sampled case) his adjoint will be his inverse in a way that allow us to define the Adjoint Transform and use it as the Inverse Transform.

\insertequation[\label{signaldecom3}]{\vec{\beta} = \Phi^{T} \cdotp \vec{s}}

\newp Sadly for unevenly-sampled data we usually want to compute a frequency grid higher than the time grid producing a oversampling Dictionary, and even if we choose the same number of frequencies than times, there is no guarantee that the resulting Fourier Matrix will be unitary and invertible in a way that his adjoint transform be his Inverse Transform. Because of this, we need to look for a way to compute the Pseudo-inverse of the dictionary ($\Phi^{+}$) instead of the Inverse, usualy we can compute this by Single Value Decompostion, where the left decomposition (known as ):

\insertequation[\label{pseudoinverse}]{\Phi^{+}}

\textbf{\texttt{UNCOMPLETE}}

\section{Signal Detection}{\label{sec:signalDetect}}
\newp Once we know how to interprete correctly some datas in frequency domain, using Non Unifor Fast Fourier Transform or Signal Decomposition, we are ready to work on the signal detection, i.e., on the search of a known signal in the observed data. This idea comes from the idealization of our data as an underlying signal plus additive \textbf{Wide-Sense Stationary} (WSS) Gaussian noise, then it is posible, through use of linear filters, to find what is this underlaying signal by comparing the observed data to known waveforms. Linears filters are defined as convolution of input data $x(t)$ with an impulse response filter wich is just the time reverse of the template waveform $h(t)$, for this we define the product $(x|h)$:

\insertequation[\label{signalDetect:linearFilter}]{(x|h)_{t_0} = [x(t) * h(-t)](t_0) = \int_{-\infty}^{\infty} \tilde{x}(f)\tilde{h}^{*}(f) e^{2\pi i f t_0} df}

\newp Here we have used eq. (\ref{corr_as_conv}) to write this a correlation. It's also important to notice that in practice we should do a $causal$ time reverse for the impulse response $h(-t) \longrightarrow h(T-t)$ where $T$ is the total duration of the template waveform, but if we always use templates of same duration as the data window, then this factor is not important due to the periodicity in Fourier Transform leadin to $h(T-t)=h(-t)$.

\newp And the optimal Filter is just find the filter that optimize a given quantity. The theory that we are going to use correspond to the \textbf{Matched Filter} (MF) which maximize the \textbf{Signal-to-Noise Ratio} (SNR) of the observed data, and in order to do this we need to consider an input signal to the filter composed of \textbf{only White Gaussian Noise} which is a WSS Gaussian noise but with constant spectral density.

\newp Obviously the raw data detected by any deviced will have mostly colored Gaussian Noise, this means, a WSS Gaussian noise with non-constant spectral density, then it is necessary to do a previous filter which is called \textit{Whitening Filter} (WF) to clear the noise.

\subsection{Whitening Filter}{\label{sec:signalDetect::ssec:WF}}

\newp For an observed data $x(t)$ of additive WSS Gaussian colored noise $n(t)$, which could have or not an underlying signal $s(t)$, we define the whitening Filter with impulse response $h_w(-t)$ as the optimal filter when its output due to the input noise $n(t)$ is white, with variance $\sigma^{2}$. This can be write as:

\insertequation[\label{whiteningFilter:whitenData}]{(x|h_w)_{t_0} = (s|h_w)_{t_0} + (n|h_w)_{t_0} \text{such that} var = \mathbb{E}[(n|h_w)^{2}] = \sigma^{2}}

\newp Working a litle with this variance we will get:

\insertequation[\label{whiteningFilter:noiseVarAsCovarFunct}]{
\begin{aligned}[b]
\mathbb{E}[|(n|h_w)|^{2}]	& = \mathbb{E}[(n|h_w)(n|h_w)^{*}] \\
						& = \mathbb{E}\left[\int_{-\infty}^{\infty} n(t_0 + t_1)h_w^{*}(t_1) dt_1 \left(\int_{-\infty}^{\infty} n(t_0 + t_2) h_w^{*}(t_2) dt_2 \right)^{*}\right] \\
						& = \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} h_w^{*}(t_1) \mathbb{E}[n(t_0 + t_1) n(t_0 + t_2)^{*}] h_w(t_2) dt_1 dt_2 \\
\end{aligned}}

\newp In order to work with the covariance $\mathbb{E}[n(t_0 + t_1) n(t_0 + t_2)^{*}]$ we need to set some considerations, first the noise used will be a 0 mean random noise, in a way that $\mathbb{E}[n(t)] = 0$, second, the noise will be invariant in time (stationary) so $n(t + t_0) = n(t)$, with this que covariance will be reduced to:

\insertequation[\label{whiteningFilter:noiseCovar}]{\mathbb{E}[n(t_0 + t_1) n(t_0 + t_2)^{*}] = \mathbb{E}[n(t_1)n^{*}(t_2)] = k_n(t_1, t_2) = k_n(\tau)}

\newp Where we have used $\tau = t_1 - t_2$, $k_n(\tau)$ correspond to the auto-correlation function of the noise. \textbf{Bochner's theorem} states that the covariance function of a stationary process can be represented as the fourier transform of a positive finite measure\cite{gauss-procss}:

\insertequation[\label{whiteningFilter:noiseCovTransf}]{k_n(\tau) = \int S_n(f)e^{2\pi i f \tau} df}

\newp This transform is defined as the two-sided power spectral density being a real and positive function with no zero values. Using this on eq. (\ref{whiteningFilter:noiseVarAsCovarFunct}) we get:

\insertequation[\label{whiteningFilter:noiseVar}]{
\begin{aligned}[b]
\mathbb{E}[|(n|h_w)|^{2}]	& = \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} h_W^{*}(t_1) k_n(t_1 - t_2) h_w(t_2) dt_1 dt_2 \\
						& =\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} h(t_2) \left( \int_{-\infty}^{\infty} S_n(f)e^{2\pi i f (t_1 - t_2)} df\right) h_w^{*}(t_1)  dt_1 dt_2 \\
						& = \int_{-\infty}^{\infty} \left( \int_{-\infty}^{\infty} h_w^{*}(t_1)e^{2\pi i f t_1} dt_1 \right) S_n(f) \left( \int_{-\infty}^{\infty} h_w(t_2) e^{-2\pi i f t_2} dt_2 \right) df \\
						& = \int_{-\infty}^{\infty}\tilde{h}_w^{*}(f) e^{-2\pi i f T} S_n(f) \tilde{h_w}(f) e^{2\pi i f T} df \\
						& = \int_{-\infty}^{\infty}\tilde{h}_w^{*}(f) S_n(f) h_w(f) df \\
						& = \int_{-\infty}^{\infty}S_n(f) |\tilde{h}_w(f)|^{2} df \\
\end{aligned}}

\newp With this, we define the Transfer function of the Whitening Filter impulse response as:

\insertequation[\label{whiteningFilter:optimalWhiteningFilter}]{S_n(f)|\tilde{h}_w(f)|^{2} = S_w(f) \:\: \textbf{such that} \:\: \sigma^{2} = \int_{-\infty}^{\infty} S_w(f)}

\newp this directly leads to:

\insertequation[\label{whiteningFilter:optimalWhiteningImpulseResponse}]{|\tilde{h}_w(f)|^{2} = S_w(f)/S_n(f)}

\newp In a window of a finite duration with a finite number of measures $N$ and a White Variance of $\sigma_w^{2}$, then $S_w(f) = \sigma_w^{2}/N = N_0$ and then:

\insertequation[\label{whiteningFilter:optimalWhiteningImpulseResponseValue}]{|\tilde{h}_w(f)|^{2} = \frac{N_0}{S_n(f)}}

\newp This definition of whitening will leads to a cleared signal of White noise with variance 1 and mean 0. After this, if we want to do another linear filter, we will need to normalize the impulse response by this whitening optimal filter too in order to get match the template waveform correctly.

\subsection{Matched Filter}{\label{sec:signalDetect::ssec:MF}}
\newp Matched filter correspond to find the optimal linear filter for maximizing the Signal To Noise Ratio (SNR) in the presence of additive white noise of a given variance $\sigma_w$ and 0 mean. For this we considers some data $x(t)$ with finite-energy which can be pure noise $x(t) = n(t)$ or a signal in addition to noise $x(t) = s(t) + n(t)$, this is taken as the input to a filter with impulse response $h_{\MF}(-t)$ and use the linear filter product defined in eq. (\ref{signalDetect:linearFilter})

\newp If we do a MF using two different templates waveform, with different energy, it make hard to say what lineal filter works better because our impulse response functions are unormalized. This is clearly seen from the noise, on the input the noise is white but on the output the noise is convolved with the template and thus is not white anymore, this introduce a change on variance:

\insertequation[\label{matchedFilter:variance}]{\sigma_{\MF} = \mathbb{E}[|(x|h_{\MF})|^{2}] = N_0 \IL |\tilde{h}_{\MF}(f)|^{2} df}

\newp Where we have used the variance of eq. (\ref{whiteningFilter:noiseVar}) with the constant white noise PSD $S_w(f) = N_0$. Then we just need to divide the matched filter by square root of this variance in order to get a normed SNR of normal distribution $\mathcal{N}(\mu_{MF} / \sigma_{MF}, 1)$ where $\mu_{MF}$ is:

\insertequation[\label{matchedFilter:mean}]{\mu_{\MF} = \mathbb{E}[(x|h_{\MF})] = \mathbb{E}[(n|h_{\MF})] + \mathbb{E}[(s|h_{\MF})] = 0 + (s|h_{\MF})_{t_0}}

\newp Here we have that $\mathbb{E}[(s|h_{\MF})] = (s|h_{\MF})_{t_0}$ because the linear filter product itself is an expected value of the output filter. This expected value is for a given $t_0$ where we supose it is produced the maximum SNR response which is writen as:

\insertequation[\label{matchedFilter:SNR}]{SNR_{t_0} = \frac{(x|h_{\MF})_{t_0}}{\sigma_{\MF}}}

\newp And the optimal matched filter is when the expected underlying waveform that comes with noise on the input signals is just equal to the template waveform. from eq. (\ref{matchedFilter:mean}) we can use $x(t)$ instead of $s(t)$ to estimate $\mu_{\MF}$

\subsubsection{scopes and limitations}{\label{sec:signalDetect::ssec:MF::sssec:scopes}}

\newp Here we like to define limits on the behavior of the SNR, first we consider the situation when the input data is only white noise. Using the Cauchy-Schwarz inequality  on eq. (\ref{matchedFilter:SNR}) with $|\tilde{x}(f)|^{2} \thickapprox \lambda S_n(f)$, where $\lambda$ depends on the normalization and method used to estimate the PSD, we get:

\insertequation[\label{matchedFilterScopes:limitOnlyNoise}]{|SNR_{t_0}| = \frac{|(x|h)_{t_0}|}{|\sigma_{\MF}|} = \frac{|(n|h)_{t_0}|}{|\sigma_{\MF}|} \leq \frac{\sqrt{\lambda \sigma_{w}} \sqrt{\int_{-\infty}^{\infty} |\tilde{h}_{\MF}(f)|^{2} df}}{\sqrt{N_0 \int_{-\infty}^{\infty} |\tilde{h}_{\MF}(f)|^{2} df}} \begin{array}{c} \text{Discrete} \\ = \\ case
\end{array} \sqrt{\frac{\lambda N}{\sigma_w}}}

\newp Here we have writen $N_0$ as the values on the discrete case $N_0 = \sigma_w^{2} / N$ of $N$ samples, we are going to continue using this but we will write the equations on continuous case. Eq. (\ref{matchedFilterScopes:limitOnlyNoise}) is a limit for every SNR produced when the input data is only white noise.

\newp Another important limit is to find the maximum value that a SNR can achieve for any type of input data (with or without underlying signal), again, using the Cauchy-Schwarz inequality, we get in this case:

\insertequation[\label{matchedFilterScopes:limitGeneral}]{|SNR_{t_0}| \leq \frac{\sqrt{\int_{-\infty}^{\infty} |\tilde{x}(f)|^{2} df} \sqrt{\int_{-\infty}^{\infty} |\tilde{h}_{\MF}(f)|^{2} df}}{\sqrt{N_0 \int_{-\infty}^{\infty} |\tilde{h}_{\MF}(f)|^{2} df}} \begin{array}{c} \text{Discrete} \\ = \\ case \end{array} \sqrt{\frac{E_x N}{\sigma_w^{2}}}}

\newp Where $E_x = \IL |\tilde{h}_{\MF}(f)|^{2} df = \IL |h_{\MF}(t)|^{2} dt$ is the energy of the input data, here we see that the maximum value possible of the SNR only depend on the input data (energy, number of samples and white noise variance) and not on the template waveform.

\subsection{Whitened Matched Filter}{\label{sec:signalDetect::ssec:WMF}}

\newp We can (and must) use the Whitening Filter (WF) together with the Matched Filter (MF) for our (guessed) input signal with colored WSS noise. Here we just need to apply WF to both the input data and the inpulse response on the MF:
\insertequation[\label{whitenedMatchedFilter:filterConvolution}]{(x|h_{\MF})^{\WM}_{t_0} = [\lbrace x * h_w\rbrace * \lbrace h_{\MF} * h_{w} \rbrace] = [x_w * h_{\WM}]}

\newp Using the definition of lineal filter on eq. (\ref{signalDetect:linearFilter}) with eq. (\ref{convolution2}), this will be:

\insertequation[\label{whitenedMatchedFilter:filterProduct}]{\wmf{x}{h} = \IL \tilde{x}(f) \tilde{h}_{\MF}^{*}(f)|\tilde{h}_w(f)|^{2} e^{2\pi i f t_0} df}

\newp Where $|\tilde{h}_w(f)|^{2}$ is given by eq. (\ref{whiteningFilter:optimalWhiteningImpulseResponseValue}) this leads to:

\insertequation[\label{whitenedMatchedFilter:filterProductValue}]{\wmf{x}{h} = N_0\IL \frac{\tilde{x}(f) \tilde{h}^{*}(f)}{S_n(f)} e^{2\pi i f t_0} df}

\newp This also affect the variance and the expected (value) remains with same definition as shown in eq. (\ref{matchedFilter:mean}) but for the product defined in eq. (\ref{whitenedMatchedFilter:filterProductValue}):

\insertequation[\label{whitenedMatchedFilter:VarianceFilter}]{\sigma_{\WM}^{2} = \mathbb{E}[|\wmf{n}{h}|^{2}] = N_0 \IL |\tilde{h}_{\MF}(f)|^{2}|\tilde{h}_{w}(f)|^{2} df = N_0^{2} \IL \frac{|\tilde{h}_{\MF}(f)|^{2}}{S_n(f)} df}

\insertequation[\label{whitenedMatchedFilter:ExpectedValueFilter}]{\mu_{\WM} = \mathbb{E}[\wmf{x}{h}] = \mathbb{E}[\wmf{x-n}{h}]}

\newp Here (and in the next sections) for simplicity we will write $\wmf{x}{h} = (x|h)$, $\sigma_{\WM} = \sigma$ and $\mu_{\WM} = \mu$.Thus the SNR for the Whitened Matched Filter (writen as $\rho$ for simplicity) will be:

\insertequation[\label{whitenedMatchedFilter:SNR-WMF}]{\rho_{t_0} = \frac{(x|h)}{\sigma}}

\newp This division doesn't depend on $N_0$ so clearly this matched filter doesnt work with White noise (and in next we will ignore this factor), instead, consider any WSS Gaussian noise. For this definition, the limit values of the matched filter defined in eq. (\ref{matchedFilterScopes:limitOnlyNoise}) and (\ref{matchedFilterScopes:limitGeneral}) should be the same but due to numerical computations, could be not exactly the same. This SNR has the same distribution than before $\mathcal{N}(\mu / \sigma, 1)$.

\section{Statistical Theory of Signal Detection}{\label{statTheoSigDet}}
\newp The observed signal carried an additive noise introduced by the detector and is usually modeled as a Wide-Sense Stationary (WSS) Gaussian random process, this means, the problem of extracting the signal from the noise (as shown in section (\ref{sec:signalDetect}) can be interpreted as a statistical one. The presence of signal changes the statistical characteristics of the data observed $x(t)$, affecting his Probability Density Function (PDF).
\newp All the theory that will be described here are explained more extense in \citep{ligo-gauss-case, mit-course-sigSysAndInfer, ligo-1992}.
\newp We are going to work with statistical hypothesis testing and then define a likelihood ratio with a threshold value in order to decide if there is really a signal or not.

\subsection{Hypothesis test}

\newp The main idea here is to consider two scenarios (binary hypothesis)  one is the \textit{null hypothesis} $H_0$ which consider data absent of signal and \textit{alternative hypothesis} $H_1$ which consider a signal present.

\insertgather{\label{hypoTest:h0} H_0 : x(t) = n(t) \\
\label{hypoTest:h1} H_1: x(t) = n(t) + s(t)}

\newp The \textit{Binary Hypothesis test} or \textit{binary decision rule} consider 4 scenarios which can be representes as a confusion matrix. Two of these fourth posibilities are consider errors:
\begin{itemize}
\item $P(H_0|H_0)$, this is the probability of choose $H_0$ when $H_0$ is true. It's a \textbf{correct decision}.
\item $P(H_1|H_0)$, probability of choose $H_1$ when $H_0$ is true. This is a Type I error and it's called \textbf{false alarm probability}.
\item $P(H_0|H_1)$, probability of choose $H_0$ when $H_1$ is true. Type II error known as \textbf{False dismissal probability}.
\item $P(H_1|H_1)$, probability of choose $H_1$ when $H_1$ is true. It's a \textbf{correct decision}.
\end{itemize}
\newp The type II error is also the \textit{probability of detection} of the signal. For the Test scenario, the type I error is called \textit{significance of the test} and type II error is \textit{power of the test}.
\newp Next, we need to find a test that is optimal. The main idea here, no mather what approach we use, is to calculate the \textbf{likelihood ratio}.
\subsection{likelihood ratio}
\newp From the binary hypothesis test, for a given measure $x$, we will have a probability $P(H_0|x)$ of $H_0$ being true and probability $P(H_1|x)$ of $H_1$ being true, oviously, if $H_1$ is actually true, then $P(H_1|x) > P(H_0|x)$, same for case of $H_0$ being true, this can be writen in the compact form:

\insertequation[\label{likelihood:desicion}]{P(H_1|x) \begin{array}{c}H_1 \\ > \\ < \\ H_0 \end{array} P(H_0|x)}

\newp Here is implicit that when $P(H_0|x) = P(H_1|x)$ we cannot know what hypothesis is correct. These conditional probabilities are using the posterior probability of the hypotesis, this is different from the \textit{a priori} probabilities of every hypothesis $P(H_0)$ and $P(H_1)$. In order to evaluate the \textit{posteriori} probabilites in (\ref{likelihood:desicion}) we use the Bayes' rule:

\insertequation[\label{likelihood:prioriDesicion}]{\frac{p_1f(x|H_1)}{f(x)} \begin{array}{c}H_1 \\ > \\ < \\ H_0 \end{array} \frac{p_0f(x|H_0)}{f(x)}}

\newp Where $f(x|H_j)$ is just the conditional PDF and $f(x)$ is the PDF of measure the observed data, since is the same for both hypothesis, we will not consider his values. On eq. (\ref{likelihood:prioriDesicion}) we have considered that $f(x|H_j) > 0$ for $j=0,1$. Next we can just rewrite this equation in order to get the likelihood ratio $\Lambda(x)$:

\insertequation[\label{likelihood:ratio}]{\Lambda(x) = \frac{f(x|H_1)}{f(x|H_0)} \begin{array}{c}H_1 \\ > \\ < \\ H_0 \end{array} \frac{p_0}{p_1} = \eta}

\newp Where $\eta$ is a \textbf{threshold} that can be defined in several ways, one can estimate the optimal case usign bayesian inference or Neyman-Pearson (NP) detection between many others. For now we will use the NP theory to define the detection and threshold.

\subsection{Neyman-Pearson Detection}
\newp This approach consider a context in which we want to maximize the probability of detection defined as:

\insertequation[\label{npDetection:probOfDetect}]{P_D = P(H_1|H_1) = \int_{D_1} f(x|H_1) dx}

\newp While keeping the probability of false alarm:

\insertequation[\label{npDetection:probOfFalseAlarm}]{P_{FA} = P(H_1|H_0) = \int_{D_1} f(x|H_0) dx = \alpha}

\newp Below a pre-specified leve $\alpha$. The values in $D_1$ are all values for which the likelihood function is higher than a given threshold $\eta$. Then the threshold is picked to provide the largest possible $P_D$ while ensuring that $P_{FA}$ is not larger than the pre-specified level. The smaller the $\eta$, the largest the desicion region $D_1$ and the value of $P_D$, but $P_{FA}$ grows as well. We can just pick the smaller threshold that satisfied the given bound of $P_{FA}$.

\subsection{NP Detection for Whitened Matched Filter}

\newp Remembering that we consider the noise as a WSS White Gaussian random process, we can modelate this noise as a normal distribution of 0 mean ($\mu = 0$) and a given variance $\sigma$ ($\mathcal{N}(0, \sigma^{2})$), thus,  for both hypothesis from eq. (\ref{hypoTest:h0}) and (\ref{hypoTest:h1}) $n(t)$ distribute as:

\insertequation[\label{npDetWMF:noiseDistr}]{n(t) \thicksim N(0, \sigma^{2}) \longrightarrow f(x) = \frac{1}{\sqrt{2 \pi \sigma^{2}}}e^{-\frac{x^{2}}{2\sigma^{2}}}}

\newp When the matched filter as defined in sec. (\ref{sec:signalDetect::ssec:WMF}) is applied, our hypothesis tests are:

\insertgather{\label{npDetWMF:H0} H_0: \:\: (x|h) = (n|h) \thicksim \mathcal{N}(0, \sigma) \\
\label{npDetWMF:H1} H_1 \:\: (x|h) = (n|h) + (s|h) \thicksim \mathcal{N}(\mu, \sigma)}

\newp then, for the SNR we normalize by square root of the variance, leading to hypothesis tests:

\insertgather{\label{npDetWMF:H0SNR} H_0: \:\: \frac{(x|h)}{\sigma} = \frac{(n|h)}{\sigma} \thicksim \mathcal{N}(0, 1) \\
\label{npDetWMF:H1SNR} H_1 \:\: \frac{(x|h)}{\sigma} = \frac{(n|h)}{\sigma} + \frac{(s|h)}{\sigma} \thicksim \mathcal{N}(\frac{\mu}{\sigma}, 1)}

\newp Here the product $(s|h)$ is optimal when the signal $s(t)$ is equal to the impulse response shifted by the optimal time of match $s(t) = h(t_0 - t)$, using the base definition of linear filter (\ref{signalDetect:linearFilter}) writen as a simple convolution (\ref{convolution1}), with the parseval's equation (\ref{parseval}) and the equations defined in sec. (\ref{sec:signalDetect::ssec:WMF}):

\insertequation[\label{npDetWMF:optimalMean}]{(s|h) = [h(t_0 - t) * h(-t)](t_0) = \IL h(t_0 - (t_0 - t))h(-t) dt = \IL |\tilde{h}(f)|^{2} df = (h|h)_{t_0=0} = \sigma^{2}}

\newp The expected value of the optimal filter is just the variance of the output filter. And the Probability Density Function (PDF) of normal distributions is:

\insertequation[\label{npDetWMF:PDF}]{f(\rho) = \frac{1}{\sqrt{2\pi}\sigma} e^{-\frac{(\rho - \mu)^{2}}{2\sigma^{2}}}}

\newp Using this PDF as a conditional PDF with the hypothesis from eq. (\ref{npDetWMF:H0}) and (\ref{npDetWMF:H0}), where $\mu = (s|h)$ for $H_1$, the likelihood ratio (\ref{likelihood:ratio}) is writen as:

\insertequation[\label{npDetWMF:likelihoodRatio}]{
\begin{aligned}[b]
\Lambda\lbrace(x|h)\rbrace & = \frac{(\sqrt{2\pi})^{-1} exp\left( -\frac{((x|h) - (s|h))^{2}}{2\sigma^{2}}\right)}{(\sqrt{2\pi})^{-1} exp\left( -\frac{(x|h)^{2}}{2\sigma^{2}}\right)} \\
			   & = exp\left(\frac{2(x|h)(s|h) - (s|h)^{2}}{2 \sigma^{2}} \right) \\
			   & =  exp\left(\frac{(s|h)}{\mathbb{E}[(n|h)^{2}]} \lbrace (x|h) - \frac{1}{2}(s|h) \rbrace \right)
\end{aligned}}

\newp Te factor $(s|h) / \mathbb{E}[(n|h)^{2}]$ on the optimal case is simplified resulting in:

\insertequation[\label{npDetWMF:likelihoodRatioSimplified}]{\Lambda\lbrace(x|h)\rbrace = exp\left( \lbrace (x|h) - \frac{1}{2}(h|h) \rbrace \right)}

\newp This likelihood ratio is the same that the one used in \cite{ligo-gauss-case, ligo-1992} even if we derived the formulas from different perspectives. If we simplify this likelihood ratio by doing a natural logarithm, we get:

\insertequation[\label{npDetWMF:logLikelihoodRatio}]{ln[\Lambda\lbrace(x|h)\rbrace] =  \lbrace (x|h) - \frac{1}{2}(h|h) \rbrace \begin{array}{c}H_1 \\ > \\ < \\ H_0 \end{array} ln\left[ \frac{p_0}{p_1} \right] = ln[\eta]}

\newp And here que can write a critical criteria over Threshold to decide what hypothesis is correct:

\insertequation[\label{npDetWMF:criteriaOverThreshold}]{(x|h)\begin{array}{c}H_1 \\ > \\ < \\ H_0 \end{array} ln[\eta] + \frac{1}{2}(h|h) = \eta'}

\newp Here we see that we can write the optimal threshold if we know what is the optimal and the \textit{a priori} probability of $H_0$ and $H_1$ (or at least is ratio), but this is not know for the mosts cases, so we need to estimate a threshold for a given constrains. Under the NP criteria, we choose this threshold $\eta'$ to maximize $P_D$ and satisfy the desired false alarm $P_{FA}$.

\newp We know that the solution for eq. (\ref{npDetection:probOfFalseAlarm}) is a \textit{complementary error function}:

\insertequation[\label{npDetWMF:falseAlarmSolution}]{P_{FA} = \int_{\eta'}^{\infty} \frac{1}{\sqrt{2\pi}\sigma} e^{-x^{2}/(2\sigma^{2})} dx = \text{erfc}\left( \frac{\eta'}{\sigma}\right)}
\newp Where $erfc()$ means the complementary error function. Then we can solve for $\eta'$ which leads to:

\insertequation[\label{npDetMF:thresholdFromFalseAlarm}]{\eta' = erfc^{-1}(P_{FA}) \sigma }

\newp With this threshold defined, we calculate the probability of detection (\ref{npDetection:probOfDetect}) which solution is:

\insertequation[\label{npDetWMF:probOfDetectSolution}]{P_{D} = \int_{\eta'}^{\infty} \frac{1}{\sqrt{2\pi}\sigma} e^{-(x - \mu)^{2}/(2\sigma^{2})} dx = \text{erfc}\left( \frac{\eta' - \mu}{\sigma}\right)}

\newp In terms of the False Alarm:

\insertequation[\label{npDetWMF:probOfDetectForGivenFalseAlarm}]{P_D = \text{erfc}\left(\frac{\sigma \text{erfc}^{-1}(O_{FA}) - \mu}{\sigma} \right) = \text{erfc}[\text{erfc}^{-1}(P_{FA}) - \mu / \sigma] = \text{erfc}\left[\text{erfc}^{-1}(P_{FA}) - \frac{(x|h)_{t_0}}{\sqrt{(h|h)_{t_0=0}}} \right]}

\newp This is the final value that we are going to use to determine if our SNR represent a detection or not, the parameters needed for this are the False alarm value $P_{FA}$ and the values calculated for the filter used to estimate the mean and variance. We can also define the counterpart, the probability of False alarm for a given Probability of detection:

\insertequation[\label{npDetWMF:probOfFalseAlarForGivenProbOfDetect}]{P_{FA} = \text{erfc}\left[\text{erfc}^{-1}(P_D) + \frac{(x|h)_{t_0}}{\sqrt{(h|h)_{t_0=0}}}\right]}

\newp What of these two probabilities we want to compute depend on what kind of event we observe.

\newpage
\nocite{*}
%\bibliography{references, revtex-custom}
%\bibliographystyle{plain}
%\printbibliography
\begin{multicols}{2}[\printbibheading]
\printbibliography[heading=none]
\end{multicols}


%!TEX root = /home/renaud/Documents/EPL/tfe/latex/tfe.tex
\section{Assessing the robustness of a partition} \label{subsec:robustness}
We present here two mechanisms commonly used to assess the relevance of a particular partition. One simple way is to consider that a robust partition should not be altered by a small modification of the quality function. Such a modification could be for example a perturbation of the Markov time $t$ at which the partition has been found. From this point of view, robust partitions correspond to \textit{plateaux} in the community curve of the graph. In other words, robust partitions should be persistent over a wide interval of Markov time.

\begin{sloppypar}
The second indicator of the robustness of a partition that we will take into account in this work follows from considering that a robust partition is one that is persistent to small modifications of the optimization algorithm. The central tool to quantify this approach of the robustness of a partition is the \textit{normalized variation of information} \cite{meilua2007comparing}, which is a popular way to compare two partitions. Let $p(\C)$ be the probability for a node to be in community $\C$, i.e. $p(\C) = n_\C/n$ where $n_\C$ is the number of nodes in community $\C$. The variation of information between partitions $\P_1$ and $\P_2$ is defined as
\begin{equation} \label{eq:clustering_VI}
	\VI(\P_1,\P_2) := \frac{H(\P_1,\P_2)-H(\P_1)-H(\P_2)}{\log (n)} = \frac{H(\P_1|\P_2)+H(\P_2|\P_1)}{\log (n)},
\end{equation}
where $\log(n)$ is a normalization factor;
\begin{equation}
	H(\P) := -\sum_{\C \in \P} p(\C) \log[p(\C)]
\end{equation}
is the Shannon entropy; $H(\P_1,\P_2)$ is the Shannon entropy of the joint probability $p(\C_1,\C_2)$ that a node belongs both to a community $\C_1$ of $\P_1$ and to a community $\C_2$ of $\P_2$. We have
\begin{equation}
	p(\C_1,\C_2) = \frac{n_{\C_1 \cap\, \C_2}}{n},
\end{equation}
and
\begin{equation}
	H(\P_1,\P_2) := -\sum_{\C_1 \in \P_1} \sum_{\C_2 \in \P_2} p(\C_1,\C_2) \log[p(\C_1,\C_2)].
\end{equation}
Similarly, $H(\P_1|\P_2)$ is the conditional Shannon entropy of partition $\P_1$ given $\P_2$, which is defined in a standard way from the joint distribution: $p(\C_1|\C_2) = p(\C_1,\C_2)/p(\C_2) = n_{\C_1\cap\, \C_2}/n_{\C_2}$, and the expression of $H(\P_1|\P_2)$ follows straightforwardly. The latter can be interpreted as the additional information needed to describe $\P_1$ once $\P_2$ is known. This measure of the difference between two partitions is then used as follows: for each Markov time, an ensemble of Louvain optimizations of stability are performed, starting from different random initial node ordering.\footnote{Remember that the problem being $\mathcal{NP}$-hard, we rely on a heuristic algorithm --- the Louvain method --- that finds a good partition for a given Markov time, but not necessarily the optimal partition. Hence the partition found may differ if a different initial condition is provided.} The normalized variation of information allows then to quantify how different the optimized partitions are. Therefore, a low variation of information indicates optimized partitions that are very similar to each others, and thus that a small modification of the algorithm barely alter the partition. From the point of view of the field of dynamical system, robust partitions have thus an attractor with a large basin of attraction for the optimization method.
\end{sloppypar}


\section{Surface Integrals}
\noindent
We can parameterize any surface as $\vec{r}(u,v) = \langle x(u,v), y(u,v), z(u,v) \rangle$ (or using some other coordinate system). This is because we are simple taking one 2D surface, the uv-plane, and transforming it into another surface in such a way that areas ear each other in the uv-plane are near each other on the surface. This fact that areas stay close to each other allows us to make statement about the complicated surface while working with the simple uv-plane.

[INSERT IMAGE]

\noindent
The change of the surface in the $u$ direction is $\frac{\partial\hat{r}}{\partial u}\mathrm{d}u$\\
The change of the surface in the $v$ direction is $\frac{\partial\hat{r}}{\partial v}\mathrm{d}v$\\
So, the area of the surface in relation to $u$ and $v$ is the area of the parallelogram spanned by these two surface: a cross product.
\begin{equation*}
	\mathrm{d}s = \norm{\left< \frac{\partial\vec{r}}{\partial u}\mathrm{d}u \right> \times \left< \frac{\partial\vec{r}}{\partial v}\mathrm{d}v \right>} = \norm{\vec{r_u} \times \vec{r_v}}\mathrm{d}u\mathrm{d}v
\end{equation*}

\begin{definition}
	For a surface $S$ parameterized by $\vec{r}(u,v)$ and $(u,v)\subset D$, the surface area is
	\begin{equation*}
		A(S) = \iint\limits_{S}{\mathrm{d}s} = \iint\limits_{D}{\norm{\vec{r_u} \times \vec{r_v}}\mathrm{d}u\mathrm{d}v}
	\end{equation*}
\end{definition}

\input{./lineSurfaceIntegrals/surfaceIntegralsScalarFunctions}
\input{./lineSurfaceIntegrals/surfaceIntegralsVFs}


\input{../../assignment-header}

%========================================================================
\title{Assignment 1:  Matlab Basics}
\date{Assigned:  Jan. 18  ---  Due:  Jan. 25}
\author{Optimal Control For Robotics}
%========================================================================
\begin{document} %
\maketitle
%=================================================

\section*{Problem 1: fun with random sequences  (25 pts)}

In this assignment you will write a computer program that generates
a sequence of points in a plane, which will converge to a well-known fractal.
There are two parts to this problem, each with slightly
different rules for generating the sequence of points, resulting in two different fractals.

\par
In addition to submitting plots for each part (on a single figure), you will
also be asked to submit your code and detail how long the assignment took you
to complete. Make sure that your code is well documented (use comments!) and
clearly written so that someone else can make sense of it.

\subsection*{Part One:  a special triangle}
Start by selecting three control points ($A$, $B$, $C$)
that are uniformly spaced around the edge of a circle.\\
Next, generate a sequence of points $P_0 ... P_N$
using a random number generator and the control points. \\
Use $\alpha = \tfrac{1}{2}$ and
the choice of $A$, $B$, or $C$ should be randomly selected with equal probability.

\begin{align}
  P_0 &= A \\
  P_{k+1} &= \alpha \cdot P_k + (1 - \alpha) \cdot \texttt{RandomChoice}(A, B, C)
\end{align}

Create a plot to visualize the sequence, using the following guidelines:
\vspace{-0.3em} \begin{itemize}  \setlength\itemsep{0em}
\item plot the circle as a thin curved black line
\item plot each control point ($A$, $B$, $C$) as a small red circle
\item plot the initial point $P_0$ as a small green ``X''
\item plot $P_1 ... P_N$ as tiny blue dots, where N is 5000
\item make sure that the axes are scaled so that the circle looks like a circle
\end{itemize}

\subsection*{Part Two:  a special rectangle}

Repeat the entire procedure from Part One, but make two changes:
\vspace{-0.6em} \begin{itemize}  \setlength\itemsep{0em}
  \item Select four uniformly spaced control points around the edge of the circle, instead of three.
  \item Set $\alpha = \tfrac{1}{3}$
\end{itemize}

\subsection*{Deliverables:}

\begin{enumerate}
  \item Submit your matlab code as a single file:  \texttt{prob\_01\_studentName.m}
  \item Submit a single figure (with two sub-plots) as a pdf:  \texttt{prob\_01\_studentName.pdf}
  \item Create a short write-up for the problem:   \texttt{prob\_01\_studentName.txt}
\end{enumerate}

\vspace{-1.0em} \begin{itemize}  \setlength\itemsep{0em} \setlength\itemindent{18pt}
  \item Header: full name, studentName, date, problem name and number
  \item List any other students that you worked with.
  \item How long did this problem take you?
  \item Briefly describe or outline your code (roughly 50 -- 100 words).
\end{itemize}

\subsection*{Comments:}
  \vspace{-0.3em} \begin{itemize}  \setlength\itemsep{0em}
    \item In part one you generated the \textit{Sierpinski Triangle} fractal.
          This algorithm is just one of many ways to create this fractal.
    \item This problem was inspired by the numberphile video \quotes{Chaos Game}: \\
          \url{https://www.youtube.com/watch?v=kbKtFN71Lfs}
    \item See the Hints section at the end of the assignment for
          how to save a Matlab figure into a pdf and
          how to include multiple matlab functions in a single file.
\end{itemize}

\section*{Problem 2: simple plots and derivatives (25 pts)}

In this problem you will generate a single figure with a set of six-subplots.
The figure is described below, along with the functions that you will be plotting.
As with the previous problem, please keep track of the time you spend on this problem
and make your code well-documented and organized.
You may use any method that you like to compute the derivatives of $x(t)$ and $y(t)$.

\begin{equation}
x(t) = \big(1 + (t-2)^2 \big) \cdot \sin(3 t)
\end{equation}

\begin{equation}
y(t) = t^3 - 6 t^2 + 2 t + 5
\end{equation}

\begin{equation}
  \dot{z}(t) \equiv \frac{d}{dt} z(t)
  \quad \quad
  \ddot{z}(t) \equiv \frac{d^2}{dt^2} z(t)
\end{equation}

\subsection*{What to plot:}

\vspace{-0.6em} \begin{itemize}  \setlength\itemsep{0em} \setlength\itemindent{18pt}
\item plot $x(t)$, $\dot{x}(t)$, and $\ddot{x}(t)$ on the domain $t \in [0,5]$
\item plot $y(t)$, $\dot{y}(t)$, and $\ddot{y}(t)$ on the domain $t \in [0,5]$
\item the plots for $x(t)$ and its derivatives should be in the left column
\item the plots for $y(t)$ and its derivatives should be in the right column
\item $x(t)$ and $y(t)$ should be in top two plots
\item $\dot{x}(t)$ and $\dot{y}(t)$ should be in middle two plots
\item $\ddot{x}(t)$ and $\ddot{y}(t)$ should be in bottom two plots
\item Plot a blue circle at the point where $y(t)$ reaches its maximum value on the domain $t \in [0,5]$
\item Plot a red ``X'' at the point where $y(t)$ reaches its minimum value on the domain $t \in [0,5]$
\item All plots should have both axes labeled and a title.
\end{itemize}

\subsection*{Deliverables:}

\begin{enumerate}
  \item Submit your matlab code as a single file:  \texttt{prob\_02\_studentName.m}
  \item Create a single figure, with six sub-plots:  \texttt{prob\_02\_studentName.pdf}
  \item Create a short write-up for the problem:   \texttt{prob\_02\_studentName.txt}
\end{enumerate}

\vspace{-1.0em} \begin{itemize}  \setlength\itemsep{0em} \setlength\itemindent{18pt}
  \item Header: full name, studentName, date, problem name and number
  \item List any other students that you worked with.
  \item How long did this problem take you?
  \item Briefly describe your code (roughly 50 -- 100 words).
  \item Show work for derivative calculations (use pseudo-code, \textit{e.g.} $\dot{x} \to $ \texttt{dx}).
  \item How did you compute the minimum and maximum values?
\end{itemize}

\pagebreak

\subsection*{Matlab Hints:}

\vspace{1.0em}

You can export a matlab figure to a .pdf file using the following function:
\lstinputlisting{../../../../codeLibrary/utilities/saveFigureToPdf.m}

\vspace{1.0em}

You can use multiple functions in a single file:
\lstinputlisting{prob_01_studentName.m}

%=================================================
\end{document}


\documentclass[notitlepage]{problem-solving}

\def\ord{\operatorname{ord}}

\title{Sums Convergent under the $p$-adic Norm}
\date{June 2016}
\author{Matt McCarthy}

\addbibresource{p-adic-sum.bib}
\nocite{*}

\begin{document}

\maketitle

\begin{problem*}
	Show that
	\[
		\sum_{n=0}^\infty 2^n = -1
	\]
	under the 2-adic metric.
\end{problem*}

\section{Background}

In order to talk about limits, we first need to understand the concept of a metric.
\begin{definition}[Metric Space]
	Let $X$ be a non-empty set and let $d:X\times X\rightarrow \RR$ be a function.
	Then $d$ is a \textit{metric} on $X$ if all of the following hold.
	\begin{enumerate}
		\item For all $x,y\in X$, $d(x,y)\geq 0$.
		\item For all $x,y\in X$, $d(x,y) = 0$ iff $x=y$.
		\item For all $x,y\in X$, $d(x,y)=d(y,x)$.
		\item For all $x,y,z\in X$, $d(x,z)\leq d(x,y)+d(y,z)$ (triangle inequality).
	\end{enumerate}
	If $d$ is a metric on $X$, then we say $(X,d)$ forms a \textit{metric space}.
\end{definition}
After defining the metric space, we can consider whether or not a sequence in that space converges.
\begin{definition}[Convergent Sequence]
	Let $(X,d)$ be a metric space and let $(a_n)_{n\in\NN}\subseteq X$ and let $a\in X$.
	We say $a_n$ is a \textit{converges} to $a$ if for all $\varepsilon>0$, there exists an $N\in\NN$ such that $d(a_n,a)<\varepsilon$ for all $n\geq N$.
	If such an $a$ exists, we say $a_n$ is \textit{convergent in $X$}.
\end{definition}
Furthermore, if a sequence is convergent in a metric space, it is also Cauchy in that space.
\begin{definition}[Cauchy Sequence]
	Let $(X,d)$ be a metric space and let $(a_n)_{n\in\NN}\subseteq X$.
	We say $a_n$ is a \textit{Cauchy sequence} if for all $\varepsilon>0$, there exists an $N\in\NN$ such that $d(a_n,a_m)<\varepsilon$ for all $n,m\geq N$.
\end{definition}
While convergent implies Cauchy, the other way does not always hold.
For example in $\QQ$ under the Euclidean metric ($d(x,y)=|x-y|$), the sequence defined by $(1+1/n)^n$ is Cauchy but not convergent.
However, if we move to $\RR$, $(1+1/n)^n$ converges to $e$.
Cauchy sequences are sequences that \textit{should} be convergent in our space.
If they are not, then we need to move to what is called the completion of the metric space.
\begin{definition}[Complete Metric Space]
	Let $(X,d)$ be a metric space.
	We say $X$ is \textit{complete} if all Cauchy sequences in $X$ converge in $X$.
\end{definition}
\begin{thm}
	Let $(X,d)$ be a metric space.
	Then $X$ has a unique completion, $C(X,d)$, up to isometry.
	Furthermore, this completion is isometric to the space $(B(X),D)$ where $B(X)$ is the set of all bounded functions from $X$ to $\RR$, and $D(f,g)=\sup_{x\in X} |f(x)-g(x)|$.
\end{thm}

Now that we have enough background in analysis, lets talk about the $p$-adic numbers.
Before we can define the $p$-adic's, we need to introduce the $p$-adic ordinal and $p$-adic absolute value first.
\begin{definition}[$p$-adic Ordinal]
	Let $p$ be a prime and let $a\in\ZZ$ be nonzero.
	Then the \textit{$p$-adic ordinal} of $a$, denoted $\ord_p a$, is defined as
	\[
		\ord_p a = \max\set{n\,:\, p^n | a}.
	\]
	Furthermore, for any nonzero $x=b/c\in\QQ$,
	\[
		\ord_p x = \ord_p a - \ord_p b.
	\]
\end{definition}
Using the definition of $p$-adic ordinal, we now provide the definition of the $p$-adic absolute value.
\begin{definition}[$p$-adic absolute value]
	Let $p$ be a prime and $x\in\QQ$.
	Then the \textit{$p$-adic absolute value of $x$}, denoted $|x|_p$, is defined as
	\[
		|x|_p =
		\begin{cases}
			p^{-\ord_p x} & x\neq 0\\
			0 & x=0.
		\end{cases}
	\]
\end{definition}
While an absolute value is not a metric in and of itself, it generates a metric.
Similarly to how we use $|x-y|$ as a metric, we use $|x-y|_p$ as the $p$-adic metric.
With this, we can define the $p$-adic numbers.
\begin{definition}[$p$-adic Numbers]
	The set of \textit{$p$-adic numbers} is the completion of the metric space $(\QQ,|\cdot|_p)$.
\end{definition}
Additionally, $\QQ_p$ satisfies a stronger version of the triangle inequality.
\begin{thm}[Strong Triangle Inequality]
	For all $x,y\in\QQ_p$, $|x+y|_p\leq\max\set{|x|_p,|y|_p}$.
\end{thm}

\section{Solution}

\begin{problem*}
	Show that
	\[
		\sum_{n=0}^\infty 2^n = -1
	\]
	under the 2-adic norm.
\end{problem*}

To start our solution, we prove that if a sequence in $\QQ_p$ converges to 0 under the $p$-adic metric, the infinite sum of all of its terms is convergent in $\QQ_p$.
While this result is not true in $\RR$ (e.g. the harmonic series), it works in $\QQ_p$ because of the strong triangle inequality.

\begin{thm}
	Let $(a_n)_{n\in\NN}\subseteq\QQ_p$ such that $a_n\rightarrow 0$.
	Then $\sum_{n=0}^\infty a_n$ converges in $\QQ_p$.
\end{thm}
\begin{proof}
	Let $\varepsilon > 0$ be given.
	Since $a_n\rightarrow 0$, there exists an $N\in\NN$ such that $|a_n|<\varepsilon$.
	Force $n\geq m\geq N$.
	Consider $|\sum_{k=0}^n a_k - \sum_{k=0}^m|_p$.
	\[
		\abs{\sum_{k=0}^n a_k - \sum_{k=0}^m}_p a_k = \abs{\sum_{k=m+1}^n a_k}_p
	\]
	Thus, by the strong triangle inequality,
	\[
		\abs{\sum_{k=0}^n a_k - \sum_{k=0}^m a_k}_p \leq \max\limits_{m+1\leq k\leq n}\set{|a_k|_p}.
	\]
	However, we know that for each $k\geq N$, $|a_k|_p<\varepsilon$.
	Therefore,
	\[
		\abs{\sum_{k=0}^n a_k - \sum_{k=0}^m a_k}_p < \varepsilon
	\]
	and $\paren{\sum_{k=0}^n a_k}_{n\in\NN}$ is Cauchy in $\QQ_p$.
	Since $\QQ_p$ is defined as a complete metric space, $\paren{\sum_{k=0}^n a_k}_{n\in\NN}$ converges in $\QQ_p$.
\end{proof}

Now, in order to show that $\sum_{n=0}^\infty 2^n$ converges with respect to the $p$-adic metric, we show that $2^n\rightarrow 0$ with respect to the $p$-adic metric.

\begin{lemma}
	Under the 2-adic metric, $2^n\rightarrow 0$.
\end{lemma}
\begin{proof}
	Let $\varepsilon > 0$ be given.
	Without loss of generality, assume $\varepsilon < 1$.
	Thus, $\lg\varepsilon < 0$.
	Take $N > -\lg\varepsilon$ and let $n\geq N$.
	Then
	\[
		|2^n|_2 = 2^{-n} \leq 2^{-N} < 2^{\lg\varepsilon} = \varepsilon.
	\]
	Therefore, $2^n\rightarrow 0$ under $|\cdot|_2$.
\end{proof}

From here, we do some algebraic manipulation to find the limit point.

\begin{proposition}
	In $\QQ_2$,
	\[
		\sum_{n=0}^\infty 2^n = -1.
	\]
\end{proposition}
\begin{proof}
	Since $2^n\rightarrow 0$, $\sum_{n=0}^\infty 2^n$ converges to some $S\in\QQ_p$.
	Consider $S$.
	\[
		S= \sum_{n=0}^\infty 2^n = 1 + \sum_{n=1}^\infty 2^n = 1+ 2\sum_{n=1}^\infty 2^{n-1} = 1+ 2 \sum_{n=0} 2^n = 1+ 2S
	\]
	Solving for $S$ yields,
	\[
		S = -1.
	\]
\end{proof}

\printbibliography

\end{document}


%!TEX root = modelguide.tex

\appendix

\chapter{Mathematical Review}
\label{MathematicalReview:sec}

This appendix reviews some of the mathematical concepts used in this
manual.

\section{Rotation transforms}
\label{Rotations:sec}

Rotation matrices are used to describe the orientation of 3D
coordinate frames in space, and to transform vectors between these
coordinate frames.

\begin{figure}[t]
\begin{center}
 \iflatexml
   \includegraphics[width=2in]{images/rotationAB}
 \else
   \includegraphics[width=2in]{images/rotationAB}
 \fi
\end{center}
\caption{Two coordinate frames A and B rotated with respect
to each other.}
\label{rotationAB:fig}
\end{figure}

Consider two 3D coordinate frames A and B that are rotated with
respect to each other (Figure \ref{rotationAB:fig}).  The orientation
of B with respect to A can be described by a $3 \times 3$ rotation
matrix $\R_{BA}$, whose columns are the unit vectors giving the
directions of the rotated axes $\x'$, $\y'$, and $\z'$ of B with
respect to A.

$\R_{BA}$ is an {\it orthogonal} matrix, meaning that
its columns are both perpendicular and mutually
orthogonal, so that
%
\begin{equation}
\R_{BA}^T \, \R_{BA} = \I
\end{equation}
%
where $\I$ is the $3 \times 3$ identity matrix. The inverse
of $\R_{BA}$ is hence equal to its transpose:
%
\begin{equation}
\R_{BA}^{-1} = \R_{BA}^T.
\label{RABinv:eqn}
\end{equation}
%
Because $\R_{BA}$ is orthogonal, $|\det \R_{BA}| = 1$, and because it
is a rotation, $\det \R_{BA} = 1$ (the other case, where $\det \R_{BA}
= -1$, is not a rotation but a {\it reflection}).  The 6 orthogonality
constraints associated with a rotation matrix mean that in spite of
having 9 numbers, the matrix only has 3 degrees of freedom.

Now, assume we have a 3D vector $\v$, and consider its coordinates
with respect to both frames A and B.  Where necessary, we use a
preceding superscript to indicate the coordinate frame with respect to
which a quantity is described, so that ${}^A\v$ and ${}^B\v$ and
denote $\v$ with respect to frames A and B, respectively.  Given the
definition of $\R_{AB}$ given above, it is fairly straightforward to
show that
%
\begin{equation}
{}^A\v = \R_{BA} \, {}^B\v
\label{vectorTransform:eqn}
\end{equation}
%
and, given (\ref{RABinv:eqn}), that
%
\begin{equation}
{}^B\v = \R_{BA}^T \, {}^A\v.
\label{vectorInvTransform:eqn}
\end{equation}
%
Hence in addition to describing the orientation of B with respect to A,
$\R_{BA}$ is also a transformation matrix that maps vectors in B
to vectors in A.

It is straightforward to show that
%
\begin{equation}
\R_{BA}^{-1} = \R_{BA}^T = \R_{AB}.
\label{rotationInverse:eqn}
\end{equation}
%

\begin{figure}[t]
\begin{center}
 \iflatexml
   \includegraphics[width=4.5in]{images/rotationsABC}
 \else
   \includegraphics[width=4.5in]{images/rotationsABC}
 \fi
\end{center}
\caption{Schematic illustration of three coordinate frames A, B, and C
and the rotational transforms relating them.}
\label{rotationsABC:fig}
\end{figure}

A simple rotation by an angle $\theta$ about one of the basic
coordinate axes is known as a {\it basic} rotation. The three
basic rotations about x, y, and z are:
%
\begin{equation*}
\R_x(\theta) = \matl 1 & 0 & 0 \\
               0 & \cos(\theta) & -\sin(\theta) \\
               0 & \sin(\theta) & \cos(\theta)
		 \matr,
\end{equation*}
%
\begin{equation*}
\R_y(\theta) = \matl \cos(\theta) & 0 & \sin(\theta) \\
               0 & 1 & 0 \\
               -\sin(\theta) & 0 & \cos(\theta)
                 \matr,
\end{equation*}
%
\begin{equation*}
\R_z(\theta) = \matl \cos(\theta) & -\sin(\theta) & 0 \\
               \sin(\theta) & \cos(\theta) & 0 \\
               0 & 0 & 1
                 \matr.
\label{elementaryRotations:sec}
\end{equation*}
%

Next, we consider transform composition. Suppose we have three
coordinate frames, A, B, and C, whose orientation are related to each other by
$\R_{BA}$, $\R_{CB}$, and $\R_{CA}$ (Figure
\ref{transformsABC:fig}).  If we know $\R_{BA}$ and $\R_{CA}$,
then we can determine $\R_{CB}$ from
%
\begin{equation}
\R_{CB} = \R_{BA}^{-1} \, \R_{CA}.
\label{rotationCB:eqn}
\end{equation}
%
This can be understood in terms of vector transforms. $\R_{CB}$
transforms a vector from C to B, which is equivalent to first
transforming from C to A,
%
\begin{equation}
{}^A\v = \R_{CA} \, {}^C\v,
\end{equation}
%
and then transforming from A to B:
%
\begin{equation}
{}^B\v = \R_{BA}^{-1} \, {}^A\v = \R_{BA}^{-1} \; \R_{CA} {}^C\v = \R_{CB} \, {}^C\v.
\end{equation}
%
Note also from (\ref{rotationInverse:eqn}) that $\R_{CB}$ can be
expressed as
%
\begin{equation}
\R_{CB} = \R_{AB} \, \R_{CA}.
\label{rotationCBII:eqn}
\end{equation}
%

In addition to specifying rotation matrix components explicitly,
there are numerous other ways to describe a rotation.
Three of the most common are:

\begin{description}

\item[Roll-pitch-yaw angles]\mbox{}

There are 6 variations of roll-pitch-yaw angles. The one used in
ArtiSynth corresponds to older robotics texts (e.g., Paul, Spong) and
consists of a roll rotation $r$ about the z axis, followed by a pitch
rotation $p$ about the new y axis, followed by a yaw rotation $y$
about the new x axis. The net rotation can be expressed by the
following product of basic rotations: $\R_z(r) \, \R_y(p) \, \R_x(y)$.

\item[Axis-angle]\mbox{}

An axis angle rotation parameterizes a rotation as a rotation by
an angle $\theta$ about a specific axis $\u$. Any rotation
can be represented in such a way as a consequence of Euler's rotation
theorem.

\item[Euler angles]\mbox{}

There are 6 variations of Euler angles. The one used in ArtiSynth
consists of a rotation $\phi$ about the z axis, followed by a rotation
$\theta$ about the new y axis, followed by a rotation $\psi$ about the
new z axis. The net rotation can be expressed by the following product
of basic rotations: $\R_z(\phi) \, \R_y(\theta) \, \R_z(\psi)$.

\end{description}

\section{Rigid transforms}
\label{RigidTransforms:sec}

Rigid transforms are used to specify both the transformation of points
and vectors between coordinate frames, as well as the relative
position and orientation between coordinate frames.

\begin{figure}[ht]
\begin{center}
 \iflatexml
   \includegraphics[width=3.25in]{images/framesAB}
 \else
   \includegraphics[width=3.25in]{images/framesAB}
 \fi
\end{center}
\caption{A position vector $\p_{BA}$ and rotation matrix
$\R_{BA}$ describing the position and orientation of frame B
with respect to frame A.}
\label{framesAB:fig}
\end{figure}

Consider two 3D coordinate frames in space, A and B (Figure
\ref{framesAB:fig}). The translational position of B with respect to A
can be described by a vector $\p_{BA}$ from the origin of A to the
origin of B (described with respect to frame A). Meanwhile, the
orientation of B with respect to A can be described by the $3 \times
3$ rotation matrix $\R_{BA}$ (Section \ref{Rotations:sec}).  The
combined position and orientation of B with respect to A is known as
the {\it pose} of B with respect to A.

\begin{figure}[t]
\begin{center}
 \iflatexml
   \includegraphics[width=3.75in]{images/pointsAB}
 \else
   \includegraphics[width=3.75in]{images/pointsAB}
 \fi
\end{center}
\caption{Point vectors ${}^A\q$ and ${}^B\q$ describing
the position of a point $\q$ with respect to frames A and B.}
\label{pointsAB:fig}
\end{figure}

Now, assume we have a 3D point $\q$, and consider its coordinates with
respect to both frames A and B (Figure \ref{pointsAB:fig}). Given the
pose descriptions given above, it is fairly straightforward to show
that
%
\begin{equation}
{}^A\q = \R_{BA} \, {}^B\q + \p_{BA},
\label{pointTransform:eqn}
\end{equation}
%
and, given (\ref{RABinv:eqn}), that
%
\begin{equation}
{}^B\q = \R_{BA}^T \, ({}^A\q - \p_{BA}).
\label{pointInvTransform:eqn}
\end{equation}
%

If we extend our points into a 4D {\it homogeneous} coordinate space
with the fourth coordinate $w$ equal to 1, i.e.,
%
\begin{equation}
\q^* \equiv \matl \q \\ 1 \matr,
\end{equation}
%
then (\ref{pointTransform:eqn}) and
(\ref{pointInvTransform:eqn}) can be simplified to
%
\begin{equation*}
{}^A\q^* = \T_{BA} \, {}^B\q^* \quad \text{and} \quad
{}^B\q^* = \T_{BA}^{-1} \, {}^A\q^*
\end{equation*}
%
where
%
\begin{equation}
\T_{BA} = \matl \R_{BA} & \p_{BA} \\ 0 & 1 \matr
\end{equation}
%
and
%
\begin{equation}
\T_{BA}^{-1} = \matl \R_{BA}^T & -\R_{BA}^T \p_{BA} \\ 0 & 1 \matr.
\end{equation}
%
$\T_{BA}$ is the $4 \times 4$ {\it rigid transform matrix} that
transforms points from B to A and also describes the pose of
B with respect to A (Figure \ref{transformAB:fig}).

\begin{figure}[t]
\begin{center}
 \iflatexml
   \includegraphics[width=3in]{images/transformAB}
 \else
   \includegraphics[width=3in]{images/transformAB}
 \fi
\end{center}
\caption{The transform matrix $\T_{BA}$ from B to A.}
\label{transformAB:fig}
\end{figure}

It is straightforward to show that $\R_{BA}^T$ and $-\R_{BA}^T
\p_{BA}$ describe the orientation and position of $A$ with respect to
$B$, and so therefore
%
\begin{equation}
\T_{BA}^{-1} = \T_{AB}.
\label{transformInverse:eqn}
\end{equation}
%

Note that if we are transforming a vector $\v$ instead of a point
between B and A, then we are only concerned about relative orientation
and the vector transforms (\ref{vectorTransform:eqn}) and
(\ref{vectorInvTransform:eqn}) should be used instead.
However, we can express these using $\T_{BA}$ if
we embed vectors in a homogeneous coordinate space
with the fourth coordinate $w$ equal to 0, i.e.,
%
\begin{equation}
\v^* \equiv \matl \v \\ 0 \matr,
\end{equation}
%
so that
%
\begin{equation*}
{}^B\v^* = \T_{BA} \, {}^A\v^* \quad \text{and} \quad
{}^A\v^* = \T_{BA}^{-1} \, {}^B\v^*.
\end{equation*}
%

\begin{figure}[t]
\begin{center}
 \iflatexml
   \includegraphics[width=4.5in]{images/transformABC}
 \else
   \includegraphics[width=4.5in]{images/transformABC}
 \fi
\end{center}
\caption{Three coordinate frames A, B, and C and the transforms
relating each one to the other.}
\label{transformsABC:fig}
\end{figure}

Finally, we consider transform composition. Suppose we have three
coordinate frames, A, B, and C, each related to the other by
transforms $\T_{BA}$, $\T_{CB}$, and $\T_{CA}$ (Figure
\ref{transformsABC:fig}).  Using the same reasoning used to derive
(\ref{rotationCB:eqn}) and (\ref{rotationCBII:eqn}), it is easy to
show that
%
\begin{equation}
\T_{CB} = \T_{BA}^{-1} \; \T_{CA} = \T_{AB} \; \T_{CA}.
\end{equation}
%

\section{Affine transforms}
\label{AffineTransforms:sec}

An {\it affine transform} is a generalization of a rigid transform, in
which the rotational component $\R$ is replaced by a general $3 \times
3$ matrix $\A$. This means that an affine transform implements a
generalized basis transformation combined with an offset of the origin
(Figure \ref{affineAB:fig}). As with $\R$ for rigid transforms, the
columns of $\A$ still describe the transformed basis vectors $\x'$,
$\y'$, and $\z'$, but these are generally no longer orthonormal.

\begin{figure}[ht]
\begin{center}
 \iflatexml
    \includegraphics[width=3.25in]{images/affineAB}
 \else
    \includegraphics[width=3.25in]{images/affineAB}
 \fi
\end{center}
\caption{A position vector $\p_{BA}$ and a general matrix $\A_{BA}$
describing the affine position and basis transform of frame B with respect to
frame A.}
\label{affineAB:fig}
\end{figure}

Expressed in terms of homogeneous coordinates,
the affine transform $\X_{AB}$ takes the form
%
\begin{equation}
\X_{BA} = \matl \A_{BA} & \p_{BA} \\ 0 & 1 \matr
\end{equation}
%
with
%
\begin{equation}
\X_{BA}^{-1} = \matl \A_{BA}^{-1} & -\A_{BA}^{-1} \p_{BA} \\ 0 & 1 \matr.
\end{equation}
%
As with rigid transforms, when an affine transform is applied to a
vector instead of a point, only the matrix $\A$ is applied and the
translation component $\p$ is ignored.

Affine transforms are typically used to effect transformations that
require stretching and shearing of a coordinate frame.  By the polar
decomposition theorem, $\A$ can be factored into a regular
rotation $\R$ plus a symmetric shearing/scaling matrix $\P$:
%
\begin{equation}
\A = \R \, \P
\end{equation}
%
Affine transforms can also be used to perform reflections, in which
$\A$ is orthogonal (so that $\A^T \, \A = \I$) but with $\det \A =
-1$.

\section{Rotational velocity}

\begin{figure}[ht]
\begin{center}
 \iflatexml
    \includegraphics[width=3.25in]{images/angularvelAB}
 \else
    \includegraphics[width=3.25in]{images/angularvelAB}
 \fi
\end{center}
\caption{Frame B rotating with respect to frame A.}
\label{angularvelAB:fig}
\end{figure}

Given two 3D coordinate frames A and B, the rotational, or {\it
angular}, velocity of B with respect to A is given by a 3D vector
$\Bom_{BA}$ (Figure \ref{angularvelAB:fig}). $\Bom_{BA}$
is related to the derivative of $\R_{BA}$ by
%
\begin{equation}
\dot\R_{BA} = [{}^A\Bom_{BA}] \R_{BA} = \R_{BA} [{}^B\Bom_{BA}]
\end{equation}
%
where ${}^A\Bom_{BA}$ and ${}^B\Bom_{BA}$ indicate $\Bom_{BA}$ with
respect to frames $A$ and $B$ and $[ \Bom ]$ denotes the $3 \times 3$
cross product matrix
%
\begin{equation}
[ \Bom ] \equiv
\matl
0 & -\omega_z & \omega_y \\
\omega_z & 0 & -\omega_x \\
-\omega_y & \omega_x & 0 \\
\matr.
\label{xprodmatrix:eqn}
\end{equation}
%

If we consider instead the velocity of $A$ with respect to $B$, it is
straightforward to show that
%
\begin{equation}
\Bom_{AB} = -\Bom_{BA}.
\end{equation}
%

\section{Spatial velocities and forces}
\label{SpatialVelocitiesAndForces:sec}

Given two 3D coordinate frames A and B, the {\it spatial velocity},
or {\it twist},
$\hat\v_{BA}$ of B with respect to A is given by the 6D
composition of the translational velocity $\v_{BA}$ of the
origin of B with respect to A and the angular velocity $\Bom_{BA}$:
%
\begin{equation}
\hat\v_{BA} \equiv \matl \v_{BA} \\ \Bom_{BA} \matr.
\end{equation}
%
Similarly, the {\it spatial force}, or {\it wrench}, $\hat\f$ acting
on a frame B is given by the 6D composition of the translational force
$\f_B$ acting on the frame's origin and the moment $\Btau$, or torque,
acting through the frame's origin:
%
\begin{equation}
\hat\f_B \equiv \matl \f_B \\ \Btau_B \matr.
\end{equation}
%

\begin{figure}[ht]
\begin{center}
 \iflatexml
   \includegraphics[width=3.25in]{images/rigidAB}
 \else
   \includegraphics[width=3.25in]{images/rigidAB}
 \fi
\end{center}
\caption{Two frames A and B rigidly connected within a rigid body
and moving with respect to a third frame C.}
\label{rigidAB:fig}
\end{figure}

If we have two frames $A$ and $B$ rigidly connected within a rigid
body (Figure \ref{rigidAB:fig}), and we know the spatial velocity
$\hat\v_{BC}$ of $B$ with respect to some third frame $C$, we may wish
to know the spatial velocity $\hat\v_{AC}$ of $A$ with respect to $C$.
The angular velocity components are the same, but the translational
velocity components are coupled by the angular velocity and the offset
$\p_{BA}$ between $A$ and $B$, so that
%
\begin{equation*}
\v_{AC} = \v_{BC} + \p_{BA} \times \Bom_{BC}.
\end{equation*}
%
$\hat\v_{AC}$ is hence related to $\hat\v_{BC}$ via
\begin{equation*}
\matl \v_{AC} \\ \Bom_{AC} \matr =
\matl \I & [\p_{BA}] \\ 0 & \I \matr \,
\matl \v_{BC} \\ \Bom_{BC} \matr.
\end{equation*}
%
where $[\p_{BA}]$ is defined by
(\ref{xprodmatrix:eqn}).

The above equation assumes that all quantities are expressed
with respect to the same coordinate frame.
If we instead consider $\hat\v_{AC}$ and $\hat\v_{BC}$ to be represented
in frames $A$ and $B$, respectively, then
we can show that
%
\begin{equation}
{}^A\hat\v_{AC} = \X_{BA} \, {}^B\hat\v_{BC},
\end{equation}
%
where
%
\begin{equation}
\X_{BA} \equiv
\matl \R_{BA} & [\p_{BA}] \R_{BA} \\ 0 & \R_{BA} \matr.
\label{XvelAB:eqn}
\end{equation}
%
The transform $\X_{BA}$ is easily formed from the components of the
rigid transform $\T_{BA}$ relating $B$ to $A$.

The spatial forces $\hat\f_A$ and $\hat\f_B$ acting on frames $A$ and
$B$ within a rigid body are related in a similar way, only with
spatial forces, it is the moment that is coupled through the moment
arm created by $\p_{BA}$, so that
%
\begin{equation*}
\Btau_{A} = \Btau_{B} + \p_{BA} \times \f_{B}.
\end{equation*}
%
If we again assume that $\hat\f_A$ and $\hat\f_B$
are expressed in frames $A$ and $B$, we can show that
%
\begin{equation}
{}^A\hat\f_A = \X^*_{BA} \, {}^B\hat\f_B,
\end{equation}
%
where
%
% LaTeXML fails to interpret the [ bracket when directly after \\
\iflatexml
\begin{equation}
\X^*_{BA} \equiv
\matl \R_{BA} & 0 \\  ~[\p_{BA}]\R_{BA} & \R_{BA} \matr.
\label{XforceAB:eqn}
\end{equation}
\else
\begin{equation}
\X^*_{BA} \equiv
\matl \R_{BA} & 0 \\  [\p_{BA}] \R_{BA} & \R_{BA} \matr.
\label{XforceAB:eqn}
\end{equation}
\fi

\section{Spatial inertia}
\label{SpatialInertia:sec}

Assume we have a rigid body with mass $m$ and a coordinate frame
located at the body's center of mass.  If $\v$ and $\Bom$ give the
translational and rotational velocity of the coordinate frame, then
the body's linear and angular momentum $\p$ and $\L$ are given by
%
\begin{equation}
\p = m \v \quad \text{and} \quad \L = \J \Bom,
\label{momenta:eqn}
\end{equation}
%
where $\J$ is the $3 \times 3$ {\it rotational inertia} with respect
to the center of mass. These relationships can be combined into a
single equation
%
\begin{equation}
\hat\p = \M \hat\v,
\label{momentum:eqn}
\end{equation}
%
where $\hat\p$ and $\M$ are the {\it spatial momentum} and
{\it spatial inertia}:
%
\begin{equation}
\hat\p \equiv \matl \p \\ \L \matr, \qquad
\M \equiv \matl m \I & 0 \\ 0 & \J \matr.
\end{equation}
%
The spatial momentum satisfies Newton's second law, so that
%
\begin{equation}
\hat\f = \frac{d \hat\p}{dt} = \M \frac{d \hat\v}{dt} + \dot\M \hat\v,
\end{equation}
%
which can be used to find the acceleration of a body in response to a
spatial force.

When the body coordinate frame is {\it not} located at the center of
mass, then the spatial inertia assumes the more complicated form
%
\begin{equation}
\matl
m \I & -m [\c] \\ m [\c] & \J - m[\c][\c]
\matr,
\end{equation}
%
where $\c$ is the center of mass and $[\c]$ is defined by
(\ref{xprodmatrix:eqn}).

Like the rotational inertia, the spatial inertia is always symmetric
positive definite if $m > 0$.


\section{Force Constants}\index{Force constants}\index{Internal!coordinate!force constants}\label{fc}
Internal coordinate force constants, $f_{ic}$, can be derived from the
Cartesian coordinates and the Cartesian force constant matrix by use of:
$$
f_{ic}(l) = \sum_j\sum_k
\frac{d({\rm c.c.}_j)}{d({\rm i.c.}_l)}
\frac{d^2E}{d({\rm c.c.}_j)d({\rm c.c.}_k)}
\frac{d({\rm c.c.}_k)}{d({\rm i.c.}_l)}.
$$
As with the gradients, the calculation of $\frac{d({\rm c.c.}_j)}{d({\rm
i.c.}_l)}$ is quite simple, and in done in routine JCARIN.

During the testing of this function, a minor fault in the conventional
force calculation was revealed.  To reduce any error introduced by
finite arithmetic, the Hessian matrix is symmetrized before the
vibrational frequencies and normal coordinates are calculated.  This is
done by \hyperref[pageref]{operating on the Hessian matrix}{ as
described on p.~}{}{sym_force}.

In addition to the requirement that the symmetry of the Hessian should be the
same as that of the nuclear coordinates, a second requirement is that the
diagonal elements of the Hessian should be equal to the negative of the sum of
the off-diagonal elements, that is, that:
$$
F_{ii} = -\sum_{j\ne i}F_{ij}.
$$
During the testing of the internal force constants, very small variations in
the force constants were found where no variation was expected.  This was
traced back to a failure of the above expression.  To correct this, the
diagonal terms of the force constant matrix were modified.  This resulted in a
perfect equivalence of equivalent force constants.  An incidental benefit would
be that the associated error in the calculated frequencies would be eliminated.


\section{Learning Theory}
\subsection{Bias and Variance}
\subsection{Notation}
\subsubsection{Union Bound}
\subsubsection{Hoeffding Inequality For Bernoulli Variables}
\subsection{Training Error}
For a given classifier $h$, we define the training error $\widehat{\epsilon}(h)$,
also known as the empirical risk or empirical error, to be:
\begin{equation} \widehat{\epsilon}(h) = \frac{1}{m}\sum_{i=1}^m 1_{\{h(x^{(i)})\neq y^{(i)}\}} \end{equation}
\subsection{Probably Approximately Correct (PAC)}
PAC learning is a framework with the following set of assumptions:
% \begin{enumerate}
% \end{enumerate}
\subsection{Hypothesis Classes}
\subsubsection{Shattering}
\subsubsection{Upper Bound Theorem}
\subsubsection{VC Dimension}
\subsubsection{Vapnik Theorem}


\subsection{Mixed logit (random coefficients)}

\subsubsection{Introduction}

In our standard model we have:

\(z_{ij}=\beta x_{ij} +\epsilon_{ij} \)

If we allow the parameters to vary for each individual we have:

\(z_{ij}=\beta_i x_{ij} +\epsilon_{ij} \)

The probability of choosing \(j\) now depends on the distribution of \(\beta \).

In the IID case we had:

\(P_{ij}=\dfrac{e^{\beta x_{ij}}}{\sum_k e^{\beta x_{ik}}}\)

Rather than evaluate this at a single point \(\beta \) we integrate.

\(P_{ij}=\int \dfrac{e^{\beta x_{ij}}}{\sum_k e^{\beta x_{ik}}}f(\beta )d\beta \)

If \(\beta \) is degenerate this reduces to the standard logit model.


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}

\begin{document}

\newtheorem{theorem}{Theorem}
\newtheorem{definition}{Definition}

\title{Deriving Laplace's Equation in Spherical Coordinates}
\author{Justin Meiners}

\maketitle

\section{Laplace's Equation}

Laplace's equation states
$$\nabla\cdot\nabla f=0$$.

For 3D coordinates this is written as:
$$\frac{\partial^{2}}{\partial x^{2}}+\frac{\partial^{2}}{\partial y^{2}}+\frac{\partial^{2}}{\partial z^{2}}=0$$

But, these are cartesian coordinates, we want to transform this into spherical coordinates.

\section{Spherical Coordinate System}

The 3 dimensional Laplace's equation can be transformed into spherical coordinates, by changing variables. This procedure is similar to change of variables for spherical integrals in multivariable calculus.

A 3D sphere can be parameterized by three variables: radius, $\theta$ and $\phi$:

\begin{equation}
\begin{split}
x&=r\sin\phi\cos\theta \\
y&=r\sin\phi\sin\theta \\
z&=r\cos\phi \\
\theta&=\arctan(\frac{y}{x}) \\
\phi&=arccos(\frac{z}{r}) \\
r^{2}&=x^{2}+y^{2}+z^{2} \\
\end{split}
\end{equation}

\section{First Derivative}

First derivative for x

$$\frac{\partial}{\partial x}=\frac{\partial r}{\partial x}\frac{\partial}{\partial r} + \frac{\partial \theta}{\partial x}\frac{\partial}{\partial \theta} + \frac{\partial \phi}{\partial x}\frac{\partial}{\partial \phi}$$

$$2r\frac{\partial r}{\partial x}=2x$$
$$\frac{\partial r}{\partial x}=\frac{x}{r}=\sin\phi\cos\theta$$

$$\frac{\partial \theta}{\partial x}=\frac{1}{1+\frac{y^{2}}{x^{2}}}(\frac{-y}{x^{2}})=\frac{-y}{x^{2}+y^{2}}=\frac{-\sin(\theta)}{r\sin(\phi)}$$

$$\frac{\partial \phi}{\partial x}=\frac{-1}{\sqrt{1-\frac{z^{2}}{r^{2}}}}z(\frac{-1}{2})(x^{2}+y^{2}+z^{2})^{-3/2}(2x)$$
$$=\frac{zx}{\sqrt{1-\cos^{2}\phi}r^{3}}=\frac{\cos\phi\cos\theta}{r}$$

First derivative for y

$$\frac{\partial}{\partial y}=\frac{\partial r}{\partial y}\frac{\partial}{\partial r}+\frac{\partial \theta}{\partial y}\frac{\partial}{\partial \theta}+\frac{\partial \phi}{\partial y}\frac{\partial}{\partial \phi}$$

$$2r\frac{\partial r}{\partial y}=2y$$
$$\frac{\partial r}{\partial y}=\frac{y}{r}=\sin(\phi)\sin(\theta)$$
$$\frac{\partial \theta}{\partial y}=\frac{1}{1+\frac{y^{2}}{x^{2}}}(\frac{1}{x})=\frac{x}{x^{2}+y^{2}}=\frac{\cos\theta}{r\sin\phi}$$
$$\frac{\partial \phi}{\partial y}=\frac{-1}{\sqrt{1-\frac{z^{2}}{r^{2}}}}z(-1/2)(x^{2}+y^{2}+z^{2})^{-3/2}2y$$
$$=\frac{r\cos{\phi}r\sin{\phi}\sin{\theta}}{\sin{\phi}(r^{3})}=\frac{\cos(\phi)\sin(\theta)}{r}$$

First derivative for z

$$\frac{\partial}{\partial z}=\frac{\partial r}{\partial z}\frac{\partial}{\partial r}+\frac{\partial \theta}{\partial z}\frac{\partial}{\partial \theta}+\frac{\partial \phi}{\partial z}\frac{\partial}{\partial \phi}$$

$$2r\frac{\partial r}{\partial z}=2z$$
$$\frac{\partial r}{\partial z}=\frac{z}{r}=\cos(\phi)$$
$$\frac{\partial \theta}{\partial z}=0$$
$$\frac{\partial \phi}{\partial z}=\frac{-1}{\sqrt{1-\frac{z^2}{r^2}}}(\frac{r - 1/2(x^{2}+y^{2}+z^{2})^{-3/2}(2z)(z)}{r^{2}})$$
$$=\frac{-1}{\sqrt{1-\cos^{2}\phi}}(\frac{1}{r}-\frac{z^{2}}{r{^3}})$$
$$=\frac{-1}{\sin \phi}(\frac{1}{r}-\frac{z^{2}}{r^{3}})=\frac{-1}{r\sin\phi}+\frac{\cos^{2}\phi}{r\sin(\phi)}$$
$$=\frac{1-\sin^{2}\phi-1}{r\sin\phi}=\frac{-\sin\phi}{r}$$

Here are all the first derivatives together:
$$\frac{\partial}{\partial x}=\sin\phi\cos\theta\frac{\partial }{\partial r}-\frac{\sin\theta}{r\sin\phi}\frac{\partial}{\partial \theta} + \frac{\cos\phi\cos\theta}{r}\frac{\partial}{\partial \phi}$$
$$\frac{\partial}{\partial y}=\sin\phi\sin\theta\frac{\partial}{\partial r}+\frac{\cos\theta}{r\sin\phi}\frac{\partial}{\partial \theta}+\frac{\cos\phi\sin\theta}{r}\frac{\partial}{\partial \phi}$$
$$\frac{\partial}{\partial z}=\cos\phi\frac{\partial}{\partial r} - \frac{\sin\phi}{r}\frac{\partial}{\partial\phi}$$

\section{Second Derivative}

We must now take the second deriative by applying each of the first derivatives to themselves.

$$\frac{\partial^{2}}{\partial x^{2}} + \frac{\partial^{2}}{\partial y^{2}} + \frac{\partial^{2}}{\partial z^{2}}=\frac{\partial}{\partial x}(\frac{\partial}{\partial x}) + \frac{\partial}{\partial y}(\frac{\partial}{\partial y}) + \frac{\partial}{\partial z}(\frac{\partial}{\partial z})$$

Because of the chain rule, each term in the first derivative needs to be applied to each other term. For each $\frac{\partial}{\partial x}$, $\frac{\partial}{\partial y}$, there will be 9 terms and for $\frac{\partial}{\partial z}$ there will be 4 terms. Each term will require the product rule, so double them, for $(9+9+4)2=44$ total.

With so many terms, it would be too unwieldy to start taking derivatives in order. Instead, we will organize them by differential terms, and then piece them together. Also, instead of computing the product rule, we will compute one part of the product rule at a time. As we will see, many of the terms will drop out anyway.

Second order terms (8 total):

$$\frac{\partial^{2}}{\partial r^{2}}:\sin^{2}\phi\cos^{2}\theta+\sin^{2}\phi\sin^{2}\theta+\cos^{2}\phi$$
$$=\sin^{2}\phi(1)+\cos^{2}\phi=1$$

$$\frac{\partial^{2}}{\partial \theta^{2}}:\frac{\sin^{2}\theta}{r^{2}\sin^{2}\phi}+\frac{\cos^{2}\theta}{r^{2}\sin^{2}\phi}+0$$
$$=\frac{1}{r^{2}\sin^{2}\phi}$$

$$\frac{\partial^{2}}{\partial \phi^{2}}:\frac{\cos^{2}\phi\cos^{2}\theta}{r^{2}} + \frac{\cos^{2}\phi\sin^{2}\theta}{r^{2}} + \frac{\sin^{2}\phi}{r^{2}}$$
$$=\frac{\cos^{2}\phi(1)+\sin^{2}\phi}{r^{2}}=\frac{1}{r^{2}}$$

Mixed partial terms (14 total):

$$\frac{\partial^{2}}{\partial r \partial\theta}: \frac{2\sin\phi\cos\theta(-\sin\theta)}{r\sin\phi}+\frac{2\sin\phi\sin\theta(\cos\theta)}{r\sin\phi}+0=0$$
$$\frac{\partial^{2}}{\partial r \partial\phi}: \frac{2\sin\phi\cos\phi\cos^{2}\theta}{r} + \frac{2\cos\phi\sin\phi\sin^{2}\theta}{r} -\frac{2\sin\phi\cos\phi}{r}$$
$$=\frac{2\sin\phi\cos\theta(1) - 2\sin\phi\cos\theta}{r}=0$$

$$\frac{\partial^{2}}{\partial\theta \partial\phi}: \frac{-2\sin\theta\cos\phi\cos\theta}{r^{2}\sin\phi} + \frac{2\cos\theta\cos\phi\sin\theta}{r^{2}\sin\phi}=0$$

First order terms (22 total):

$$\frac{\partial}{\partial r}: \sin\phi\cos\theta\frac{\partial}{\partial r}(\sin\phi\cos\theta) + \frac{-\sin\theta}{r\sin\phi}\frac{\partial}{\partial \theta}(\sin\phi\cos\theta) + \frac{\cos\phi\cos\theta}{r}\frac{\partial}{\partial\phi}(\sin\phi\cos\theta) $$
$$+\sin\phi\sin\theta\frac{\partial}{\partial r}(\sin\phi\sin\theta)+\frac{\cos\theta}{r\sin\phi}\frac{\partial}{\partial \theta}(\sin\phi\sin\theta)+\frac{\cos\phi\sin\theta}{r}\frac{\partial}{\partial\phi}(\sin\phi\sin\theta)$$
$$+\cos\phi\frac{\partial}{\partial r}(\cos\phi) - \frac{\sin\phi}{r}\frac{\partial}{\partial \phi}(\cos\phi)$$

$$=0+\frac{\sin^{2}\theta}{r}+\frac{\cos^{2}\phi\cos^{2}\theta}{r} + 0 + \frac{\cos^{2}\theta}{r} + \frac{\cos^{2}\phi\sin^{2}\theta}{r}+0+\frac{\sin^{2}\phi}{r}$$

$$=\frac{1}{r}+\frac{\cos^{2}\phi +\sin^{2}\phi}{r}=\frac{2}{r}$$

$$\frac{\partial}{\partial \theta}: \sin\phi\cos\theta\frac{\partial}{\partial r}(\frac{-\sin\theta}{r\sin\phi}) + \frac{-\sin\theta}{r\sin\phi}\frac{\partial}{\partial \theta}(\frac{-\sin\theta}{r\sin\phi}) + \frac{\cos\phi\cos\theta}{r}\frac{\partial}{\partial \phi}(\frac{-\sin\theta}{r\sin\phi})$$
$$+ \sin\phi\sin\theta\frac{\partial}{\partial r}(\frac{\cos\theta}{r\sin\phi}) + \frac{\cos\theta}{r\sin\phi}\frac{\partial}{\partial\theta}(\frac{\cos\theta}{r\sin\phi}) + \frac{\cos\phi\sin\theta}{r}\frac{\partial}{\partial \phi}(\frac{\cos\theta}{r\sin\phi})$$
$$=0$$

$$\frac{\partial}{\partial \phi}: \sin\phi\cos\theta\frac{\partial}{\partial r}(\frac{\cos\phi\cos\theta}{r}) - \frac{\sin\theta}{r\sin\phi}\frac{\partial}{\partial\theta}(\frac{\cos\phi\cos\theta}{r}) + \frac{\cos\phi\cos\theta}{r}\frac{\partial}{\partial\phi}(\frac{\cos\phi\cos\theta}{r})$$
$$+\sin\phi\sin\theta\frac{\partial}{\partial r}(\frac{\cos\phi\sin\theta}{r}) + \frac{\cos\theta}{r\sin\phi}\frac{\partial}{\partial\theta}(\frac{\cos\phi\sin\theta}{r}) + \frac{\cos\phi\sin\theta}{r}\frac{\partial}{\partial\phi}(\frac{\cos\phi\sin\theta}{r})$$
$$+\cos\phi\frac{\partial}{\partial r}(\frac{-\sin\phi}{r})-\frac{\sin\phi}{r}\frac{\partial}{\partial\phi}(\frac{-\sin\phi}{r})$$
$$=\frac{-\sin\phi\cos\phi\cos^{2}\theta}{r^{2}}+\frac{\sin^{2}\theta\cos\phi}{r^{2}\sin\phi}-\frac{\cos\phi\sin\phi\cos^{2}\theta}{r^{2}}$$
$$-\frac{\sin\phi\cos\phi\sin^{2}\theta}{r^{2}}+\frac{\cos\phi\cos^{2}\theta}{r^{2}\sin\phi}-\frac{\sin\phi\cos\phi\sin^{2}\theta}{r^{2}}$$
$$+\frac{\cos\phi\sin\phi}{r^{2}}+\frac{\cos\phi\sin\phi}{r}$$
$$=\frac{-\sin\phi\cos\phi(1)}{r^{2}}+\frac{\cos\phi(1)}{r^{2}\sin\phi} - \frac{\sin\phi\cos\phi(1)}{r^{2}} + \frac{2\cos\phi\sin\phi}{r^{2}}$$
$$=\frac{\cot\phi}{r^{2}}$$

\section{Final Equation}

Adding up each of the second derivative terms gives us the final equation.

$$\frac{\partial^{2}}{\partial r^{2}} + \frac{2}{r}\frac{\partial}{\partial r} + \frac{1}{r^{2}\sin^{2}\phi}\frac{\partial^{2}}{\partial\theta^{2}} + \frac{1}{r^{2}}\frac{\partial^{2}}{\partial \phi^{2}}+\frac{\cot\phi}{r^{2}}\frac{\partial}{\partial \phi}=0$$

\end{document}


\section{Functions of Random Variables, Derived Distributions}

(This section needs revision)

I find changes of variables to be the easiest to understand by writing down a joint probability distribution using a delta function to express the conditional probability for the new variables based on the old variables. The next step is to perform a change of variables in the delta function, and integrate. The delta function means that the relationship between the old and the new variables is deterministic. If the relationship is non-deterministic you can use whatever expression for the conditional probability is appropriate.

Let's say that you want to know the probability distribution of the function $\vec{u} = \mathbf{H}(\vec{x})$ of some random variable $\vec{x}$ for which the probability distribution is known.

I personally find it least confusing to approach this problem by thinking about the joint probability distribution, and then obtaining $p(\vec{u})$ through marginalization:

\begin{equation}
p(\vec{u}) = \int dx^n p(\vec{x},\vec{u}) = \int dx^n p_x(\vec{x}) p_u(\vec{u}|\vec{x})
\end{equation}

Since $\vec{u}$ is a deterministic function of $\vec{x}$, the conditional probabiltiy $p_u(\vec{u}|\vec{x}) = \delta(\vec{u} - \mathbf{H}(\vec{x}))$. Explicitly:

\begin{equation}
p(\vec{u}) = \int dx^n p_x(\vec{x}) \delta(\vec{u} - \mathbf{H}(\vec{x}))
\end{equation}

What this does, is to integrate over all the points $p_x(\vec{x})$ where the argument of the delta function is zero. The important thing is that care needs to be taken when the argument of the delta function is itself a function. In that case, a change of variables has to be performed, so that this is no longer the case. On wikipedia, this is done by defining the new variable $du = |\frac{d}{dx}g(x)|dx$, from which follows:

\begin{equation}
\delta(g(x)) = \sum_{x_0} \frac{\delta(x-x_0)}{|g'(x_0)|}
\end{equation}

Where it is necessary to sum over each point $x_0$ for which $g(x_0) = 0$.

Which makes sense. Except, if you rewrite in terms of the new variable, you get an expression that does not strike me as necessarily the same:

\begin{equation}
\int f(x) \delta(g(x))dx = \sum_{n}\int f(g_n^{-1}(u)) \delta(u) \left|\frac{d}{du}g^{-1}(u)\right|du
\end{equation}

Where the sum $n$ is over all functions $g_n^{-1}$ that satisfy $g(g_n^{-1}(u)) = u$. For example, if $u = g(x) = sin(x)$ then $g_n^{-1} = asin(u) + n2\pi$, for any integer $n$. I feel more comfortable with the second path. The answer to my confusion is most likely the inverse function theorem (cf. section \ref{sec:inverse_function_theorem}). In other words: this section needs revision! But hey, I flagged it.

Note that a delta function with a vector argument can be written as a product of the delta functions along each dimension.

Define a new set of variables $\vec{a} = \vec{u} - \mathbf{H}(\vec{x})$. It follows that $\vec{x} = \mathbf{H_n^{-1}}(\vec{u}-\vec{a})$ and the volume element $dx^n = \left| \frac{d}{d\vec{a}}\mathbf{H}_n^{-1}(\vec{u}-\vec{a})\right|da^n$. Here, $H^{-1}_n$ are all the functions that satisfy $\mathbf{H}(\mathbf{H}_n^{-1}(\vec{u})) = \vec{u}$. The integral becomes:

\begin{equation}
p(\vec{u}) = \int da^n p_x(\mathbf{H}^{-1}_n(\vec{u}-\vec{a}))\left| \frac{d}{d\vec{a}}\mathbf{H}_n^{-1}(\vec{u}-\vec{a})\right| \delta(\vec{a})
\end{equation}

At this point it is safe to evaluate the delta function integral. The result is:

\begin{equation}
p(\vec{u}) = p_x(\mathbf{H}^{-1}_n(\vec{u})\left| \frac{d}{d\vec{a}}\mathbf{H}_n^{-1}(\vec{u}-\vec{a})\right|_{(\vec{a}=0)}
\end{equation}

\subsection{Example: Sum of Random Variables}
If the map $\mathbf{H}$ is not bijective, for example because $\vec{u}$ has lower dimensionality than $\vec{x}$, then a bijective map can be artifically constructed by introducing additional variables that are then also marginalized out. For example, if the goal is to calculate the probability of measuring some sum of random variables $s$, then you can define:

\begin{equation}
u_0 = s - \sum(x_i)\\
u_1 = x_1\\
u_2 = x_2\\
\vdots\\
u_{(n-1)} = x_{n-1}
\end{equation}

The inverse is:

\begin{equation}
x_1 = u_1\\
x_2 = u_2\\
\vdots\\
x_n = s - u_0 - \sum_{i<n} x_i
\end{equation}

The argument in the delta function is transformed $\delta(s - \sum(x_i)) \rightarrow \delta(u_0)$. The determinant of the Jacobian $|J| = |\frac{d}{d\vec{u}} H^{-1}(\vec{u})|$ is given by $[\frac{d}{du_1} \mathbf{H^{-1}},\frac{d}{du_2} \mathbf{H^{-1}},...,\frac{d}{du_{n-1}} \mathbf{H^{-1}}]$. That's an $nxn$ matrix where the first row is all $-1$, the lower left is an $(n-1)x(n-1)$ identity matrix, and the lower right is a $(n-1)x1$ vector of 0s. The determinant is one. Consequently the probability distribution of measuring a sum $s$ is given by:

\begin{equation}
p(s)= \int dx^{n-1} p_x(x_1,x_2,x_3,...,s-\sum_{x<n}x_i)
\end{equation}

Which turns out to be the convolution when the variables are independent.

\subsection{Example: Lower Dimensional Random Variable}

This was already the case for the sum of several random variables, in which case the dimensionality of the problem was reduced from many to one.

Consider the case many to fewer. Again, you just need to perform a change of variables. The new set of variables needs to have the same dimensionality as the old set of variables. The rest should follow pretty obviously.

Linear example: $\vec{y} = \mathbf{M}\vec{x}$ where $y$ has 2 dimensions and $x$ has 3.

Transform:

\begin{equation}
u_1 = y_1 - \vec{M_1}\cdot\vec{x}\\
u_2 = y_2 - \vec{M_2}\cdot\vec{x}\\
u_3 = x_3
\end{equation}

You can write this in terms of some invertible matrix $\mathbf{W} = [\mathbf{M},[0,0,1]]$ as $\vec{u} = \mathbf{W}\vec{x}$, so that $dx^n = |\mathbf{W}^{-1}|du^n|$. The integral is then:

\begin{equation}
p(y) = \int du^n p_x(\mathbf{W}^{-1}(\vec{y}-\vec{u}))\delta(u_1)\delta(u_2)|\mathbf{W}^{-1}|
\end{equation}