\documentclass[10pt,a4paper,notitlepage]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{array}
\usepackage{float}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{float}
\usepackage{bm}
\begin{document}
\title{Project CFD--Lab\\ \textit{Free Surface Simulation}}
\author{\textbf{Group Number: 1}\\Ioannis Kouroudis, Lukas Krenz, Andreas Reiser,  \\ \\\textbf{Supervisor}\\Friedrich Menhorn}
\maketitle
\section*{Abstract}
In industry there are many applications that require the simulation of a system of unmixable fluids with vastly different kinematic viscosity, separated by a discrete interface.  This problem is commonly referred to as a free surface flow problem. In effect, the kinematic viscosity difference causes the first fluid, which is usually the industrially relevant one, to exert force on the second one, receiving a negligible reaction in return. The most common configuration of this sort is a liquid-gas system and is frequently encountered in applications such as dam breaking scenarios, material processes and wave simulations.
In this particular project, the phenomenon is simulated  using a Lattice Boltzmann approach combined with a volume of fluid method.

\section{Implementation \textit{\&} Theory}
The algorithm generally works by keeping track of the mass exchange between cells.
To do this, we save the mass $m$ for each cell and convert the cells if they are either full or empty.
Between the filled and empty cells we insert interface cells.
The program was written in C\texttt{++} utilising OpenMP parallelization.
We followed the algorithm described in~\cite{thurey2007physically}.

\subsection{Gravity }
We modify the standard collision process to include the gravity force $\bm{g}$~\cite{thurey2006optimization}
\begin{equation}\label{eq:BGK}
f_{i}^*(\bm{x},t) =f_i (\bm{x}, t) - \frac{1}{\tau} \left( f_i(\bm{x}, t) - f_{i}^{eq}\rho(\bm{x},t),\mathbf{u}(\bm{x},t)+\tau \bm{g}) \right).
%f_{i}(x,t+\Delta t)=(1-\omega)f_{i}^{eq}+\omega f_{i}^{eq}(\rho(\mathbf{x},t),\mathbf{u}(\mathbf{x},t)+\frac{\bm{g}}{\omega}).
\end{equation}

\subsection{Mass exchange}
In order to keep track of the interface, two additional variables have to be stored for each cell, namely the mass $m$ and the fluid fraction $\varepsilon_i = \frac{m_{i}}{\rho_{i}}$.
The mass exchanged between an interface and a fluid cell can be calculated directly using their distributions
\begin{equation} \label{eq:fluid-interface}
\Delta m_{i}(\bm{x},t)=f_{\hat{i}}(\bm{x}+\bm{e}_{i},t)-f_{i}(\bm{x},t),
\end{equation}
where $\hat{i}$ is the inverse velocity of $i$.
The interface--interface mass exchange
\begin{equation} \label{eq:intint}
\Delta m_{i}(\bm{x},t+\Delta t)=s_{e} \frac{\varepsilon(\bm{x}+\Delta t \bm{e}_{i},t)+\varepsilon(\bm{x},t)}{2}
\end{equation}
takes the fluid fraction of both cells into account.
To avoid ``lonely'' interface cells that have no empty or filled neighbours, we use the appropriate \(s_e\) from table \ref{table:artifacts} in equation \eqref{eq:intint}.
Because the mass exchange is a symmetric process, the used scheme is explicitly mass preserving.

\begin{table}[ht]
\centering
\hspace*{-2cm} % dirty hack to centre table
\begin{tabular}{lllll}
 		&  standard cell at $(\bm{x}+e_{i})$	& no fluid nb at $(\bm{x}+\bm{e}_{i})$     & no empty nb at $(\bm{x}+\bm{e}_{i})$    \\
 standard cell at $(\bm{x})$		&$f_{\hat{i}}(\bm{x}_{nb},t)-f(\bm{x},t)$	&  $f_{\hat{i}}(\bm{x}_{nb},t)$		&$-f_{i}(\bm{x},t)$		    \\
 no fluid neighbours at $(\bm{x})$	&$-f_{i}(\bm{x},t)$			&  $f_{\hat{i}}(\bm{x}_{nb},t)-f(\bm{x},t)$	&$-f_{i}(\bm{x},t)$		     \\
 no empty neighbours at $(\bm{x})$	&$f_{\hat{i}}(\bm{x}_{nb},t)$	&  $f_{\hat{i}}(\bm{x}_{nb},t)$		&$f_{\hat{i}}(\bm{x}_{nb},t)-f(\bm{x},t)$
\end{tabular}
\caption{\label{table:artifacts}Interface--Interface mass streaming coefficient $s_{e}$ with $x_b$ as the neighbouring cell.}
\end{table}

\subsection{Boundary conditions}
Because interface cells have empty cells as neighbours, they do not have a full set of distributions.
This is why we need to reconstruct all distributions coming from empty cells during the streaming step.
The physical perspective of this condition is that the gas has a far greater kinematic viscosity and is consequently pushed without resistance by the liquid.
Furthermore we use the atmospheric pressure ($\rho_{A} = 1$) in place of the density in the equilibrium function.
If $\bm{e}_{i}$ comes from an empty cell into a interface cell the following distribution is constructed:
\begin{equation} \label{eq:dist}
f'_{\hat{i}} (\bm{x}, t + \Delta t) = f^{eq}_{\hat{i}}(\rho_A, \mathbf{u})+f^{eq}_{{i}}(\rho_A, \mathbf{u})-f_{i}(\mathbf{x},t).
\end{equation}

However, constructing these distributions along only one side of the interface would result in a force imbalance.
To rectify this, we additionally reconstruct distributions coming in along the surface normal, which we approximate by central differences
\begin{equation} \label{eq:normal}
  \mathbf{n}=\frac{1}{2}\begin{bmatrix} \varepsilon(x_{j-1,k,l})-\varepsilon(x_{j+1,k,l})\\\varepsilon(x_{j,k-1,l})-\varepsilon(x_{j,k+1,l})
    \\
    \varepsilon(x_{j,k,l-1})-\varepsilon(x_{j,k,l+1} ) \end{bmatrix},
\end{equation}
where $\varepsilon(x_{j,k,l})$ is the fluid fraction of the cell at coordinates $(j,k,l)$.
If $\mathbf(\bm{n} \cdot \bm{e_i})>0$, $f_{i}$ is reconstructed according to equation \eqref{eq:dist}

\subsection{Flag Update}
After the mass calculation step, all interface cells that have filled or emptied are assigned temporary flag as filled or emptied cells respectively.
To determine this, the criterion
\begin{equation} \label{eq:filled}
m(\bm{x},t+ \Delta t)>(1+ \text{offset }) \cdot \rho(\bm{x},t+\Delta t),
\end{equation}
\begin{equation} \label{eq:empty}
m(\bm{x},t+ \Delta t)<(0- \text{offset }) \cdot \rho(\bm{x},t+\Delta t),
\end{equation}
is used, with an offset of $10^{-3}$, to avoid oscillating changes of the cell status~\cite{thurey2007physically}.

We then convert all empty neighbours of filled cells to interface cells and all neighbouring emptied cells back to interface cells.
After this is done, all fluid neighbours of emptied cells are converted to interface cells.
Formerly empty cells that became interface cells are initialised with the average density and velocity of their fluid, interface and filled neighbours.
We need to use temporary flags because otherwise the update rule would be order dependent.

\subsection{Excess Mass }
Newly converted cells have an excess mass $m^{\text{ex}}$.
Fluid cells should have mass equal to their density, empty cells should have mass zero.
All excess mass is distributed to all interface neighbours.
For reasons of stability, we weigh the updates according to the position of the current interface
\begin{equation} \label{eq:exmass}
\Delta m(\bm{x}+\bm{e}_{i})=m^{\text{ex}}\frac{v_{i}}{v_{\text{total}}},
\end{equation}
where $v_{i}$ is the weight of the neighbouring cell and $v_{\text{total}}$ is the sum of all weights.
The weights are calculated by
\begin{align}
  \label{eq:excessMassWeights}
 v_i^f &=  \max(0, \bm{n} \cdot \bm{e_i}),\\
 v_i^e &= - \min(0, \bm{n} \cdot \bm{e_i}),
\end{align}
for filled and emptied cells respectively.

Again it is important to update the mass of all cells at the same time.
After the excess mass has been distributed, all flags are converted from the temporary to the finalised ones.

\subsection{Adaptive time step \textit{\&} turbulence model}
If velocities become larger than $1/3$ distributions can become negative thus impacting the stability.
To avoid instabilities due to high velocities and to acquire speedup during periods of rest we use an adaptive time step.
We use a critical velocity of $\bm{u_\text{crit}} = 1/6$, which is half the lattice speed of sound.

To adapt the time step, we calculate the maximum velocity $\bm{u}_{\text{max}}$ of the simulation and modify the time step if needed.
If $ \Vert \bm{u}_{max} \Vert > \xi \bm{u}_{\text{crit}}$ the new timestep becomes $\Delta t_{\text{new}}= \xi \Delta t_{\text{old}}$ and if
$\Vert \bm{u}_{max} \Vert < \xi \bm{u}_\text{crit}$ the new timestep is updated to $\Delta t_{\text{new}} = \Delta t_{\text{old}} / \xi$,
where $\xi = 4/5$ is the rescaling factor~\cite{thurey2005interactive}.
The timestep is used to compute a new value of the relaxation parameter $\tau$
\begin{equation} \label{eq:omega}
\tau_{\text{new}}= \frac{\Delta t_{\text{new}}}{\Delta t_{\text{old}}} (\tau_{\text{old}}- 0.5) +0.5
\end{equation}
which is then used in the collision process.

After each time step change, all time dependent values must be rescaled, e.g.~density, mass, velocity and gravity.
Additionally all distributions must be rescaled.
The details of the rescaling procedure are omitted to preserve space, the interested reader is referred to chapter 6 of~\cite{thurey2007physically}.

For very small time steps or viscosities, $\tau$ is close to $0.5$, thus leading to instabilities in the simulation.
To rectify this, we use a Smagorinsky turbulence model.
For this we calculate the local stress tensor $S$ by
\begin{equation} \label{eq:stress}
\Pi_{\alpha,\beta}=\sum^{19}_{i=1}e_{i\alpha}e_{i\beta}(f_{i}-f_{i}^{eq}),
\end{equation}
where $\alpha$ and $\beta$ are the components of the lattice velocities.
 The intensity $S$ is then given by
\begin{equation} \label{eq:intensity}
S=\frac{1}{6C^{2}}(\sqrt{\nu^{2}+18C^{2}\sqrt{\Pi_{\alpha,\beta} \Pi_{\alpha,\beta}}-\nu},
\end{equation}
where summation over $\alpha$ and $\beta$ is implied and $C$ is a constant usually set to values close to $0.03$~\cite{thurey2007physically}.
Finally we set the local relaxation time to
\begin{equation} \label{eq:relax}
\tau_s=3(\nu+C^{2}S)+\frac{1}{2},
\end{equation}
where $\nu$ is the global viscosity.
Intuitively, the turbulence model increases the local viscosity for cells according to the stress exerted onto them.

\section{Results \textit{\&} Scenarios}
Two simple scenarios were chosen which produced both physically correct and visually pleasing results.
\subsection{Dam Breaking}
The dam breaking was simulated by a large vertical mass of fluid that is released and affected by gravity.
\begin{figure}[H]
\centering
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{dam/1.png}
\end{subfigure}%
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{dam/2.png}
\end{subfigure}
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{dam/3.png}
\end{subfigure}
\centering
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{dam/4.png}
\end{subfigure}%
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{dam/5.png}
\end{subfigure}
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{dam/6.png}
\end{subfigure}
\centering
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{dam/7.png}
\end{subfigure}%
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{dam/8.png}
\end{subfigure}
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{dam/9.png}
\end{subfigure}
\caption{Breaking Dam}
\label{fig:dam}
\end{figure}
\subsection{Falling Drop}
A drop was left to fall in a pool of stationary water. The results look comparable to other sources and resemble the expected physical behaviour.
\begin{figure}[H]
\centering
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{drop/1.png}
\end{subfigure}%
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{drop/2.png}
\end{subfigure}
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{drop/3.png}
\end{subfigure}
\centering
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{drop/4.png}
\end{subfigure}%
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{drop/5.png}
\end{subfigure}
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{drop/6.png}
\end{subfigure}
\centering
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{drop/7.png}
\end{subfigure}%
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{drop/8.png}
\end{subfigure}
\begin{subfigure}{0.25\textwidth}
  \includegraphics[width=1.0\linewidth]{drop/9.png}
\end{subfigure}
\caption{Falling Drop}
\label{fig:drop}
\end{figure}

\bibliographystyle{alpha}
\bibliography{refs}

\end{document}\grid

\grid
\grid


\problemname{Inverted Deck}

\illustration{0.2}{elf.jpg}{\href{https://commons.wikimedia.org/wiki/File:DnD_Elven.png}{Elf} by LadyofHats via Wikimedia Commons, public domain}%
\noindent
As a huge fan of the popular collectible card game Numinous Wilds: the Elven Reign Chronicles (NWERC), you have a large collection of cards which you carefully organise by their rarity.
One day you notice that someone has touched your collection, and that some of the cards are now out of order.
The most natural suspect, of course, is your little brother Billy, who was absolutely $100\%$ forbidden from playing with your cards.
After a few minutes of interrogation Billy confesses that he indeed took a few consecutive cards from the middle of the stack, but he swears that he put them back in exactly the same order as they were.
You suspect that Billy, being so young, may have simply mistakenly reversed the order of the cards that he took.
Now you want to check your theory and decide if you can find the batch of cards that Billy took to play with.

Is it possible to restore the order of the cards into non-decreasing order of their rarity by reversing
just one contiguous batch of cards?

\section*{Input}

The input consists of:

\begin{itemize}
  \item One line containing an integer $n$ ($1 \le n \le 10^6$), the number of cards in your collection.
  \item One line containing $n$ integers $v_1, \ldots, v_n$ ($1 \le v_{i} \le 10^9$ for all $i$), the current order of the cards' rarity values.
\end{itemize}

\section*{Output}

If the cards can be sorted by reversing exactly one contiguous subsequence of the list, then output the $1$-based start and end indices of such a subsequence.
Otherwise, output ``\texttt{impossible}''.
If there are multiple valid solutions you may output any one of them.


\section{de211}

Function: 2-to-4 decoder/demultiplexer

Terminals: (A, B, Y0, Y1, Y2, Y3, vss, vdd)

IEC-symbol:
%figuur de211.iec.eps
\begin{figure}[bth]
\callpsfig{de211.iec.eps}{width=.31\textwidth}
\end{figure}

Function table:
\begin{table}[bth]
\begin{tabular}{|cc||cccc|}
\hline
B	&A	&Y0	&Y1	&Y2	&Y3\\
\hline
L	&L	&H	&L	&L	&L\\
L	&H	&L	&H	&L	&L\\
H	&L	&L	&L	&H	&L\\
H	&H	&L	&L	&L	&H\\
\hline
\end{tabular}
\vspace{1cm}

Propagation and load dependent delays:\\

\begin{tabular}{|c|c|c|c|c|}
\hline
Parameter               &From            &To   &Typ    &Unit\\
\hline
T$_{PLH}$               &A     		&Y0,Y2  &0.5    &ns\\
T$_{PHL}$               &A     		&Y0,Y2  &0.3    &ns\\
T$_{PLH}$               &A     		&Y1,Y3  &0.8    &ns\\
T$_{PHL}$               &A     		&Y1,Y3  &0.8    &ns\\
T$_{PLH}$               &B	    	&Y0,Y1  &0.5    &ns\\
T$_{PHL}$               &B	    	&Y0,Y1  &0.3    &ns\\
T$_{PLH}$               &B	    	&Y2,Y3  &0.8    &ns\\
T$_{PHL}$               &B	    	&Y2,Y3  &0.8    &ns\\
\hline
$\Delta$T$_{PLH}$       &any           &any   &2.2    &ns/pF\\
$\Delta$T$_{PHL}$       &any           &any   &1.0    &ns/pF\\
\hline
C$_{in}$                &A   		&vss    &0.38   &pF\\
C$_{in}$                &B             	&vss    &0.37   &pF\\
\hline
\end{tabular}
\end{table}

Equivalent chip area: 14

\begin{figure}[bth]
circuit:\\

\callpsfig{de211.cir.eps}{width=1\textwidth}
\end{figure}

\begin{figure}[bth]
layout:\\

\callpsfig{de211.lay.eps}{height=4in}
\end{figure}

\clearpage


\documentclass[aspectratio=149]{beamer}

\input{../../shared_slides.tex}

% reference: https://yuxinchen2020.github.io/ele522_optimization/lectures/variance_reduction.pdf

\title{Variance reduction for stochastic gradient methods}
\date{\today}

\begin{document}
\maketitle
\frame{\tableofcontents}

\section{Introduction}%

\begin{frame}
  \frametitle{The finite sum probem}
  A common Task in (supervised) machine learning:
  \begin{equation}
    \min_{x\in \R^d} f(x) := \frac{1}{n} \sum_{i=1}^{n} \underbrace{f_i(x)}_{\textcolor{blue}{\text{loss for $i$-th sample}}} + \underbrace{\psi(x)}_{\textcolor{blue}{\text{regularizer}}}
  \end{equation}
  where the $i$-th sample is $(a_i, y_i)$.

  \textbf{Examples:}
  \begin{itemize}
    \item linear regression: $f_i(x) = {(a_i^T x -y_i)}^2$, and $\psi=0$
    \item logistic regression: $f_i(x) = \log(1+e^{-y_i a_i^T x})$, and $\psi=0$
          ``sigmoid function'' and logistic loss.
    \item Lasso: $f_i$ as for linear regression but $\psi(x) = \Vert x \Vert_1$
    \item SVM: $f_i(x) = \max \{0 , 1 - y_i a_i^T x\}$ and $\psi(x)= \Vert x \Vert^2$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Gradient descent}
  \begin{algorithm}[H]
    \caption{(batch) GD}
    \begin{algorithmic}[1]
      \For{$k = 1,2, \dots$}
      \State{ $x_{k+1} = x_k  - \alpha_k \nabla f(x_k) $}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}

  \begin{itemize}
    \item gradient can be computed via
          \begin{equation}
            \nabla f(x) = \nabla \left(\sum_{i=1}^{n}f_i(x)\right) = \sum_{i=1}^{n} \nabla f_i(x_k)
          \end{equation}
    \item good convergence properties
    \item can be \textbf{expensive} if $n$ is large!
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Stochastic gradient descent}
  \begin{algorithm}[H]
    \caption{SGD}\label{sgd}
    \begin{algorithmic}[1]
      \For{$k = 1,2, \dots$}
      \State{pick $i_k$ uniform at random in $[n]$}
      \State{$x_{k+1} = x_k  - \alpha_k \nabla f_{i_k}(x_k)$}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}

  We already noticed that:
  \begin{itemize}
    \item unbiased: $\E[\nabla f_{i_k}(x)] = \sum_{i=1}^{n} \P[i=i_k] \nabla f_i(x) = \sum_{i=1}^{n} \frac{1}{n} \nabla f_i(x)$
    \item large stepsizes fail to suppress variability in the stoch.\ gradients $\rightarrow$ leads to oscillations
    \item decreasing stepsizes mitigate this problem but \textbf{slows down} convergence (too \emph{conservative})
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Recall SGD (template)}
  \begin{equation}
    x_{k+1} = x_k - \alpha_k g_k
  \end{equation}
  \begin{itemize}
    \item $g_k$ is an unbiased estimator of the true gradient $\nabla F(x_k)$
    \item convergence depends on \textbf{variance} $\E [ \Vert g_k - \nabla F(x_k) \Vert ] \le \sigma_g$ \\
          (not strictly necessary)
    \item vanilla SGD uses $g_k = \nabla f_{i_k}(x_k)$ \\
          \textbf{issue:} $\sigma_g$ is non-negligible even close to the solution
    \item \textbf{Q:} can we choose $g_k$ in a different way to reduce variability?
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Minibatching}
  \begin{algorithm}[H]
    \caption{minibatch SGD}
    \begin{algorithmic}[1]
      \For{$k = 1,2, \dots$}
      \State{pick $I_k$ random subset of $[n]$ with $\vert I_k \vert = b $}
      \State{$x_{k+1} = x_k  - \alpha_k \sum_{i \in I_k} \nabla f_{i}(x_k)$}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}
  \begin{itemize}
    \item typically we make a (uniform) \emph{random} choice $i_k \in [n] = \{1, \dots, n\}$\\
          (or random reshuffling)
    \item by increasing the size to a \textbf{random subset} $I_k \subset [n]$ of size $b \ll n$ we can
          \begin{itemize}
            \item decrease variance
            \item increase cost only moderately,
            \item no improvement in the rate
          \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{A simple idea}
  Consider
  \begin{itemize}
    \item estimator $X$ for parameter $\mu$ ($\E[X]=\mu$ and $\Var[X]=\sigma^2$)
    \item want to keep unbiased but reduce variance
    \item find $Y$ such that $\E[Y] = 0$ but $\Cov (X,Y)$ is large and $\tilde{X}=X-Y$
    \item remains unbiased

          \begin{equation}
            \Var[\tilde{X}] = \Var[X]+\Var[Y]-2\Cov[X,Y]
          \end{equation}
    \item can be much smaller than $\Var[X]$ if $X,Y$ are highly correlated
  \end{itemize}
\end{frame}

\section{SAG}%
\label{sec:}

\begin{frame}
  \frametitle{Stochastic average gradient (SAG), 2013}
  \begin{itemize}
    \item maintain table containing gradients $g_i$ of $f_i$
    \item at step $k = 1,2, \dots$ pick random $i_k \in [n]$ and
          \begin{equation}
            g_i^k := \nabla f_{i_k}(x^{k})
          \end{equation}
          set $g_{i}^k = g_i^{k-1}$ for all $i\neq i_k$ (remain the same)
    \item Update
          \begin{equation}
            x^{k+1} = x^k - \alpha_k \frac{1}{n} \sum_{i=1}^{n} g_i^k
          \end{equation}
    \item assuming gradients do not change too much along trajectory
    \item gradient estimator \textcolor{red}{no longer unbiased}
    \item Isn't it expensive to average these gradients?
          \begin{equation}
            x^{k+1} = x^k -u \alpha_k  \Big( \frac{g_{i_k}^k}{n} - \frac{g_{i_k}^{k-1}}{n} + \underbrace{\frac{1}{n}\sum_{i=1}^{n} g_i^k}_{\text{old table average}} \Big)
          \end{equation}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SAG variance reduction}
  Gradient estimator in SAG:
  \begin{equation}
    x^{k+1} = x^k - \alpha_k \frac{1}{n} \Big( \underbrace{g_{i_k}^k}_X - \underbrace{g_{i_k}^{k-1} - \sum_{i=1}^{n} g_i^k}_{Y} \Big)
  \end{equation}
  \begin{itemize}
    \item While $\E[X] = \nabla f(x^k)$, but $\E[Y]\neq 0$ $\rightarrow $ biased estimator
    \item $X$ and $Y$ are correlated as $X-Y \to 0$  as
    \item $x^k$ and $x_{k-1}$ both converge to $x^*$ and
    \item the last term converges to $\nabla f(x^*)= 0$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Convergence}
  As always, initialization plays a role: $D^2 := \Vert x^0 -x^* \Vert^2$.

  \begin{equation}
    \begin{aligned}
      \text{SAG:}& \quad \frac{n}{k}(f(x^0)-f^*) + \frac{L}{k}D^2 \\
      \text{GD:}& \quad \frac{L}{k}D^2 \\
      \text{SGD:}& \quad \frac{L}{\sqrt{k}}D^2 \\
    \end{aligned}
  \end{equation}
  \begin{itemize}
    \item Achieves linear convergence in the \emph{strongly} convex setting.
    \item proofs are difficult (and computer-aided)
  \end{itemize}

  \begin{center}
    Same gradient oracle cost as SGD, but same converge rate as GD.
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{SAG.png}
    \caption{\label{fig:label} }
  \end{figure}

\end{frame}

\section{SAGA}%
\label{sec:}

\begin{frame}
  \frametitle{SAGA, 2014}
  Very similar:
  \begin{itemize}
    \item maintain table containing gradients $g_i$ of $f_i$
    \item at step $k = 1,2, \dots$ pick random $i_k \in [n]$ and
          \begin{equation}
            g_i^k := \nabla f_{i_k}(x^{k})
          \end{equation}
          set $g_{i}^k = g_i^{k-1}$ for all $i\neq i_k$ (remain the same)
    \item Update
          \begin{equation}
            x^{k+1} = x^k -u \alpha_k  \Big( g_{i_k}^k - g_{i_k}^{k-1} + \frac{1}{n}\sum_{i=1}^{n} g_i^k \Big)
          \end{equation}
    \item estimator now \textbf{unbiased}!
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{For Comparison}
  SAGA gradient estimate:
  \begin{equation}
    g_{i_k}^k - g_{i_k}^{k-1} + \frac{1}{n}\sum_{i=1}^{n} g_i^k
  \end{equation}

  SAGA gradient estimate:
  \begin{equation}
    \frac{1}{n}g_{i_k}^k - \frac{1}{n}g_{i_k}^{k-1} + \frac{1}{n}\sum_{i=1}^{n} g_i^k
  \end{equation}

\end{frame}

\begin{frame}
  \frametitle{}

  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{SAGA.png}
    % \caption{\label{fig:label} }
  \end{figure}
  Highlights the low variance of SAG.
\end{frame}

\section{SVRG}%

\begin{frame}
  \frametitle{Stochastic Variance Reduced Gradient (SVRG), 2013}

  \begin{algorithm}[H]
    \caption{SVRG}\label{}
    \begin{algorithmic}[1]
      \For{$k=1,2, \dots$}
      \State{Set $x^1 = \tilde{x} = \tilde{x}^{k}$}
      \State{Compute $\tilde{\mu} := \nabla f(\tilde{x})$ \hfill \textcolor{blue}{//update snapshot}}
      \For{$l=1,2, \dots, m$ }\hfill \textcolor{blue}{//$m$ iterations per epoch}
      \State{pick $i_l$ uniform at random in $[n]$}
      \State{Set $x^{l+1} = x^l - \alpha (\nabla f_{i_l}(x^l) - \nabla f_{i_l}(\tilde{x}) + \tilde{\mu})$}
      \EndFor{}
      \State{$\tilde{x}^{k+1}= x^{m+1}$}
      \EndFor{}
    \end{algorithmic}
  \end{algorithm}

  \begin{itemize}
    \item Does \textbf{not} need to \textbf{store} full table of gradients.
    \item requires \emph{batch} gradient computation every \emph{epoch}
    \item per iteration cost is comparable to that of SGD if $m \ge n$
    \item convergence rates similar to SAGA, but simpler analysis.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SVRG}
  \textbf{key idea:} by storing old point we can
  \begin{equation}
    \underbrace{\nabla f_{i_k}(x^k) - \nabla f_{i_k}(x^{\text{old}})}_{\text{$\to 0$ if $x\approx x^{\text{old}}$}} + \underbrace{\nabla f(x^{\text{old}})}_{\text{$\to 0$ if $x^{\text{old}}$}}
  \end{equation}
  \begin{itemize}
    \item is an unbiased estimate of $\nabla f(x^k)$
    \item converges to $0$ (meaning reduced variability) if $x^k\approx x^{\text{old}} \approx x^*$
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{SVRG: Theorem}

  \begin{theorem}
    Assume each $f_i$ is convex and $L$-smooth, and $F$ is $\mu$-strongly convex.
    Choose $m$ large enough s.t. $\rho = \frac{1}{\mu \alpha (1-2L \alpha)m} + \frac{2L \alpha}{1-2L \alpha}< 1$, then
    \begin{equation}
      \E[F(x_s^{old})- F(x^*)] \le \rho^s [F(x_0^{old})-F(x^*)]
    \end{equation}
  \end{theorem}
  Computational cost:
  \begin{itemize}
    \item per epoch: $(m+n)$
    \item complexity: $(n+ \kappa) \log(1/\epsilon)$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SVRG: Convergence Proof}
  Denote $g_s^k = \nabla f_{i_l}(x_s^k) - \nabla f_{i_l}(x_s^{old}) + \nabla F(x_s^{old})$. Conditioning on everythong prior to $x_s^{k+1}$ we get
  \begin{equation}
    \begin{aligned}
      \MoveEqLeft \E [\Vert x_s^{k+1} - x^* \Vert^2] = \E [\Vert x_s^{k}- \alpha g_s^k - x^* \Vert^2] \\
      &= \Vert x_s^k-x^* \Vert^2 - 2 \alpha {(x_s^k-x^* )}^T \E[g_s^t] + \alpha^2 \E[\Vert g_s^k \Vert^2] \\
      &= \Vert x_s^k-x^* \Vert^2 - 2 \alpha {(x_s^k-x^* )}^T \textcolor{blue}{\nabla F(x_s^t)} + \alpha^2 \E[\Vert g_s^k \Vert^2] \\
      &\le \Vert x_s^k-x^* \Vert^2 - 2 \alpha (F(x_s^k) - F(x^*))+ \alpha^2 \E[\Vert g_s^k \Vert^2]
    \end{aligned}
  \end{equation}

  \begin{itemize}
     \item \textbf{key step:} control $\E[\Vert g_s^k \Vert^2]$
   \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{SVRG: convergence Proof}
  \begin{lemma}%
    \begin{equation}
      \E[\Vert g_s^k \Vert^2] \le 4L [ F(x_s^k)- F(x^*) + F(x_s^{old}-F(x^*)) ]
    \end{equation}

  \end{lemma}

\end{frame}

\begin{frame}
  \frametitle{Comparison}

  % \begin{tabular}{l | c}
  %   SVRG & $(n + \kappa) \log(1/\epsilon)$\\
  %   GD   & $n \kappa \log(1/\epsilon)$ \\
  %   SGD  & $\kappa^2 / \epsilon$
  % \end{tabular}

  \begin{tabular}{c | c | c}
    SVRG  & GD & SGD \\
    \hline
 $(n + \kappa) \log(1/\epsilon)$ & $n \kappa \log(1/\epsilon)$ & $\kappa^2 / \epsilon$
  \end{tabular}

  % \begin{itemize}
  %   \item SVRG:\ $(n + \kappa) \log(1/\epsilon)$
  %   \item GD:\ $n \kappa \log(1/\epsilon)$
  %   \item SGD:\ $\kappa^2 / \epsilon$
  % \end{itemize}

\end{frame}

% talk about \sqrt{n} improvement
% talk about loopless svrg?

\end{document}


\subsubsection{Spline form}
\label{sec:twobodyjastrowspline}

The two-body spline Jastrow function is the most commonly used two-body Jastrow for solids. This form
was first described and used in \cite{EslerKimCeperleyShulenburger2012}.
Here $u_{ab}$ is an interpolating 1D B-spline (tricublc spline on a linear grid) between
zero distance and $r_{cut}$. In 3D periodic systems,
the default cutoff distance is the Wigner Seitz cell radius. For other periodicities, including isolated
molecules, the $r_{cut}$ must be specified.  $r_i$ and $r_j$ are typically electron positions.  The cusp
condition as $r_i$ approaches $r_j$ is set by the relative spin of the electrons.

\FloatBarrier
\paragraph{Input specification}

\FloatBarrier
\begin{table}[h!]
\begin{center}
\begin{tabular}{l c c c l }
\hline
\multicolumn{5}{l}{Correlation element} \\
\hline
\bfseries Name & \bfseries Datatype & \bfseries Values & \bfseries Defaults & \bfseries Description \\
\hline
SpeciesA & Text & U or d & (Required) & Quantum species target \\
SpeciesB & Text & U or d & (Required) & Quantum species target \\
Size & Integer & $> 0$ & (Required) & Number of coefficients \\
Rcut & Real & $> 0$ & See below & Distance at which the correlation goes to 0 \\
Spin & Text & Yes or no & No & Spin-dependent Jastrow factor \\
\hline
\multicolumn{5}{l}{Elements}\\ \hline
& Coefficients & & & \\ \hline
\multicolumn{5}{l}{Contents}\\ \hline
& (None)  & & &  \\ \hline
\end{tabular}
%\end{tabular*}
\end{center}
\end{table}

\FloatBarrier

Additional information:
\begin{itemize}
\item \ishell{speciesA, speciesB} The scale function u(r) is defined for species pairs uu and ud.
There is no need to define ud or dd since uu=dd and ud=du.  The cusp condition is computed internally
based on the charge of the quantum particles.
\end{itemize}

\begin{table}[h]
\begin{center}
\begin{tabular}{l c c c l }
\hline
\multicolumn{5}{l}{Coefficients element} \\
\hline
\bfseries Name & \bfseries Datatype & \bfseries Values & \bfseries Defaults & \bfseries Description \\
\hline
Id & Text & & (Required) & Unique identifier \\
Type & Text & Array & (Required) & \\
Optimize & Text & Yes or no & Yes & If no, values are fixed in optimizations \\
\hline
\multicolumn{5}{l}{Elements}\\ \hline
(None) & & & \\ \hline
\multicolumn{5}{l}{Contents}\\ \hline
 (No name) & Real array & & Zeros & Jastrow coefficients \\ \hline
\end{tabular}
%\end{tabular*}
\end{center}
\end{table}

\paragraph{Example use cases}
\label{sec:2bjsplineexamples}

Specify a spin-dependent function with four parameters for each channel.  In this case, the cusp is set at
a radius of 4.0 bohr (rather than to the default of the Wigner Seitz cell radius).  Also, in this example,
the coefficients are set to not be optimized during an optimization step.

\begin{lstlisting}[style=QMCPXML]
<jastrow name="J2" type="Two-Body" function="Bspline" print="yes">
  <correlation speciesA="u" speciesB="u" size="8" rcut="4.0">
    <coefficients id="uu" type="Array" optimize="no"> 0.2309049836 0.1312646071 0.05464141356 0.01306231516</coefficients>
  </correlation>
  <correlation speciesA="u" speciesB="d" size="8" rcut="4.0">
    <coefficients id="ud" type="Array" optimize="no"> 0.4351561096 0.2377951747 0.1129144262 0.0356789236</coefficients>
  </correlation>
</jastrow>
\end{lstlisting}


\documentclass[margin=5mm, varwidth]{standalone}
\usepackage[paperwidth=9in,top=1in,bottom=1in,right=1in,left=1in,heightrounded]{geometry}
\usepackage{niftydiags}

\begin{document}
\begin{pycode}
import numpy as np
def rad(deg):
    return deg * np.pi / 180
def roundCoord(point):
    return [ round(x, 2) for x in point ]
def tuplefy(points):
    return ( tuple(roundCoord(p)) for p in points)
\end{pycode}

\section{(Excursion) vector components as linear combinations of vectors.}

\items{
\item Every vector can be written as a combination of scalars and unit vectors
\item $\vec{v} = \vektor{a\\b} = a \cdot \vektor{1\\0} + b \cdot \vektor{0\\1}$
\item $\hat{e}_1 = \vektor{1\\0}$ and $\hat{e}_2 = \vektor{0\\1}$ are canonical base vectors in $R^2$ \\
      For $R^n$: $\hat{e}_i = \vektor{0 \\ \vdots\\ 1 \\ \vdots \\ 0}$ $i$th component being 1 all other 0
\item Generally: $\vec{v} = \vektor{x_1 \\ x_2 \\ \vdots \\ x_n} = \sum\limits_{i=1}^{n}{x_i \cdot e_i}$
\item Points on the unit cirlce (c=1):
}

\newcommand\cosdiag{0.7071}
\newcommand\lenax{1.2}
\begin{tikzpicture}[scale=2]
\coordinate (A) at (0, 0);
\coordinate (B) at (\cosdiag, 0);
\coordinate (C) at (\cosdiag, \cosdiag);

\draw[very thin, <->] (-\lenax, 0) -- (\lenax, 0) node[right] {$x$};
\draw[very thin, <->] (0, -\lenax) -- (0, \lenax) node[above]{$y$};
\draw[dashed] (0,0) circle (1);

\newcommand\rad{0.35}
\draw[fill=green!30, very thin] (0,0) -- (45:\rad) arc (45:0:\rad);
\draw (0.22, 0.09) node {$\alpha$};

\newcommand\ninetyrad{\rad /2}
\draw[very thin] (\cosdiag,\ninetyrad) arc (90:180:\ninetyrad);
\fill(\cosdiag - \ninetyrad / 2.5, \ninetyrad / 2.5) circle (0.02);

\draw (A) -- (B) node[midway, below] {$a$};
\draw (B) -- (C) node[midway, right] {$b$} node[above, right] {$\vec{v}$};
\draw (C) -- (A) node[midway, above left] {$c$};
\end{tikzpicture}
\begin{pycode}
deg=0
rotdeg=45
v=(np.cos(rad(deg)), np.sin(rad(deg)) )
v2=(-v[1],v[0])
a=np.multiply(np.cos(rad(rotdeg)), v)
b=np.multiply(np.sin(rad(rotdeg)), v2)
v3=a+b
alphaLoc = np.multiply(0.3, ( np.cos(rad(deg+rotdeg/2)), np.sin(rad(deg+rotdeg/2)) ) )
a, b, v3, alphaLoc = tuplefy([a, b, v3, alphaLoc])
\end{pycode}
\begin{pysub}
    \begin{tikzpicture}[scale=2]
        \coordinate (O) at (0, 0);
        \coordinate (A) at !{v};
        \coordinate (B) at !{v2};
        \coordinate (C) at !{v3};
        \draw[very thin, <->] (-\lenax, 0) -- (\lenax, 0) node[right] {$x$};
        \draw[very thin, <->] (0, -\lenax) -- (0, \lenax) node[above]{$y$};
        \draw[dashed, very thin] (0,0) circle (1);

        \draw[fill=green!30, very thin] (0,0) -- (!{deg}:0.4) arc (!{deg}:!{deg}+!{rotdeg}:0.4);
        \draw !{alphaLoc} node {$\alpha$};

        \draw[-latex, thick] (O) -- +(A) node[right, below] {$\hat{e}_1$};
        \draw[-latex, dashed, thick] (O) -- +(B) node[above, left] {$\hat{e}_2$};
        \draw[-latex, dashed, thick] (O) -- +(C) node[right] {$\vec{v}$};

        \draw[dashed, very thin] !{b} -- +!{a};
        \draw[dashed, very thin] !{a} -- +!{b};

        \draw[-latex, thick] (O) -- +!{a} node[midway, below] {$\cos(\alpha) \, \hat{e}_1$};
        \draw[-latex, thick] (O) -- +!{b} node[midway, left] {$\sin(\alpha) \, \hat{e}_2$};
    \end{tikzpicture}
\end{pysub}
\items{
\item $\vec{v} = \vektor{a\\b} = \vektor{\cos(\alpha)\\ \sin(\alpha)} = \cos(\alpha) \cdot \vektor{1\\0} +\sin(\alpha) \cdot \vektor{0\\1} = \cos(\alpha) \cdot \hat{e}_1 +\sin(\alpha) \cdot \hat{e}_2$
}
\section{(Excursion) Orthogonal complement of a vector ($90^\circ$ Rotation CCW)}
\items{
\item Set up orthogonal vector to obtain up-vector for new coordinate frame
}
{ \large
\begin{tikzpicture}[scale=1.5, ultra thin]
\draw[step=0.5cm, dashed, lightgray] (-.99,-0.3) grid (1.49, 1.49);
\draw[<->, gray] (0, -0.3) -- (0, 1.5cm);
\draw[<->, gray] (-0.99, 0) -- (1.5cm, 0);
\draw[very thick, -latex] (0, 0) -- (1, 0.5) node[above] {$\vec{v}$};% = \begin{pmatrix} x \\ y\end{pmatrix}$};
\draw[dashed] (0, .5) -- (1, .5) node[midway, above] {$x$};
\draw[dashed] (1, 0) -- (1, .5) node[midway, right] {$y$};
\draw[very thick, dashed, -latex] (0, 0) -- (-0.5, 1) node[left] {$\vec{v}^{R}$};
\draw[dashed] (0, 1) -- (-.5, 1) node[midway, above] {$-y$};
\draw[dashed] (-.5, 0) -- (-.5, 1) node[midway, left] {$x$};
\end{tikzpicture} }
$\vec{v}^{R} =  \vektor{x\\y} ^{R} = \vektor{-y\\x}$
\\
\items{
\item Derive $\vec{v}^R$: \\
$\gleichung{
    \vec{v}^{R}&=&  \vektor{x \\ y} ^{R}= \vektor{\cos(\alpha) \\ \sin(\alpha)} ^{R}=\vektor{ \cos(\alpha + 90^\circ) \\ \sin(\alpha + 90^\circ) }
    \\
    &=& \vektor{ \cos(\alpha) \cos(90^\circ) - \sin(\alpha) \sin(90^\circ)  \\ \cos(\alpha) \sin(90^\circ) + \sin(\alpha) \cos(90^\circ) }
    \\
    &=&\vektor{ \cos(\alpha) \cdot 0 - \sin(\alpha) \cdot 1   \\ \cos(\alpha) \cdot 1 + \sin(\alpha) \cdot 0 }
    = \vektor{ - \sin(\alpha) \\ \cos(\alpha) } = \underline{\vektor{- y \\ x }}
}$
\item $\vec{v}^{R}$ is perpendicular to $\vec{v}$: $ \vektor{x \\ y} \bullet \vektor{-y \\ x} = \underline{-xy + yx = 0}$
}

\section{From 2D vector arithmetic to 2D rotation matrix}

\items {
\item Apply $\cos$ and $\sin$ to new coordinate frame with axes $\vec{v}$ and $\vec{v}^R$ to obtain rotated vector $\vec{v}'$
}

\begin{pycode}
deg=25
rotdeg=45
v=(np.cos(rad(deg)), np.sin(rad(deg)) )
v2=(-v[1],v[0])
a=np.multiply(np.cos(rad(rotdeg)), v)
b=np.multiply(np.sin(rad(rotdeg)), v2)
v3=a+b
alphaLoc = np.multiply(0.3, ( np.cos(rad(deg+rotdeg/2)), np.sin(rad(deg+rotdeg/2)) ) )
a, b, v3, alphaLoc = tuplefy([a, b, v3, alphaLoc])
\end{pycode}
\begin{pysub}
    \begin{tikzpicture}[scale=2]
        \coordinate (O) at (0, 0);
        \coordinate (A) at !{v};
        \coordinate (B) at !{v2};
        \coordinate (C) at !{v3};
        \draw[very thin, <->] (-\lenax, 0) -- (\lenax, 0) node[right] {$x$};
        \draw[very thin, <->] (0, -\lenax) -- (0, \lenax) node[above]{$y$};
        \draw[dashed, very thin] (0,0) circle (1);

        \draw[fill=green!30, very thin] (0,0) -- (!{deg}:0.4) arc (!{deg}:!{deg}+!{rotdeg}:0.4);
        \draw !{alphaLoc} node {$\alpha$};

        \draw[-latex, thick] (O) -- +(A) node[above] {$\vec{v}$};
        \draw[-latex, dashed, thick] (O) -- +(B) node[above] {$\vec{v}^{R}$};
        \draw[-latex, dashed, thick] (O) -- +(C) node[above] {$\vec{v}'$};

        \draw[dashed, very thin] !{b} -- +!{a};
        \draw[dashed, very thin] !{a} -- +!{b};

        \draw[-latex, thick] (O) -- +!{a} node[midway, right] {$\cos(\alpha) \, \vec{v}$};
        \draw[-latex, thick] (O) -- +!{b} node[midway, left] {$\sin(\alpha) \, \vec{v}^{R}$};
    \end{tikzpicture}
\end{pysub}

\items {
\item $\vec{v}' = \cos(\alpha) \, \vec{v} + \sin(\alpha) \, \vec{v}^{R}$
\item rearrange to matrix arithmetic to apply transformation to arbitrary vector \\
$\gleichung{
    v' &=& \cos(\alpha) \, \vektor{x \\ y}  + \sin(\alpha) \, \vektor{-y \\ x}
    &=& \vektor{\cos(\alpha) \, x \\ \cos(\alpha) \, y}  + \vektor{ -\sin(\alpha) \, y \\ \sin(\alpha) \, x}
    \\
    &=& \vektor{\cos(\alpha) \, x  -\sin(\alpha) \, y \\ \sin(\alpha) \, x + \cos(\alpha) \, y}
    &=& \underline{\mymat{\cos(\alpha) \, & -\sin(\alpha) \, \\ \sin(\alpha) \, & \cos(\alpha) \,} } \, \vektor{x \\ y}
}$
}

\section{3D Angle-Axis Rotation(Rodrigues' rotation formula)}

\items{
\item Set up parametric plane equation placed orthogonal to rotation axis(normal) and rotate vector along plane. \\
\item Points (x,y) on plane P (${R^3}$): \\ $P(x, y) =  \vec{c} + x \, \vec{a} +  y \, \vec{b}$ with $\vec{a} \bullet \vec{b} = 0$ \\
Circle on plane: $P(\cos(\theta), \sin(\theta)) =  \vec{c} + \cos(\theta) \, \vec{a} +  \sin(\theta) \, \vec{b}$
}

\newcommand\unit{.8}
\newcommand\hgt{.6}
\newcommand\myscale{4}
\begin{tikzpicture}[scale=\myscale, x={({cos(20)},{-sin(20)},0)},z={({-sin(40)},{-cos(40)},0)}]
    \draw[dashed] (-\unit/2,\hgt,\unit/2) -- (-\unit/2,\hgt,-\unit/2) -- (\unit/2,\hgt,-\unit/2) -- (\unit/2,\hgt,\unit/2) -- cycle;
%    \draw[-latex] (0,0,0) -- (0, 1, 0) node[above] {$\hat{n}$};
%    \draw[-latex] (0,0,0) -- (.4, .6, 0) ;%node[right] {$\vec{v}$};
    \draw[-latex] (0, .6, 0) -- (.4, .6, 0) node[midway, above right] {$\vec{a}$};
    \draw[-latex] (0, 0.6, 0) -- (0, 0.6, -.4) node[above right] {$\vec{b}$};
    \draw[-latex] (0, 0, 0) -- (0, .6, 0) node [below left] {$\vec{c}$};
\end{tikzpicture}
\begin{tikzpicture}[scale=\myscale, x={({cos(20)},{-sin(20)},0)},z={({-sin(40)},{-cos(40)},0)}]
%    \draw[-latex, very thick] (0,0,0) -- (0, 1, 0) node[above] {$\hat{n}$};
%    \draw[-latex] (0,0,0) -- (.4, .6, 0);% node[right] {$\vec{v}$};
    \newcommand\rad{.4}
    \draw[fill=green!30, dashed] (\rad, \hgt, 0) \foreach \X in {0,-5,...,-45} { -- ({\rad/2*cos(\X)},\hgt,{\rad/2*sin(\X)}) } --  (0, \hgt, 0);
    \draw[dashed] (\rad, \hgt, 0) \foreach \X in {0,10,...,360} { -- ({\rad*cos(\X)},\hgt,{\rad*sin(\X)}) };

    \draw[-latex] (0, .6, 0) -- (.4, .6, 0) node[below] {$\vec{a}$};
    \draw[-latex] (0, 0.6, 0) -- (0, 0.6, -.4) node[above right] {$\vec{b}$};
    \draw[-latex] (0, 0, 0) -- (0, .6, 0) node [below left] {$\vec{c}$};

    \draw[-latex, dashed] (0, .6, 0) -- ({.4*cos(-45)}, .6, {.4*sin(-45)}) node[above right] {$P(\cos(\theta),\sin(\theta))$};

    \draw[dashed] (0, .6, {.4*sin(-45)}) -- ({.4*cos(-45)}, .6, {.4*sin(-45)});
    \draw[dashed] ({.4*cos(-45)}, .6, 0) -- ({.4*cos(-45)}, .6, {.4*sin(-45)});
    \draw[-latex, thick] (0, .6, 0) -- (0, .6, {.4*sin(-45)}) node[midway, left] {$\sin(\theta) \, \vec{b}$};
    \draw[-latex, thick] (0, .6, 0) -- ({.4*cos(-45)}, .6, 0) node[midway, below] {$\cos(\theta) \, \vec{a}$};

%    \draw[dashed] (0, .6, 0) node {};
%    \draw[dashed] (0, .6, 0)node {};

    \draw (0.15, \hgt ,-0.05) node []{$\theta$};
\end{tikzpicture}

\items{
    \item With $\hat{n}$ rotation axis and normal to the plane of rotation\\
    { \small(the rotation axis vector $\hat{n}$ needs to be normalized, because of the projection operation)}
    \item $\vec{v}$ vector to be rotated around axis vector $\hat{n}$
    \item Construct plane of rotation with rejection(projection) of $\vec{v}$ onto $\hat{n}$ and cross product of $\vec{n}$ with $\hat{v}$
}

\begin{tikzpicture}[scale=\myscale, x={({cos(20)},{-sin(20)},0)},z={({-sin(40)},{-cos(40)},0)}]
    \newcommand\rad{.25}
    \draw[-latex, very thick] (0,0,0) -- (0, 1, 0) node[above] {$\hat{n}$};
    \draw[-latex, very thick] (0,0,0) -- (.4, .6, 0) node[right] {$\vec{v}$};
    \draw (\rad, \hgt, 0) \foreach \X in {0,-5,...,-45} { -- ({\rad*cos(\X)},\hgt,{\rad*sin(\X)}) };
    \draw[dashed] (0, .6, 0) -- (.4, .6, 0);
    \draw[-latex, dashed] (0, .6, 0) -- ({.4*cos(-45)}, .6, {.4*sin(-45)}) node[above] {$\vec{v'}$};
\end{tikzpicture}
\begin{tikzpicture}[scale=\myscale, x={({cos(20)},{-sin(20)},0)},z={({-sin(40)},{-cos(40)},0)}]
    \draw[dashed] (-\unit/2,\hgt,\unit/2) -- (-\unit/2,\hgt,-\unit/2) -- (\unit/2,\hgt,-\unit/2) -- (\unit/2,\hgt,\unit/2) -- cycle;
    \draw[-latex, thick] (0,0,0) -- (0, 1, 0) node[above] {$\hat{n}$};
    \draw[-latex, thick] (0,0,0) -- (.4, .6, 0) node[right] {$\vec{v}$};
    \draw[-latex, dashed] (0, .6, 0) -- (.4, .6, 0) node[midway, above right] {$\vec{a}$};
    \draw[-latex, thick] (0, 0, 0) -- (0, .6, 0) node [below left] {$\vec{c}$};
    \draw[-latex, thick] (0, 0.6, 0) -- (0, 0.6, -.4) node[above right] {$\vec{b}$};
\end{tikzpicture}
\begin{tikzpicture}[scale=\myscale, x={({cos(20)},{-sin(20)},0)},z={({-sin(40)},{-cos(40)},0)}]
    \draw[dashed] (-\unit/2,\hgt,\unit/2) -- (-\unit/2,\hgt,-\unit/2) -- (\unit/2,\hgt,-\unit/2) -- (\unit/2,\hgt,\unit/2) -- cycle;
    \draw[-latex, very thick] (0,0,0) -- (0, 1, 0) node[above] {$\hat{n}$};
    \draw[-latex, very thick] (0,0,0) -- (.4, .6, 0) node[right] {$\vec{v}$};
    \draw[-latex, dashed] (0, .6, 0) -- (.4, .6, 0) node[midway, above right] {$\vec{v}_{\perp}$};
    \draw[-latex,  thick] (0, 0, 0) -- (0, .6, 0) node [below left] {$\vec{v}_{\parallel}$};
    \draw[-latex,  thick] (0, 0.6, 0) -- (0, 0.6, -.4) node[above right] {$\hat{n} \times \vec{v}$};
\end{tikzpicture}
\begin{tikzpicture}[scale=\myscale, x={({cos(20)},{-sin(20)},0)},z={({-sin(40)},{-cos(40)},0)}]
    \draw[-latex, very thick] (0,0,0) -- (0, 1, 0) node[above] {$\hat{n}$};
    \draw[-latex, very thick] (0,0,0) -- (.4, .6, 0) node[right] {$\vec{v}$};
    \draw[-latex, dashed] (0, .6, 0) -- (.4, .6, 0) node[midway, above] {$\cos_\theta \vec{v}_{\perp}$};
    \draw[-latex,  thick] (0, 0, 0) -- (0, .6, 0) node [below left] {$\vec{v}_{\parallel}$};
    \draw[-latex,  thick] (0, 0.6, 0) -- (0, 0.6, -.4) node[above right] {$\sin_\theta \hat{n} \times \vec{v}$};
    \newcommand\rad{.4}
    \draw[dashed] (\rad, \hgt, 0) \foreach \X in {0,10,...,360} { -- ({\rad*cos(\X)},\hgt,{\rad*sin(\X)}) };
\end{tikzpicture}

\items{
\item $\vec{v} = \vec{v_\perp} + \vec{v_\parallel}$
\item $\vec{v}_\parallel$ part of $\vec{v}$ parallel to $\hat{n}$ (Projection of $\vec{v}$ onto $\hat{n}$)\\
      $\vec{c} \rightarrow \vec{v_\parallel} = ( \hat{n} \bullet \vec{v} ) \hat{n}$
\item $\vec{v}_\perp$ part of $\vec{v}$ perpendicular to $\hat{n}$ \\
      $\vec{a} \rightarrow \vec{v_\perp} = \vec{v} - \vec{v_\parallel} = \vec{v} - ( \hat{n}\bullet \vec{v}) \hat{n}$
\item vector in plane perpendicular to $\vec{v}$ and $\hat{n}$\\
      $\vec{b} \rightarrow \hat{n} \times \vec{v} = \hat{n} \times \vec{v}_\perp $ \\
\item $\vec{v'} = \vec{v}_\parallel + \cos_\theta \, \vec{v}_\perp + \sin_\theta \, \hat{n} \times \vec{v}$
}

\subsection{Derivation of Rodrigues Rotation Matrix:}

\items{

\item Express vector arithmetic terms with matrix arthimic equivalents: \\
{ \small(Generally you can convert any vector arithmetic terms by transforming the base vectors of a space or the column vectors of a matrix. In case of the projection of a vector we can rearrange the operation to get it directly) }

\item[]

\item $\mathbf{K}$ is skew symmetric matrix which can used calculate the cross product of $\hat{n}$ with $\vec{v}$: \\
$\mathbf{K} \, \vec{v} = \hat{n} \times \vec{v} = [\hat{n}]_{\times} \, \vec{v}$ \\
$\mathbf{K} = [\hat{n}]_{\times} = \mymat{&&\\ \hat{n}\times\hat{e}_1 & \hat{n}\times\hat{e}_2 & \hat{n}\times\hat{e}_3 \\ && } \\
=
\mymat{
\vektor{ n_y \cdot 0 - n_z \cdot 0 \\ n_z \cdot 1 - n_x \cdot 0 \\ n_x \cdot 0 - n_y \cdot 1 } &
\vektor{ n_y \cdot 0 - n_z \cdot 1 \\ n_z \cdot 0 - n_x \cdot 0 \\ n_x \cdot 1 - n_y \cdot 0 } &
\vektor{ n_y \cdot 1 - n_z \cdot 0 \\ n_z \cdot 0 - n_x \cdot 1 \\ n_x \cdot 0 - n_y \cdot 0 }
}
=
\mymat{ 0 & -n_z & n_y \\ n_z & 0 & -n_x \\ -n_y & n_x & 0 } $

\item[]

\item $\mathbf{P}$ is the matrix which multiplied with projects a vector $\vec{v}$ onto normal $\hat{n}$:\\
$\mathbf{P} \, \vec{v} = (\vec{v} \bullet \hat{n}) \, \hat{n} = (\hat{n} \bullet \vec{v}) \hat{n} = (\hat{n}^T \hat{v})\hat{n} = \hat{n}\hat{n}^T \vec{v} $\\
$\gleichung{
& & \, \mymat { n_x & n_y & n_z } \\
\mathbf{P} = \hat{n} \hat{n}^\textsf{T} =  & \mymat{ n_x \\ n_y \\ n_z } &
  \mymat { n_x^2 & n_x n_y & n_x n_z  \\
    n_x n_y & n_y^2 & n_y n_z  \\
    n_x n_z & n_y n_z & n_z^2  \\ }
}$

\item[]

\item rearrange vector arithmetic rotation formula to matrix arithmetic (and factorize $\vec{v}$) \\
$\gleichung{
    \vec{v'}
      & = \vec{v}_\parallel + \cos(\theta) \, \vec{v}_\perp + \sin(\theta) \, \hat{n} \times \vec{v} &|& \text{def. } \vec{v}_\parallel, \vec{v}_\perp\\
      & =  (\vec{v} \bullet \hat{n} ) \hat{n} + \cos(\theta) \left(\vec{v} - ( \vec{v} \bullet \hat{n} ) \hat{n} \right) + \sin(\theta) \, \hat{n} \times \vec{v} &|& (\vec{v} \bullet \hat{n}) \hat{n} = (\hat{n}^T \hat{v})\hat{n} = \hat{n}\hat{n}^T \vec{v}\\
      & =  \hat{n}\hat{n}^{T} \vec{v} + \cos(\theta) \left(\vec{v} - \hat{n}\hat{n}^{T} \vec{v} \right) + \sin(\theta) \, [\hat{n}]_{\times} \vec{v} &|& \mathbf{P} = \hat{n}\hat{n}^T, \mathbf{K} = [\hat{n}]_\times, \mathbf{I} \, \vec{v} = \vec{v} \\
      & =  \mathbf{P} \, \vec{v} + \cos(\theta) \left(\mathbf{I} \, \vec{v} - \mathbf{P} \, \vec{v} \right) + \sin(\theta) \, \mathbf{K} \, \vec{v} &|& \text{factorize } \vec{v}: \mathbf{A} \vec{v} + \mathbf{B} \vec{v} = [ \mathbf{A} + \mathbf{B} ] \vec{v} \\
      & =  [\, \mathbf{P} + \cos(\theta) \left(\mathbf{I} - \mathbf{P} \right) + \sin(\theta) \, \mathbf{K} \, ] \, \vec{v} \\
      & = \mathbf{R}(\hat{n}, \theta) \, \vec{v}
    }$

\item[]

\item Matrix for rotating arbitrary vector around axis $\hat{n}$ with angle $\theta$: \\
$\mathbf{R}(\hat{n}, \theta) =  \mathbf{P} + \cos(\theta) \left(\mathbf{I} - \mathbf{P} \right) + \sin(\theta) \, \mathbf{K}$ \\

$\mathbf{R}(\hat{n}, \theta) =  \mymat { n_x^2 & n_x n_y & n_x n_z  \\ n_x n_y & n_y^2 & n_y n_z  \\ n_x n_z & n_y n_z & n_z^2  \\ } + \cos(\theta) \left( \mymat{ 1 & 0 & 0 \\ 0 & 1 & 0 \\ 0 & 0 & 1 } - \mymat { n_x^2 & n_x n_y & n_x n_z  \\ n_x n_y & n_y^2 & n_y n_z  \\ n_x n_z & n_y n_z & n_z^2  \\ } \right) + \sin(\theta) \, \mymat{ 0 & -n_z & n_y \\ n_z & 0 & -n_x \\ -n_y & n_x & 0 } $

}

\end{document}

%\draw \foreach \X [remember=\X as \lastX (initially 0)] in {0,10,...,360} { ({\unit*cos(\lastX)},0,{\unit*sin(\lastX)}) -- ({\unit*cos(\X)},0,{\unit*sin(\X)}) };


\section{Objects}
We now start with the characterization of objects in our system. The $Object$ predicate says that an object will be a record which supports methods and inheritance; of course all the object operators are expected to work in conjunction with the rest of the system. Since objects will reference themselves, to avoid recursive type definitions we give a predicate that allows us to freely add or remove some constructor from an object:
\begin{lstlisting}
class Recursive o where
  type Rec o :: *
  to :: o -> Rec o
  from :: Rec o -> o
\end{lstlisting}

An object is required to support the $Recursive$ predicate:
\begin{lstlisting}
class (Record o st s, Recursive o, ro~Rec o) => Object o st s where
  type Base o :: *
  get_base :: Ref st s o -> Ref st s (Base o)

  type Method ro st :: * -> * -> *
  (<=|) :: Ref st s r -> Label r (Method ro st a b) -> a -> st s b
\end{lstlisting}

With respect to inheritance, it looks clear how we can instance coercion to take advantage (and make access more uniform) of the $base$ operations:
\begin{lstlisting}
instance Object o st s => Coercible (Ref st s o) (Ref st s (Base o)) where
  coerce = get_base
\end{lstlisting}

We also add a further predicate that characterizes inheritance:
\begin{lstlisting}
class Inherits a b

instance (Object o st s) => Inherits o (Base o)
\end{lstlisting}

Notice that methods are defined with a different operator than the selection operator for records. This can be addressed as follows: we define a new predicate for selecting something from a reference through a label:
\begin{lstlisting}
class Selectable st t s a where
  type Selection s t a :: *
  (<=) :: Ref st s t -> Label t a -> Selection s t a
\end{lstlisting}

We instance this predicate twice, one for field selection and one for method selection. Before doing so, though, we must be careful to disambiguate the last parameter in the record definition. For this purpose we add a $Field$ type function that represents a placeholder type that will distinguish fields from methods and inherited types:
\begin{lstlisting}
class (RefSt st s) => Record r st s where
  type Label r :: * -> *
  type FieldSlot a :: *
  (<==) :: Ref st s r -> Label r (FieldSlot a) -> Ref st s a
\end{lstlisting}

In the case of simple mutable records we can easily add a trivial constructor for $FieldSlot$ such as:
\begin{lstlisting}
data Field a = Field a
\end{lstlisting}

Now we can instance the selection predicate:
\begin{lstlisting}
instance Record st r s => Selectable st r s (FieldSlot a) where
  type Selection s r (Field a) = Ref st s a
  (<=) = (<==)

instance (Object st o s, ro~Rec o) => Selectable st o s (Method ro a b) where
  type Selection s o (Method ro a b) = a -> st s b
  (<=) = (<=|)
\end{lstlisting}

The final operation we wish to support is that of selecting a method or a field directly from any of the inherited classes of an object without explicit coercions:
\begin{lstlisting}
instance (Object st o s, bo~Base o, Selectable st bo s a) => Selectable st o s a where
  type Selection s o a = Selection s bo a
  (<=) = get_base . (<=)
\end{lstlisting}

Before Giuseppe's PhD/Monads/ObjectiveMonad/MonadicObjects/trunk/tex v2/6.objects.tex


\documentclass[colorlinks=true,pdfstartview=FitV,linkcolor=blue,
            citecolor=red,urlcolor=magenta]{ligodoc}

\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{longtable}
\usepackage{rotating}
\usepackage[usenames,dvipsnames]{color}
\usepackage{fancyhdr}
\usepackage{subfigure}
\usepackage{hyperref}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows,fit}
\ligodccnumber{T}{19}{00287}{}{v1}% \ligodistribution{AIC, ISC}

\title{Data Clustering Techniques for the Correlation of Environmental Noise to Signals in LIGO Detectors}

\author{Jacob Bernhardt, Anamaria Effler, Rana Adhikari}

\begin{document}

\input{include/motivation}

\section{Interim Report 1}

In the first three weeks of the project, clustering scripts and BLRMS-generating scripts were developed and tested.
Testing the clustering code on a dataset with known sources of noise was thought to be a good first-order check of its efficacy.

\subsection{$k$-means clustering with histories}
The $k$-means algorithm was used to cluster the two hours of minute-trend seismic BLRMS preceding each point in time.
Thus, each coordinate in the clustering sub-space for a channel was as follows:
\begin{equation}
  \left\{s(t_0),s(t_{-1}),s(t_{-2})\cdots,s(t_{-n})\right\}
\end{equation}
with $s(t)$ the seismometer velocity at time $t$.
Each dimension corresponded to ``channel value a specific number of minutes ago''.
This allowed trends over time to be matched together in a phase-agnostic way.

\begin{figure}[h]
  \tikzstyle{block} = [rectangle, draw, text width=6em, text centered, rounded corners, minimum height=4em]
  \begin{tikzpicture}[node distance = 9em, auto]
    \node [block] (dl) {get minute-trend data from NDS};
    \node [block, right of=dl] (input) {create input matrix};
    \node [block, right of=input] (compute) {compute $k$-means clusters};
    \node [block, right of=compute] (save) {save labels};
    \draw [->] (dl) -- (input);
    \draw [->] (input) -- (compute);
    \draw [->] (compute) -- (save);
  \end{tikzpicture}
  \caption{Clustering script flowchart.}
\end{figure}

For a total clustering duration of 30 days, using the seismometers attached to ETMY, ETMX, and ITMY, in minute-trend half-order-of-magnitude BLRMS bands from 30 mHz to 30 Hz, the following known noise events were easily identified using a ``2-hour history'' $k$-means method (see Figures~\ref{fig:eq}-\ref{fig:anth}):
\begin{itemize}
\item earthquakes ($0.01\to0.1$ Hz)
\item microseisms ($0.1\to1$ Hz)
\item anthropogenic noise ($1\to10$ Hz)
\end{itemize}

Some differentiation between subcategories of events in the same frequency band but of different timescales (e.g. earthquakes vs. wind; train vs. noise from cars) was lacking.

The length of the history was initially thought to have an effect on the timescales of identifiable events; experimentation (namely, trying 30-minute and 6-hour histories on the same data) showed that this is not really true.

The next idea was that there were too many different types of features in too large a space for events with a small number of points, like the trains, to be separated out.
The test was re-run with only anthropogenic seismic BLRMS bands at the end stations, which yielded very clear distinction between the anthropogenic noise types (Figure~\ref{fig:trains}).

A test\footnote{\url{https://scikit-learn.org/stable/auto_examples/cluster/plot_cluster_comparison.html}} which swaps out the $k$-means algorithm for others implemented in \texttt{sklearn} is being executed to probe the geometry of the clusters.
At the time of submission, the job has not completed.

\newcommand\rightfigures[2]{
  \begin{figure}
  \begin{minipage}[t]{0.3\textwidth}
    #2
  \end{minipage}\hfill
  \begin{minipage}[c]{0.67\textwidth}
    \begin{tabular}{c}
      #1
    \end{tabular}
  \end{minipage}
\end{figure}
}

\newcommand\leftfigures[2]{
\begin{figure}
  \begin{minipage}[c]{0.67\textwidth}
    \begin{tabular}{c}
      #1
    \end{tabular}
  \end{minipage}\hfill
  \begin{minipage}[t]{0.3\textwidth}
    #2
  \end{minipage}
\end{figure}
}

\rightfigures
{
  \includegraphics[width=\textwidth]{assets/report1/earthquakes-day2_075-L1:ISI-GND_STS_ITMY_BLRMS_30M_100Mmean,m-trend.png}\\
  \includegraphics[width=\textwidth]{assets/report1/earthquakes-day1475_05-L1:ISI-GND_STS_ITMY_BLRMS_30M_100Mmean,m-trend.png}\\
  \includegraphics[width=\textwidth]{assets/report1/earthquakes-day16_1-L1:ISI-GND_STS_ITMY_BLRMS_30M_100Mmean,m-trend.png}\\
  \includegraphics[width=\textwidth]{assets/report1/earthquakes-day24_075-L1:ISI-GND_STS_ITMY_BLRMS_30M_100Mmean,m-trend.png}
}{
  \caption{Examples of earthquakes. These vary most of all seismic noise events, so most of the extra, unrequired clusters tend to pick details out in these. For instance, two different kinds of earthquakes were clustered (here orange and purple predominantly).}\label{fig:eq}
}

\leftfigures
{
  \includegraphics[width=\textwidth]{assets/report1/microseism-day4_4-L1:ISI-GND_STS_ITMY_BLRMS_100M_300Mmean,m-trend.png}\\
  \includegraphics[width=\textwidth]{assets/report1/microseism-day8_4-L1:ISI-GND_STS_ITMY_BLRMS_100M_300Mmean,m-trend.png}\\
  \includegraphics[width=\textwidth]{assets/report1/microseism-day22_4-L1:ISI-GND_STS_ITMY_BLRMS_100M_300Mmean,m-trend.png}
}{
  \caption{Examples of microseisms identified by the $k$-means algorithm. Because the algorithm was run with more clusters than needed, several clusters are assigned to this class of event. Running with less clusters fixes this issue, but details are sometimes missed.}
}

\rightfigures
{
  \includegraphics[width=\textwidth]{assets/report1/daynight-day3_3-L1:ISI-GND_STS_ITMY_BLRMS_1_3mean,m-trend.png}\\
  \includegraphics[width=\textwidth]{assets/report1/daynight-day12_9-L1:ISI-GND_STS_ITMY_BLRMS_1_3mean,m-trend.png}
}{
  \caption{Day-night anthropogenic noise variation is identified by the $k$-means algorithm. These clusters are depicted gray.}
}

\leftfigures
{
  \includegraphics[width=\textwidth]{assets/report1/trains-day8_1-L1:ISI-GND_STS_ITMY_BLRMS_1_3mean,m-trend.png}\\
  \includegraphics[width=\textwidth]{assets/report1/30m-trains-day8_1-L1:ISI-GND_STS_ITMY_BLRMS_1_3mean,m-trend.png}
}{
  \caption{Trains are clustered with the day/night anthropogenic noise. Shortening the history window from 2 hours (upper) to 30 minutes (lower)  helps to clarify this.}\label{fig:anth}
}

\rightfigures
{
  \includegraphics[width=\textwidth]{assets/report1/trains-day8_1-L1:ISI-GND_STS_ETMY_BLRMS_1_3mean,m-trend.png}\\
  \includegraphics[width=\textwidth]{assets/report1/trains-day8_3-L1:ISI-GND_STS_ETMY_BLRMS_1_3mean,m-trend.png}
}{
  \caption{Here, the clustering space was reduced to only the anthropogenic noise bands. This enabled identification of trains (red) separate from day/night noise (blue).}\label{fig:trains}
}

\subsection{BLRMS-generating script}
A script to generate minute-trend BLRMS from raw PEM channels was created. This will enable BLRMS clustering for those channels not already in BLRMS frames.

Some thought was put into this script.
It is designed to post-process a channel in real time, using strides and HDF5 appending to provide a ``streaming mode''.
The BLRMS-generating function is an implementation of a general post-processing interface, a \texttt{python3.7} dataclass which is fed INI options upon construction (see Figure~\ref{fig:pp}).
Any data processing function which maps an input channel to an output channel and has tons of configuration parameters can take advantage of the streaming mode developed in this script by implementing the \texttt{PostProcessor} interface.

\begin{figure}[h]
  \tikzstyle{block} = [rectangle, draw, text width=6em, text centered, rounded corners, minimum height=4em]
  \tikzstyle{section} =[rectangle, draw, inner sep=1.125em, dashed]
  \begin{tikzpicture}[node distance = 9em, auto]
    \node [block] (dl) {get raw data from NDS};
    \node [block, below of=dl] (strides) {compute strides};
    \node [block, right of=strides] (next) {get next stride};
    \node [block, below of=next] (stride) {extract stride};
    \node [block, right of=stride] (compute) {compute spectogram};
    \node [block, right of=compute] (add) {add power within requested bands};
    \node [block, right of=add] (save) {stitch stride into .hdf5 file};
    \node [block, below of=add] (shape) {read .hdf5 dataset shape};
    \node [block, below of=save] (offset) {compute offset};
    \node [section, label=below:BLRMS \texttt{PostProcessor}, fit=(add) (compute)] (blrms) {};
    \node [block, above of=blrms, node distance = 18em, dashed] (load) {load INI section};
    \draw [->] (dl) -- (strides);
    \draw [->] (strides) -- (next);
    \draw [->] (next) -- (stride);
    \draw [->] (stride) -- (compute);
    \draw [->] (compute) -- (add);
    \draw [->] (add) -- (save);
    \draw [->] (stride) |- (shape);
    \draw [->] (shape) -- (offset);
    \draw [->] (offset) -- (save);
    \draw [->] (save) |- (next);
    \draw [->, dashed] (load) -- (blrms);
  \end{tikzpicture}
  \caption{States of the ``streaming post-processor'' script.}\label{fig:pp}
\end{figure}

Considerable effort was expended to keep both the BLRMS and clustering scripts \texttt{GWpy}-compliant.
Enabling HDF5 appending in \texttt{GWpy} savefiles and modifying \texttt{GWpy} plots seem like common tasks for users of \texttt{GWpy}, but they needed some hackery and keyword-argument abuse (i.e. making use of possibly unintended library behavior) to work properly.

\subsection{Next Steps}
The next step is to cluster with accelerometer, microphone, and DARM BLRMS.
There are a number of strategies that will be employed in light of the past few weeks' results.

For instance, the clustering algorithm appears to gravitate toward differences across a channel parameter that is more varied in the input.
The initial run with 6 BLRMS bands and 3 sensors distinguished between noise in different bands more than noise in the same band but in different sensors (i.e. trains vs. day/night noise).
When the number of sensors was similar to the number of bands, the sensor-specific noise was more clearly parsed out.

For unknown noise, it will thus be important to run multiple tests on the same data which focus on each varying parameter.
That is, it may be best to cluster all sensors in one band, and all bands of one sensor, separately, rather than doing it all together.

In addition, much will rest on appropriate BLRMS band selection.
Instead of simply using half-order-of-magnitude spacing, it may be worthwhile to make a program that searches for time variance in the channel by frequency, so as to pick the most varying, and possibly charactaristic, bands for the channel.
In his BLRMS implementation, Vajente includes hard-coded DARM bands, so it may be smart to stick with those for DARM\cite{vajente}.

After experiencing the most naive presentation of the clusters, namely, time series plots with colored points, it occurs to us that there should be a way to programmatically identify the pattern that characterizes each cluster, to enable looking into mechanisms and solutions for the noise.
This will probably involve getting the frequency of occurrence of the cluster, as well as its average dimensions (in time and amplitude), and ratios of presence in bands and sensors.

\input{include/bibliography.tex}

\end{document}


\documentclass[12pt]{article}
%Sets size of page and margins
\textheight 21cm
\textwidth 15cm

\title{Manual for CONLAW versions 1,2,3,4}
\author{Thomas Wolf \\
        Department of Mathematics \\
        Brock University \\
        St.Catharines \\
        Ontario, Canada L2S 3A1 \\
        twolf@brocku.ca}

\begin{document}
\maketitle
\section{Purpose}
The procedures {\tt CONLAW1, CONLAW2, CONLAW3, CONLAW4}
try to find conservation laws
for a given single/system of differential equation(s) (ODEs or PDEs)
\begin{equation}
     u^{\alpha}_J = w^{\alpha}(x,u^{\beta},...,u^{\beta}_K,...)  \label{a1}
\end{equation}
{\tt CONLAW1} tries to find the conserved current $P^i$ by solving
\begin{equation}
   \mbox{Div}\;\,P = 0  \;\;\; \mbox{modulo}\;\; (\ref{a1})
\label{a2}
\end{equation}
directly. {\tt CONLAW3} tries to find $P^i$ and the characteristic
functions (integrating factors) $Q^{\nu}$ by solving
\begin{equation}
\mbox{Div}\,P = \sum_{\nu} Q^{\nu}\cdot (u^{\nu}_J - w^{\nu}) \label{a3}
\end{equation}
identically in all $u$-derivatives.
Applying the Euler operator (variational derivative) for each $u^{\nu}$
on (\ref{a3}) gives a zero left hand side and therefore
conditions involving only $Q^{\nu}$. {\tt CONLAW4} tries to solve these
conditions identically in all $u$-derivatives and to compute $P^i$
afterwards. {\tt CONLAW2}
does substitutions based on (\ref{a1}) before solving these
conditions on $Q^{\nu}$
and therefore computes adjoined symmetries. These are
completed, if possible, to conservation laws by computing $P^i$ from
the $Q^{\nu}$.

All four procedures have the same syntax. They have two parameters,
both are lists. The first parameter specifies the equations (\ref{a1}),
the second specifies the computation to be done. One can either specify an
ansatz for $P^i, Q^{\nu}$ or investigate a general situation, only specifying
the order of the characteristic functions or the conserved current. For a
more detailed description see the file {\tt conca.tex}.

The file {\tt CONLAW0.RED} contains subroutines used in all four versions.

\section{The Syntax}
The procedures {\tt CONLAWi} $\;\;$ i=1,2,3,4 are called through \\
{\tt CONLAWi({\it problem,runmode}); } \\
where i=1,2,3,4. Both parameters {\it problem, runmode} are lists.
The first specifies the DEs to be investigated: \\
{\it problem} ... \{{\it equations, ulist, xlist}\}
\begin{tabbing}
\hspace{0.5cm}
 {\it equations}\= ... \= list of equations,
              each has the form {\tt df(ui,..)=...} where \\
       \>  \> the LHS (left hand side) {\tt df(ui,..)} is selected such that \\
       \>  \>  - The RHS (right h.s.) of an equations must not include     \\
       \>  \>$\;\;$ the derivative on the LHS nor a derivative of it.  \\
       \>  \>  - The LHS of any equation should not occur in any other\\
       \>  \>$\;\;$ equation nor any derivative of the LHS.  \\
       \>  \> If CONLAW3 or CONLAW4 are run where no substitutions\\
       \>  \> are made then the LHS of equations can be {\tt df(ui,..)**n=...}\\
       \>  \> where {\tt n} is a number. No difference is made between\\
       \>  \> equations and constraints. \\
\hspace{0.5cm}
 {\it ulist} \>  ... \> list of function names, which can be chosen freely \\
             \>      \> the number of functions and equations need not
be equal \\
\hspace{0.5cm}
 {\it xlist}  \>  ... \> list of variable names, which can be chosen freely
\end{tabbing}
The second parameter specifies the calculation to be done.  \\
{\it runmode} ... \{{\it minord, maxord, expl, flist, inequ}\}
\begin{tabbing}
\hspace{0.5cm}
     {\it minord} \= ... \= the minimum of the highest order of
                          derivatives of u \\
             \> \>        - in {\tt p\_t} for CONLAW1 where {\tt t} is the first variable in\\
             \> \>        \ \  {\tt xlist} and \\
             \> \>        - in {\tt q\_j} for CONLAW2,CONLAW3,CONLAW4  \\
\hspace{0.5cm}
     {\it maxord} \> ... \> the maximum of the highest order of
                          derivatives of u \\
             \> \>        - in {\tt p\_i} for CONLAW1 where {\tt t} is the first variable in\\
             \> \>        \ \  {\tt xlist} and  \\
             \> \>        - in {\tt q\_j} for CONLAW2,CONLAW3,CONLAW4 \\
\hspace{0.5cm}
     {\it expl}       \> ... \> (t/nil) whether or not the charac.\
                          functions {\tt q\_i} or conserved \\
             \> \>        current may depend explicitly on
                          the variables of xlist \\
\hspace{0.5cm}
     {\it flist}      \> ... \> a list of unknown functions in any ansatz for
                          {\tt p\_i,q\_j}, \\
        \>  \>            also all parameters and parametric functions in
                          the equation \\
        \>  \>            that are to be calculated such that
                          conservation laws exist,\\
        \>  \>            if there are no such unknown functions then
                          {\it flist} is the \\
        \>  \>            empty list: \verb+{}+ \\
\hspace{0.5cm}
     {\it inequ}      \> ... \> a list of expressions non of which
                          may be identically \\
        \>  \>            zero for the conservation
                          law to be found, if there is no such \\
        \>  \>            expression
                          then {\it inequ} is an empty list: \verb+{}+
\end{tabbing}
The procedures {\tt CONLAWi} return a list of conservation laws
\verb+ {+$C_1,C_2,\ldots$\verb+}+, if no
non-trivial conservation law is found they return the empty list \verb+{}+.
Each $C_i$ representing a conservation law
has the form  \verb+ {{+$P^1,P^2,\ldots$\verb+},{+$Q^1,Q^2,\ldots$\verb+}}+.

An ansatz for a conservation law can be formulated by specifying one
or more of the components $P^i$ for {\tt CONLAW1}, one or more of the
functions $Q^{\mu}$ for {\tt CONLAW2, CONLAW4} or one or more of
$P^i, Q^{\mu}$ for {\tt CONLAW3}. The $P^i$ are input as {\tt p\_i} where
{\tt i} in {\tt p\_i} stands for a variable name, and the $Q^{\mu}$ are
input as {\tt q\_i} where {\tt i} stands for an index - the number of the
equation in the input list {\it equations} with which {\tt q\_i} is
multiplied.

There is a restriction in the structure of all the expressions for
{\tt p\_i, q\_j} that are specified: they must be homogeneous linear
in the unknown functions or constants which appear in these expressions.
The reason for this restriction is not
for CRACK to be able to solve the resulting overdetermined system but for
{\tt CONLAWi} to be able afterwards to extract the individual conservation
laws from the general solution of the determining conditions.

All such unknown functions and constants must be
listed in {\it flist} (see above). The dependencies of such functions
must be defined before calling {\tt CONLAWi}. This is done with the
command {\tt DEPEND}, e.g. {\tt DEPEND f,t,x,u\$ } to specify $f$ as a
function of $t,x,u$. If one wants to have $f$ as a function of derivatives
of $u(t,x)$, say $f$ depending on $u_{txx}$, then one can {\it not} write \\
\verb+    DEPEND f,df(u,t,x,2)$ + \\
but instead \\
\verb+    DEPEND f,u!`1!`2!`2$ + \\
if {\it xlist} has been specified as \verb+ {t,x}+, because {\tt t} is the
first variable and {\tt x} is the second variable in {\it xlist} and {\tt u}
is differentiated ones wrt.\ {\tt t} and two times wrt.\ {x} we therefore get
\verb+ u!`1!`2!`2+. The character {\tt !} is the exempt character to allow
special characters like ` to occur in an identifier name.

It is possible to add extra conditions like PDEs for $P^i, Q^{\mu}$ as a list
{\tt cl\_condi} of expressions that shall vanish. \hspace{6pt}
\noindent Remarks:
\begin{itemize}
\item The input to {\tt CONLAW1, CONLAW2, CONLAW3, CONLAW4} is the
same apart from:
\begin{itemize}
\item an ansatz for $Q^{\nu}$ is ignored in {\tt CONLAW1}
\item an ansatz for $P^i$ is ignored in {\tt CONLAW2, CONLAW4}
\item the meaning of mindensord, maxdensord is different in {\tt CONLAW1}
on one hand and {\tt CONLAW2,CONLAW3,CONLAW4} on the other (see above).
\end{itemize}

\item
It matters how the differential equations are input, i.e. which
derivatives are eliminated. For example, the Korteweg - de Vries
equation if input in the form $u_{xxx}=-uu_x-u_t$ instead of
$u_t=-uu_x-u_{xxx}$
in {\tt CONLAW1} and choosing {\tt maxdensord}=1 then
$P^i$ will be of at most first order, Div $P$ of second order and $u_{xxx}$
will not be substituted and no non-trival conservation laws can be found.
This does not mean that one will not find low order conservation laws at
all with the substitution $u_{xxx}$ one only has to go to
{\tt maxdensord}=2 with longer computations as a consequence compared to the
input $u_t=-uu_x-u_{xxx}$ where {\tt maxdensord}=0 is enough to find
non-trivial conservation laws.

\item The drawback of using $u_t=\ldots$ compared with $u_{xxx}=\ldots$
is that when seeking all conservation laws up to some order
then one has to investigate a higher order ansatz, because with each
substitution $u_t=-u_{xxx}+\ldots$ the order increases by 2.
For example, if all conservation laws of order up to two in $Q^{\nu}$
are to be determined then in order to include a $u_{tt}$-dependence the
dependence of $Q^{\nu}$ on $u_x$ up to $u_{6x}$ has to be considered.

\item Although for any equivalence class of conserved currents $P^i$
differing only by a curl, there exist characteristic functions $Q^{\mu}$,
this need not be true for any particular $P^i$.
Therefore one cannot specify a known density $P^i$ for {\tt CONLAW3}
and hope to calculate the remaining $P^j$ and the corresponding $Q^{\mu}$
with {\tt CONLAW3}. What one
can do is to use {\tt CONLAW1} to calculate the remaining components
$P^j$ and from this a trivial conserved
density $R$ and characteristic functions $Q^{\nu}$ are computed such that
\[ \mbox{Div}\,(P-R) = \sum_{\nu} Q^{\nu}\cdot (u^{\nu}_J - w^{\nu}). \]

\item The $Q^{\mu}$ are uniquely determined only modulo $\Delta=0$.
If one makes an ansatz for $Q^{\mu}$ then this freedom should be removed
by having
the $Q^{\mu}$ independent of the LHS's of the equations and independent of
derivatives of the LHS's of them. If the $Q^{\mu}$ were allowed to depend on
anything, then (\ref{a3}) could simply be solved for one $Q^{\nu}$ in terms of
arbitrary $P^j$ and other arbitrary $Q^{\rho}$, giving $Q^{\nu}$ that are
singular for solutions of the differential equations as the expression
of the differential equation would appear in the denominator of $Q^{\nu}$.

\item Any ansatz for $P^i,Q^{\nu}$ should
as well be independent of the LHS's of the equations (\ref{a1}) and
independent of derivatives of the LHS's of (\ref{a1}).

\item If in equation (\ref{a3}) the right hand side is of order $m$ then
from the conserved current $P^i$ a trivial conserved current can be
subtracted such that the remaining conserved current is at most of order $m$.
If the right hand side is linear in the highest derivatives of order
$m$ then subtraction of a trivial conserved current can even achieve a
conserved current of order $m-1$. The relevance of this result is that
if the right hand side is known to be linear in the highest derivatives
then for $P^i$ an ansatz of order $m-1$ is only necessary. To take
advantage of this relation
if the right hand side is known to be linear in the highest
derivatives, a flag {\tt quasilin\_rhs} can be set to {\tt t} (see
below).

\end{itemize}

\section{Flags, parameters}

\verb+    LISP (PRINT_:= NIL/0/1/ ...)$+ \\
\verb+print_=nil+ suppresses all CRACK output, for \verb+print_=n+
($n$ an integer)
{\tt CRACK} prints only equations with at most $n$ factors in its terms.

%\verb+ LISP (LOGOPRINT_:=t/nil)$+ \\
%to print/not print a logo at the start of {\tt CONLAWi}

\verb+ CRACKHELP()$+ \\
to show other flags controling the solution of the overdetermined PDE-system,

\verb+ OFF BATCH_MODE$+ \\
to solve the system of conditions with {\tt CRACK} interactively.

\verb+ LISP(QUASILIN_RHS:=T)$+ \\
reduces in the ansatz for $P^i$ the order to $m-1$ if the order of the
right hand side is $m$. This can be used to speed up computations if
the right hand side is known to be linear in the highest derivatives
(see the note above).

\section{Requirements}

{\tt REDUCE 3.6} and
the files {\tt CRACK.RED, CONLAW0.RED}, one of the files
{\tt CONLAW1.RED, CONLAW2.RED, CONLAW3.RED, CONLAW4.RED} depending which
program should be used and all files {\tt CR*.RED} which
are read in from {\tt CRACK.RED}.

One either has to read in files with \\
\verb+ IN "crack.red","conlaw0.red","conlaw1.red"$+ \\
\noindent (and appropriate paths) or compile them before with

\begin{verbatim}
    FASLOUT "crack"$
    IN "crack.red"$
    FASLEND$
    FASLOUT "conlaw0"$
    IN "conlaw0.red"$
    FASLEND$
    FASLOUT "conlaw1"$
    IN "conlaw1.red"$
    FASLEND$
    BYE$
\end{verbatim}
and load them afterwards with \verb+ LOAD crack,conlaw0,conlaw1$+ \\
\verb+conlaw2, conlaw3, conlaw4+ are treated like {\tt conlaw1}.

\section{Examples}
Below a {\tt CRACK}-procedure {\tt nodepnd} is used to clean up
after each run and delete all dependencies of each
function in the list of functions in the argument of {\tt nodepnd}.
More details concerning these examples are given when running
the file {\tt conlaw.tst}.

\verb+ lisp(print_:=nil);  +   to suppress output from CRACK
\begin{itemize}
\item a single PDE:
\begin{verbatim}
   depend u,x,t$
   conlaw1({{df(u,t)=-u*df(u,x)-df(u,x,3)}, {u}, {t,x}},
           {0, 1, t, {}, {}})$
   nodepnd {u}$
\end{verbatim}

\item a system of equations:
\begin{verbatim}
   depend u,x,t$
   depend v,x,t$
   conlaw1({{df(u,t)=df(u,x,3)+6*u*df(u,x)+2*v*df(v,x),
             df(v,t)=2*df(u,x)*v+2*u*df(v,x)           },
            {u,v}, {t,x}},
           {0, 1, t, {}, {}})$
   nodepnd {u,v}$
\end{verbatim}

\item a system of equations with ansatz:
\begin{verbatim}
   depend u,x,t$
   depend v,x,t$
   depend r,t,x,u,v,u!`2,v!`2$
   q_1:=r*df(u,x,2)$
   conlaw2({{df(u,t)=df(v,x),
             df(v,t)=df(u,x) }, {u,v}, {t,x}},
           {2, 2, t, {r}, {r}})$
   nodepnd {u,v,r}$
\end{verbatim}

\item for the
determination of parameters, such that conservation laws exist:
\begin{verbatim}
   depend u,x,t;
   conlaw1({{df(u,t)=-df(u,x,5)-a*u**2*df(u,x)
                     -b*df(u,x)*df(u,x,2)-c*u*df(u,x,3)},
            {u}, {t,x}},
           {0, 1, t, {a,b,c}, {}});
   nodepnd {u};
\end{verbatim}

\item for first integrals of an ODE-system including the determination of
parameter values s,b,r such that conservation laws exist:
\begin{verbatim}
   depend {x,y,z},t;
   depend a1,x,t;
   depend a2,y,t;
   depend a3,z,t;

   p_t:=a1+a2+a3;
   conlaw2({{df(x,t) = - s*x + s*y,
             df(y,t) = x*z + r*x - y,
             df(z,t) = x*y - b*z},
            {x,y,z},{t}
           },
           {0,0,t,{a1,a2,a3,s,r,b},{}});
   nodepnd {x,y,z,a1,a2,a3};
\end{verbatim}

\end{itemize}

\end{document}