\subsection{Fuel salt reprocessing system}

\begin{frame}
  \frametitle{Fuel salt reprocessing system overview: gas separation}
  Gaseous fission products (e.g., Xe, Kr) must be removed from the fuel salt
  to avoid reactor poisoning ($\sigma_{a,^{135}Xe}=10^6\dots10^7$b).

      \begin{columns}
      	\column[t]{4.0cm}
    \begin{block}{Noble gas removal}
      \begin{enumerate}
      	\item[\textcolor{blue}{\textbullet}] bubble generator injects He
      	bubbles in the salt stream
      	\item[\textcolor{green}{\textbullet}] noble gases migrate to the He
      	bubbles
      	\item[\textcolor{red}{\textbullet}] gas separator discharges the
      	poison-rich bubbles
      \end{enumerate}
    \end{block}

     	\column[t]{8cm}
  \begin{figure}[t]
	  \centering
	  		\vspace{-4mm}
		\includegraphics[width=1.03\textwidth]{./images/msbr_gas_separation.pdf}
	\caption{Schematic flow diagram of the \gls{MSBR} gas separation system
	(figure reproduced from Robertson \emph{et al.}
	\cite{robertson_conceptual_1971}).}
    \end{figure}

	\end{columns}
\end{frame}

\begin{frame}
  \frametitle{Mathematical model for gas separation efficiency}
  		\vspace{-1mm}
Xenon removal efficiency ($\epsilon_{Xe}$) in a gas separation system is
\cite{peebles_removal_1968, sada_gas-liquid_1987}:
\begin{align}
& \qquad\qquad \epsilon_{Xe} = \frac{1-e^{-\beta}}{1+\alpha} \nonumber \\
\alpha &= \frac{RTQ_{L}}{HQ_{G}} \nonumber \\
\beta &= K_L \frac{6}{d_b} \frac{Q_G}{Q_G+G_L} \frac{A_C L (1+\alpha)}{Q_{L}}
\nonumber \\
Q_{L}&= \mbox{volumetric salt flow rate [$m^3/s$]} \nonumber \\
Q_{G}&= \mbox{volumetric helium flow rate [$m^3/s$]} \nonumber \\
H &= \mbox{Henry's law constant [$Pa\cdot mol^{-1}\cdot L$]} \nonumber \\
d_b &= \mbox{helium bubble diameter [m]} \nonumber \\
K_L &= \mbox{liquid phase mass transfer coefficient [m/s].} \nonumber
\end{align}
		\vspace{-5mm}
  \begin{figure}[t]
	\includegraphics[width=0.77\textwidth]{./images/pipeline_contactor.png}
	\vspace{-2mm}
	\caption{Flow diagram for gas separator (figure reproduced from Peebles
		\emph{et al.} \cite{peebles_removal_1968}).}
\end{figure}

\end{frame}

\begin{frame}
\frametitle{Fuel processing system overview: TAP concept}
\begin{textblock*}{12.4cm}(0.25cm,1.7cm) % {block width} (coords)
\begin{figure}[htp!] % replace 't' with 'b' to
	\begin{columns}
		\column{0.65\linewidth}
			\hspace{+3mm}
		\includegraphics[height=0.88\textheight]{../dissertation/figures/ch4/tap_primary_loop.png}

		\column{0.3\linewidth}
		\caption{Simplified \gls{TAP} primary loop design including off-gas
		system
		(blue), nickel filter (orange) and liquid metal extraction system
		(green) \cite{transatomic_power_transatomic_2019}.}
	\end{columns}
\end{figure}
\end{textblock*}
\end{frame}

\subsection{SaltProc tool design}

\begin{frame}
\frametitle{SaltProc class architecture}
	\begin{itemize}
		\item \textit{Simulation} class
			\begin{itemize}
				\item Manages simulation process
				\item Stores data into the HDF5 database
				\item Tracks time, power level
			\end{itemize}
		\item \textit{Depcode} class
			\begin{itemize}
				\item Contains attributes and methods for reading user's input
				\item Creates input files for depletion code
				\item Parses depletion code output
			\end{itemize}
		\item \textit{Process} class
			\begin{itemize}
				\item Represents fuel processing system component
				\item Contains attributes of the component ($\vec{\epsilon}$,
				throughput rate)
				\item Tracks waste stream
			\end{itemize}
		\item \textit{MaterialFlow} class
			\begin{itemize}
				\item Instances of that class represents the material flowing between processes
			\end{itemize}
	\end{itemize}
		\vspace{1mm}
	\begin{figure}[ht!] % replace 't' with 'b' to
		\centering
		\begin{overprint}
		\onslide<1>\centerline{\includegraphics[width=0.6\textwidth]{../dissertation/figures/ch2/materialflow.pdf}}
		\onslide<2>\centerline{\includegraphics[width=0.6\textwidth]{../dissertation/figures/ch2/tap_materialflow.pdf}}
		\end{overprint}
		\vspace{-2mm}
		\caption{Schematic for passing material data between fuel processing
		system components.}
	\end{figure}

\end{frame}

\begin{frame}
\frametitle{SaltProc flowchart}
\begin{textblock*}{12.4cm}(0.07cm,1.7cm) % {block width} (coords)
\begin{figure}[ht!] % replace 't' with 'b' to \centering
	\centering
	\includegraphics[width=\textwidth]{../dissertation/figures/ch2/saltproc_flowchart.pdf}
		\vspace{-4mm}
	\caption{SaltProc v1.0 Python package flowchart with example of object
	instances.}
\end{figure}
\end{textblock*}

\end{frame}

\begin{frame}
\frametitle{Multi-component fuel reprocessing system model in SaltProc}

\begin{figure}[htp!] % replace 't' with 'b' to
	\centering
	\vspace{-2mm}
	\begin{overprint}
	\onslide<1>\includegraphics[height=0.85\textheight]{./images/tap_saltproc_var_eps.png}
		\vspace{-2mm}
    \caption{\textcolor{cyan}{\gls{TAP}} reprocessing scheme for
	SaltProc demonstration.}
	\onslide<2>\includegraphics[height=0.85\textheight]{./images/msbr_saltproc_var_eps.png}
		\vspace{-2mm}
	\caption{\textcolor{red}{\gls{MSBR}} reprocessing scheme for
	SaltProc demonstration.}
	\end{overprint}
\end{figure}

\end{frame}

\begin{frame}[fragile]
\frametitle{DOT-code describing reprocessing system as a directed graph}
\small
\begin{verbatim}
digraph fuel {
==============================================================================
core_outlet -> sparger [label="100%"]
sparger -> waste_sparger [label="60% of Xe, Kr, H"]
sparger -> entrainment_separator [label="100%"]
entrainment_separator -> nickel_filter [label="100%"]
entrainment_separator -> waste_entrainment_separator [label="97% of Xe, Kr, H"]
nickel_filter -> bypass [label="90%"]
bypass -> heat_exchanger [label="90%"]
nickel_filter -> waste_nickel_filter [label="100% of noble metals"]
nickel_filter -> liquid_metal [label="10%"]
liquid_metal -> heat_exchanger [label="10%"]
liquid_metal -> waste_liquid_metal [label="57% of seminoble metals & RE"]
heat_exchanger -> core_inlet [label="100%"]
LEU_feed -> core_inlet
==============================================================================
# Optional parameters to prettify plots
\end{verbatim}
\end{frame}


% Created 2022-02-16 Wed 14:39
% Intended LaTeX compiler: pdflatex
\documentclass[12pt, reqno, oneside]{amsbook}
              \usepackage[letterpaper, width=6.5in, height=9in]{geometry}
\usepackage[framemethod=TikZ, skipabove=10pt, skipbelow=10pt, backgroundcolor=black!3, roundcorner=4pt, linewidth=1pt]{mdframed}
\BeforeBeginEnvironment{minted}{\begin{mdframed}}
\AfterEndEnvironment{minted}{\end{mdframed}}
\numberwithin{equation}{chapter}
\usepackage{appendix}
\usepackage{url}

\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage[newfloat]{minted}
\usepackage{caption}
\author{Yi Zhang}
\date{\today}
\title{\texttt{Fazang}: A Reverse-mode Automatic differentiation tool in Fortran\\\medskip
\large User's Guide \\  (Version 0.1.0)}
\hypersetup{
 pdfauthor={Yi Zhang},
 pdftitle={\texttt{Fazang}: A Reverse-mode Automatic differentiation tool in Fortran},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 27.2 (Org mode 9.4.4)},
 pdflang={English}}
\begin{document}

\begin{titlepage}
\maketitle
Copyright 2022, Yi Zhang
\newline
\newline
\newline

\today
\tableofcontents
\end{titlepage}

\chapter{Introduction}
\label{sec:orgb4421eb}
\texttt{Fazang} is a reverse-mode automatic differentiation (AD) tool. The
project is heavily influenced by \texttt{Stan/Math} \cite{Carpenter:2015}, a project the author
is also involved in. \texttt{Fazang} is intended to support general scientific
computing in Fortran beyond Bayesian inference and Markov Chain
Monte Carlo that \texttt{Stan/Math} is designed for.

User should be aware that the project is at early stage and still
under development. For any questions, suggestions, and
contributions, please visit the project at \url{https://github.com/yizhang-yiz/fazang}.
\chapter{Quick Start}
\label{sec:orgcc6cf98}
Currently \texttt{Fazang} has been tested on Linux and MacOS platform, with
Fortran compiler Intel Fortran 19.0.1+ and GNU Fortran 11.2.0+.

After downloading \texttt{Fazang}, user can use \texttt{meson} to build the library.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{bash}
git clone git@github.com:yizhang-yiz/fazang.git
cd fazang && mkdir build && cd build
meson compile
\end{minted}
This generates a shared library at \texttt{build/src/}. User needs to link
this library when building an application. This can be done in
\texttt{meson} by setting
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{python}
executable('app_name', files('path/to/app_file.F90'), dependencies : fazang_dep)
\end{minted}

\texttt{Fazang} provides a user-facing derived type \texttt{var}. This is the
type for the dependent and independent variables of which the
adjoint (derivative) will be calculated.

For example, consider the log of
the Gaussian distribution density with mean \(\mu\) and
standard deviation \(\sigma\)
\begin{equation}\label{eq:lnormal_example}
  f(\mu, \sigma) = \log{\left(
      \frac{1}{\sigma\sqrt{2\pi}} \exp\left(
        -\frac{1}{2}\left(\frac{y-\mu}{\sigma}\right)^2
      \right)
    \right)}
\end{equation}
The following programe calculates \(\frac{df}{d\mu}\) and
      \(\frac{df}{d\sigma}\) at \(y=1.3\), \(\mu=0.5\), and \(\sigma=1.2\).
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
program log_demo
  use fazang ! load Fazang library

  implicit none

  real(rk) :: y
  type(var) :: f, sigma, mu

  ! data
  y = 1.3d0

  ! independent variables
  mu = var(0.5d0)
  sigma = var(1.2d0)

  ! dependent
  f = var(-0.5d0 * log(2 * pi))
  f = f - log(sigma)
  f = f - 0.5d0 * ((y - mu) / sigma) ** 2.d0;

  ! use grad() to calculate df/d(mu) and df/d(sigma). Each var's
  ! derivative (also called adjoint) can be access through var%adj().

  call f%grad()
  write(*, *) "df/d(mu): ", mu%adj()
  write(*, *) "df/d(sigma): ", sigma%adj()
end program log_demo
\end{minted}

\chapter{Use \texttt{Fazang}}
\label{sec:org446bb05}
\texttt{Fazang} uses \texttt{var} type to record numerical and gradient
operations. The type supports three functions
\begin{itemize}
\item \mintinline[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}{var%val()} : returns value
\item \mintinline[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}{var%adj()} : returns derivative, henceforth referred as \emph{adjoint}.
\item \mintinline[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}{var%grad()} : takes gradient operation with respect to the current \mintinline[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}{var} variable.
\end{itemize}
\section{Constructors}
\label{sec:orgbe12852}
\texttt{var} can be constructed using overloaded \texttt{var} interface.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
real(real64) :: a, b(3), c(2, 3)
real(real64) :: new_a, new_b(3), new_c(2, 3)
type(var) :: x, y(3), z(2, 3)
! ...
x = var()                 ! x%val() == 0.d0
x = var(a)                ! x%val() == a
y = var(b)                ! y%val() == b
z = var(c)                ! z%val() == c
\end{minted}
\section{Assignment}
\label{sec:org8058780}
\texttt{var} can be assigned from consistent \texttt{var} and \texttt{real(real64)}.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
! ....
x = new_a                 ! x%val() == new_a
y = new_b                 ! y%val() == new_b
z = new_c                 ! z%val() == new_c
\end{minted}

\section{Gradient}
\label{sec:orgfd6291a}
   \label{sec:gradient}
Consider a variable \(z\) calculated by the composition of a series of operations
\begin{equation*}
z = f_1(z_1), \quad z_1 = f_2(z_2), \quad \dots, \quad z_{n-1} = f_n(z_n).
\end{equation*}
For \(z_i, i = 1, \dots, n\) we refer \(dz/d{z_i}\) as the \emph{adjoint} of \(z_i\),
denoted by \(z_i^{\text{adj}}\).
The chain rule says the adjoints can be calculated recursively \cite{griewank_evaluating_2008},
\begin{equation*}
z^{\text{adj}} = 1, \quad
z_1^{\text{adj}} = z^{\text{adj}} \frac{df_1}{dz_1}, \quad
\dots, \quad
z_i^{\text{adj}} = z_{i-1}^{\text{adj}} \frac{df_i}{dz_i}.
\end{equation*}

We often refer each \((f_i, z_i)\) pair as a
\emph{node}, and \(z_i\) the \emph{operand} of operation \(f_i\). The above recursion through the nodes requires a way to store
and visit the \emph{callstack} of nodes.  It is embodied in \texttt{Fazang} by the \texttt{var\%grad()}
function. When \texttt{z\%grad()} is called, \texttt{z}'s adjoint is set to 1, and
every other \texttt{var} variable is transversed with its adjoint updated. In
order to calculate the adjoint with respect to another variable, user
must \texttt{call set\_zero\_all\_adj()} first to reset all adjoints to zero.

An alternative to invoke gradient calculation is to define the
dependent as a function and feed it to \texttt{Fazang}'s \texttt{gradient}
function. Take Eq.\eqref{eq:lnormal_example} for example, we can first
define the function for \(f(\mu, \sigma)\).
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
module func
  use fazang ! load Fazang library
  implicit none

  real(rk), parameter :: y = 1.3d0

contains
  type(var) function f(x)
    type(var), intent(in) :: x(:)
    type(var) :: mu, sigma
    mu = x(1)
    sigma = x(2)
    f = -0.5d0 * log(2 * pi) - log(sigma) - 0.5d0 * ((y - mu) / sigma) ** 2.d0;
  end function f

end module func
\end{minted}
Then we can supply function \texttt{f} as a procedure argument.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
program log_demo2
  use iso_c_binding
  use fazang
  use func

  implicit none

  real(real64) :: fx(3), x(2)
  x = [0.5d0, 1.2d0]

  fx = gradient(f, x)
  write(*, *) "f(x): ", fx(1)
  write(*, *) "df/d(x(1)): ", fx(2)
  write(*, *) "df/d(x(2)): ", fx(3)
end program log_demo2
\end{minted}
The output of \texttt{gradient(f, x)} is an array of size \texttt{1 + size(x)}, with
first component being the function value, and the rest the partial
derivatives.

Note that the above approach of using \texttt{gradient} function does not
involve explicitly setting up \texttt{var} variables. \texttt{Fazang} achieves this
by using a \emph{nested} AD envionment.

\section{Nested AD envionment}
\label{sec:orgf79e864}
   \label{sec:nested}
Let us take a look of the internals of \texttt{Fazang}'s \texttt{gradient} function.
The \texttt{dependent\_function} interface requires \(f\) to follow the above
example's signature, and \texttt{x} is the \texttt{real64} array of independent variables.
We then create the \texttt{var} version of \texttt{x} and introduct it to \texttt{f}. The
evaluation result is saved in \texttt{f\_var} variable. The adjoints are
obtained by calling \texttt{f\_var\%grad()}. Unlike what we have seen, the
above process happens within a pairing \texttt{begin\_nested()} and
\texttt{end\_nested()} calls.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
function gradient(f, x) result (f_df)
  procedure(dependent_function) :: f
  real(real64), intent(in) :: x(:)
  real(real64) :: f_df(1 + size(x))
  type(var) :: x_var(size(x)), f_var

  call begin_nested()

  x_var = var(x)
  f_var = f(x_var)
  f_df(1) = f_var%val()
  call f_var%grad()
  f_df(2:(1+size(x))) = x_var%adj()

  call end_nested()
end function gradient
\end{minted}
When we use these two functions, all the \texttt{var} variables created
in between are "temporary", in the sense that the values and adjoints
of these variables are no longer available after \texttt{call
end\_nested()}. User can use this function pair to construct a local
gradient evaluation procedure.

\section{Jacobian}
\label{sec:orgdde6fc2}
   \label{sec:jacobian}
Similar to \texttt{gradient}, using the same nested technique \texttt{Fazang}
provides a \texttt{jacobian} function that calculates the Jacobian matrix of
\texttt{f}, a multivariate function \(f: \mathbb{R}^m \rightarrow \mathbb{R}^n\) for
an input array \texttt{x} of dimension \texttt{m}.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
function jacobian(f, n, x) result (f_df)
\end{minted}
The input function must follow the interface
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
abstract interface
   function jac_dependent_function (x, n) result (fx)
     import :: var
     integer, intent(in) :: n
     type(var), intent(in) :: x(:)
     type(var) :: fx(n)
   end function jac_dependent_function
end interface
\end{minted}
where \texttt{n} is the output dimension. Like \texttt{gradient}, the output \texttt{f\_df}
has dimension \(n\times(m+1)\), with the first column being the function
results and the rest columns the adjoints.

\section{Functions}
\label{sec:org2b87706}
Numeric functions supported by \texttt{Fazang} are listed in Appendix \ref{appendix:func}. All unary and
binary functions are \texttt{elemental}. The binary functions allow mixed
argument types, namely, either argument can be \texttt{real64} type while the
other the \texttt{var} type.

Probability distributions supported by \texttt{Fazang} are list in Appendix \ref{appendix:likelihood}.

\section{Ordinary differential equations}
\label{sec:orge24bbe8}
\texttt{Fazang} supports ODE solutions through CVODES from SUNDIALS library
\cite{hindmarsh2005sundials}. One can solve ODE like this.

\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
! user defined ODE
module ode_mod
  use fazang
  use, intrinsic :: iso_c_binding
  implicit none

  real(rk), parameter :: params(2) = [0.2d0, 0.1d0]

contains
  ! user defined right-hand-side
  subroutine eval_rhs(t, y, fy)
    real(c_double), intent(in) :: t, y(:)
    real(c_double), intent(inout) :: fy(size(y))
    fy(1) = y(2)
    fy(2) = t * y(1) * sum(params%val())
  end subroutine eval_rhs
end module ode_mod

program cvodes_solve_data
  use ode_mod                   ! import ODE
  use fazang                    ! import Fazang

  implicit none

  real(rk) :: yt(2, 3)          ! output array
  real(rk) :: y0(2)             ! initial condition
  type(cvodes_tol) :: tol       ! basic solver control

  ! use BDF method with given relative tolerance, absolute tolerance,
  ! and max number of steps between outputs
  tol = cvodes_tol(CV_BDF, 1.d-10, 1.d-10, 1000_8)

  ! initial condition
  y0 = [1.2d0, 1.8d0]

  ! solve the ODE with initial time 0.d0 and
  ! output time 1.d0, 2.d0, 3.d0
  yt = cvodes_sol(0.d0, y0, [1.d0, 2.d0, 3.d0], eval_rhs, tol)

end program cvodes_solve_data
\end{minted}
In the above example, we first define an ODE following \texttt{Fazang}'s
interface on the RHS. The defined RHS function \texttt{eval\_rhs} will be later used as an
argument to \texttt{cvodes\_sol}. In addition to initial condition, one must
also define an object for solver control. Such an object must be of a
type that \texttt{extend} the \texttt{cvodes\_options} abstract type. Here we use \texttt{Fazang}'s
basic type \texttt{cvodes\_tol}, which gives: the integration scheme (\texttt{CV\_BDF}
for BDF method or \texttt{CV\_ADAMS} for Adams-Moulton method), relative
tolerance, absolution tolerance, and the maximum number of steps
allowed between output.

Then to the solver interface \texttt{cvodes\_sol} we give the initial time,
initial condition, array for output time, the RHS subroutine, and the
solver control. It returns a 2D array with each column at a requested
output time.

\subsection{Forward sensitivity}
\label{sec:org5755bfa}
Combining the sensitivity capability of \texttt{CVODES} and AD from \texttt{Fazang},
we can solve for ODE sensitivity with respect to given parameters
without explicitly supplying Jacobian. For that the user-defined ODE
must include an additional RHS definition with \texttt{var} parameters,
following \texttt{Fazang}'s RHS interface.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
module ode_mod
  use fazang
  use, intrinsic :: iso_c_binding
  implicit none

  real(rk), parameter :: omega = 0.5d0
  real(rk), parameter :: d1 = 1.0d0
  real(rk), parameter :: d2 = 1.0d0

contains
  ! right-hand-side for data input
  subroutine eval_rhs(t, y, fy)
    implicit none
    real(c_double), intent(in) :: t, y(:)
    real(c_double), intent(inout) :: fy(size(y))
    fy(1) = y(2)
    fy(2) = sin(omega * d1 * d2 * t)
  end subroutine eval_rhs

  ! right-hand-side for var input with parameters
  ! y, p, and output fy must all be of var type
  subroutine eval_rhs_pvar(t, y, fy, p)
    implicit none
    real(c_double), intent(in) :: t
    type(var), intent(in) :: y(:), p(:)
    type(var), intent(inout) :: fy(size(y))
    fy(1) = y(2)
    fy(2) = sin(p(1) * p(2) * p(3) * t)
  end subroutine eval_rhs_pvar
end module ode_mod
\end{minted}
Now we can solve the defined ODE in a similar way.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
program cvodes_demo
  use ode_mod
  use fazang
  implicit none

  type(var) :: yt(2, 3)
  type(cvodes_tol) :: tol
  real(rk), parameter :: ts(3) = [1.2d0, 2.4d0, 4.8d0]
  real(rk), parameter :: y00(2) = [0.2d0, 0.8d0]
  type(var) :: param(3)
  real(rk) :: y0(2), ga(2)
  integer :: i, j

  y0 = y00                      ! init condition
  param = var([omega, d1, d2])  ! parameters
  tol = cvodes_tol(CV_BDF, 1.d-10, 1.d-10, 1000_8)

  yt = cvodes_sol(0.d0, y0, ts, param, eval_rhs,&
       & eval_rhs_pvar, tol)
! ...
end program cvodes_demo
\end{minted}
Note that now the call to \texttt{cvodes\_sol} includes additional argument
\texttt{param} as the sensitivity parameters, as well as the RHS function for
\texttt{var} inputs. The sensitivities are obtained the same way by calling
\texttt{grad} and \texttt{adj} functions.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
call yt(1, 1) % grad()
write(*, *) "dy_1/ d_omega at time ts(1):", param(1)%adj()
\end{minted}

\subsection{Functions}
\label{sec:orgf1d1100}
\begin{enumerate}
\item Data solution
\label{sec:org271df10}
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
function cvodes_sol(t, y, ts, rhs, cvs_options) result(yt)
    real(real64), intent(in) :: t          ! initial time
    real(real64), intent(inout) :: y(:)    ! initial condition
    real(real64), intent(in) :: ts(:)      ! output time
    procedure(cvs_rhs_func) :: rhs         ! RHS definition (see below)
    class(cvodes_options), intent(in) :: cvs_options ! solver control
    real(real64) :: yt(size(y), size(ts))  ! solution
\end{minted}

\item Sensitivity solution  with respect to the initial condition
\label{sec:orgf19dcad}
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
function cvodes_sol(t, y, ts, rhs, rhs_yvar, cvs_options) result(yt)
    real(real64), intent(in) :: t          ! initial time
    type(var), intent(inout) :: y(:)       ! initial condition
    real(real64), intent(in) :: ts(:)      ! output time
    procedure(cvs_rhs_func) :: rhs         ! data-only RHS (see below)
    procedure(cvs_rhs_func_yvar) :: rhs_yvar ! var-type RHS (see below)
    class(cvodes_options), intent(in) :: cvs_options  ! solver control
    type(var) :: yt(size(y), size(ts))     ! solution
\end{minted}

\item Sensitivity solution  with respect to the parameters
\label{sec:org64f810b}
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
function cvodes_sol(t, y, ts, param, rhs, rhs_pvar, cvs_options) result(yt)
    real(real64), intent(in) :: t          ! initial time
    real(real64), intent(inout) :: y(:)    ! initial condition
    real(real64), intent(in) :: ts(:)      ! output time
    type(var), target, intent(in) :: param(:) ! parameters
    procedure(cvs_rhs_func) :: rhs         ! data-only RHS (see below)
    procedure(cvs_rhs_func_pvar) :: rhs_pvar ! var-type RHS (see below)
    class(cvodes_options), intent(in) :: cvs_options  ! solver control
    type(var) :: yt(size(y), size(ts))     ! solution
\end{minted}

\item Interfaces for different solvers
\label{sec:org58311fb}
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
abstract interface
   subroutine cvs_rhs_func(t, y, fy)
     import c_double
     real(c_double), intent(in) :: t, y(:)
     real(c_double), intent(inout) :: fy(size(y))
   end subroutine cvs_rhs_func

   subroutine cvs_rhs_func_yvar(t, y, f)
     import c_double, var
     real(c_double), intent(in) :: t
     type(var), intent(in) :: y(:)
     type(var), intent(inout) :: f(size(y))
   end subroutine cvs_rhs_func_yvar

   subroutine cvs_rhs_func_pvar(t, y, f, p)
     import c_double, var
     real(c_double), intent(in) :: t
     type(var), intent(in) :: y(:), p(:)
     type(var), intent(inout) :: f(size(y))
   end subroutine cvs_rhs_func_pvar
end interface
\end{minted}

\item Solver controls
\label{sec:org06a915d}
The last argument of the solver call is a solver control
object. User-defined type must be able to follow \texttt{CVODES} user guide
to modify \texttt{CVODES} memory object, by extending the abstract type \texttt{cvodes\_options}.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
type, abstract :: cvodes_options
   integer :: cv_method = -1
 contains
   procedure(set_cvodes), deferred :: set
end type cvodes_options

abstract interface
   subroutine set_cvodes(this, mem)
     import c_ptr, cvodes_options
     class(cvodes_options), intent(in) :: this
     type(c_ptr), intent(inout) :: mem ! CVODES memory
   end subroutine set_cvodes
end interface
\end{minted}
One can follow \texttt{Fazang} 's tolerance control type as an example.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
  type, extends(cvodes_options) :: cvodes_tol
     real(c_double) :: rtol, atol
     integer(c_long) :: max_nstep
  contains
    procedure :: set
 end type cvodes_tol

contains

  subroutine set(this, mem)
    class(cvodes_tol), intent(in) :: this
    type(c_ptr), intent(inout) :: mem ! CVODES memory
    integer :: ierr
! call cvodes functions
    ierr = FCVodeSStolerances(mem, this % rtol, this % atol)
    ierr = FCVodeSetMaxNumSteps(mem, this % max_nstep)
  end subroutine set
\end{minted}
\end{enumerate}

\chapter{Design}
\label{sec:org85ae2ef}
The core of any reverse-mode automatic differentiation is the data
structure to store and visit the callstack. \texttt{Fazang} achieves this
through two derived types, \texttt{tape} and \texttt{vari}.

\section{\texttt{tape} data structure}
\label{sec:org2bb7ef1}
A \texttt{tape} is an \texttt{int32} array emulating a stack, with an integer marker \texttt{head} pointing to the
head to the current stack top.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
type :: tape
     integer(ik) :: head = 1
     integer(ik), allocatable :: storage(:)
!...
\end{minted}
Each time a new AD node is created,
space in \texttt{storage} is allotted to store the node's
\begin{itemize}
\item value \(f_i(z_i)\),
\item adjoint \(z_{i-1}^{\text{adj}}\),
\item number of \texttt{var} operands of \(f_i\),
\item The \texttt{var} operands' index in the same \texttt{tape} array,
\item number of \texttt{real64} operands of \(f_i\),
\item The \texttt{real64} operands' value.
\end{itemize}

Since a node's value, adjoint, and data
operands are \texttt{real64}, they are first converted to \texttt{int32} using
\texttt{transfer} function before stored in the \texttt{tape} array, so that each such
a value occupies two \texttt{storage} entries. After each
allotation, the \texttt{head} is moved to point to the next empty slot in
the array after saving its current value to a \texttt{vari} type variable
for future retrieval.

\section{\texttt{vari} type}
\label{sec:org611a41b}
The \texttt{vari} type is simply a proxy of a node's storage location in the tape
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
type :: vari
  integer(ik) :: i = 0
  procedure(chain_op), pass, pointer :: chain
contains
   !....
\end{minted}
where \texttt{i} is the index to the beginning of a node's storage, and the
\texttt{chain} procedure encodes the node's operation
\(f_i\). \texttt{chain} follows an interface that describes the chain rule
operation
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
abstract interface
   subroutine chain_op(this)
     import :: vari
     class(vari), intent(in) :: this
   end subroutine chain_op
end interface
\end{minted}
An alternative to integer index is to a \texttt{pointer} to the according
enry in the \texttt{tape} array. However, we will need to expand the
\texttt{storage} when it is filled up, and \texttt{Fazang} does this by doubling the
\texttt{storage} size and use \texttt{move\_alloc} to
restore the original values. Since there is no guarantee that \texttt{move\_alloc}
will keep the original memory, a pointer to the original address would
be corrupted.

As a \texttt{Fazang} program steps forward, a series of \texttt{vari} variables are
generated, with their \emph{values} calculated and stored. This is called
a \emph{forward pass}. The generated \texttt{vari} variables in the forward pass are
stored in array \texttt{varis}. Each entry in \texttt{varis} is a dependent
(operation output) of one or more previous entries.

\section{\texttt{var} type}
\label{sec:orgf8b9d4f}
The user-facing \texttt{var} type serves as proxy to \texttt{vari}. Each \texttt{var}
stores the index of a \texttt{vari} in the \texttt{varis} array.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
type :: var
   integer(int32) :: vi
 contains
   procedure :: val
   procedure :: adj
   procedure :: grad
   procedure :: set_chain
end type var
\end{minted}
After the forward pass, when adjoints are desired, we call \texttt{grad} or
\texttt{gradient} procedure. This initiates a \emph{backward pass}, in which  the
\texttt{varis} array is traversed backward
so that each \texttt{vari}'s \texttt{chain} procedure is called to update the
operand adjoints.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
subroutine grad(this)
  class(var), intent(in) :: this
  integer i
  call callstack % varis (this%vi) % init_dependent()
  do i = callstack % head - 1, 1, -1
     call callstack % varis(i) % chain()
  end do
end subroutine grad
\end{minted}
Here \texttt{callstack} is the module variable that encapsulate \texttt{tape} and
\texttt{varis} arrays.

\section{Nested tape}
\label{sec:orgde2db32}
\texttt{Fazang} use \texttt{begin\_nested()} and \texttt{end\_nested()} to record and
terminate a nested tape. With \texttt{call begin\_nested()} \texttt{Fazang} records
the current \texttt{tape} and \texttt{varis} array head. When \texttt{end\_nested()} is
called, the storage between the recorded head and current head are
wiped, and the head is moved back to the recorded location. Multiple
levels of nested envionment are supported this way.

\chapter{Add operation functions}
\label{sec:org0c57838}
Adding an operation \(f_i\) involves creating functions for forward
pass and backward pass. Let us first use \texttt{log} function as a simple
example.

First, we create a \texttt{log\_v} function for the forward pass.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
impure elemental function log_v(v) result(s)
  type(var), intent(in) :: v
  type(var) :: s
  s = var(log(v%val()), [v])
  call s%set_chain(chain_log)
end function log_v
\end{minted}
The function generates a new \texttt{var} variable \texttt{s} using a special
constructor \texttt{var(value, array of operands)} which stores the value as
well as the single operand \texttt{v}'s index (in the \texttt{tape} \texttt{storage}
array). It also points \texttt{s}'s chain to a dedicated procedure \texttt{chain\_log}.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
subroutine chain_log(this)
  class(vari), intent(in) :: this
  real(rk) :: adj(1), val(1)
  val = this%operand_val()
  adj(1) = this%adj() / val(1)
  call this%set_operand_adj(adj)
end subroutine chain_log
\end{minted}
To understand this function, recall the recursion in Section \ref{sec:gradient},
assume the \texttt{log} operation is node \(i\), then \(f_i=\log(\dot)\) and
\(z_i\) is the operand \texttt{v}, and the new \texttt{var} \texttt{s} would be
\(z_{i-1}\). During the backward pass when the node is visited, \texttt{chain\_log}
first retrieves current \((z_i, z_i^{\text{adj}})\)
using \texttt{operand\_val()} and \texttt{operand\_adj()}, then updates
\(z_i^{\text{adj}}\) with an additional
\begin{equation*}
z_{i-1}^{\text{adj}} \frac{df_i}{dz_i} = z_{i-1}^{\text{adj}}\frac{d\log(z_i)}{dz_i}=\frac{z_{i-1}^{\text{adj}}}{z_i}.
\end{equation*}

Adding a binary operation \(f_i(z_i^{(1)}, z_i^{2})\) is slightly more complex, as we will need to
address possibly different scenarios when \(z_i^{(1)}\) and \(z_i^{(2)}\)
are either \texttt{var} or \texttt{real64}. Let us use overloaded division \texttt{operator(/)} as an example.

With
\begin{equation*}
f_i(z_i^{(1)}, z_i^{2}) = z_i^{(1)} / z_i^{(2)}
\end{equation*}
we need to account for
\begin{itemize}
\item both \(z_i^{(1)}\) and \(z_i^{2}\) are \texttt{var}'s
\item \(z_i^{(1)}\) is \texttt{var}, \(z_i^{2}\) is \texttt{real64},
\item \(z_i^{(1)}\) is \texttt{real64}, \(z_i^{2}\) is \texttt{var},
\end{itemize}

For the first scenario, we create
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
impure elemental function div_vv(v1, v2) result(s)
  type(var), intent(in) :: v1, v2
  type(var) :: s
  s = var(v1%val() / v2%val(), [v1, v2])
  call s%set_chain(chain_div_vv)
end function div_vv
\end{minted}
Similar to the \texttt{log} example, we create a new \texttt{s} with both operands
stored. In the corresponding \texttt{chain} procedure, we need update
the adjoints of both \texttt{v1} and \texttt{v2}.
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
subroutine chain_div_vv(this)
  class(vari), intent(in) :: this
  real(rk) :: adj(2), val(2)
  val = this%operand_val()
  adj(1) = this%adj()/val(2)
  adj(2) = - this%val() * this%adj()/val(2)
  call this%set_operand_adj(adj)
end subroutine chain_div_vv
\end{minted}

For the second scenario, we create
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
impure elemental function div_vd(v, d) result(s)
  type(var), intent(in) :: v
  real(rk), intent(in) :: d
  type(var) :: s
  s = var(v%val() / d, [v], [d])
  call s%set_chain(chain_div_vd)
end function div_vd
\end{minted}
Again we create a new \texttt{var} \texttt{s}. But this time
we use another constructor \texttt{var(value, var operands, data
operands)} to store value, \texttt{var} operand \texttt{v}, and \texttt{real64}
operand \texttt{d}. In the corresponding backward pass \texttt{chain} procedure, not
      only we need retrieve \texttt{var} operand \texttt{v} but also data operand
      \texttt{d}, as the new adjoint of \(z_i^{(1)}\) is
\begin{equation*}
z_i^{(1)\text{new adj}} = z_i^{(1)\text{old adj}} + z_{i-1}^{\text{adj}}\frac{df_i}{dz_i^{(1)}}
= z_i^{(1)\text{old adj}} + z_{i-1}^{\text{adj}}\frac{1}{dz_i^{(2)}}
\end{equation*}
So with \texttt{v} as \(z_i^{(1)}\) and \texttt{d} as \(z_i^{(2)}\) we have
\begin{minted}[breaklines=true,fontsize=\footnotesize,breakanywhere=true]{fortran}
subroutine chain_div_vd(this)
  class(vari), intent(in) :: this
  real(rk) d(1), adj(1)
  d = this%data_operand()
  adj(1) = this%adj() / d(1)
  call this%set_operand_adj(adj)
end subroutine chain_div_vd
\end{minted}

The third scenario is treated similarly.

\appendix
\chapter{\texttt{Fazang} Functions \label{sec:func_list}}
\label{sec:org3f9b4b7}
\label{appendix:func}
\begin{center}
\begin{tabular}{lll}
Function & Argument(s) & Operation\\
\hline
\texttt{sin} & scalar or array & same as intrinsic\\
\texttt{cos} & scalar or array & same as intrinsic\\
\texttt{tan} & scalar or array & same as intrinsic\\
\texttt{asin} & scalar or array & same as intrinsic\\
\texttt{acos} & scalar or array & same as intrinsic\\
\texttt{atan} & scalar or array & same as intrinsic\\
\texttt{log} & scalar or array & same as intrinsic\\
\texttt{exp} & scalar or array & same as intrinsic\\
\texttt{sqrt} & scalar or array & same as intrinsic\\
\texttt{erf} & scalar or array & same as intrinsic\\
\texttt{erfc} & scalar or array & same as intrinsic\\
\texttt{abs} & scalar or array & same as intrinsic\\
\texttt{norm2} & 1D array & same as intrinsic\\
\texttt{hypot} & scalars or arrays & same as intrinsic\\
\texttt{sinh} & scalar of array & same as intrinsic\\
\texttt{cosh} & scalar of array & same as intrinsic\\
\texttt{tanh} & scalar of array & same as intrinsic\\
\texttt{asinh} & scalar of array & same as intrinsic\\
\texttt{acosh} & scalar of array & same as intrinsic\\
\texttt{atanh} & scalar of array & same as intrinsic\\
\texttt{log\_gamma} & scalar or array & same as intrinsic\\
\texttt{square} & scalar or array & For input \texttt{x}, calculate \texttt{x**2}\\
\texttt{inv} & scalar or array & For input \texttt{x}, calculate \texttt{1/x}\\
\texttt{inv\_square} & scalar or array & For input \texttt{x}, calculate \texttt{1/x**2}\\
\texttt{inv\_sqrt} & scalar or array & For input \texttt{x}, calculate \texttt{1/sqrt(x)}\\
\texttt{logit} & scalar or array & For input \texttt{x}, calculate \texttt{log(x/(1-x))}\\
\texttt{inv\_logit} & scalar or array & For input \texttt{x}, calculate \texttt{1/(1+exp(-x))}\\
operator (\texttt{+}) & scalars or arrays & same as intrinsic\\
operator (\texttt{-}) & scalars or arrays & same as intrinsic\\
operator (\texttt{*}) & scalars or arrays & same as intrinsic\\
operator (\texttt{/}) & scalars or arrays & same as intrinsic\\
operator (\texttt{**}) & scalars & same as intrinsic\\
\texttt{sum} & 1D array & same as intrinsic\\
\texttt{dot\_product} & 1D arrays & same as intrinsic\\
\texttt{log\_sum\_exp} & 1D array & For input \texttt{x}, calculate \texttt{log(sum(exp((x))))}\\
\texttt{matmul} & 2D arrays & same as intrinsic\\
\end{tabular}
\end{center}

\chapter{\texttt{Fazang} Probability distributions \label{sec:likelihood_list}}
\label{sec:org558cd1e}
\label{appendix:likelihood}
\section{Normal distribution}
\label{sec:org1a8138a}
\begin{equation}
\text{Normal}(y, \mu, \sigma) = \prod_{i=1}^n\frac{1}{\sqrt{2\pi}\sigma}\exp{
  \left(
    -\frac{1}{2}
    \left(
      \frac{y_i-\mu}{\sigma}
    \right)^2
  \right)
},\qquad
\forall y\in \mathbb{R}^n, \mu\in \mathbb{R}, \sigma\in \mathbb{R}^+.
\end{equation}

\begin{itemize}
\item \texttt{normal\_lpdf(y, mu, sigma)}
\begin{itemize}
\item \texttt{y}: \texttt{real64} array.
\item \texttt{mu}: \texttt{real64} or \texttt{var}.
\item \texttt{sigma}: \texttt{real64} or \texttt{var}.
\item Return: the \texttt{log} of \(\text{Normal}(y, \mu, \sigma)\).
\end{itemize}
\end{itemize}

\section{LogNormal distribution}
\label{sec:org8579ae6}
\begin{equation}
\text{LogNormal}(y, \mu, \sigma) = \prod_{i=1}^n\frac{1}{\sqrt{2\pi}\sigma}\frac{1}{y_i}\exp{
  \left(
    -\frac{1}{2}
    \left(
      \frac{\log{y_i}-\mu}{\sigma}
    \right)^2
  \right)
},\quad
\forall y\in (\mathbb{R}^+)^n, \mu\in \mathbb{R}, \sigma\in \mathbb{R}^+.
\end{equation}

\begin{itemize}
\item \texttt{lognormal\_lpdf(y, mu, sigma)}
\begin{itemize}
\item \texttt{y}: \texttt{real64} array.
\item \texttt{mu}: \texttt{real64} or \texttt{var}.
\item \texttt{sigma}: \texttt{real64} or \texttt{var}.
\item Return: the \texttt{log} of \(\text{LogNormal}(y, \mu, \sigma)\).
\end{itemize}
\end{itemize}
\section{{\bfseries\sffamily TODO} additional distributions}
\label{sec:orgfacca40}

\bibliographystyle{plain}
\bibliography{ref}
\end{document}


% This is part of the TFTB Reference Manual.
% Copyright (C) 1996 CNRS (France) and Rice University (US).
% See the file refguide.tex for copying conditions.

\markright{anafsk}
\section*{\hspace*{-1.6cm} anafsk}

\vspace*{-.4cm}
\hspace*{-1.6cm}\rule[0in]{16.5cm}{.02cm}
\vspace*{.2cm}

{\bf \large \sf Purpose}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
Frequency Shift Keyed (FSK) signal.
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Synopsis}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
[y,iflaw] = anafsk(N)
[y,iflaw] = anafsk(N,ncomp)
[y,iflaw] = anafsk(N,ncomp,nbf)
\end{verbatim}
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Description}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
        {\ty anafsk} simulates a phase coherent Frequency Shift Keyed (FSK)
        signal. This signal is a succession of complex sinusoids of {\ty ncomp}
        points each and with a normalized frequency uniformly chosen
        between {\ty nbf} distinct values between 0.0 and 0.5.  Such signal is
        only 'quasi'-analytic.\\

\hspace*{-.5cm}\begin{tabular*}{14cm}{p{1.5cm} p{8.5cm} c}
Name & Description & Default value\\
\hline
        {\ty N }    & number of points\\
        {\ty ncomp} & number of points of each component & {\ty N/5}\\
        {\ty nbf}   & number of distinct frequencies     & {\ty 4}  \\
 \hline {\ty y }    & signal\\
        {\ty iflaw} & instantaneous frequency law  \\
\hline
\end{tabular*}

\end{minipage}
\vspace*{1cm}

{\bf \large \sf Example}
\begin{verbatim}
         [signal,ifl]=anafsk(512,64,5);
         subplot(211); plot(real(signal));
         subplot(212); plot(ifl);
\end{verbatim}
\vspace*{.5cm}

{\bf \large \sf See Also}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
anabpsk, anaqpsk, anaask.
\end{verbatim}
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Reference}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
[1] W. Gardner {\it Introduction to Random Processes, with Applications to
Signals and Systems}, 2nd Edition, McGraw-Hill, New-York, p. 357 ,1990.
\end{minipage}


\chapter{Specific Implementation}
\label{chap-spec-impl}

\begin{minted}{haskell}
data Tree a = Leaf a
            | Node (Leaf a) a (Leaf a)
\end{minted}

\begin{minted}{haskell}
sumTree :: Tree Int -> Int
sumTree (Leaf x)     = x
sumTree (Node l x r) = x + (sumTree l) + (sumTree r)
\end{minted}

Computing a value of a data structure can easily be defined in Haskell, but every time there is a small change in the \texttt{Tree}, the entire \texttt{Tree} needs to be recomputed. This is inefficient, because most of the computations have already been performed in the previous computation.

To prevent recomputation of already computed values, the technique memoization is introduced. Memoization is a technique where the results of computational intensive tasks are stored and when the same input occurs, the result is reused.

The comparison of two values in Haskell is done with the \texttt{Eq} typeclass, which implements the equality operator \inlinehaskell{(==) :: a -> a -> Bool}. So, an example implementation of the \texttt{Eq} typeclass for the \texttt{Tree} datatype would be:

\begin{minted}{haskell}
instance Eq a => Eq (Tree a) where
  Leaf x1       == Leaf x2       = x1 == x2
  Node l1 x1 r1 == Node l2 x2 r2 = x1 == x2 && l1 == l2 && r1 == r2
  _             == _             = False
\end{minted}

The problem with using this implementation of the \texttt{Eq} typeclass for Memoization is that for every comparison of the \texttt{Tree} datatype the equality is computed. This is inefficient because the equality implementation has to traverse the complete \texttt{Tree} data structure to know if the \texttt{Tree}'s are equal.

To efficiently compare the \texttt{Tree} datatypes, we need to represent the structure in a manner which does not lead to traversing to the complete \texttt{Tree} data structure. This can be accomplished using a \texttt{hash} function. A hash function is a process of transforming a data structure into an arbitrary fixed-size value, where the same input always generates the same output.

One of the disadvantages of using hashes is \textit{hash collisions}. Hash collisions happen when two different pieces of data have the same hash. This is because a hash function has a limited amount of bits to represent every possible combination of data. Using the formula $p = \epsilon^{\frac{-k(k-1)}{2N}}$ from \citetitle{hashcoll2011}\cite{hashcoll2011}, we can calculate a $50\%$ chance of getting a hash collision with a collection of $k$. The hash function CRC-32 needs a collection of 77163 hash values. The hash function MD5 needs a collection of $\num{5.06e9}$ hash values. And, the hash function SHA-1 needs a collection of $\num{1.42e24}$ hash values. As a result of, we can say that for most popular hash functions, hash collisions are negligible.

\begin{minted}{haskell}
class Hashable a where
  hash :: a -> Hash

instance Hashable a => Hashable (Tree a) where
  hash (Leaf x)     = concatHash [hash "Leaf", hash x]
  hash (Node l x r) = concatHash [hash "Node", hash x, hash l, hash r]
\end{minted}

The hashes can then be used to efficiently compare two \texttt{Tree} data structures, without having to traverse the entire \texttt{Tree} data structure. To keep track of the intermediate results of the computation, we store the results in a \texttt{Map}. A \texttt{Map}, also known as a dictionary, is an implementation of mapping a key to a value. In our next example the \texttt{Hash} is the key and the value is the intermediate result.

\begin{minted}{haskell}
sumTreeInc :: Tree Int -> (Int, Map Hash Int)
sumTreeInc l@(Leaf x)     = (x, insert (hash l) x empty)
sumTreeInc n@(Node l x r) = (y, insert (hash n) y (ml <> mr))
  where
    y = x + xl + xr
    (xl, ml) = sumTreeInc l
    (xr, mr) = sumTreeInc r
\end{minted}

Then after the first computation over the entire \texttt{Tree}, we can recompute the \texttt{Tree} using the previously created \texttt{Map}. Thus, when we recompute the \texttt{Tree}, we first look in the \texttt{Map} if the computation has already been performed then return the result. Otherwise, compute the result and store it in the \texttt{Map}.

\question{Maybe add the more efficient implementation of merging maps?}
\begin{minted}{haskell}
sumTreeIncMap :: Map Hash Int -> Tree Int -> (Int, Map Hash Int)
sumTreeIncMap m l@(Leaf x) = case lookup (hash l) m of
  Just x  -> (x, m)
  Nothing -> (x, insert (hash l) x empty)
sumTreeIncMap m n@(Node l x r) = case lookup (hash n) m of
  Just x  -> (x, m)
  Nothing -> (y, insert (hash n) y (ml <> mr))
    where
      y = x + xl + xr
      (xl, ml) = sumTreeIncMap m l
      (xr, mr) = sumTreeIncMap m r
\end{minted}

Generating a hash for every computation over the data structure is time-consuming and unnecessary, because most of the \texttt{Tree} data structure stays the same. The work of \citeauthor{miraldo2019efficient}\cite{miraldo2019efficient} inspired the use of the Merkle Tree. A Merkle Tree is a data structure which integrates the hashes within the data structure.

\section{Merkle Tree (\texttt{TreeH})}
First we introduce a new datatype \texttt{TreeH}, which contains a \texttt{Hash} for every constructor in \texttt{Tree}. Then to convert the \texttt{Tree} datatype into the \texttt{TreeH} datatype, the structure of the Tree is hashed and stored into the datatype using the \texttt{merkle} function.

\begin{minted}{haskell}
data TreeH a = LeafH Hash a
             | NodeH Hash (Leaf a) a (Leaf a)
\end{minted}

\begin{minted}{haskell}
merkle :: Tree Int -> TreeH Int
merkle l@(Leaf x) = LeafH (hash l) x
merkle (Node l x r) = NodeH h l' x r'
  where
    h = hash ["Node", x, getHash l', getHash r']
    l' = merkle l
    r' = merkle r
\end{minted}

The precomputed hashes can then be used to easily create a \texttt{Map}, without computing the hashes every time the \texttt{sumTreeIncH} function is called.

\begin{minted}{haskell}
sumTreeIncH :: TreeH Int -> (Int, Map Hash Int)
sumTreeIncH (LeafH h x)     = (x, insert h x empty)
sumTreeIncH (NodeH h l x r) = (y, insert h y (ml <> mr))
  where
    y = x + xl + xr
    (xl, ml) = sumTreeInc l
    (xr, mr) = sumTreeInc r
\end{minted}

The problem with this implementation is, that when the \texttt{Tree} datatype is updated, the entire \texttt{Tree} needs to be converted into a \texttt{TreeH}, which is linear in time. This can be done more efficiently, by only updating the hashes which are impacted by the changes. Which means that only the hashes of the change and the parents need to be updated.

The first intuition to fixing this would be using a pointer to the value that needs to be changed. But because Haskell is a functional programming language, there are no pointers. Luckily, there is a data structure which can be used to efficiently update the data structure, namely the Zipper\cite{huet1997zipper}.

\input{sections/specific_implementation/zipper.tex}


\section{Varieties of Hypergraphs}
\p{Any notion of hypergraphs contrasts with an
underlying graph model, such that some element treated as
singular or unstructured in regular graphs becomes a multiplicity
or compound structure in the hypergraph.  So for example, an
edge generalizes to a hyperedge with more than two incident
nodes (which means, for directed graphs, more than
one source and/or target node).  Likewise, nodes might generalize
to complex structures containing other nodes (including,
potentially, nested graphs).  The \q{elements} of a graph
are nodes and edges, but also (for labeled/weighted
and/or directed graphs) things like labels, weights
(such as probability metrics associated with edges), and directions
(in the distinction between incoming and outgoing edges incident to any node).
Potentially, any of these elements can be transformed
from a unit to a plural structure, a process
I will call \i{diversification}.  That is,
a hypergraph emerges from a graph by \q{diversifying}
some elements, rendering as multiplicities what
had previously been a single entity \mdash{} nodes become node-sets, edges
become grouped into larger aggregates, labels generalize to
complex structures (which I will call \q{annotations}),
etc.  Different avenues of diversification give
rise to different varieties of hypergraphs, as
I will review in the next several paragraphs.
}
\p{\begin{description}\item[Hyperedges]  Arguably the most common model of
hypergraphs involves generalizing edges to hyperedges,
each of which (potentially) connect more than two nodes.
For directed graphs,
directed hyperedges have a \q{source} node-set and a \q{target} node-set,
either of which can (potentially) have more than one node.
Note that this is actually a form of node-diversification
\mdash{} there is still (in this genre of hypergraph) just one
edge at a time, but its incident node set (or source and
target node-sets) are sets and not single nodes.  Another way
of looking at directed hyperedges is to see source and target
node-sets as integral complex parts, or \q{hypernodes}.
So a (directed) hypergraph with hyperedges can also
be seen as generalizing ordinary graphs by replacing
nodes with hypernodes (such that one hypernode
spans a \i{set} of \q{normal} nodes).

\item[Recursive Graphs]  Whereas hyperedges embody a relatively
simple node-diversification \mdash{} nodes replaced by node-sets
\mdash{} so-called \q{recursive} graphs allow compound nodes (hypernodes)
to contain entire nested graphs.  Edges in this case can still
connect two hypernodes as in ordinary graphs, but the
hypernodes internally contain other graphs,
with their own (on another \q{level}) nodes and edges.

\item[Link Grammar]   Since \i{labeled}
graphs are an important model for computational models,
we can also consider generalizations of labels to be
compound structures rather than numeric or string labels
(or, as in the Semantic Web, \q{predicate} terms,
drawn from an Ontology, in \q{Subject-Predicate-Object}
triples).  Compound labels (which I will generically call
double- or multiple-annotations) are encountered
in several different branches of mathematics and other fields.
\vspace*{.3em}\\\hspace*{3em}
In particular, compound annotations can represent the
rules, justifications, or \q{compatibility} which allows
two nodes to be connected.   In this case the annotation
may contain information about both incident nodes.
This general phenomenon (I will mention further examples
below) can be called a \q{diversification}
on \i{labels}, transforming from labels as single units
to labels as multi-part records.
\item[Hypergraph Categories]  Directed hyperedges have source-
and target-\i{sets} of nodes (possibly ordered).  Allowing
these sets to be \i{empty} yields structures similar to
hypergraph categories in recent mathematical treatments, such
as \cite{BrendanFong}, \cite{AleksKissinger}.  Hypergraphs
in this characterization might be used to study software applications
and real-time monitoring systems.  Hyperedges would then
mark information flows between different sites \i{inside} the
system.  A directed hyperedge \i{without} a source node-sets then
models information \i{entering} the system \i{ab initio}.
Likewise hyperedges without target node-sets could model
empirical effects produced by the system, which from one
perspective embodies information \q{leaving} the system.
\vspace*{.3em}\\\hspace*{3em}
Analogous models can be realized by, in lieu of
hyperedges \i{without} source or target node-sets at all,
designating special hypernodes representing empty
node-sets; or hypernodes providing greater detail about
origin- or destination-points for paths modeling
information flows.  In software and some CyberPhysical
systems, information \q{enters} the system via
\q{events} \mdash{} that is, \i{events} are introduced as a
primitive concept alongside \i{procedures}, though
events themselves are not \q{implemented} (they have no
\q{body}).  While procedures call other procedures,
events \i{trigger} calls via event-to-procedure
\i{connections}.
Events which are not generated by components \i{in} the
system then correspond to hyperedges without source-nodes,
but the situation can be more precisely modeled via a
special class of event hypernodes (analogous to
events, as a distinct signature-bearing element from
procedures, in event-driven programming;
cf. \cite{JenniferPaykin},
\cite{PaykinKrishnaswami}, \cite{WolfgangJeltsch}).
\item[Edges Incident to Edges]  A further generalization
permits edges to \q{point at} other edges
\cite[p. 10]{BalintMolnar}; \cite[p. 13]{BenGoertzel}.
The rationale for corresponding constructions is that
an edge \mdash{} qua relation between two or more
components \mdash{} is itself a fact, datapoint, or
assertion, and as such can be on some perspectives a
singular element in a space of information.
The Semantic Web, for example, includes \q{reified}
edges (Subject/Predicate/Object triples).
Reification permits properties to be stated about edges,
such as provenance data (who asserts the edge and on
what evidence) and context (the assertion is believed true
at a specific time or in a given context).
It is plausible then to allow reified edges
to be treated as nodes incident to other edges.  As
with hyperedges possessing empty source or target
sets, the associated paradigm can be interpreted through
a special class of hypernode: we can certainly
consider nodes which refer to edges (perhaps via the
latter's label or annotation).  Such \q{referring}
is not necessarily an explicit further edge in graphs
but could instea be \q{semantic} \mdash{} that is, an edge being the
\q{value} associated with a node (or likewise
one value in a multi-element hypernode).
\item[Channels]   Hyperedges, which connect multiple
nodes, are still generally seen as single edges
(in contrast to multigraphs which allow multiple
edges between two nodes).  Analogous to the grouping
of nodes into hypernodes, we can also consider structures
where several different edges are unified into a larger
totality, which I will call a \i{channel}.
In the canonical case, a directed graph can group
edges into composites (according to more fine-grained criteria
than just distinguishing incoming and outgoing edges) which share a
source or target node.  The set of incoming and/or outgoing  edges
to each node may be partitioned into distinct
\q{channels}, so that at one scale of consideration the network
structure can be analyzed via channels rather than via
single edges (this shows why channels mark edge \i{diversification}
as well as edge \i{aggregation}, because the conceptual role
of edges is potentially replaced by channels, which are composed
of numerous edges).
\vspace*{.3em}\\\hspace*{3em}
As a concrete case,
consider graphs used to model Object-Oriented programming
languages, where a single node can represent a
single function call.  The incoming edges are then
\q{input parameters}, and outgoing edges are procedural
results or \q{outputs}.  In the Object-Oriented paradigm, however,
input parameters are organized into two groups: in addition
to any number of \q{conventional} arguments, there is
a single \this{} or \self{} object which has a distinct  semantic
status (\visavis{} name resolution, function visibility, and
polymorphism).  This calls, under the graph model, for splitting
incoming edges into two \q{channels}, one representing regular
parameters and a separate channel for the distinguished or
\q{receiver} object-value.
\end{description}
}
\p{In each of these formulations, what makes a graph \q{hyper} is the presence of
supplemental information \q{attached} to parts of an underlying (potentially
labeled, directed)
graph.  For a concrete example, consider a case from linguistics \mdash{} specifically,
morphosyntactic agreement
between grammatically linked words, which involves details
matched between both \q{ends} of a word-to-word \q{link}
(part-of-speech, plural/singular, gender, case/declension,
etc.).  According to the theory of Link Grammar,
words are associated with \q{connectors} (a related useful terminology,
derived more in a Cognitive-Linguistic context, holds that
words carry \q{expectations} which must be matched by other words they
could connect to).\footnote{Link Grammar proper was developed by Davy Temperley and Daniel Sleator
\cite{TemperleySleator}, \cite{GoertzelPLN}, and computer
scientists such as Kenneth Holmqvist and Matt Selway have adopted
concepts from Link Grammar as part of projects formalizing
Cognitive Linguistics in the tradition associated (notably) with
Ronald Langacker \cite{HolmqvistDiss}, \cite{MattSelway}
The OpenCog system is an example of Link Grammar integrated with
hypergraphs concerned with data models and procedural
execution, embodied in the AtomSpace data management component
\cite{GoertzelEtAl}, \cite{RuitingLian}, \cite[pp. 55ff]{GoertzelP2}.
}  The word \i{many}, for instance, as in \i{many dogs},
carries an implicit expectation to be paired with a plural noun.  The actual
syntactic connection \mdash{} as would be embodied by a graph-edge when a graph
formation is employed to model parse structures \mdash{} therefore depends on
both the expectations on one word in a pair (whichever acts as a modifier,
like \i{many}) and the \q{lexicomorphic} details of its \q{partner}
(\q{dog}, as a lexical item, being a noun, and \i{dogs} being
in plural form).
}
\p{In Link Grammar terminology, both the expectations on
one word and the lexical and morphological state of a second are called
\q{connectors}; a proper linkage between two words is then a \i{connection}.
For each connection there is accordingly two sets of relevant information,
which might be regarded as a generalization on edge-labeling wherein edges
could have two or more labels.  Furthermore, the assertion of multi-part annotations
on edges permits edges themselves to be grouped and categorized: aside from several
dozen recognized link varieties between words (which can be treated as conventional
edge-labels drawn from a taxonomy, consistent with ordinary labeled graphs),
edge-annotations in this framework mark patterns of semantic and syntactic agreement
in force between word pairs (not just foundational grammatic matching, like
gender and number \mdash{} singular/plural \mdash{} but more nuanced compatibility at the
boundary between syntax and semantics, such as the stipulation that a noun in a
locative position must have some semantic interpretation as a place or
destination).  Insofar as these agreement-patterns carry over to other word-pairs,
annotations mark linguistic criteria that tie together multiple edges
in the guise of signals that a specific parse-graph (out of the space
of possible graphs that could be formed from a sentence's word set) is correct.
}
\p{Ordinary directed graphs
(not necessarily hypergraphs) already have some sense of grouping edges together,
given that incoming and outgoing edges are distinguished;
but this indirect association between edges does not
internally yield a concordant grouping of the nodes
at the sources of edges all pointing to one target node
(or analogously the targets of one source node).
In the theory of
hypergraph \i{categories}, hypernodes come into play insofar as representation
calls for the nodes \q{across} (i.e., at the other end of)
incoming/outgoing edges to be pulled together.
We distinguish an \i{incoming node-set} from an \i{outgoing node-set};
then we might treat these sets as integral \q{bodies} of inputs
(respectively, outputs):
\begin{cquote}The term hypergraph category was introduced recently ... in reference
to the fact that these special commutative Frobenius monoids provide precisely the
structure required for their string diagrams to be directed graphs with \sq{hyperedges}:
edges connecting any number of inputs to any number of outputs. ...
We then think of morphisms in a hypergraph category as hyperedges
\cite[p. 13]{BrendanFongThesis}.
\end{cquote}
For a general mapping of
categories to graphs where edges represent morphisms, this represents a
generalization on the notion of \i{edges} themselves.  Suppose morphisms are
intended to model computational procedures in a general sense (say, as
morphisms in an ambient program state).  Because procedures can have any
number (even zero) of both inputs and outputs, this implies a generalization
wherein directed edges can have zero, one, or multiple source (and respectively
target) nodes.  A corresponding hypergraph form is one where hyperedges have
source- and target- hypernodes, but each hypernode models variant-sized sets
of further \q{inner} nodes (or \q{hyponodes}); here the empty set can be
a hypernode with no hyponodes.  Elsewhere, apparently an equivalent
structure is called a \q{trivial system}:
\begin{cquote}Monoidal categories admit an elegant and powerful graphical notation
[wherein] an object $A$ is denoted by a wire [and] [a]
morphism \fAB{} is represented by a box.  The trivial system
$I$ is the empty diagram.  Morphisms \uIA{} and \vAI{}
... are referred to as \textbf{states} and \textbf{effects}
\cite[p. 4]{InteractingConceptualSpaces}.
\end{cquote}
The system $I$ \mdash{} which we can also see as a hypernode with an empty
(hypo-)node set \mdash{} may embody a procedure which has no internal
algorithmic or calculational structure (at least relative to the
domain of analysis where we might represent computer code).  In
CyberPhysical Systems, a function which just produces a
value (with no input and no intermediate computations) can also
be called an \i{observation}, perhaps a direct reading from a
physical \i{sensor} (accordingly, as in the above excerpt, a \i{state}).
Dually, a procedure which performs no evident calculation and produces
no output value, but has a CyberPhysical \i{effect}, can be called an
\q{actuation}, potentially connected to a CyberPhysical \i{actuator}
(an example of a sensor would be a thermostat, and an example of an
actuator would be a device which can activate/deactivate a furnace
and/or cooling system).
}
\p{In these examples I have cited hypergraphs in a linguistic (Link Grammar)
and a mathematical (hypergraph categories) context.  These
share some parallels insofar as a core motivation is to generalize and add
structure to edges, either freeing edges from limts on
node-arity (even allowing edges to be \q{unattached}),
or supplying edges with structured (potentially multi-part) annotations.
}
\p{A somewhat different conception of hypergraphs is found in database
systems such as HypergraphDB.  Graphs in that context express what have
been called \i{recursive} graphs, wherein a hypernode \q{contains} or
\q{designates} its own graph.  The basic idea is that for each (hyper-)node we
can associate a separate (sub-)graph.  This can actually work two
ways, yielding a distinction between (I'll say) \i{nested}
and \i{cross-referencing} graphs.  In a \i{cross-referencing} graph,
subgraphs or other collections of graph elements (nodes, edges, and/or
annotations) can be given unique identifiers or designations and, as
a data point, associated with a separate node.  Consider a case where
nodes refer to typed values from a general-purpose type system; insofar
as subgraphs themselves may be represented as typed values, a node could
reference a subgraph by analogy to any other value (textual, numeric,
nominal/enumerative data, etc.).  Here the (hyper)node does not
\q{contain} but \i{references} a subgraph; the added structure involves
subgraphs themselves being incorporated into the universe of values
which nodes may quantify over.  Conversely, \i{nested} hypergraphs
model hypernodes which have other graphs \q{inside} them, thereby creating
an ordering among nodes (we can talk of nodes at one level belonging
to graphs which are contained in nodes at a higher level).  Such constructions
may or may not allow edges across nodes at different levels.
}
\p{Note that nested hypergraphs can be seen as a special case of cross-referencing
graphs, where each hypernode \nodeN{} is given an index \ix{},
with the restriction that when  \nodeN{} designates (e.g., via
its corresponding typed value) a subgraph \sg{}, all of
\sg{}'s hypernodes have index \ixminusone{}.  Cross-referencing
graphs, in turn, can be seen as special cases of an overall
space of hypergraphs wherein hypernodes are paired with
typed values from some suitably expressive type system \tys{}.
If \tys{} includes higher-order types \mdash{} especially, lists and
other \q{collections} types which become concrete types in
conjunction with another type (as in \i{list} qua generic
becomes the concrete type \i{list of integers}), then
hypernodes acquire aggregate structure in part by
acquiring values whose types encompass multiple
other values.  I will use the term \i{procedural} hypergraphs
to discuss structures that model node diversification
via mapping hypernodes to collections-types
(with the possibility for hypernodes
to \q{expand} or \q{contract} as values are inserted into or
removed from the collection).
}
\p{Cross-referencing also potentially introduces a variant derivation of
hypergraphs which proceeds by accumulating graph elements (nodes,
edges, and labels or annotations) into higher-scale posits, rather
than defining inner structures on elements.  Specifically, as a
complimentary operation to diversification, consider \q{aggregation}
of graph elements: the option to take a set of (hyper)nodes,
(hyper)edges, and/or annotations as a typed value which can then
be assigned to a (hyper)node.  In such a manner, higher-level
structures can be notated with respect to graphs, which is one
way to model phenomena such as \i{contexts} \mdash{} the kind of multi-scale
patterns that are considered endemic to practical domains
like the Semantic Web.  While it is informally acknowledged that
a single-level interpretation of the Semantic Web is misleading
\mdash{} the Semantic Web is not an undifferentiated mesh of connections, but
rather an aggregation of data from many sources, which implies the existence of
localization, contextualization, and other \q{emergent} structure \mdash{}
there is no definitive protocol for actually representing this emergent
structure.  This issue, in turn, is one of arguments
for hypergraphs in lieu of ordinary graphs as general-purpose
data representations.
}
\p{The multi-scale, contextualized nature of the Semantic Web also points toward
a conceptual duality in \i{how} graphs represent data.  On the one hand, graph
structures \mdash{} especially in the case of the Semantic Web, which builds off of
internet technology in general \mdash{} represent \i{relationships} between points
or structures of data in some sense; in familiar web terms, the relata linked
by graph edges are often \q{resources}, designated by unique web addresses.
A more theoretical model might take the information \q{residing} at
graph nodes as typed values.  But in either case a given node may stand in
for an aggregate of information \mdash{} a single web resources may contain a
theoretically unlimited supply of data, and a typed value can be of a list
or tuple type internally containing its own body of information.
Consequently, the full stock of data embodied in a graph may not lie
primarily in the graph structure itself, but rather distributed among
its nodes (that is, among nodes' associated data).
}
\p{Conversely, graph structures (with suitable semantic specifications) are
also considered to be media for serializing arbitrary data structures, which
implies representing all details, at all levels of hierarchical organization,
via graph structures.  Insofar as nodes embody their own information spaces,
such internal structuration must then be mapped to their own graphs,
as part of a workflow to project arbitrary structured data onto a canonical
format.  The theoretical corollary to this idea would be that node data
(via associated typed values) has its own internal representation;
that it is, every typed value has a corresponding graph structure
that may be \q{contained} within a higher-scale node.
}
\p{Different varieties of hypergraph forms complicate this picture
because the structuring elements of hypergraphs include
aggregate data within nodes as well as the space of
edges and incidence relations.  Given the structures I have referred to
as \q{procedural} hypergraphs, nodes can encompass multiple
internal values so long as they have a suitably well-defined
internal structure.  We can analyze these possibilities
by defining, for each hypernode, an \q{interface} or list of
operations available for updating hypernodes' associated values.
Which operations are proper depends on a hypernode's type:
hypernodes associated with a single unstructured value
should have one basic update operation, while nodes with
list-like types would have operations to insert (and remove)
values at different positions.  In general, procedural
hypergraphs should differentiate between \i{atomic} hypernodes
with one associated value; \i{tuple} hypernodes with a
fixed array of values (potentially of multiple types);
and \i{collections} hypernodes encompassing lists of
values subject to append/insert/remove actions
(in which case inserted values' types should
have a predictable pattern;
consider key- and value-type alternation in
associative arrays).
}

\p{Separate and apart from operations modifying hypernodes' values, there
are also conventional graph operations \mdash{} adding and removing
nodes and edges.  In combination, the graph-oriented and node-oriented
operations present a variegated interface for manipulating
procedural hypergraphs.  Such an interface then serves as a rigorous
characterization of the overall hypergraph model \mdash{} the structuring
elements expressed by enumerating graphs' transformation operations
represent the particular features of each specific hypergraph
variety.
}
\p{In the case of procedural hypergraphs, many of these transformations
are not graph-related, per se, but derive from hypernodes' collections or tuple
types.  I contend that this is a useful property of procedural hypergraphs
for reasons I alluded to in the introduction \mdash{} hypergraphs (or at least the
data represented with them) need to work in a variety of computational contexts.
The relatively unstructured form of graph data is not always appropriate
from one context to another; the list-of-values or value-tuple structures
embodied as hypernode data may be more consistent with internal representations
in database or language-runtime engines, for example.  Procedural hypergraphs
are appropriately flexible in that some data is modeled at the graph level
proper while other data is modeled as lists, tuples, and similar data structures
within individual nodes.
}
\p{In many practical contexts graph structures are not implicitly used at all;
the importance of Semantic Web-style representation is for intermediary
structures, where information is routed among different environments
(database, applications, serialization, and so forth).  Transformations
between hypergraphs can be a central process in generic transformations
between data structures proper to different contexts.  In effect, where
there is a general need for data transforms in routing between
contexts \mdash{} e.g., database to application runtime \mdash{} the relevant
transform logic is specifically
mappings between hypergraphs, with each hypergraph possessing a structure
optimized for being initialized from one context (or for generating data
used in another context).  This progression may involve restructuring
wherein information modeled at the \q{inter-hypernode} level \mdash{}
which we can call \i{hypernode} data \mdash{} tends to be migrated
to the level \i{inside} hypernodes, which we can call \i{hyponode} data.
}
\p{In broad outline, then, hypergraph transforms will in many scenarios
progress from relatively
casual structures \mdash{} with a preponderance of information expressed as
\i{hypernode} data \mdash{} to more constrained structures.  The underlying
pattern in such instances is mapping
\i{hypernode} to \i{hyponode} data \mdash{} i.e., mapping data from structures
\i{between} hypernodes to those \i{within} hypernodes \mdash{} where
the hyponode data is regulated by hypernodes' types.
}
\thindecoline{}
\p{In this section I identified different additions through which graph
structures generalize to hypergraphs; a general-purpose hypergraph
engine would need to support each of these variations, which entails
enabling the complete repository of transform-operations applicable to
different hypergraph varieties.  This includes generalizing
edges to hyperedges by \q{diversifying} nodes to encompass mutiple values;
representing nodes' internal structure in terms of data structures
such as lists, tuples, and nested graphs; and generalizing edge-labels
to annotations which may have multiple parts.  I have not yet discussed
the possibility of grouping hyperedges into higher-level structures
(what I called \q{channels}), but I will return to
those details in a later section.
}
\p{For the remainder of this paper, I will attend especially to
hypergraphs modeling (and subsequently executing) computer code,
because constructing a working runtime engine which runs source
code, via hypergraph intermediaries, demonstrates a variety of concepts
applicable to hypergraphs in general.  I will discuss a workflow
connecting parsers, a runtime \q{virtual machine}, intermediate representations
for hypergraphs, and inter-graph transforms.  Collectively
this workflow implicates many of the capabilities which would be
requisite for a general-purpose hypergraph software ecosystem.
}
\p{Interested readers who would like to observe a concrete
unfolding of this workflow are invited to download the
code base accompanying this paper, where readers can examine
the operations of parsers and code generators working with
a built-in, general-purpose hypergraph library.  The demo
is fully integrated with \Qt{} Creator, a \Cpp{} Integrated Development
Environment, and has no further dependencies (assuming users
have a working \Qt{} and \Cpp{} compiler; \Qt{} is a popular \Cpp{} application-development
framework).  The dataset includes instructions for
experimenting with the hypergraph library via \Qt{} Creator and examining
runtime structures by executing demonstration scripts in Debug mode.
}


\section{Method}

\subsection{Dataset}
The datasets originate from a study on the mechanism behind head direction sense \cite{projectdata}.
Five adult male and two female mice were implanted with recording electrodes under anesthesia \cite{projectdata}.
Silicon probes were mounted on movable drives for recording of neural activity in the anteriour thalamus in all mice, while in additional three of the mice probes were also mounted in the post-subiculum (PoS) \cite{projectdata}. For the remaning four mice with probes only mounted in the anteriour thalamus, additional probes were mounted in the hippocampal CA1 pyramidal layer for accurate sleep scoring \cite{projectdata}.
The head directions (HD) were tracked using two light-emitting-diodes (LEDs) mounted to the back of the head and recored using a videocamera with a frame rate of 30fps. The data was then resampled to 39Hz by the aquisition system \cite{projectdata}.
Data from two mice were explored, mouse 12 with probes only in the anteriour thalamus and mouse 28 with additional probes in the PoS. Only the awake dataset where used.
\subsection{Tuning curves}
The electrode spikes can be encoded as timestamps synchronized with the tracked head direction. The timestamps are binned into timebins with a time resolution of
$$\Delta t = \frac{1}{39Hz} \approx 25.64ms$$
corresponding to the sampling frequency of $39$Hz set by the aquisition system.
Each bin corresponds to the number of cell firings over a period of $\Delta t$ time, and each sample has a corresponding head angle.
Dividing by the sampling time yields the number of spikes per unit time in each timebin.

The continious range of possible head directions,
$\theta \in [0, 2\pi)$
, can be discretized into a desired number of edges $N_{HD}$ with fixed spacing such that
\begin{equation}
\theta_k \in \{\frac{2 \pi k}{N_{HD}}| k \in \mathbb{N}, 0 \leq k \leq N_{HD} \}
\end{equation}
and each tracked head direction sample is binned using the rule
\begin{equation}
    f_{\theta}(\theta) =  \begin{cases}
        \theta_0, & \text{if } \theta_0 \leq \theta < \theta_{1} \\
        \theta_1, & \text{if } \theta_1 \leq \theta < \theta_{2} \\
         & \vdots \\
        \theta_k, & \text{if } \theta_k \leq \theta < \theta_{k+1}
    \end{cases} \quad \text{for} \quad 0 \leq k < N_{HD}
\end{equation}
The number of cell firings were binned accordingly to create an overview of number of spikes per head angle.
The firing rate can be found by dividing the sum of spikes for each head direction bin by the number of head direction samples in the same bin.
\begin{equation} \label{eq:firing_rate}
    \lambda_k = \frac{\sum_i S_i}{\sum_i 1} \quad i \in \{t | t \in \mathbb{N}, 0 \leq t \leq N, f_\theta(\theta_t) = \theta_k\}
\end{equation}
where $S_i$ is the number of spikes in timestep $i$, $\theta_t$ is the head angle in timestep $t$ and $N$ is the number of timesteps. Tuning curves are created by plotting the firing rate $\lambda_k$ against the head direction $\theta_k$ in MATLAB or other similar tools.

\subsection{Mutual information}
Mutual information can be used to quantify how much knowing the state of one variable can tell us about another, i.e. how much information is shared between the variables.
The amount of information is quantified as bits (or Shannons), and originates from information theory. A bit can be thought of as the answer to a yes/no question.
The mutual information can be calculated using
\begin{equation}\label{eq:mutual_info}
    I(X;Y) = \sum_{i, j} Pr(y_i,x_j) \log_2(\frac{Pr(y_i, x_j)}{Pr(y_i)Pr(x_j)}) = \sum_{i,j}Pr(y_i|x_j)Pr(x_j) \log_2(\frac{Pr(y_i | x_j)Pr(x_j)}{Pr(y_i)Pr(x_j)})
\end{equation}
During a sufficiently short period of time $\Delta t$ the cell spikes can be thought of as a binary (bernouilly) variable, either spiking once or not at all. \cite{mutualinfo} The probability of a cell spiking can be then be calculated using
\begin{align} \label{eq:probs}
    Pr(S=1 | X = k) &= \lambda_k \Delta t & Pr(S=1) = \lambda \Delta t = \Delta t \sum_k \lambda_k
\end{align}
where $\lambda_k$ is the mean fire rate in bin $k$ and $\lambda$ is the mean firing rate over all bins.
By inserting \cref{eq:probs} into \cref{eq:mutual_info} it can be shown that the mutal information can be calculated using a discrete approximation \cite{mutualinfo}.
\begin{equation} \label{eq:mutinfo_disc}
    I \approx \Delta t \sum_k \lambda_k \log_2(\frac{\lambda_k}{\lambda})Pr(X = k)
\end{equation}
It can be rewritted to compute the mutual information per unit time, which may be easier to interpret.
\begin{equation} \label{eq:mutinfo_disc_per_time}
    \frac{I}{\Delta t} \approx \sum_k \lambda_k \log_2(\frac{\lambda_k}{\lambda})Pr(X = k)
\end{equation}
The prior distribution of head angles can be estimated by
\begin{equation}
    \hat{Pr}(X = k) = \frac{\sum_i 1}{N}  \quad i \in \{t | t \in \mathbb{N}, 0 \leq t \leq N, f_\theta(\theta_t) = \theta_k\}
\end{equation}
i.e. the proportion of samples corresponding to bin $k$.

Due to the approximations, \cref{eq:mutinfo_disc_per_time} may lead to slight negative values. Negative information does not contain any meaning and any values below zero is therefore considered to be approximation errors, and is manually forced to zero.

\subsection{Principal Component Analysis}
To reduce the feature space from many potentially unimportant variables to only a few important ones, principal analysis can be used.
The goal of PCA is to create new variables using linear transformations of the original variables, creating a new basis for the vector space. More specifically the goal is to create new variables which describe as much of the variance in the original data as possible.
The amount of variance explained by each variable is sorted descendingly with the first principal component explaining the most, and where each new variables are orthogonal to the rest. MATLAB can be used to perform PCA analysis using the \texttt{pca} command by providing a $NxP$ matrix where $N$ is the number of samples and $P$ is the number of features. The output PCA \texttt{scores} contains the original samples represented by the new basis.
A color coded scatter plot can be generated from the first few principal components.

\subsection{Polar coordinates}
Polar coordinates can be used to represent cartesian datapoints as a rotation and a distance. If the samples are  distributed around a circle, this can help visualize the samples in a linear space of rotations. The MATLAB function \texttt{cart2pol} can be used to transform from cartesian to polar coordinates.